From 944a0ac318a7c3758ddb46f8d972c7f322c1f540 Mon Sep 17 00:00:00 2001 From: Joey Date: Wed, 6 Jan 2021 14:03:59 -0700 Subject: [PATCH 0001/2472] DELETE requests should always have a content-length header Not having DELETE in this list is causing hanging when trying to close webdriver sessions in [halonium](https://github.com/halonium/halonium/issues/10) and likely any other implementation of the webdriver protocol. Both at least chromedriver and geckodriver are affected by this issue. --- lib/pure/httpclient.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 3093f5564..a6d7edf33 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -980,7 +980,7 @@ proc requestAux(client: HttpClient | AsyncHttpClient, url, httpMethod: string, var data: seq[string] if multipart != nil and multipart.content.len > 0: data = await client.format(multipart) - elif httpMethod in ["POST", "PATCH", "PUT"] or body.len != 0: + elif httpMethod in ["POST", "PATCH", "PUT", "DELETE"] or body.len != 0: client.headers["Content-Length"] = $body.len when client is AsyncHttpClient: From 04b765c16d4a53c03f8d41c9b9ad97ac4ac8189e Mon Sep 17 00:00:00 2001 From: inv2004 Date: Thu, 7 Jan 2021 05:09:02 +0300 Subject: [PATCH 0002/2472] Jsonutils: pass opt for object in object (#16615) * jsonutils: fromJson forward opt param fix * jsonutils: object in object test + fix: opt pass --- lib/std/jsonutils.nim | 2 +- tests/stdlib/tjsonutils.nim | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim index 4dca024c1..b9e47bd70 100644 --- a/lib/std/jsonutils.nim +++ b/lib/std/jsonutils.nim @@ -119,7 +119,7 @@ template fromJsonFields(newObj, oldObj, json, discKeys, opt) = when key notin discKeys: if json.hasKey key: numMatched.inc - fromJson(val, json[key]) + fromJson(val, json[key], opt) elif opt.allowMissingKeys: # if there are no discriminant keys the `oldObj` must always have the # same keys as the new one. Otherwise we must check, because they could diff --git a/tests/stdlib/tjsonutils.nim b/tests/stdlib/tjsonutils.nim index f56c32719..06d01a9fb 100644 --- a/tests/stdlib/tjsonutils.nim +++ b/tests/stdlib/tjsonutils.nim @@ -123,6 +123,9 @@ template fn() = a: int b: string c: float + type Bar = object + foo: Foo + boo: string var f: seq[Foo] try: fromJson(f, parseJson """[{"b": "bbb"}]""") @@ -131,6 +134,9 @@ template fn() = doAssert true fromJson(f, parseJson """[{"b": "bbb"}]""", Joptions(allowExtraKeys: true, allowMissingKeys: true)) doAssert f == @[Foo(a: 0, b: "bbb", c: 0.0)] + var b: Bar + fromJson(b, parseJson """{"foo": {"b": "bbb"}}""", Joptions(allowExtraKeys: true, allowMissingKeys: true)) + doAssert b == Bar(foo: Foo(a: 0, b: "bbb", c: 0.0)) block testHashSet: testRoundtrip(HashSet[string]()): "[]" From 4754806fb5d6afb034dc2d27959353ec579b08e4 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Thu, 7 Jan 2021 07:39:56 +0000 Subject: [PATCH 0003/2472] Fixes the asynchttpserver example some more (#16599) I dislike this example a lot (busy looping for FDs to be closed is a very poor waste of resources) but at least with these changes it's a little bit better. --- lib/pure/asynchttpserver.nim | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index d9f5a3a0f..29a695379 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -18,29 +18,28 @@ runnableExamples: # This example will create an HTTP server on port 8080. The server will # respond to all requests with a `200 OK` response code and "Hello World" # as the response body. Run locally with: - # `nim doc --doccmd:-d:nimAsynchttpserverEnableTest --lib:lib lib/pure/asynchttpserver.nim` + # `nim doc --doccmd:-d:nimAsyncHttpServerEnableTest --lib:lib lib/pure/asynchttpserver.nim` import asyncdispatch - if defined(nimAsynchttpserverEnableTest): + if defined(nimAsyncHttpServerEnableTest): proc main {.async.} = const port = 8080 var server = newAsyncHttpServer() proc cb(req: Request) {.async.} = echo (req.reqMethod, req.url, req.headers) - let headers = {"Date": "Tue, 29 Apr 2014 23:40:08 GMT", - "Content-type": "text/plain; charset=utf-8"} + let headers = {"Content-type": "text/plain; charset=utf-8"} await req.respond(Http200, "Hello World", headers.newHttpHeaders()) echo "test this with: curl localhost:" & $port & "/" - server.listen Port(port) + server.listen(Port(port)) while true: if server.shouldAcceptRequest(): await server.acceptRequest(cb) else: # too many concurrent connections, `maxFDs` exceeded - poll() + # wait 500ms for FDs to be closed + await sleepAsync(500) - asyncCheck main() - runForever() + waitFor main() import asyncnet, asyncdispatch, parseutils, uri, strutils import httpcore From bab0aa6ecf8b91f9befaebefd0d4d5adeaac3ac3 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Thu, 7 Jan 2021 01:48:02 -0600 Subject: [PATCH 0004/2472] add math.signbit (#16592) --- changelog.md | 3 +++ compiler/vmops.nim | 6 ++++++ lib/pure/math.nim | 27 ++++++++++++++++++++++++++- tests/stdlib/tmath.nim | 10 ++++++++++ 4 files changed, 45 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 306cf54d3..4e9b8c162 100644 --- a/changelog.md +++ b/changelog.md @@ -93,6 +93,9 @@ with other backends. see #9125. Use `-d:nimLegacyJsRound` for previous behavior. + +- Added `math.signbit`. + ## Language changes - `nimscript` now handles `except Exception as e`. diff --git a/compiler/vmops.nim b/compiler/vmops.nim index 3e859d3d7..504a352b5 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -16,6 +16,9 @@ from math import sqrt, ln, log10, log2, exp, round, arccos, arcsin, when declared(math.copySign): from math import copySign +when declared(math.signbit): + from math import signbit + from os import getEnv, existsEnv, dirExists, fileExists, putEnv, walkDir, getAppFilename from md5 import getMD5 from sighashes import symBodyDigest @@ -174,6 +177,9 @@ proc registerAdditionalOps*(c: PCtx) = when declared(copySign): wrap2f_math(copySign) + when declared(signbit): + wrap1f_math(signbit) + wrap1s(getMD5, md5op) proc `mod Wrapper`(a: VmArgs) {.nimcall.} = diff --git a/lib/pure/math.nim b/lib/pure/math.nim index a6a3676b9..bcda68afe 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -56,7 +56,7 @@ import std/private/since {.push debugger: off.} # the user does not want to trace a part # of the standard library! -import bitops, fenv +import std/[bitops, fenv] when defined(c) or defined(cpp): proc c_isnan(x: float): bool {.importc: "isnan", header: "".} @@ -65,6 +65,8 @@ when defined(c) or defined(cpp): proc c_copysign(x, y: cfloat): cfloat {.importc: "copysignf", header: "".} proc c_copysign(x, y: cdouble): cdouble {.importc: "copysign", header: "".} + proc c_signbit(x: SomeFloat): cint {.importc: "signbit", header: "".} + func binom*(n, k: int): int = ## Computes the `binomial coefficient `_. runnableExamples: @@ -156,6 +158,29 @@ func isNaN*(x: SomeFloat): bool {.inline, since: (1,5,1).} = when defined(js): fn() else: result = c_isnan(x) +when defined(js): + proc toBitsImpl(x: float): array[2, uint32] = + asm """ + const buffer = new ArrayBuffer(8); + const floatBuffer = new Float64Array(buffer); + const uintBuffer = new Uint32Array(buffer); + floatBuffer[0] = `x`; + `result` = uintBuffer + """ + +proc signbit*(x: SomeFloat): bool {.inline, since: (1, 5, 1).} = + ## Returns true if `x` is negative, false otherwise. + runnableExamples: + doAssert not signbit(0.0) + doAssert signbit(-0.0) + doAssert signbit(-0.1) + doAssert not signbit(0.1) + when defined(js): + let uintBuffer = toBitsImpl(x) + result = (uintBuffer[1] shr 31) != 0 + else: + result = c_signbit(x) != 0 + func copySign*[T: SomeFloat](x, y: T): T {.inline, since: (1, 5, 1).} = ## Returns a value with the magnitude of `x` and the sign of `y`; ## this works even if x or y are NaN or zero, both of which can carry a sign. diff --git a/tests/stdlib/tmath.nim b/tests/stdlib/tmath.nim index e5cb58eba..62fdcd19f 100644 --- a/tests/stdlib/tmath.nim +++ b/tests/stdlib/tmath.nim @@ -304,6 +304,16 @@ block: template main = # xxx wrap all under `main` so it also gets tested in vm. + block: # signbit + doAssert not signbit(0.0) + doAssert signbit(-0.0) + doAssert signbit(-0.1) + doAssert not signbit(0.1) + + doAssert not signbit(Inf) + doAssert signbit(-Inf) + doAssert not signbit(NaN) + block: # isNaN doAssert NaN.isNaN doAssert not Inf.isNaN From 89a21e4ec71e705833d2aacd069e291cf41a19c6 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Thu, 7 Jan 2021 02:38:31 -0600 Subject: [PATCH 0005/2472] oids: switch from PRNG to random module (#16203) * switch from PRNG to random module * fix the regression * comments + tests * runnableExamples * make oids better --- lib/pure/oids.nim | 39 +++++++++++++++++++-------------------- tests/stdlib/toids.nim | 6 ++++++ 2 files changed, 25 insertions(+), 20 deletions(-) create mode 100644 tests/stdlib/toids.nim diff --git a/lib/pure/oids.nim b/lib/pure/oids.nim index d3a7c4fb6..957a8193a 100644 --- a/lib/pure/oids.nim +++ b/lib/pure/oids.nim @@ -12,24 +12,24 @@ ## produce a globally distributed unique ID. This implementation was extracted ## from the Mongodb interface and it thus binary compatible with a Mongo OID. ## -## This implementation calls ``math.randomize()`` for the first call of +## This implementation calls `initRand()` for the first call of ## ``genOid``. -import hashes, times, endians +import hashes, times, endians, random type - Oid* = object ## an OID + Oid* = object ## An OID. time: int32 ## fuzz: int32 ## count: int32 ## proc `==`*(oid1: Oid, oid2: Oid): bool = - ## Compare two Mongo Object IDs for equality + ## Compares two Mongo Object IDs for equality. return (oid1.time == oid2.time) and (oid1.fuzz == oid2.fuzz) and (oid1.count == oid2.count) proc hash*(oid: Oid): Hash = - ## Generate hash of Oid for use in hashtables + ## Generates hash of Oid for use in hashtables. var h: Hash = 0 h = h !& hash(oid.time) h = h !& hash(oid.fuzz) @@ -44,7 +44,7 @@ proc hexbyte*(hex: char): int = else: discard proc parseOid*(str: cstring): Oid = - ## parses an OID. + ## Parses an OID. var bytes = cast[cstring](addr(result.time)) var i = 0 while i < 12: @@ -52,6 +52,7 @@ proc parseOid*(str: cstring): Oid = inc(i) proc oidToString*(oid: Oid, str: cstring) = + ## Converts an oid to `str` which must have space allocated for 25 elements. const hex = "0123456789abcdef" # work around a compiler bug: var str = str @@ -66,35 +67,33 @@ proc oidToString*(oid: Oid, str: cstring) = str[24] = '\0' proc `$`*(oid: Oid): string = + ## Converts an oid to string. result = newString(24) oidToString(oid, result) -proc rand(): cint {.importc: "rand", header: "", nodecl.} -proc srand(seed: cint) {.importc: "srand", header: "", nodecl.} - -var t = getTime().toUnix.int32 -srand(t) var - incr: int = rand() - fuzz: int32 = rand() + t = getTime().toUnix.int32 + seed = initRand(t) + incr: int = seed.rand(int.high) + +let fuzz = cast[int32](seed.rand(high(int))) proc genOid*(): Oid = - ## generates a new OID. + ## Generates a new OID. + runnableExamples: + doAssert ($genOid()).len == 24 + if false: doAssert $genOid() == "5fc7f546ddbbc84800006aaf" t = getTime().toUnix.int32 - var i = int32(atomicInc(incr)) + var i = cast[int32](atomicInc(incr)) bigEndian32(addr result.time, addr(t)) result.fuzz = fuzz bigEndian32(addr result.count, addr(i)) proc generatedTime*(oid: Oid): Time = - ## returns the generated timestamp of the OID. + ## Returns the generated timestamp of the OID. var tmp: int32 var dummy = oid.time bigEndian32(addr(tmp), addr(dummy)) result = fromUnix(tmp) - -when not defined(testing) and isMainModule: - let xo = genOid() - echo xo.generatedTime diff --git a/tests/stdlib/toids.nim b/tests/stdlib/toids.nim new file mode 100644 index 000000000..f162dbe57 --- /dev/null +++ b/tests/stdlib/toids.nim @@ -0,0 +1,6 @@ +import std/oids + + +block: # genOid + let x = genOid() + doAssert ($x).len == 24 From 0e7902b976fbd0e566f0fb5f4915695aca406451 Mon Sep 17 00:00:00 2001 From: PMunch Date: Thu, 7 Jan 2021 16:09:57 +0100 Subject: [PATCH 0006/2472] Implements streams for sockets (#15729) --- changelog.md | 2 + lib/std/socketstreams.nim | 181 ++++++++++++++++++++++++++++++++ tests/stdlib/tsocketstreams.nim | 64 +++++++++++ 3 files changed, 247 insertions(+) create mode 100644 lib/std/socketstreams.nim create mode 100644 tests/stdlib/tsocketstreams.nim diff --git a/changelog.md b/changelog.md index 4e9b8c162..afe49f1f3 100644 --- a/changelog.md +++ b/changelog.md @@ -91,6 +91,8 @@ - `math.round` now is rounded "away from zero" in JS backend which is consistent with other backends. see #9125. Use `-d:nimLegacyJsRound` for previous behavior. +- Added `socketstream` module that wraps sockets in the stream interface + diff --git a/lib/std/socketstreams.nim b/lib/std/socketstreams.nim new file mode 100644 index 000000000..5c882858d --- /dev/null +++ b/lib/std/socketstreams.nim @@ -0,0 +1,181 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2021 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module provides an implementation of the streams interface for sockets. +## It contains two separate implementations, a +## `ReadSocketStream <#ReadSocketStream>`_ and a +## `WriteSocketStream <#WriteSocketStream>`_. +## +## The `ReadSocketStream` only supports reading, peeking, and seeking. +## It reads into a buffer, so even by +## seeking backwards it will only read the same position a single time from the +## underlying socket. To clear the buffer and free the data read into it you +## can call `resetStream`, this will also reset the position back to 0 but +## won't do anything to the underlying socket. +## +## The `WriteSocketStream` allows both reading and writing, but it performs the +## reads on the internal buffer. So by writing to the buffer you can then read +## back what was written but without receiving anything from the socket. You +## can also set the position and overwrite parts of the buffer, and to send +## anything over the socket you need to call `flush` at which point you can't +## write anything to the buffer before the point of the flush (but it can still +## be read). Again to empty the underlying buffer you need to call +## `resetStream`. +## +## Examples +## ======== +## +## .. code-block:: Nim +## import std/socketstreams +## +## var +## socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) +## stream = newReadSocketStream(socket) +## socket.sendTo("127.0.0.1", Port(12345), "SOME REQUEST") +## echo stream.readLine() # Will call `recv` +## stream.setPosition(0) +## echo stream.readLine() # Will return the read line from the buffer +## stream.resetStream() # Buffer is now empty, position is 0 +## echo stream.readLine() # Will call `recv` again +## stream.close() # Closes the socket +## +## .. code-block:: Nim +## +## import std/socketstreams +## +## var socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) +## socket.connect("127.0.0.1", Port(12345)) +## var sendStream = newWriteSocketStream(socket) +## sendStream.write "NOM" +## sendStream.setPosition(1) +## echo sendStream.peekStr(2) # OM +## sendStream.write "I" +## sendStream.setPosition(0) +## echo sendStream.readStr(3) # NIM +## echo sendStream.getPosition() # 3 +## sendStream.flush() # This actually performs the writing to the socket +## sendStream.setPosition(1) +## sendStream.write "I" # Throws an error as we can't write into an already sent buffer + +import net, streams + +type + ReadSocketStream* = ref ReadSocketStreamObj + ReadSocketStreamObj* = object of StreamObj + data: Socket + pos: int + buf: seq[byte] + WriteSocketStream* = ref WriteSocketStreamObj + WriteSocketStreamObj* = object of ReadSocketStreamObj + lastFlush: int + +proc rsAtEnd(s: Stream): bool = + return false + +proc rsSetPosition(s: Stream, pos: int) = + var s = ReadSocketStream(s) + s.pos = pos + +proc rsGetPosition(s: Stream): int = + var s = ReadSocketStream(s) + return s.pos + +proc rsPeekData(s: Stream, buffer: pointer, bufLen: int): int = + let s = ReadSocketStream(s) + if bufLen > 0: + let oldLen = s.buf.len + s.buf.setLen(max(s.pos + bufLen, s.buf.len)) + if s.pos + bufLen > oldLen: + result = s.data.recv(s.buf[oldLen].addr, s.buf.len - oldLen) + if result > 0: + result += oldLen - s.pos + else: + result = bufLen + copyMem(buffer, s.buf[s.pos].addr, result) + +proc rsReadData(s: Stream, buffer: pointer, bufLen: int): int = + result = s.rsPeekData(buffer, bufLen) + var s = ReadSocketStream(s) + s.pos += bufLen + +proc rsReadDataStr(s: Stream, buffer: var string, slice: Slice[int]): int = + var s = ReadSocketStream(s) + result = slice.b + 1 - slice.a + if result > 0: + result = s.rsReadData(buffer[slice.a].addr, result) + inc(s.pos, result) + else: + result = 0 + +proc wsWriteData(s: Stream, buffer: pointer, bufLen: int) = + var s = WriteSocketStream(s) + if s.pos < s.lastFlush: + raise newException(IOError, "Unable to write into buffer that has already been sent") + if s.buf.len < s.pos + bufLen: + s.buf.setLen(s.pos + bufLen) + copyMem(s.buf[s.pos].addr, buffer, bufLen) + s.pos += bufLen + +proc wsPeekData(s: Stream, buffer: pointer, bufLen: int): int = + var s = WriteSocketStream(s) + result = bufLen + if result > 0: + if s.pos > s.buf.len or s.pos == s.buf.len or s.pos + bufLen > s.buf.len: + raise newException(IOError, "Unable to read past end of write buffer") + else: + copyMem(buffer, s.buf[s.pos].addr, bufLen) + +proc wsReadData(s: Stream, buffer: pointer, bufLen: int): int = + result = s.wsPeekData(buffer, bufLen) + var s = ReadSocketStream(s) + s.pos += bufLen + +proc wsAtEnd(s: Stream): bool = + var s = WriteSocketStream(s) + return s.pos == s.buf.len + +proc wsFlush(s: Stream) = + var s = WriteSocketStream(s) + discard s.data.send(s.buf[s.lastFlush].addr, s.buf.len - s.lastFlush) + s.lastFlush = s.buf.len + +proc rsClose(s: Stream) = + {.cast(tags: []).}: + var s = ReadSocketStream(s) + s.data.close() + +proc newReadSocketStream*(s: Socket): owned ReadSocketStream = + result = ReadSocketStream(data: s, pos: 0, + closeImpl: rsClose, + atEndImpl: rsAtEnd, + setPositionImpl: rsSetPosition, + getPositionImpl: rsGetPosition, + readDataImpl: rsReadData, + peekDataImpl: rsPeekData, + readDataStrImpl: rsReadDataStr) + +proc resetStream*(s: ReadSocketStream) = + s.buf = @[] + s.pos = 0 + +proc newWriteSocketStream*(s: Socket): owned WriteSocketStream = + result = WriteSocketStream(data: s, pos: 0, + closeImpl: rsClose, + atEndImpl: wsAtEnd, + setPositionImpl: rsSetPosition, + getPositionImpl: rsGetPosition, + writeDataImpl: wsWriteData, + readDataImpl: wsReadData, + peekDataImpl: wsPeekData, + flushImpl: wsFlush) + +proc resetStream*(s: WriteSocketStream) = + s.buf = @[] + s.pos = 0 + s.lastFlush = 0 diff --git a/tests/stdlib/tsocketstreams.nim b/tests/stdlib/tsocketstreams.nim new file mode 100644 index 000000000..0cf952810 --- /dev/null +++ b/tests/stdlib/tsocketstreams.nim @@ -0,0 +1,64 @@ +discard """ + output: ''' +OM +NIM +3 +NIM +NIM +Hello server! +Hi there client! +'''""" +import std/socketstreams, net, streams + +block UDP: + var recvSocket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) + var recvStream = newReadSocketStream(recvSocket) + recvSocket.bindAddr(Port(12345), "127.0.0.1") + + var sendSocket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) + sendSocket.connect("127.0.0.1", Port(12345)) + var sendStream = newWriteSocketStream(sendSocket) + sendStream.write "NOM\n" + sendStream.setPosition(1) + echo sendStream.peekStr(2) + sendStream.write "I" + sendStream.setPosition(0) + echo sendStream.readStr(3) + echo sendStream.getPosition() + sendStream.flush() + + echo recvStream.readLine() + recvStream.setPosition(0) + echo recvStream.readLine() + recvStream.close() + +block TCP: + var server = newSocket() + server.setSockOpt(OptReusePort, true) + server.bindAddr(Port(12345)) + server.listen() + + var + client = newSocket() + clientRequestStream = newWriteSocketStream(client) + clientResponseStream = newReadSocketStream(client) + client.connect("127.0.0.1", Port(12345)) + clientRequestStream.writeLine("Hello server!") + clientRequestStream.flush() + + var + incoming: Socket + address: string + server.acceptAddr(incoming, address) + var + serverRequestStream = newReadSocketStream(incoming) + serverResponseStream = newWriteSocketStream(incoming) + echo serverRequestStream.readLine() + serverResponseStream.writeLine("Hi there client!") + serverResponseStream.flush() + serverResponseStream.close() + serverRequestStream.close() + + echo clientResponseStream.readLine() + clientResponseStream.close() + clientRequestStream.close() From cbf227d9493be1e82f3abec1d019ce397163635c Mon Sep 17 00:00:00 2001 From: haxscramper Date: Thu, 7 Jan 2021 21:14:50 +0300 Subject: [PATCH 0007/2472] [FIX] Update fusion master commit hash (#16630) --- koch.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/koch.nim b/koch.nim index 5da262884..95c9c6161 100644 --- a/koch.nim +++ b/koch.nim @@ -11,7 +11,7 @@ const NimbleStableCommit = "8f7af860c5ce9634af880a7081c6435e1f2a5148" # master - FusionStableCommit = "319aac4d43b04113831b529f8003e82f4af6a4a5" + FusionStableCommit = "372ee4313827ef9f2ea388840f7d6b46c2b1b014" when not defined(windows): const From 0da4cb93d1924d99a91491c2ec6124d9ec2a413b Mon Sep 17 00:00:00 2001 From: rockcavera Date: Thu, 7 Jan 2021 15:16:26 -0300 Subject: [PATCH 0008/2472] minor fix (#16624) --- tests/stdlib/tbitops.nim.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/stdlib/tbitops.nim.cfg b/tests/stdlib/tbitops.nim.cfg index f0d7668a7..013e8d38e 100644 --- a/tests/stdlib/tbitops.nim.cfg +++ b/tests/stdlib/tbitops.nim.cfg @@ -1 +1 @@ --d:noUndefinedBitOps +-d:noUndefinedBitOpts From 796498525a3d1197cd6468e4eb45b1fb47a702af Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 7 Jan 2021 20:26:40 +0100 Subject: [PATCH 0009/2472] IC: next steps (#16550) * cleanups * ast.nim: cleanups * IC: no more sym.tab field, stored externally in the module graph * nimble compiles again * rodfiles: store bitwidth of integers and the endianness in the cookie because we serialize 'int' directly * rodfiles: added compilerproc and export sections * rodfiles: added all the missing sections * rodfiles: track the missing information * IC: architecture for lazy loading of proc bodies * make tests green again * completed the lazy loading of proc bodies * symbol lookup integration, part 1 * symbol lookup integration, part 2 * symbol lookup integration, part 3 * make tcompilerapi work again * rodfiles: fixed config change handling --- compiler/ast.nim | 19 - ...nicalizer.nim => canonicalizer_unused.nim} | 0 compiler/ccgexprs.nim | 4 +- compiler/ccgtypes.nim | 12 +- compiler/commands.nim | 17 +- compiler/evaltempl.nim | 2 +- compiler/ic/packed_ast.nim | 23 - compiler/ic/rodfiles.nim | 48 +- compiler/ic/to_packed_ast.nim | 433 +++++++++++++----- compiler/idents.nim | 5 + compiler/importer.nim | 27 +- compiler/jsgen.nim | 6 +- compiler/lookups.nim | 56 +-- compiler/magicsys.nim | 7 +- compiler/modulegraphs.nim | 102 ++++- compiler/modules.nim | 70 +-- compiler/nimeval.nim | 13 +- compiler/packagehandling.nim | 3 +- compiler/passes.nim | 8 +- compiler/plugins/itersgen.nim | 4 +- compiler/pragmas.nim | 4 + compiler/sem.nim | 3 +- compiler/semdata.nim | 22 +- compiler/semexprs.nim | 28 +- compiler/semfields.nim | 2 +- compiler/semgnrc.nim | 2 +- compiler/seminst.nim | 4 +- compiler/semstmts.nim | 12 +- compiler/semtempl.nim | 2 +- compiler/semtypes.nim | 4 +- compiler/sighashes.nim | 2 +- compiler/suggest.nim | 77 ++-- compiler/transf.nim | 15 +- compiler/vm.nim | 2 +- compiler/vmgen.nim | 13 +- nimsuggest/nimsuggest.nim | 2 +- 36 files changed, 653 insertions(+), 400 deletions(-) rename compiler/{canonicalizer.nim => canonicalizer_unused.nim} (100%) diff --git a/compiler/ast.nim b/compiler/ast.nim index ccf4fe497..de4ab5b77 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -833,18 +833,6 @@ type procInstCache*: seq[PInstantiation] gcUnsafetyReason*: PSym # for better error messages wrt gcsafe transformedBody*: PNode # cached body after transf pass - of skModule, skPackage: - # modules keep track of the generic symbols they use from other modules. - # this is because in incremental compilation, when a module is about to - # be replaced with a newer version, we must decrement the usage count - # of all previously used generics. - # For 'import as' we copy the module symbol but shallowCopy the 'tab' - # and set the 'usedGenerics' to ... XXX gah! Better set module.name - # instead? But this doesn't work either. --> We need an skModuleAlias? - # No need, just leave it as skModule but set the owner accordingly and - # check for the owner when touching 'usedGenerics'. - usedGenerics*: seq[PInstantiation] - tab*: TStrTable # interface table for modules of skLet, skVar, skField, skForVar: guard*: PSym bitsize*: int @@ -1455,8 +1443,6 @@ proc copySym*(s: PSym; id: ItemId): PSym = result.typ = s.typ result.flags = s.flags result.magic = s.magic - if s.kind == skModule: - copyStrTable(result.tab, s.tab) result.options = s.options result.position = s.position result.loc = s.loc @@ -1474,13 +1460,10 @@ proc createModuleAlias*(s: PSym, id: ItemId, newIdent: PIdent, info: TLineInfo; result.ast = s.ast #result.id = s.id # XXX figure out what to do with the ID. result.flags = s.flags - system.shallowCopy(result.tab, s.tab) result.options = s.options result.position = s.position result.loc = s.loc result.annex = s.annex - # XXX once usedGenerics is used, ensure module aliases keep working! - assert s.usedGenerics.len == 0 proc initStrTable*(x: var TStrTable) = x.counter = 0 @@ -1915,8 +1898,6 @@ template incompleteType*(t: PType): bool = template typeCompleted*(s: PSym) = incl s.flags, sfNoForward -template getBody*(s: PSym): PNode = s.ast[bodyPos] - template detailedInfo*(sym: PSym): string = sym.name.s diff --git a/compiler/canonicalizer.nim b/compiler/canonicalizer_unused.nim similarity index 100% rename from compiler/canonicalizer.nim rename to compiler/canonicalizer_unused.nim diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index b8a8d21b0..440ecdc7f 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1292,7 +1292,7 @@ proc rawGenNew(p: BProc, a: var TLoc, sizeExpr: Rope; needsInit: bool) = genAssignment(p, a, b, {}) else: let ti = genTypeInfoV1(p.module, typ, a.lode.info) - if bt.destructor != nil and not isTrivialProc(bt.destructor): + if bt.destructor != nil and not isTrivialProc(p.module.g.graph, bt.destructor): # the prototype of a destructor is ``=destroy(x: var T)`` and that of a # finalizer is: ``proc (x: ref T) {.nimcall.}``. We need to check the calling # convention at least: @@ -2204,7 +2204,7 @@ proc genDestroy(p: BProc; n: PNode) = else: discard "nothing to do" else: let t = n[1].typ.skipTypes(abstractVar) - if t.destructor != nil and t.destructor.ast[bodyPos].len != 0: + if t.destructor != nil and getBody(p.module.g.graph, t.destructor).len != 0: internalError(p.config, n.info, "destructor turned out to be not trivial") discard "ignore calls to the default destructor" diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index ab12bad1e..798eaf7f9 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -36,14 +36,6 @@ proc mangleField(m: BModule; name: PIdent): string = if isKeyword(name): result.add "_0" -when false: - proc hashOwner(s: PSym): SigHash = - var m = s - while m.kind != skModule: m = m.owner - let p = m.owner - assert p.kind == skPackage - result = gDebugInfo.register(p.name.s, m.name.s) - proc mangleName(m: BModule; s: PSym): Rope = result = s.loc.r if result == nil: @@ -1313,11 +1305,11 @@ proc genTypeInfo2Name(m: BModule; t: PType): Rope = it = it[0] result = makeCString(res) -proc isTrivialProc(s: PSym): bool {.inline.} = s.ast[bodyPos].len == 0 +proc isTrivialProc(g: ModuleGraph; s: PSym): bool {.inline.} = getBody(g, s).len == 0 proc genHook(m: BModule; t: PType; info: TLineInfo; op: TTypeAttachedOp): Rope = let theProc = t.attachedOps[op] - if theProc != nil and not isTrivialProc(theProc): + if theProc != nil and not isTrivialProc(m.g.graph, theProc): # the prototype of a destructor is ``=destroy(x: var T)`` and that of a # finalizer is: ``proc (x: ref T) {.nimcall.}``. We need to check the calling # convention at least: diff --git a/compiler/commands.nim b/compiler/commands.nim index 9fb9b7e6e..adb85bdd7 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -801,14 +801,15 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; expectNoArg(conf, switch, arg, pass, info) helpOnError(conf, pass) of "symbolfiles": discard "ignore for backwards compat" - of "incremental": - case arg.normalize - of "on": conf.symbolFiles = v2Sf - of "off": conf.symbolFiles = disabledSf - of "writeonly": conf.symbolFiles = writeOnlySf - of "readonly": conf.symbolFiles = readOnlySf - of "v2": conf.symbolFiles = v2Sf - else: localError(conf, info, "invalid option for --incremental: " & arg) + of "incremental", "ic": + if pass in {passCmd2, passPP}: + case arg.normalize + of "on": conf.symbolFiles = v2Sf + of "off": conf.symbolFiles = disabledSf + of "writeonly": conf.symbolFiles = writeOnlySf + of "readonly": conf.symbolFiles = readOnlySf + of "v2": conf.symbolFiles = v2Sf + else: localError(conf, info, "invalid option for --incremental: " & arg) of "skipcfg": processOnOffSwitchG(conf, {optSkipSystemConfigFile}, arg, pass, info) of "skipprojcfg": diff --git a/compiler/evaltempl.nim b/compiler/evaltempl.nim index 691d33a2c..a85314ac2 100644 --- a/compiler/evaltempl.nim +++ b/compiler/evaltempl.nim @@ -187,7 +187,7 @@ proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym; ctx.instID = instID[] ctx.idgen = idgen - let body = tmpl.getBody + let body = tmpl.ast[bodyPos] #echo "instantion of ", renderTree(body, {renderIds}) if isAtom(body): result = newNodeI(nkPar, body.info) diff --git a/compiler/ic/packed_ast.nim b/compiler/ic/packed_ast.nim index 546e495c5..cbfb0c5d1 100644 --- a/compiler/ic/packed_ast.nim +++ b/compiler/ic/packed_ast.nim @@ -16,28 +16,6 @@ import std / [hashes, tables, strtabs, md5] import bitabs import ".." / [ast, options] -const - localNamePos* = 0 - localExportMarkerPos* = 1 - localPragmaPos* = 2 - localTypePos* = 3 - localValuePos* = 4 - - typeNamePos* = 0 - typeExportMarkerPos* = 1 - typeGenericParamsPos* = 2 - typePragmaPos* = 3 - typeBodyPos* = 4 - - routineNamePos* = 0 - routineExportMarkerPos* = 1 - routinePatternPos* = 2 - routineGenericParamsPos* = 3 - routineParamsPos* = 4 - routineResultPos* = 5 - routinePragmasPos* = 6 - routineBodyPos* = 7 - const nkModuleRef* = nkNone # pair of (ModuleId, SymId) @@ -55,7 +33,6 @@ type TypeId* = PackedItemId const - nilTypeId* = PackedItemId(module: LitId(0), item: -1.int32) nilItemId* = PackedItemId(module: LitId(0), item: -1.int32) const diff --git a/compiler/ic/rodfiles.nim b/compiler/ic/rodfiles.nim index 99ce183f2..725262120 100644 --- a/compiler/ic/rodfiles.nim +++ b/compiler/ic/rodfiles.nim @@ -18,6 +18,14 @@ type depsSection integersSection floatsSection + exportsSection + reexportsSection + compilerProcsSection + trmacrosSection + convertersSection + methodsSection + pureEnumsSection + macroUsagesSection topLevelSection bodiesSection symsSection @@ -36,26 +44,30 @@ type const RodVersion = 1 cookie = [byte(0), byte('R'), byte('O'), byte('D'), - byte(0), byte(0), byte(0), byte(RodVersion)] + byte(sizeof(int)*8), byte(system.cpuEndian), byte(0), byte(RodVersion)] + +proc setError(f: var RodFile; err: RodFileError) {.inline.} = + f.err = err + #raise newException(IOError, "IO error") proc storePrim*(f: var RodFile; s: string) = if f.err != ok: return if s.len >= high(int32): - f.err = tooBig + setError f, tooBig return var lenPrefix = int32(s.len) if writeBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix): - f.err = ioFailure + setError f, ioFailure else: if s.len != 0: if writeBuffer(f.f, unsafeAddr(s[0]), s.len) != s.len: - f.err = ioFailure + setError f, ioFailure proc storePrim*[T](f: var RodFile; x: T) = if f.err != ok: return when supportsCopyMem(T): if writeBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x): - f.err = ioFailure + setError f, ioFailure elif T is tuple: for y in fields(x): storePrim(f, y) @@ -65,11 +77,11 @@ proc storePrim*[T](f: var RodFile; x: T) = proc storeSeq*[T](f: var RodFile; s: seq[T]) = if f.err != ok: return if s.len >= high(int32): - f.err = tooBig + setError f, tooBig return var lenPrefix = int32(s.len) if writeBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix): - f.err = ioFailure + setError f, ioFailure else: for i in 0.. 0: if readBuffer(f.f, unsafeAddr(s[0]), s.len) != s.len: - f.err = ioFailure + setError f, ioFailure proc loadPrim*[T](f: var RodFile; x: var T) = if f.err != ok: return when supportsCopyMem(T): if readBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x): - f.err = ioFailure + setError f, ioFailure elif T is tuple: for y in fields(x): loadPrim(f, y) @@ -100,7 +112,7 @@ proc loadSeq*[T](f: var RodFile; s: var seq[T]) = if f.err != ok: return var lenPrefix = int32(0) if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix): - f.err = ioFailure + setError f, ioFailure else: s = newSeq[T](lenPrefix) for i in 0..= g.len: - g.setLen(m+1) - - case g[m].status - of undefined: - g[m].status = loading - let fullpath = msgs.toFullPath(conf, fileIdx) - let rod = toRodFile(conf, AbsoluteFile fullpath) - let err = loadRodFile(rod, g[m].fromDisk, conf) - if err == ok: - result = false - # check its dependencies: - for dep in g[m].fromDisk.imports: - let fid = toFileIndex(dep, g[m].fromDisk, conf) - # Warning: we need to traverse the full graph, so - # do **not use break here**! - if needsRecompile(g, conf, fid): - result = true - - g[m].status = if result: outdated else: loaded - else: - loadError(err, rod) - g[m].status = outdated - result = true - of loading, loaded: - result = false - of outdated: - result = true - # ------------------------------------------------------------------------- proc storeError(err: RodFileError; filename: AbsoluteFile) = @@ -465,7 +471,7 @@ proc storeError(err: RodFileError; filename: AbsoluteFile) = removeFile(filename.string) proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder) = - rememberConfig(encoder, encoder.config) + #rememberConfig(encoder, encoder.config) var f = rodfiles.create(filename.string) f.storeHeader() @@ -473,40 +479,50 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder) = f.storePrim encoder.m.definedSymbols f.storePrim encoder.m.cfg - f.storeSection stringsSection - f.store encoder.m.sh.strings + template storeSeqSection(section, data) {.dirty.} = + f.storeSection section + f.storeSeq data - f.storeSection checkSumsSection - f.storeSeq encoder.m.includes + template storeTabSection(section, data) {.dirty.} = + f.storeSection section + f.store data - f.storeSection depsSection - f.storeSeq encoder.m.imports + storeTabSection stringsSection, encoder.m.sh.strings - f.storeSection integersSection - f.store encoder.m.sh.integers + storeSeqSection checkSumsSection, encoder.m.includes - f.storeSection floatsSection - f.store encoder.m.sh.floats + storeSeqSection depsSection, encoder.m.imports - f.storeSection topLevelSection - f.storeSeq encoder.m.topLevel.nodes + storeTabSection integersSection, encoder.m.sh.integers + storeTabSection floatsSection, encoder.m.sh.floats - f.storeSection bodiesSection - f.storeSeq encoder.m.bodies.nodes + storeSeqSection exportsSection, encoder.m.exports - f.storeSection symsSection - f.storeSeq encoder.m.sh.syms + storeSeqSection reexportsSection, encoder.m.reexports - f.storeSection typesSection - f.storeSeq encoder.m.sh.types + storeSeqSection compilerProcsSection, encoder.m.compilerProcs + + storeSeqSection trmacrosSection, encoder.m.trmacros + storeSeqSection convertersSection, encoder.m.converters + storeSeqSection methodsSection, encoder.m.methods + storeSeqSection pureEnumsSection, encoder.m.pureEnums + storeSeqSection macroUsagesSection, encoder.m.macroUsages + + storeSeqSection topLevelSection, encoder.m.topLevel.nodes + + storeSeqSection bodiesSection, encoder.m.bodies.nodes + storeSeqSection symsSection, encoder.m.sh.syms + + storeSeqSection typesSection, encoder.m.sh.types close(f) if f.err != ok: - loadError(f.err, filename) + storeError(f.err, filename) - when true: + when false: # basic loader testing: var m2: PackedModule discard loadRodFile(filename, m2, encoder.config) + echo "loaded ", filename.string # ---------------------------------------------------------------------------- @@ -516,7 +532,25 @@ type lastLit*: LitId lastFile*: FileIndex # remember the last lookup entry. config*: ConfigRef - ident: IdentCache + cache: IdentCache + +type + ModuleStatus* = enum + undefined, + loading, + loaded, + outdated + + LoadedModule* = object + status*: ModuleStatus + symsInit, typesInit: bool + fromDisk: PackedModule + syms: seq[PSym] # indexed by itemId + types: seq[PType] + module*: PSym # the one true module symbol. + iface: Table[PIdent, seq[PackedItemId]] # PackedItemId so that it works with reexported symbols too + + PackedModuleGraph* = seq[LoadedModule] # indexed by FileIndex proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; t: PackedItemId): PType proc loadSym(c: var PackedDecoder; g: var PackedModuleGraph; s: PackedItemId): PSym @@ -546,7 +580,7 @@ proc loadNodes(c: var PackedDecoder; g: var PackedModuleGraph; of nkEmpty, nkNilLit, nkType: discard of nkIdent: - result.ident = getIdent(c.ident, g[c.thisModule].fromDisk.sh.strings[n.litId]) + result.ident = getIdent(c.cache, g[c.thisModule].fromDisk.sh.strings[n.litId]) of nkSym: result.sym = loadSym(c, g, PackedItemId(module: LitId(0), item: tree.nodes[n.int].operand)) of directIntLit: @@ -567,6 +601,31 @@ proc loadNodes(c: var PackedDecoder; g: var PackedModuleGraph; for n0 in sonsReadonly(tree, n): result.add loadNodes(c, g, tree, n0) +proc loadProcHeader(c: var PackedDecoder; g: var PackedModuleGraph; + tree: PackedTree; n: NodePos): PNode = + # do not load the body of the proc. This will be done later in + # getProcBody, if required. + let k = n.kind + result = newNodeIT(k, translateLineInfo(c, g, n.info), + loadType(c, g, n.typ)) + result.flags = n.flags + assert k in {nkProcDef, nkMethodDef, nkIteratorDef, nkFuncDef, nkConverterDef} + var i = 0 + for n0 in sonsReadonly(tree, n): + if i != bodyPos: + result.add loadNodes(c, g, tree, n0) + else: + result.add nil + inc i + +proc loadProcBody(c: var PackedDecoder; g: var PackedModuleGraph; + tree: PackedTree; n: NodePos): PNode = + var i = 0 + for n0 in sonsReadonly(tree, n): + if i == bodyPos: + result = loadNodes(c, g, tree, n0) + inc i + proc moduleIndex*(c: var PackedDecoder; g: var PackedModuleGraph; s: PackedItemId): int32 {.inline.} = result = if s.module == LitId(0): c.thisModule @@ -579,13 +638,17 @@ proc symHeaderFromPacked(c: var PackedDecoder; g: var PackedModuleGraph; info: translateLineInfo(c, g, s.info), options: s.options, position: s.position, - name: getIdent(c.ident, g[si].fromDisk.sh.strings[s.name]) + name: getIdent(c.cache, g[si].fromDisk.sh.strings[s.name]) ) template loadAstBody(p, field) = if p.field != emptyNodeId: result.field = loadNodes(c, g, g[si].fromDisk.bodies, NodePos p.field) +template loadAstBodyLazy(p, field) = + if p.field != emptyNodeId: + result.field = loadProcHeader(c, g, g[si].fromDisk.bodies, NodePos p.field) + proc loadLib(c: var PackedDecoder; g: var PackedModuleGraph; si, item: int32; l: PackedLib): PLib = # XXX: hack; assume a zero LitId means the PackedLib is all zero (empty) @@ -600,7 +663,10 @@ proc symBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph; s: PackedSym; si, item: int32; result: PSym) = result.typ = loadType(c, g, s.typ) loadAstBody(s, constraint) - loadAstBody(s, ast) + if result.kind in {skProc, skFunc, skIterator, skConverter, skMethod}: + loadAstBodyLazy(s, ast) + else: + loadAstBody(s, ast) result.annex = loadLib(c, g, si, item, s.annex) when hasFFI: result.cname = g[si].fromDisk.sh.strings[s.cname] @@ -615,7 +681,7 @@ proc symBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph; result.loc.r = rope externalName proc loadSym(c: var PackedDecoder; g: var PackedModuleGraph; s: PackedItemId): PSym = - if s == nilTypeId: + if s == nilItemId: result = nil else: let si = moduleIndex(c, g, s) @@ -626,10 +692,16 @@ proc loadSym(c: var PackedDecoder; g: var PackedModuleGraph; s: PackedItemId): P if g[si].syms[s.item] == nil: let packed = addr(g[si].fromDisk.sh.syms[s.item]) - result = symHeaderFromPacked(c, g, packed[], si, s.item) - # store it here early on, so that recursions work properly: - g[si].syms[s.item] = result - symBodyFromPacked(c, g, packed[], si, s.item, result) + + if packed.kind != skModule: + result = symHeaderFromPacked(c, g, packed[], si, s.item) + # store it here early on, so that recursions work properly: + g[si].syms[s.item] = result + symBodyFromPacked(c, g, packed[], si, s.item, result) + else: + result = g[si].module + assert result != nil + else: result = g[si].syms[s.item] @@ -654,7 +726,7 @@ proc typeBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph; result.methods.add((gen, loadSym(c, g, id))) proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; t: PackedItemId): PType = - if t == nilTypeId: + if t == nilItemId: result = nil else: let si = moduleIndex(c, g, t) @@ -672,15 +744,142 @@ proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; t: PackedItemId): else: result = g[si].types[t.item] +proc setupLookupTables(m: var LoadedModule; conf: ConfigRef; cache: IdentCache; fileIdx: FileIndex) = + m.iface = initTable[PIdent, seq[PackedItemId]]() + for e in m.fromDisk.exports: + let nameLit = e[0] + m.iface.mgetOrPut(cache.getIdent(m.fromDisk.sh.strings[nameLit]), @[]).add(PackedItemId(module: LitId(0), item: e[1])) + for re in m.fromDisk.reexports: + let nameLit = re[0] + m.iface.mgetOrPut(cache.getIdent(m.fromDisk.sh.strings[nameLit]), @[]).add(re[1]) -when false: - proc initGenericKey*(s: PSym; types: seq[PType]): GenericKey = - result.module = s.owner.itemId.module - result.name = s.name.s - result.types = mapIt types: hashType(it, {CoType, CoDistinct}).MD5Digest + let filename = AbsoluteFile toFullPath(conf, fileIdx) + # We cannot call ``newSym`` here, because we have to circumvent the ID + # mechanism, which we do in order to assign each module a persistent ID. + m.module = PSym(kind: skModule, itemId: ItemId(module: int32(fileIdx), item: 0'i32), + name: getIdent(cache, splitFile(filename).name), + info: newLineInfo(fileIdx, 1, 1)) + +proc needsRecompile(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; + fileIdx: FileIndex): bool = + let m = int(fileIdx) + if m >= g.len: + g.setLen(m+1) + + case g[m].status + of undefined: + g[m].status = loading + let fullpath = msgs.toFullPath(conf, fileIdx) + let rod = toRodFile(conf, AbsoluteFile fullpath) + let err = loadRodFile(rod, g[m].fromDisk, conf) + if err == ok: + result = false + # check its dependencies: + for dep in g[m].fromDisk.imports: + let fid = toFileIndex(dep, g[m].fromDisk, conf) + # Warning: we need to traverse the full graph, so + # do **not use break here**! + if needsRecompile(g, conf, cache, fid): + result = true + + if not result: + setupLookupTables(g[m], conf, cache, fileIdx) + g[m].status = if result: outdated else: loaded + else: + loadError(err, rod) + g[m].status = outdated + result = true + of loading, loaded: + result = false + of outdated: + result = true + +proc moduleFromRodFile*(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; + fileIdx: FileIndex): PSym = + ## Returns 'nil' if the module needs to be recompiled. + if needsRecompile(g, conf, cache, fileIdx): + result = nil + else: + result = g[int fileIdx].module + assert result != nil + +template setupDecoder() {.dirty.} = + var decoder = PackedDecoder( + thisModule: int32(module), + lastLit: LitId(0), + lastFile: FileIndex(-1), + config: config, + cache: cache) + +proc loadProcBody*(config: ConfigRef, cache: IdentCache; + g: var PackedModuleGraph; s: PSym): PNode = + let mId = s.itemId.module + var decoder = PackedDecoder( + thisModule: mId, + lastLit: LitId(0), + lastFile: FileIndex(-1), + config: config, + cache: cache) + let pos = g[mId].fromDisk.sh.syms[s.itemId.item].ast + assert pos != emptyNodeId + result = loadProcBody(decoder, g, g[mId].fromDisk.bodies, NodePos pos) + +type + RodIter* = object + decoder: PackedDecoder + values: seq[PackedItemId] + i: int + +proc initRodIter*(it: var RodIter; config: ConfigRef, cache: IdentCache; + g: var PackedModuleGraph; module: FileIndex; + name: PIdent): PSym = + it.decoder = PackedDecoder( + thisModule: int32(module), + lastLit: LitId(0), + lastFile: FileIndex(-1), + config: config, + cache: cache) + it.values = g[int module].iface.getOrDefault(name) + it.i = 0 + if it.i < it.values.len: + result = loadSym(it.decoder, g, it.values[it.i]) + inc it.i + +proc initRodIterAllSyms*(it: var RodIter; config: ConfigRef, cache: IdentCache; + g: var PackedModuleGraph; module: FileIndex): PSym = + it.decoder = PackedDecoder( + thisModule: int32(module), + lastLit: LitId(0), + lastFile: FileIndex(-1), + config: config, + cache: cache) + it.values = @[] + for v in g[int module].iface.values: + it.values.add v + it.i = 0 + if it.i < it.values.len: + result = loadSym(it.decoder, g, it.values[it.i]) + inc it.i + +proc nextRodIter*(it: var RodIter; g: var PackedModuleGraph): PSym = + if it.i < it.values.len: + result = loadSym(it.decoder, g, it.values[it.i]) + inc it.i + +iterator interfaceSymbols*(config: ConfigRef, cache: IdentCache; + g: var PackedModuleGraph; module: FileIndex; + name: PIdent): PSym = + setupDecoder() + let values = g[int module].iface.getOrDefault(name) + for pid in values: + let s = loadSym(decoder, g, pid) + assert s != nil + yield s + +proc interfaceSymbol*(config: ConfigRef, cache: IdentCache; + g: var PackedModuleGraph; module: FileIndex; + name: PIdent): PSym = + setupDecoder() + let values = g[int module].iface.getOrDefault(name) + result = loadSym(decoder, g, values[0]) - proc addGeneric*(m: var Module; c: var PackedEncoder; key: GenericKey; s: PSym) = - ## add a generic to the module - if key notin m.generics: - m.generics[key] = toPackedSym(s, m.ast, c) - toPackedNode(s.ast, m.ast, c) diff --git a/compiler/idents.nim b/compiler/idents.nim index 1e8c912d9..d2a84fd36 100644 --- a/compiler/idents.nim +++ b/compiler/idents.nim @@ -113,3 +113,8 @@ proc newIdentCache*(): IdentCache = proc whichKeyword*(id: PIdent): TSpecialWord = if id.id < 0: result = wInvalid else: result = TSpecialWord(id.id) + +proc hash*(x: PIdent): Hash {.inline.} = x.h +proc `==`*(a, b: PIdent): bool {.inline.} = + if a.isNil or b.isNil: result = system.`==`(a, b) + else: result = a.id == b.id diff --git a/compiler/importer.nim b/compiler/importer.nim index 645c03b2b..0fde13eb9 100644 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -11,7 +11,8 @@ import intsets, ast, astalgo, msgs, options, idents, lookups, - semdata, modulepaths, sigmatch, lineinfos, sets + semdata, modulepaths, sigmatch, lineinfos, sets, + modulegraphs proc readExceptSet*(c: PContext, n: PNode): IntSet = assert n.kind in {nkImportExceptStmt, nkExportExceptStmt} @@ -108,7 +109,7 @@ proc rawImportSymbol(c: PContext, s, origin: PSym; importSet: var IntSet) = proc importSymbol(c: PContext, n: PNode, fromMod: PSym; importSet: var IntSet) = let ident = lookups.considerQuotedIdent(c, n) - let s = strTableGet(fromMod.tab, ident) + let s = someSym(c.graph, fromMod, ident) if s == nil: errorUndeclaredIdentifier(c, n.info, ident.s) else: @@ -118,16 +119,16 @@ proc importSymbol(c: PContext, n: PNode, fromMod: PSym; importSet: var IntSet) = # for an enumeration we have to add all identifiers if multiImport: # for a overloadable syms add all overloaded routines - var it: TIdentIter - var e = initIdentIter(it, fromMod.tab, s.name) + var it: ModuleIter + var e = initModuleIter(it, c.graph, fromMod, s.name) while e != nil: if e.name.id != s.name.id: internalError(c.config, n.info, "importSymbol: 3") if s.kind in ExportableSymKinds: rawImportSymbol(c, e, fromMod, importSet) - e = nextIdentIter(it, fromMod.tab) + e = nextModuleIter(it, c.graph) else: rawImportSymbol(c, s, fromMod, importSet) - suggestSym(c.config, n.info, s, c.graph.usageSym, false) + suggestSym(c.graph, n.info, s, c.graph.usageSym, false) proc addImport(c: PContext; im: sink ImportedModule) = for i in 0..high(c.imports): @@ -176,18 +177,6 @@ proc importAllSymbolsExcept(c: PContext, fromMod: PSym, exceptSet: IntSet) = c.addImport ImportedModule(m: fromMod, mode: importExcept, exceptSet: exceptSet) addUnnamedIt(c, fromMod, it.id notin exceptSet) - when false: - var i: TTabIter - var s = initTabIter(i, fromMod.tab) - while s != nil: - if s.kind != skModule: - if s.kind != skEnumField: - if s.kind notin ExportableSymKinds: - internalError(c.config, s.info, "importAllSymbols: " & $s.kind & " " & s.name.s) - if exceptSet.isNil or s.name.id notin exceptSet: - rawImportSymbol(c, s, fromMod) - s = nextIter(i, fromMod.tab) - proc importAllSymbols*(c: PContext, fromMod: PSym) = c.addImport ImportedModule(m: fromMod, mode: importAll) addUnnamedIt(c, fromMod, true) @@ -256,7 +245,7 @@ proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym = message(c.config, n.info, warnDeprecated, result.constraint.strVal & "; " & result.name.s & " is deprecated") else: message(c.config, n.info, warnDeprecated, result.name.s & " is deprecated") - suggestSym(c.config, n.info, result, c.graph.usageSym, false) + suggestSym(c.graph, n.info, result, c.graph.usageSym, false) importStmtResult.add newSymNode(result, n.info) #newStrNode(toFullPath(c.config, f), n.info) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 4b369210d..f588f9555 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -887,7 +887,7 @@ proc genCaseJS(p: PProc, n: PNode, r: var TCompRes) = var v = copyNode(e[0]) inc(totalRange, int(e[1].intVal - v.intVal)) if totalRange > 65535: - localError(p.config, n.info, + localError(p.config, n.info, "Your case statement contains too many branches, consider using if/else instead!") while v.intVal <= e[1].intVal: gen(p, v, cond) @@ -1076,7 +1076,7 @@ proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) = useMagic(p, "nimCopy") # supports proc getF(): var T if x.kind in {nkHiddenDeref, nkDerefExpr} and x[0].kind in nkCallKinds: - lineF(p, "nimCopy($1, $2, $3);$n", + lineF(p, "nimCopy($1, $2, $3);$n", [a.res, b.res, genTypeInfo(p, y.typ)]) else: lineF(p, "$1 = nimCopy($1, $2, $3);$n", @@ -1426,7 +1426,7 @@ proc genSym(p: PProc, n: PNode, r: var TCompRes) = if lfNoDecl in s.loc.flags or s.magic != mNone or {sfImportc, sfInfixCall} * s.flags != {}: discard - elif s.kind == skMethod and s.getBody.kind == nkEmpty: + elif s.kind == skMethod and getBody(p.module.graph, s).kind == nkEmpty: # we cannot produce code for the dispatcher yet: discard elif sfForward in s.flags: diff --git a/compiler/lookups.nim b/compiler/lookups.nim index 3c0aabe9a..cf6f07b7c 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -11,7 +11,8 @@ import intsets, ast, astalgo, idents, semdata, types, msgs, options, - renderer, nimfix/prettybase, lineinfos, strutils + renderer, nimfix/prettybase, lineinfos, strutils, + modulegraphs proc ensureNoMissingOrUnusedSymbols(c: PContext; scope: PScope) @@ -107,8 +108,9 @@ proc localSearchInScope*(c: PContext, s: PIdent): PSym = scope = scope.parent result = strTableGet(scope.symbols, s) -proc initIdentIter(ti: var TIdentIter; marked: var IntSet; im: ImportedModule; name: PIdent): PSym = - result = initIdentIter(ti, im.m.tab, name) +proc initIdentIter(ti: var ModuleIter; marked: var IntSet; im: ImportedModule; name: PIdent; + g: ModuleGraph): PSym = + result = initModuleIter(ti, g, im.m, name) while result != nil: let b = case im.mode @@ -117,11 +119,12 @@ proc initIdentIter(ti: var TIdentIter; marked: var IntSet; im: ImportedModule; n of importExcept: name.id notin im.exceptSet if b and not containsOrIncl(marked, result.id): return result - result = nextIdentIter(ti, im.m.tab) + result = nextModuleIter(ti, g) -proc nextIdentIter(ti: var TIdentIter; marked: var IntSet; im: ImportedModule): PSym = +proc nextIdentIter(ti: var ModuleIter; marked: var IntSet; im: ImportedModule; + g: ModuleGraph): PSym = while true: - result = nextIdentIter(ti, im.m.tab) + result = nextModuleIter(ti, g) if result == nil: return nil case im.mode of importAll: @@ -134,17 +137,17 @@ proc nextIdentIter(ti: var TIdentIter; marked: var IntSet; im: ImportedModule): if result.name.id notin im.exceptSet and not containsOrIncl(marked, result.id): return result -iterator symbols(im: ImportedModule; marked: var IntSet; name: PIdent): PSym = - var ti: TIdentIter - var candidate = initIdentIter(ti, marked, im, name) +iterator symbols(im: ImportedModule; marked: var IntSet; name: PIdent; g: ModuleGraph): PSym = + var ti: ModuleIter + var candidate = initIdentIter(ti, marked, im, name, g) while candidate != nil: yield candidate - candidate = nextIdentIter(ti, marked, im) + candidate = nextIdentIter(ti, marked, im, g) iterator importedItems*(c: PContext; name: PIdent): PSym = var marked = initIntSet() for im in c.imports.mitems: - for s in symbols(im, marked, name): + for s in symbols(im, marked, name, c.graph): yield s proc allPureEnumFields(c: PContext; name: PIdent): seq[PSym] = @@ -169,15 +172,15 @@ iterator allSyms*(c: PContext): (PSym, int, bool) = dec scopeN isLocal = false for im in c.imports.mitems: - for s in im.m.tab.data: - if s != nil: - yield (s, scopeN, isLocal) + for s in modulegraphs.allSyms(c.graph, im.m): + assert s != nil + yield (s, scopeN, isLocal) proc someSymFromImportTable*(c: PContext; name: PIdent; ambiguous: var bool): PSym = var marked = initIntSet() result = nil for im in c.imports.mitems: - for s in symbols(im, marked, name): + for s in symbols(im, marked, name, c.graph): if result == nil: result = s else: @@ -214,7 +217,7 @@ proc searchInScopesFilterBy*(c: PContext, s: PIdent, filter: TSymKinds): seq[PSy if result.len == 0: var marked = initIntSet() for im in c.imports.mitems: - for s in symbols(im, marked, s): + for s in symbols(im, marked, s, c.graph): if s.kind in filter: result.add s @@ -240,6 +243,7 @@ type oimSymChoiceLocalLookup TOverloadIter* = object it*: TIdentIter + mit*: ModuleIter m*: PSym mode*: TOverloadIterMode symChoiceIndex*: int @@ -306,7 +310,7 @@ proc addDeclAt*(c: PContext; scope: PScope, sym: PSym) = proc addInterfaceDeclAux(c: PContext, sym: PSym) = if sfExported in sym.flags: # add to interface: - if c.module != nil: strTableAdd(c.module.tab, sym) + if c.module != nil: exportSym(c, sym) else: internalError(c.config, sym.info, "addInterfaceDeclAux") proc addInterfaceDeclAt*(c: PContext, scope: PScope, sym: PSym) = @@ -475,7 +479,7 @@ proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym = if m == c.module: result = strTableGet(c.topLevelScope.symbols, ident).skipAlias(n, c.config) else: - result = strTableGet(m.tab, ident).skipAlias(n, c.config) + result = someSym(c.graph, m, ident).skipAlias(n, c.config) if result == nil and checkUndeclared in flags: fixSpelling(n[1], ident, searchInScopes) errorUndeclaredIdentifier(c, n[1].info, ident.s) @@ -509,7 +513,7 @@ proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym = scope = scope.parent if scope == nil: for i in 0..c.imports.high: - result = initIdentIter(o.it, o.marked, c.imports[i], ident).skipAlias(n, c.config) + result = initIdentIter(o.mit, o.marked, c.imports[i], ident, c.graph).skipAlias(n, c.config) if result != nil: o.currentScope = nil o.importIdx = i @@ -535,7 +539,7 @@ proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym = ident).skipAlias(n, c.config) o.mode = oimSelfModule else: - result = initIdentIter(o.it, o.m.tab, ident).skipAlias(n, c.config) + result = initModuleIter(o.mit, c.graph, o.m, ident).skipAlias(n, c.config) else: noidentError(c.config, n[1], n) result = errorSym(c, n[1]) @@ -568,7 +572,7 @@ proc nextOverloadIterImports(o: var TOverloadIter, c: PContext, n: PNode): PSym var idx = o.importIdx+1 o.importIdx = c.imports.len # assume the other imported modules lack this symbol too while idx < c.imports.len: - result = initIdentIter(o.it, o.marked, c.imports[idx], o.it.name).skipAlias(n, c.config) + result = initIdentIter(o.mit, o.marked, c.imports[idx], o.it.name, c.graph).skipAlias(n, c.config) if result != nil: # oh, we were wrong, some other module had the symbol, so remember that: o.importIdx = idx @@ -578,7 +582,7 @@ proc nextOverloadIterImports(o: var TOverloadIter, c: PContext, n: PNode): PSym proc symChoiceExtension(o: var TOverloadIter; c: PContext; n: PNode): PSym = assert o.currentScope == nil while o.importIdx < c.imports.len: - result = initIdentIter(o.it, o.marked, c.imports[o.importIdx], o.it.name).skipAlias(n, c.config) + result = initIdentIter(o.mit, o.marked, c.imports[o.importIdx], o.it.name, c.graph).skipAlias(n, c.config) #while result != nil and result.id in o.marked: # result = nextIdentIter(o.it, o.marked, c.imports[o.importIdx]) if result != nil: @@ -602,12 +606,12 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym = else: o.importIdx = 0 if c.imports.len > 0: - result = initIdentIter(o.it, o.marked, c.imports[o.importIdx], o.it.name).skipAlias(n, c.config) + result = initIdentIter(o.mit, o.marked, c.imports[o.importIdx], o.it.name, c.graph).skipAlias(n, c.config) if result == nil: result = nextOverloadIterImports(o, c, n) break elif o.importIdx < c.imports.len: - result = nextIdentIter(o.it, o.marked, c.imports[o.importIdx]).skipAlias(n, c.config) + result = nextIdentIter(o.mit, o.marked, c.imports[o.importIdx], c.graph).skipAlias(n, c.config) if result == nil: result = nextOverloadIterImports(o, c, n) else: @@ -615,7 +619,7 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym = of oimSelfModule: result = nextIdentIter(o.it, c.topLevelScope.symbols).skipAlias(n, c.config) of oimOtherModule: - result = nextIdentIter(o.it, o.m.tab).skipAlias(n, c.config) + result = nextModuleIter(o.mit, c.graph).skipAlias(n, c.config) of oimSymChoice: if o.symChoiceIndex < n.len: result = n[o.symChoiceIndex].sym @@ -654,7 +658,7 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym = incl o.marked, result.id elif o.importIdx < c.imports.len: - result = nextIdentIter(o.it, o.marked, c.imports[o.importIdx]).skipAlias(n, c.config) + result = nextIdentIter(o.mit, o.marked, c.imports[o.importIdx], c.graph).skipAlias(n, c.config) #assert result.id notin o.marked #while result != nil and result.id in o.marked: # result = nextIdentIter(o.it, c.imports[o.importIdx]).skipAlias(n, c.config) diff --git a/compiler/magicsys.nim b/compiler/magicsys.nim index d700ab9a7..e79be6776 100644 --- a/compiler/magicsys.nim +++ b/compiler/magicsys.nim @@ -26,7 +26,7 @@ proc newSysType(g: ModuleGraph; kind: TTypeKind, size: int): PType = result.align = size.int16 proc getSysSym*(g: ModuleGraph; info: TLineInfo; name: string): PSym = - result = strTableGet(g.systemModule.tab, getIdent(g.cache, name)) + result = systemModuleSym(g, getIdent(g.cache, name)) if result == nil: localError(g.config, info, "system module needs: " & name) result = newSym(skError, getIdent(g.cache, name), nextSymId(g.idgen), g.systemModule, g.systemModule.info, {}) @@ -34,15 +34,12 @@ proc getSysSym*(g: ModuleGraph; info: TLineInfo; name: string): PSym = if result.kind == skAlias: result = result.owner proc getSysMagic*(g: ModuleGraph; info: TLineInfo; name: string, m: TMagic): PSym = - var ti: TIdentIter let id = getIdent(g.cache, name) - var r = initIdentIter(ti, g.systemModule.tab, id) - while r != nil: + for r in systemModuleSyms(g, id): if r.magic == m: # prefer the tyInt variant: if r.typ[0] != nil and r.typ[0].kind == tyInt: return r result = r - r = nextIdentIter(ti, g.systemModule.tab) if result != nil: return result localError(g.config, info, "system module needs: " & name) result = newSym(skError, id, nextSymId(g.idgen), g.systemModule, g.systemModule.info, {}) diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index 5544668cc..ab3ef5ab8 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -10,25 +10,11 @@ ## This module implements the module graph data structure. The module graph ## represents a complete Nim project. Single modules can either be kept in RAM ## or stored in a rod-file. -## -## The caching of modules is critical for 'nimsuggest' and is tricky to get -## right. If module E is being edited, we need autocompletion (and type -## checking) for E but we don't want to recompile depending -## modules right away for faster turnaround times. Instead we mark the module's -## dependencies as 'dirty'. Let D be a dependency of E. If D is dirty, we -## need to recompile it and all of its dependencies that are marked as 'dirty'. -## 'nimsuggest sug' actually is invoked for the file being edited so we know -## its content changed and there is no need to compute any checksums. -## Instead of a recursive algorithm, we use an iterative algorithm: -## -## - If a module gets recompiled, its dependencies need to be updated. -## - Its dependent module stays the same. -## -import ast, intsets, tables, options, lineinfos, hashes, idents, +import ast, astalgo, intsets, tables, options, lineinfos, hashes, idents, btrees, md5 -# import ic / packed_ast +import ic / to_packed_ast type SigHash* = distinct MD5Digest @@ -39,9 +25,12 @@ type converters*: seq[PSym] patterns*: seq[PSym] pureEnums*: seq[PSym] + interf: TStrTable ModuleGraph* = ref object ifaces*: seq[Iface] ## indexed by int32 fileIdx + packed: PackedModuleGraph + startupPackedConfig*: PackedConfig packageSyms*: TStrTable deps*: IntSet # the dependency graph or potentially its transitive closure. importDeps*: Table[FileIndex, seq[FileIndex]] # explicit import module dependencies @@ -64,6 +53,7 @@ type sysTypes*: array[TTypeKind, PType] compilerprocs*: TStrTable exposed*: TStrTable + packageTypes*: TStrTable intTypeCache*: array[-5..64, PType] opContains*, opNot*: PSym emptyNode*: PNode @@ -132,6 +122,62 @@ proc toBase64a(s: cstring, len: int): string = result.add cb64[a shr 2] result.add cb64[(a and 3) shl 4] +template semtab*(m: PSym; g: ModuleGraph): TStrTable = + g.ifaces[m.position].interf + +proc cachedModule(g: ModuleGraph; m: PSym): bool {.inline.} = + m.position < g.packed.len and g.packed[m.position].status == loaded + +type + ModuleIter* = object + fromRod: bool + modIndex: int + ti: TIdentIter + rodIt: RodIter + +proc initModuleIter*(mi: var ModuleIter; g: ModuleGraph; m: PSym; name: PIdent): PSym = + assert m.kind == skModule + mi.modIndex = m.position + mi.fromRod = mi.modIndex < g.packed.len and g.packed[mi.modIndex].status == loaded + if mi.fromRod: + result = initRodIter(mi.rodIt, g.config, g.cache, g.packed, FileIndex mi.modIndex, name) + else: + result = initIdentIter(mi.ti, g.ifaces[mi.modIndex].interf, name) + +proc nextModuleIter*(mi: var ModuleIter; g: ModuleGraph): PSym = + if mi.fromRod: + result = nextRodIter(mi.rodIt, g.packed) + else: + result = nextIdentIter(mi.ti, g.ifaces[mi.modIndex].interf) + +iterator allSyms*(g: ModuleGraph; m: PSym): PSym = + if cachedModule(g, m): + var rodIt: RodIter + var r = initRodIterAllSyms(rodIt, g.config, g.cache, g.packed, FileIndex m.position) + while r != nil: + yield r + r = nextRodIter(rodIt, g.packed) + else: + for s in g.ifaces[m.position].interf.data: + if s != nil: + yield s + +proc someSym*(g: ModuleGraph; m: PSym; name: PIdent): PSym = + if cachedModule(g, m): + result = interfaceSymbol(g.config, g.cache, g.packed, FileIndex(m.position), name) + else: + result = strTableGet(g.ifaces[m.position].interf, name) + +proc systemModuleSym*(g: ModuleGraph; name: PIdent): PSym = + result = someSym(g, g.systemModule, name) + +iterator systemModuleSyms*(g: ModuleGraph; name: PIdent): PSym = + var mi: ModuleIter + var r = initModuleIter(mi, g, g.systemModule, name) + while r != nil: + yield r + r = nextModuleIter(mi, g) + proc `$`*(u: SigHash): string = toBase64a(cast[cstring](unsafeAddr u), sizeof(u)) @@ -186,6 +232,7 @@ proc registerModule*(g: ModuleGraph; m: PSym) = if m.position >= g.ifaces.len: setLen(g.ifaces, m.position + 1) g.ifaces[m.position] = Iface(module: m, converters: @[], patterns: @[]) + initStrTable(g.ifaces[m.position].interf) proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph = result = ModuleGraph() @@ -202,6 +249,7 @@ proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph = result.methods = @[] initStrTable(result.compilerprocs) initStrTable(result.exposed) + initStrTable(result.packageTypes) result.opNot = createMagic(result, "not", mNot) result.opContains = createMagic(result, "contains", mInSet) result.emptyNode = newNode(nkEmpty) @@ -226,8 +274,11 @@ proc resetAllModules*(g: ModuleGraph) = initStrTable(g.exposed) proc getModule*(g: ModuleGraph; fileIdx: FileIndex): PSym = - if fileIdx.int32 >= 0 and fileIdx.int32 < g.ifaces.len: - result = g.ifaces[fileIdx.int32].module + if fileIdx.int32 >= 0: + if fileIdx.int32 < g.packed.len and g.packed[fileIdx.int32].status == loaded: + result = g.packed[fileIdx.int32].module + elif fileIdx.int32 < g.ifaces.len: + result = g.ifaces[fileIdx.int32].module proc dependsOn(a, b: int): int {.inline.} = (a shl 15) + b @@ -280,3 +331,18 @@ proc markClientsDirty*(g: ModuleGraph; fileIdx: FileIndex) = proc isDirty*(g: ModuleGraph; m: PSym): bool = result = g.suggestMode and sfDirty in m.flags + +proc getBody*(g: ModuleGraph; s: PSym): PNode {.inline.} = + result = s.ast[bodyPos] + if result == nil and g.config.symbolFiles in {readOnlySf, v2Sf}: + result = loadProcBody(g.config, g.cache, g.packed, s) + s.ast[bodyPos] = result + assert result != nil + +proc moduleFromRodFile*(g: ModuleGraph; fileIdx: FileIndex): PSym = + ## Returns 'nil' if the module needs to be recompiled. + if g.config.symbolFiles in {readOnlySf, v2Sf}: + result = moduleFromRodFile(g.packed, g.config, g.cache, fileIdx) + +proc configComplete*(g: ModuleGraph) = + rememberStartupConfig(g.startupPackedConfig, g.config) diff --git a/compiler/modules.nim b/compiler/modules.nim index a8a9c4df8..deb0174b5 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -31,43 +31,49 @@ proc getPackage(graph: ModuleGraph; fileIdx: FileIndex): PSym = pck = getPackageName(graph.config, filename.string) pck2 = if pck.len > 0: pck else: "unknown" pack = getIdent(graph.cache, pck2) - var packSym = graph.packageSyms.strTableGet(pack) - if packSym == nil: - packSym = newSym(skPackage, getIdent(graph.cache, pck2), packageId(), nil, info) - initStrTable(packSym.tab) - graph.packageSyms.strTableAdd(packSym) + result = graph.packageSyms.strTableGet(pack) + if result == nil: + result = newSym(skPackage, getIdent(graph.cache, pck2), packageId(), nil, info) + #initStrTable(packSym.tab) + graph.packageSyms.strTableAdd(result) else: - let existing = strTableGet(packSym.tab, name) - if existing != nil and existing.info.fileIndex != info.fileIndex: - when false: - # we used to produce an error: - localError(graph.config, info, - "module names need to be unique per Nimble package; module clashes with " & - toFullPath(graph.config, existing.info.fileIndex)) - else: - # but starting with version 0.20 we now produce a fake Nimble package instead - # to resolve the conflicts: - let pck3 = fakePackageName(graph.config, filename) - # this makes the new `packSym`'s owner be the original `packSym` - packSym = newSym(skPackage, getIdent(graph.cache, pck3), packageId(), packSym, info) - initStrTable(packSym.tab) - graph.packageSyms.strTableAdd(packSym) - result = packSym + # we now produce a fake Nimble package instead + # to resolve the conflicts: + let pck3 = fakePackageName(graph.config, filename) + # this makes the new `packSym`'s owner be the original `packSym` + result = newSym(skPackage, getIdent(graph.cache, pck3), packageId(), result, info) + #initStrTable(packSym.tab) + graph.packageSyms.strTableAdd(result) + + when false: + let existing = strTableGet(packSym.tab, name) + if existing != nil and existing.info.fileIndex != info.fileIndex: + when false: + # we used to produce an error: + localError(graph.config, info, + "module names need to be unique per Nimble package; module clashes with " & + toFullPath(graph.config, existing.info.fileIndex)) + else: + # but starting with version 0.20 we now produce a fake Nimble package instead + # to resolve the conflicts: + let pck3 = fakePackageName(graph.config, filename) + # this makes the new `packSym`'s owner be the original `packSym` + packSym = newSym(skPackage, getIdent(graph.cache, pck3), packageId(), packSym, info) + #initStrTable(packSym.tab) + graph.packageSyms.strTableAdd(packSym) proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; filename: AbsoluteFile) = let packSym = getPackage(graph, fileIdx) result.owner = packSym result.position = int fileIdx - graph.registerModule(result) - - initStrTable(result.tab) + #initStrTable(result.tab(graph)) when false: strTableAdd(result.tab, result) # a module knows itself # This is now implemented via # c.moduleScope.addSym(module) # a module knows itself # in sem.nim, around line 527 - strTableAdd(packSym.tab, result) + #strTableAdd(packSym.tab, result) proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym = let filename = AbsoluteFile toFullPath(graph.config, fileIdx) @@ -79,6 +85,7 @@ proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym = if not isNimIdentifier(result.name.s): rawMessage(graph.config, errGenerated, "invalid module name: " & result.name.s) partialInitModule(result, graph, fileIdx, filename) + graph.registerModule(result) proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): PSym = var flags = flags @@ -92,21 +99,20 @@ proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): P elif graph.config.projectIsCmd: s = llStreamOpen(graph.config.cmdInput) discard processModule(graph, result, idGeneratorFromModule(result), s) if result == nil: + result = moduleFromRodFile(graph, fileIdx) let filename = AbsoluteFile toFullPath(graph.config, fileIdx) - when false: - # XXX entry point for module loading from the rod file - result = loadModuleSym(graph, fileIdx, filename) - when true: + if result == nil: result = newModule(graph, fileIdx) result.flags.incl flags registerModule(graph, result) + processModuleAux() else: partialInitModule(result, graph, fileIdx, filename) - processModuleAux() + # XXX replay the pragmas here! elif graph.isDirty(result): result.flags.excl sfDirty # reset module fields: - initStrTable(result.tab) + initStrTable(result.semtab(graph)) result.ast = nil processModuleAux() graph.markClientsDirty(fileIdx) @@ -152,6 +158,8 @@ proc compileProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) = connectCallbacks(graph) let conf = graph.config wantMainModule(conf) + configComplete(graph) + let systemFileIdx = fileInfoIdx(conf, conf.libpath / RelativeFile"system.nim") let projectFile = if projectFileIdx == InvalidFileIdx: conf.projectMainIdx else: projectFileIdx conf.projectMainIdx2 = projectFile diff --git a/compiler/nimeval.nim b/compiler/nimeval.nim index b93129852..0170c9949 100644 --- a/compiler/nimeval.nim +++ b/compiler/nimeval.nim @@ -24,11 +24,8 @@ type iterator exportedSymbols*(i: Interpreter): PSym = assert i != nil assert i.mainModule != nil, "no main module selected" - var it: TTabIter - var s = initTabIter(it, i.mainModule.tab) - while s != nil: + for s in modulegraphs.allSyms(i.graph, i.mainModule): yield s - s = nextIter(it, i.mainModule.tab) proc selectUniqueSymbol*(i: Interpreter; name: string; symKinds: set[TSymKind] = {skLet, skVar}): PSym = @@ -37,14 +34,14 @@ proc selectUniqueSymbol*(i: Interpreter; name: string; assert i != nil assert i.mainModule != nil, "no main module selected" let n = getIdent(i.graph.cache, name) - var it: TIdentIter - var s = initIdentIter(it, i.mainModule.tab, n) + var it: ModuleIter + var s = initModuleIter(it, i.graph, i.mainModule, n) result = nil while s != nil: if s.kind in symKinds: if result == nil: result = s else: return nil # ambiguous - s = nextIdentIter(it, i.mainModule.tab) + s = nextModuleIter(it, i.graph) proc selectRoutine*(i: Interpreter; name: string): PSym = ## Selects a declared routine (proc/func/etc) from the main module. @@ -70,7 +67,7 @@ proc evalScript*(i: Interpreter; scriptStream: PLLStream = nil) = ## This can also be used to *reload* the script. assert i != nil assert i.mainModule != nil, "no main module selected" - initStrTable(i.mainModule.tab) + initStrTable(i.mainModule.semtab(i.graph)) i.mainModule.ast = nil let s = if scriptStream != nil: scriptStream diff --git a/compiler/packagehandling.nim b/compiler/packagehandling.nim index a781f1d51..4af0c28fa 100644 --- a/compiler/packagehandling.nim +++ b/compiler/packagehandling.nim @@ -44,7 +44,8 @@ proc fakePackageName*(conf: ConfigRef; path: AbsoluteFile): string = # in different directory get different name and they can be # placed in a directory. # foo-#head/../bar becomes @foo-@hhead@s..@sbar - result = "@m" & relativeTo(path, conf.projectPath).string.multiReplace({$os.DirSep: "@s", $os.AltSep: "@s", "#": "@h", "@": "@@", ":": "@c"}) + result = "@m" & relativeTo(path, conf.projectPath).string.multiReplace( + {$os.DirSep: "@s", $os.AltSep: "@s", "#": "@h", "@": "@@", ":": "@c"}) proc demanglePackageName*(path: string): string = result = path.multiReplace({"@@": "@", "@h": "#", "@s": "/", "@m": "", "@c": ":"}) diff --git a/compiler/passes.nim b/compiler/passes.nim index 997a10cd8..e3885540e 100644 --- a/compiler/passes.nim +++ b/compiler/passes.nim @@ -110,6 +110,12 @@ proc prepareConfigNotes(graph: ModuleGraph; module: PSym) = proc moduleHasChanged*(graph: ModuleGraph; module: PSym): bool {.inline.} = result = module.id >= 0 or isDefined(graph.config, "nimBackendAssumesChange") +proc partOfStdlib(x: PSym): bool = + var it = x.owner + while it != nil and it.kind == skPackage and it.owner != nil: + it = it.owner + result = it != nil and it.name.s == "stdlib" + proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator; stream: PLLStream): bool {.discardable.} = if graph.stopCompile(): return true @@ -131,7 +137,7 @@ proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator; while true: openParser(p, fileIdx, s, graph.cache, graph.config) - if module.owner == nil or module.owner.name.s != "stdlib" or module.name.s == "distros": + if not partOfStdlib(module) or module.name.s == "distros": # XXX what about caching? no processing then? what if I change the # modules to include between compilation runs? we'd need to track that # in ROD files. I think we should enable this feature only diff --git a/compiler/plugins/itersgen.nim b/compiler/plugins/itersgen.nim index 78c79bf59..24e26b2b7 100644 --- a/compiler/plugins/itersgen.nim +++ b/compiler/plugins/itersgen.nim @@ -9,7 +9,7 @@ ## Plugin to transform an inline iterator into a data structure. -import ".." / [ast, lookups, semdata, lambdalifting, msgs] +import ".." / [ast, modulegraphs, lookups, semdata, lambdalifting, msgs] proc iterToProcImpl*(c: PContext, n: PNode): PNode = result = newNodeI(nkStmtList, n.info) @@ -29,7 +29,7 @@ proc iterToProcImpl*(c: PContext, n: PNode): PNode = localError(c.config, n[2].info, "type must be a non-generic ref|ptr to object with state field") return - let body = liftIterToProc(c.graph, iter.sym, iter.sym.getBody, t, c.idgen) + let body = liftIterToProc(c.graph, iter.sym, getBody(c.graph, iter.sym), t, c.idgen) let prc = newSym(skProc, n[3].ident, nextSymId c.idgen, iter.sym.owner, iter.sym.info) prc.typ = copyType(iter.sym.typ, nextTypeId c.idgen, prc) diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index a79a471de..c1627fe0c 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -14,6 +14,8 @@ import wordrecg, ropes, options, strutils, extccomp, math, magicsys, trees, types, lookups, lineinfos, pathutils, linter +from ic / to_packed_ast import addCompilerProc + const FirstCallConv* = wNimcall LastCallConv* = wNoconv @@ -702,6 +704,8 @@ proc markCompilerProc(c: PContext; s: PSym) = incl(s.flags, sfCompilerProc) incl(s.flags, sfUsed) registerCompilerProc(c.graph, s) + if c.config.symbolFiles != disabledSf: + addCompilerProc(c.encoder, s) proc deprecatedStmt(c: PContext; outerPragma: PNode) = let pragma = outerPragma[1] diff --git a/compiler/sem.nim b/compiler/sem.nim index d5ae5a21d..6f3b15867 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -19,7 +19,8 @@ import lowerings, plugins/active, lineinfos, strtabs, int128, isolation_check, typeallowed -from modulegraphs import ModuleGraph, PPassContext, onUse, onDef, onDefResolveForward +from modulegraphs import ModuleGraph, PPassContext, onUse, onDef, onDefResolveForward, + systemModuleSym, semtab, getBody, someSym, allSyms when defined(nimfix): import nimfix/prettybase diff --git a/compiler/semdata.nim b/compiler/semdata.nim index da38a6fc2..f2f074447 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -268,7 +268,7 @@ proc newContext*(graph: ModuleGraph; module: PSym): PContext = result.typesWithOps = @[] result.features = graph.config.features if graph.config.symbolFiles != disabledSf: - initEncoder result.encoder, module, graph.config + initEncoder result.encoder, module, graph.config, graph.startupPackedConfig proc addIncludeFileDep*(c: PContext; f: FileIndex) = if c.config.symbolFiles != disabledSf: @@ -286,15 +286,29 @@ proc inclSym(sq: var seq[PSym], s: PSym) = proc addConverter*(c: PContext, conv: PSym) = inclSym(c.converters, conv) inclSym(c.graph.ifaces[c.module.position].converters, conv) - #addConverter(c.graph, c.module, conv) # upcoming + if c.config.symbolFiles != disabledSf: + addConverter(c.encoder, conv) proc addPureEnum*(c: PContext, e: PSym) = inclSym(c.graph.ifaces[c.module.position].pureEnums, e) + if c.config.symbolFiles != disabledSf: + addPureEnum(c.encoder, e) proc addPattern*(c: PContext, p: PSym) = inclSym(c.patterns, p) inclSym(c.graph.ifaces[c.module.position].patterns, p) - #addPattern(c.graph, c.module, p) # upcoming + if c.config.symbolFiles != disabledSf: + addTrmacro(c.encoder, p) + +proc exportSym*(c: PContext; s: PSym) = + strTableAdd(c.module.semtab(c.graph), s) + if c.config.symbolFiles != disabledSf: + addExported(c.encoder, s) + +proc reexportSym*(c: PContext; s: PSym) = + strTableAdd(c.module.semtab(c.graph), s) + if c.config.symbolFiles != disabledSf: + addReexport(c.encoder, s) proc newLib*(kind: TLibKind): PLib = new(result) @@ -489,4 +503,4 @@ proc storeRodNode*(c: PContext, n: PNode) = proc saveRodFile*(c: PContext) = if c.config.symbolFiles != disabledSf: - saveRodFile(toRodFile(c.config, c.filename.AbsoluteFile), c.encoder) + saveRodFile(toRodFile(c.config, AbsoluteFile toFullPath(c.config, FileIndex c.module.position)), c.encoder) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 1e5772189..4a04cfb6d 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1014,7 +1014,7 @@ proc semDirectOp(c: PContext, n: PNode, flags: TExprFlags): PNode = proc buildEchoStmt(c: PContext, n: PNode): PNode = # we MUST not check 'n' for semantics again here! But for now we give up: result = newNodeI(nkCall, n.info) - var e = strTableGet(c.graph.systemModule.tab, getIdent(c.cache, "echo")) + let e = systemModuleSym(c.graph, getIdent(c.cache, "echo")) if e != nil: result.add(newSymNode(e)) else: @@ -1897,7 +1897,7 @@ proc lookUpForDeclared(c: PContext, n: PNode, onlyCurrentScope: bool): PSym = if m == c.module: result = strTableGet(c.topLevelScope.symbols, ident) else: - result = strTableGet(m.tab, ident) + result = someSym(c.graph, m, ident) of nkSym: result = n.sym of nkOpenSymChoice, nkClosedSymChoice: @@ -2504,7 +2504,7 @@ proc semBlock(c: PContext, n: PNode; flags: TExprFlags): PNode = elif labl.owner == nil: labl.owner = c.p.owner n[0] = newSymNode(labl, n[0].info) - suggestSym(c.config, n[0].info, labl, c.graph.usageSym) + suggestSym(c.graph, n[0].info, labl, c.graph.usageSym) styleCheckDef(c.config, labl) onDef(n[0].info, labl) n[1] = semExpr(c, n[1], flags) @@ -2522,15 +2522,12 @@ proc semExportExcept(c: PContext, n: PNode): PNode = let exceptSet = readExceptSet(c, n) let exported = moduleName.sym result = newNodeI(nkExportStmt, n.info) - strTableAdd(c.module.tab, exported) - var i: TTabIter - var s = initTabIter(i, exported.tab) - while s != nil: + reexportSym(c, exported) + for s in allSyms(c.graph, exported): if s.kind in ExportableSymKinds+{skModule} and s.name.id notin exceptSet and sfError notin s.flags: - strTableAdd(c.module.tab, s) + reexportSym(c, s) result.add newSymNode(s, n.info) - s = nextIter(i, exported.tab) markUsed(c, n.info, exported) proc semExport(c: PContext, n: PNode): PNode = @@ -2548,15 +2545,12 @@ proc semExport(c: PContext, n: PNode): PNode = localError(c.config, a.info, errGenerated, "cannot export: " & renderTree(a)) elif s.kind == skModule: # forward everything from that module: - strTableAdd(c.module.tab, s) - var ti: TTabIter - var it = initTabIter(ti, s.tab) - while it != nil: + reexportSym(c, s) + for it in allSyms(c.graph, s): if it.kind in ExportableSymKinds+{skModule}: - strTableAdd(c.module.tab, it) + reexportSym(c, it) result.add newSymNode(it, a.info) specialSyms(c, it) - it = nextIter(ti, s.tab) markUsed(c, n.info, s) else: while s != nil: @@ -2565,7 +2559,7 @@ proc semExport(c: PContext, n: PNode): PNode = "; enum field cannot be exported individually") if s.kind in ExportableSymKinds+{skModule} and sfError notin s.flags: result.add(newSymNode(s, a.info)) - strTableAdd(c.module.tab, s) + reexportSym(c, s) markUsed(c, n.info, s) specialSyms(c, s) if s.kind == skType and sfPure notin s.flags: @@ -2575,7 +2569,7 @@ proc semExport(c: PContext, n: PNode): PNode = var e = etyp.n[j].sym if e.kind != skEnumField: internalError(c.config, s.info, "rawImportSymbol") - strTableAdd(c.module.tab, e) + reexportSym(c, e) s = nextOverloadIter(o, c, a) diff --git a/compiler/semfields.nim b/compiler/semfields.nim index 7e8fffc01..93184c568 100644 --- a/compiler/semfields.nim +++ b/compiler/semfields.nim @@ -106,7 +106,7 @@ proc semForFields(c: PContext, n: PNode, m: TMagic): PNode = # so that 'break' etc. work as expected, we produce # a 'while true: stmt; break' loop ... result = newNodeI(nkWhileStmt, n.info, 2) - var trueSymbol = strTableGet(c.graph.systemModule.tab, getIdent(c.cache, "true")) + var trueSymbol = systemModuleSym(c.graph, getIdent(c.cache, "true")) if trueSymbol == nil: localError(c.config, n.info, "system needs: 'true'") trueSymbol = newSym(skUnknown, getIdent(c.cache, "true"), nextSymId c.idgen, getCurrOwner(c), n.info) diff --git a/compiler/semgnrc.nim b/compiler/semgnrc.nim index 1f633549a..dfbb022c8 100644 --- a/compiler/semgnrc.nim +++ b/compiler/semgnrc.nim @@ -481,7 +481,7 @@ proc semGenericStmt(c: PContext, n: PNode, if sfGenSym in s.flags and s.ast == nil: body = n[bodyPos] else: - body = s.getBody + body = getBody(c.graph, s) else: body = n[bodyPos] n[bodyPos] = semGenericStmtScope(c, body, flags, ctx) closeScope(c) diff --git a/compiler/seminst.nim b/compiler/seminst.nim index 3f1cdace3..d273cac14 100644 --- a/compiler/seminst.nim +++ b/compiler/seminst.nim @@ -160,7 +160,7 @@ proc fixupInstantiatedSymbols(c: PContext, s: PSym) = pushInfoContext(c.config, oldPrc.info) openScope(c) var n = oldPrc.ast - n[bodyPos] = copyTree(s.getBody) + n[bodyPos] = copyTree(getBody(c.graph, s)) instantiateBody(c, n, oldPrc.typ.n, oldPrc, s) closeScope(c) popInfoContext(c.config) @@ -383,7 +383,7 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable, if n[pragmasPos].kind != nkEmpty: pragma(c, result, n[pragmasPos], allRoutinePragmas) if isNil(n[bodyPos]): - n[bodyPos] = copyTree(fn.getBody) + n[bodyPos] = copyTree(getBody(c.graph, fn)) if c.inGenericContext == 0: instantiateBody(c, n, fn.typ.n, result, fn) sideEffectsCheck(c, result) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index c2c59bb31..a76116b1c 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -66,7 +66,7 @@ proc semBreakOrContinue(c: PContext, n: PNode): PNode = x.info = n.info incl(s.flags, sfUsed) n[0] = x - suggestSym(c.config, x.info, s, c.graph.usageSym) + suggestSym(c.graph, x.info, s, c.graph.usageSym) onUse(x.info, s) else: localError(c.config, n.info, errInvalidControlFlowX % s.name.s) @@ -332,7 +332,7 @@ proc semIdentDef(c: PContext, n: PNode, kind: TSymKind): PSym = discard result = n.info let info = getLineInfo(n) - suggestSym(c.config, info, result, c.graph.usageSym) + suggestSym(c.graph, info, result, c.graph.usageSym) proc checkNilable(c: PContext; v: PSym) = if {sfGlobal, sfImportc} * v.flags == {sfGlobal} and v.typ.requiresInit: @@ -1055,14 +1055,14 @@ proc typeDefLeftSidePass(c: PContext, typeSection: PNode, i: int) = if pkg.isNil or pkg.kind != skPackage: localError(c.config, name.info, "unknown package name: " & pkgName.s) else: - let typsym = pkg.tab.strTableGet(typName) + let typsym = c.graph.packageTypes.strTableGet(typName) if typsym.isNil: s = semIdentDef(c, name[1], skType) onDef(name[1].info, s) s.typ = newTypeS(tyObject, c) s.typ.sym = s s.flags.incl sfForward - pkg.tab.strTableAdd s + c.graph.packageTypes.strTableAdd s addInterfaceDecl(c, s) elif typsym.kind == skType and sfForward in typsym.flags: s = typsym @@ -1088,7 +1088,7 @@ proc typeDefLeftSidePass(c: PContext, typeSection: PNode, i: int) = if not isTopLevel(c) or pkg.isNil: localError(c.config, name.info, "only top level types in a package can be 'package'") else: - let typsym = pkg.tab.strTableGet(s.name) + let typsym = c.graph.packageTypes.strTableGet(s.name) if typsym != nil: if sfForward notin typsym.flags or sfNoForward notin typsym.flags: typeCompleted(typsym) @@ -1956,7 +1956,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, if not comesFromShadowScope: excl(proto.flags, sfForward) incl(proto.flags, sfWasForwarded) - suggestSym(c.config, s.info, proto, c.graph.usageSym) + suggestSym(c.graph, s.info, proto, c.graph.usageSym) closeScope(c) # close scope with wrong parameter symbols openScope(c) # open scope for old (correct) parameter symbols if proto.ast[genericParamsPos].kind != nkEmpty: diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim index 14c3b9a11..65dc95916 100644 --- a/compiler/semtempl.nim +++ b/compiler/semtempl.nim @@ -250,7 +250,7 @@ proc semTemplSymbol(c: PContext, n: PNode, s: PSym; isField: bool): PNode = else: result = newSymNode(s, n.info) # Issue #12832 when defined(nimsuggest): - suggestSym(c.config, n.info, s, c.graph.usageSym, false) + suggestSym(c.graph, n.info, s, c.graph.usageSym, false) if {optStyleHint, optStyleError} * c.config.globalOptions != {}: styleCheckUse(c.config, n.info, s) diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 99c588657..64113a4c6 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -140,7 +140,7 @@ proc semEnum(c: PContext, n: PNode, prev: PType): PType = if result.sym != nil and sfExported in result.sym.flags: incl(e.flags, sfUsed) incl(e.flags, sfExported) - if not isPure: strTableAdd(c.module.tab, e) + if not isPure: exportSym(c, e) result.n.add symNode styleCheckDef(c.config, e) onDef(e.info, e) @@ -779,7 +779,7 @@ proc semRecordNodeAux(c: PContext, n: PNode, check: var IntSet, pos: var int, n[i][1].info else: n[i].info - suggestSym(c.config, info, f, c.graph.usageSym) + suggestSym(c.graph, info, f, c.graph.usageSym) f.typ = typ f.position = pos f.options = c.config.options diff --git a/compiler/sighashes.nim b/compiler/sighashes.nim index 9a16cc3e0..156bc66d7 100644 --- a/compiler/sighashes.nim +++ b/compiler/sighashes.nim @@ -371,7 +371,7 @@ proc symBodyDigest*(graph: ModuleGraph, sym: PSym): SigHash = if sym.ast != nil: md5Init(c) c.md5Update(cast[cstring](result.addr), sizeof(result)) - hashBodyTree(graph, c, sym.ast[bodyPos]) + hashBodyTree(graph, c, getBody(graph, sym)) c.md5Final(result.MD5Digest) graph.symBodyHashes[sym.id] = result diff --git a/compiler/suggest.nim b/compiler/suggest.nim index 73929f813..560d20b3f 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -56,10 +56,10 @@ proc findDocComment(n: PNode): PNode = elif n.kind in {nkAsgn, nkFastAsgn} and n.len == 2: result = findDocComment(n[1]) -proc extractDocComment(s: PSym): string = +proc extractDocComment(g: ModuleGraph; s: PSym): string = var n = findDocComment(s.ast) if n.isNil and s.kind in routineKinds and s.ast != nil: - n = findDocComment(s.ast[bodyPos]) + n = findDocComment(getBody(g, s)) if not n.isNil: result = n.comment.replace("\n##", "\n").strip else: @@ -117,7 +117,7 @@ proc getTokenLenFromSource(conf: ConfigRef; ident: string; info: TLineInfo): int elif sourceIdent != ident: result = 0 -proc symToSuggest(conf: ConfigRef; s: PSym, isLocal: bool, section: IdeCmd, info: TLineInfo; +proc symToSuggest(g: ModuleGraph; s: PSym, isLocal: bool, section: IdeCmd, info: TLineInfo; quality: range[0..100]; prefix: PrefixMatch; inTypeContext: bool; scope: int; useSuppliedInfo = false): Suggest = @@ -136,7 +136,7 @@ proc symToSuggest(conf: ConfigRef; s: PSym, isLocal: bool, section: IdeCmd, info if u.fileIndex == info.fileIndex: inc c result.localUsages = c result.symkind = byte s.kind - if optIdeTerse notin conf.globalOptions: + if optIdeTerse notin g.config.globalOptions: result.qualifiedPath = @[] if not isLocal and s.kind != skModule: let ow = s.owner @@ -156,20 +156,20 @@ proc symToSuggest(conf: ConfigRef; s: PSym, isLocal: bool, section: IdeCmd, info else: result.forth = "" when defined(nimsuggest) and not defined(noDocgen) and not defined(leanCompiler): - result.doc = s.extractDocComment + result.doc = extractDocComment(g, s) let infox = if useSuppliedInfo or section in {ideUse, ideHighlight, ideOutline}: info else: s.info - result.filePath = toFullPath(conf, infox) + result.filePath = toFullPath(g.config, infox) result.line = toLinenumber(infox) result.column = toColumn(infox) - result.version = conf.suggestVersion + result.version = g.config.suggestVersion result.tokenLen = if section != ideHighlight: s.name.s.len else: - getTokenLenFromSource(conf, s.name.s, infox) + getTokenLenFromSource(g.config, s.name.s, infox) proc `$`*(suggest: Suggest): string = result = $suggest.section @@ -261,7 +261,7 @@ proc fieldVisible*(c: PContext, f: PSym): bool {.inline.} = proc suggestField(c: PContext, s: PSym; f: PNode; info: TLineInfo; outputs: var Suggestions) = var pm: PrefixMatch if filterSym(s, f, pm) and fieldVisible(c, s): - outputs.add(symToSuggest(c.config, s, isLocal=true, ideSug, info, 100, pm, c.inTypeContext > 0, 0)) + outputs.add(symToSuggest(c.graph, s, isLocal=true, ideSug, info, 100, pm, c.inTypeContext > 0, 0)) proc getQuality(s: PSym): range[0..100] = if s.typ != nil and s.typ.len > 1: @@ -275,7 +275,7 @@ template wholeSymTab(cond, section: untyped) {.dirty.} = let it = item var pm: PrefixMatch if cond: - outputs.add(symToSuggest(c.config, it, isLocal = isLocal, section, info, getQuality(it), + outputs.add(symToSuggest(c.graph, it, isLocal = isLocal, section, info, getQuality(it), pm, c.inTypeContext > 0, scopeN)) proc suggestSymList(c: PContext, list, f: PNode; info: TLineInfo, outputs: var Suggestions) = @@ -346,7 +346,7 @@ proc suggestEverything(c: PContext, n, f: PNode, outputs: var Suggestions) = for (it, scopeN, isLocal) in allSyms(c): var pm: PrefixMatch if filterSym(it, f, pm): - outputs.add(symToSuggest(c.config, it, isLocal = isLocal, ideSug, n.info, 0, pm, + outputs.add(symToSuggest(c.graph, it, isLocal = isLocal, ideSug, n.info, 0, pm, c.inTypeContext > 0, scopeN)) proc suggestFieldAccess(c: PContext, n, field: PNode, outputs: var Suggestions) = @@ -365,10 +365,10 @@ proc suggestFieldAccess(c: PContext, n, field: PNode, outputs: var Suggestions) let m = c.graph.importModuleCallback(c.graph, c.module, fileInfoIdx(c.config, fullPath)) if m == nil: typ = nil else: - for it in items(n.sym.tab): + for it in allSyms(c.graph, n.sym): if filterSym(it, field, pm): - outputs.add(symToSuggest(c.config, it, isLocal=false, ideSug, n.info, 100, pm, c.inTypeContext > 0, -100)) - outputs.add(symToSuggest(c.config, m, isLocal=false, ideMod, n.info, 100, PrefixMatch.None, + outputs.add(symToSuggest(c.graph, it, isLocal=false, ideSug, n.info, 100, pm, c.inTypeContext > 0, -100)) + outputs.add(symToSuggest(c.graph, m, isLocal=false, ideMod, n.info, 100, PrefixMatch.None, c.inTypeContext > 0, -99)) if typ == nil: @@ -378,11 +378,11 @@ proc suggestFieldAccess(c: PContext, n, field: PNode, outputs: var Suggestions) # all symbols accessible, because we are in the current module: for it in items(c.topLevelScope.symbols): if filterSym(it, field, pm): - outputs.add(symToSuggest(c.config, it, isLocal=false, ideSug, n.info, 100, pm, c.inTypeContext > 0, -99)) + outputs.add(symToSuggest(c.graph, it, isLocal=false, ideSug, n.info, 100, pm, c.inTypeContext > 0, -99)) else: - for it in items(n.sym.tab): + for it in allSyms(c.graph, n.sym): if filterSym(it, field, pm): - outputs.add(symToSuggest(c.config, it, isLocal=false, ideSug, n.info, 100, pm, c.inTypeContext > 0, -99)) + outputs.add(symToSuggest(c.graph, it, isLocal=false, ideSug, n.info, 100, pm, c.inTypeContext > 0, -99)) else: # fallback: suggestEverything(c, n, field, outputs) @@ -440,27 +440,27 @@ when defined(nimsuggest): if infoB.infoToInt == infoAsInt: return s.allUsages.add(info) -proc findUsages(conf: ConfigRef; info: TLineInfo; s: PSym; usageSym: var PSym) = - if conf.suggestVersion == 1: - if usageSym == nil and isTracked(info, conf.m.trackPos, s.name.s.len): +proc findUsages(g: ModuleGraph; info: TLineInfo; s: PSym; usageSym: var PSym) = + if g.config.suggestVersion == 1: + if usageSym == nil and isTracked(info, g.config.m.trackPos, s.name.s.len): usageSym = s - suggestResult(conf, symToSuggest(conf, s, isLocal=false, ideUse, info, 100, PrefixMatch.None, false, 0)) + suggestResult(g.config, symToSuggest(g, s, isLocal=false, ideUse, info, 100, PrefixMatch.None, false, 0)) elif s == usageSym: - if conf.lastLineInfo != info: - suggestResult(conf, symToSuggest(conf, s, isLocal=false, ideUse, info, 100, PrefixMatch.None, false, 0)) - conf.lastLineInfo = info + if g.config.lastLineInfo != info: + suggestResult(g.config, symToSuggest(g, s, isLocal=false, ideUse, info, 100, PrefixMatch.None, false, 0)) + g.config.lastLineInfo = info when defined(nimsuggest): - proc listUsages*(conf: ConfigRef; s: PSym) = + proc listUsages*(g: ModuleGraph; s: PSym) = #echo "usages ", s.allUsages.len for info in s.allUsages: let x = if info == s.info and info.col == s.info.col: ideDef else: ideUse - suggestResult(conf, symToSuggest(conf, s, isLocal=false, x, info, 100, PrefixMatch.None, false, 0)) + suggestResult(g.config, symToSuggest(g, s, isLocal=false, x, info, 100, PrefixMatch.None, false, 0)) -proc findDefinition(conf: ConfigRef; info: TLineInfo; s: PSym; usageSym: var PSym) = +proc findDefinition(g: ModuleGraph; info: TLineInfo; s: PSym; usageSym: var PSym) = if s.isNil: return - if isTracked(info, conf.m.trackPos, s.name.s.len) or (s == usageSym and sfForward notin s.flags): - suggestResult(conf, symToSuggest(conf, s, isLocal=false, ideDef, info, 100, PrefixMatch.None, false, 0, useSuppliedInfo = s == usageSym)) + if isTracked(info, g.config.m.trackPos, s.name.s.len) or (s == usageSym and sfForward notin s.flags): + suggestResult(g.config, symToSuggest(g, s, isLocal=false, ideDef, info, 100, PrefixMatch.None, false, 0, useSuppliedInfo = s == usageSym)) if sfForward notin s.flags: suggestQuit() else: @@ -472,8 +472,9 @@ proc ensureIdx[T](x: var T, y: int) = proc ensureSeq[T](x: var seq[T]) = if x == nil: newSeq(x, 0) -proc suggestSym*(conf: ConfigRef; info: TLineInfo; s: PSym; usageSym: var PSym; isDecl=true) {.inline.} = +proc suggestSym*(g: ModuleGraph; info: TLineInfo; s: PSym; usageSym: var PSym; isDecl=true) {.inline.} = ## misnamed: should be 'symDeclared' + let conf = g.config when defined(nimsuggest): if conf.suggestVersion == 0: if s.allUsages.len == 0: @@ -482,15 +483,15 @@ proc suggestSym*(conf: ConfigRef; info: TLineInfo; s: PSym; usageSym: var PSym; s.addNoDup(info) if conf.ideCmd == ideUse: - findUsages(conf, info, s, usageSym) + findUsages(g, info, s, usageSym) elif conf.ideCmd == ideDef: - findDefinition(conf, info, s, usageSym) + findDefinition(g, info, s, usageSym) elif conf.ideCmd == ideDus and s != nil: if isTracked(info, conf.m.trackPos, s.name.s.len): - suggestResult(conf, symToSuggest(conf, s, isLocal=false, ideDef, info, 100, PrefixMatch.None, false, 0)) - findUsages(conf, info, s, usageSym) + suggestResult(conf, symToSuggest(g, s, isLocal=false, ideDef, info, 100, PrefixMatch.None, false, 0)) + findUsages(g, info, s, usageSym) elif conf.ideCmd == ideHighlight and info.fileIndex == conf.m.trackPos.fileIndex: - suggestResult(conf, symToSuggest(conf, s, isLocal=false, ideHighlight, info, 100, PrefixMatch.None, false, 0)) + suggestResult(conf, symToSuggest(g, s, isLocal=false, ideHighlight, info, 100, PrefixMatch.None, false, 0)) elif conf.ideCmd == ideOutline and isDecl: # if a module is included then the info we have is inside the include and # we need to walk up the owners until we find the outer most module, @@ -503,7 +504,7 @@ proc suggestSym*(conf: ConfigRef; info: TLineInfo; s: PSym; usageSym: var PSym; parentModule = parentModule.owner if parentFileIndex == conf.m.trackPos.fileIndex: - suggestResult(conf, symToSuggest(conf, s, isLocal=false, ideOutline, info, 100, PrefixMatch.None, false, 0)) + suggestResult(conf, symToSuggest(g, s, isLocal=false, ideOutline, info, 100, PrefixMatch.None, false, 0)) proc extractPragma(s: PSym): PNode = if s.kind in routineKinds: @@ -572,7 +573,7 @@ proc markUsed(c: PContext; info: TLineInfo; s: PSym) = if sfError in s.flags: userError(conf, info, s) when defined(nimsuggest): - suggestSym(conf, info, s, c.graph.usageSym, false) + suggestSym(c.graph, info, s, c.graph.usageSym, false) if {optStyleHint, optStyleError} * conf.globalOptions != {}: styleCheckUse(conf, info, s) markOwnerModuleAsUsed(c, s) @@ -658,7 +659,7 @@ proc suggestSentinel*(c: PContext) = for (it, scopeN, isLocal) in allSyms(c): var pm: PrefixMatch if filterSymNoOpr(it, nil, pm): - outputs.add(symToSuggest(c.config, it, isLocal = isLocal, ideSug, + outputs.add(symToSuggest(c.graph, it, isLocal = isLocal, ideSug, newLineInfo(c.config.m.trackPos.fileIndex, 0, -1), 0, PrefixMatch.None, false, scopeN)) diff --git a/compiler/transf.nim b/compiler/transf.nim index 461db9e89..e9ab6b47d 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -126,7 +126,7 @@ proc transformSymAux(c: PTransf, n: PNode): PNode = var tc = c.transCon if sfBorrow in s.flags and s.kind in routineKinds: # simply exchange the symbol: - b = s.getBody + b = getBody(c.graph, s) if b.kind != nkSym: internalError(c.graph.config, n.info, "wrong AST for borrowed symbol") b = newSymNode(b.sym, n.info) elif c.inlining > 0: @@ -594,7 +594,7 @@ proc findWrongOwners(c: PTransf, n: PNode) = else: for i in 0.. Date: Thu, 7 Jan 2021 20:57:11 +0100 Subject: [PATCH 0010/2472] Improve documentation for the md5 module (#16631) --- lib/pure/md5.nim | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/lib/pure/md5.nim b/lib/pure/md5.nim index ba944ba81..11c324548 100644 --- a/lib/pure/md5.nim +++ b/lib/pure/md5.nim @@ -7,11 +7,14 @@ # distribution, for details about the copyright. # -## Module for computing `MD5 checksums `_. +## Module for computing [MD5 checksums](https://en.wikipedia.org/wiki/MD5). ## -## **See also:** -## * `base64 module`_ implements a base64 encoder and decoder -## * `std/sha1 module `_ for a sha1 encoder and decoder +## **Note:** The procs in this module can be used at compile time. +## +## See also +## ======== +## * `base64 module`_ implements a Base64 encoder and decoder +## * `std/sha1 module `_ for a SHA-1 encoder and decoder ## * `hashes module`_ for efficient computations of hash values ## for diverse Nim types @@ -22,8 +25,8 @@ type MD5State = array[0..3, uint32] MD5Block = array[0..15, uint32] MD5CBits = array[0..7, uint8] - MD5Digest* = array[0..15, uint8] ## \ - ## MD5 checksum of a string, obtained with `toMD5 proc <#toMD5,string>`_. + MD5Digest* = array[0..15, uint8] + ## MD5 checksum of a string, obtained with the `toMD5 proc <#toMD5,string>`_. MD5Buffer = array[0..63, uint8] MD5Context* {.final.} = object state: MD5State @@ -180,7 +183,7 @@ proc md5Final*(c: var MD5Context, digest: var MD5Digest) {.raises: [], tags: [], proc toMD5*(s: string): MD5Digest = ## Computes the `MD5Digest` value for a string `s`. ## - ## See also: + ## **See also:** ## * `getMD5 proc <#getMD5,string>`_ which returns a string representation ## of the `MD5Digest` ## * `$ proc <#$,MD5Digest>`_ for converting MD5Digest to string @@ -202,10 +205,8 @@ proc `$`*(d: MD5Digest): string = proc getMD5*(s: string): string = ## Computes an MD5 value of `s` and returns its string representation. - ## .. note:: - ## available at compile time ## - ## See also: + ## **See also:** ## * `toMD5 proc <#toMD5,string>`_ which returns the `MD5Digest` of a string runnableExamples: assert getMD5("abc") == "900150983cd24fb0d6963f7d28e17f72" @@ -226,9 +227,9 @@ proc `==`*(D1, D2: MD5Digest): bool = proc md5Init*(c: var MD5Context) = - ## Initializes a `MD5Context`. + ## Initializes an `MD5Context`. ## - ## If you use `toMD5 proc <#toMD5,string>`_ there's no need to call this + ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this ## function explicitly. c.state[0] = 0x67452301'u32 c.state[1] = 0xEFCDAB89'u32 @@ -241,7 +242,7 @@ proc md5Init*(c: var MD5Context) = proc md5Update*(c: var MD5Context, input: cstring, len: int) = ## Updates the `MD5Context` with the `input` data of length `len`. ## - ## If you use `toMD5 proc <#toMD5,string>`_ there's no need to call this + ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this ## function explicitly. var input = input var Index = int((c.count[0] shr 3) and 0x3F) @@ -263,7 +264,7 @@ proc md5Update*(c: var MD5Context, input: cstring, len: int) = proc md5Final*(c: var MD5Context, digest: var MD5Digest) = ## Finishes the `MD5Context` and stores the result in `digest`. ## - ## If you use `toMD5 proc <#toMD5,string>`_ there's no need to call this + ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this ## function explicitly. var Bits: MD5CBits From ab5d77f9721298cad443d0d294ff57e965ff2ce1 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 7 Jan 2021 16:34:57 -0700 Subject: [PATCH 0011/2472] Change the content length calculation to match the http spec For reference: https://www.w3.org/Protocols/HTTP/1.0/draft-ietf-http-spec.html#Entity-Body --- lib/pure/httpclient.nim | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index a6d7edf33..70f3327b1 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -980,8 +980,11 @@ proc requestAux(client: HttpClient | AsyncHttpClient, url, httpMethod: string, var data: seq[string] if multipart != nil and multipart.content.len > 0: data = await client.format(multipart) - elif httpMethod in ["POST", "PATCH", "PUT", "DELETE"] or body.len != 0: - client.headers["Content-Length"] = $body.len + else: + if body.len != 0: + client.headers["Content-Length"] = $body.len + elif httpMethod notin ["GET", "HEAD"] and not client.headers.hasKey("Content-Length"): + client.headers["Content-Length"] = "0" when client is AsyncHttpClient: if not client.parseBodyFut.isNil: From 596da7f9a0d6303738d368e119e49461591bfe71 Mon Sep 17 00:00:00 2001 From: Andrew Date: Fri, 8 Jan 2021 11:29:32 +0300 Subject: [PATCH 0012/2472] happy new year 2021 again (#16638) --- compiler/options.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/options.nim b/compiler/options.nim index 4a013f8fc..6cdb5db63 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -19,7 +19,7 @@ const useEffectSystem* = true useWriteTracking* = false hasFFI* = defined(nimHasLibFFI) - copyrightYear* = "2020" + copyrightYear* = "2021" type # please make sure we have under 32 options # (improves code efficiency a lot!) From add1ccb6cb4ad4b444a5cfa3ef5573b0c1d47d09 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 8 Jan 2021 13:36:03 +0100 Subject: [PATCH 0013/2472] compiler: minor refactoring (#16633) --- compiler/main.nim | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/compiler/main.nim b/compiler/main.nim index 4c67aaea7..438f90ba6 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -211,18 +211,17 @@ proc mainCommand*(graph: ModuleGraph) = defineSymbol(conf.symbols, "nimdoc") body - block: ## command prepass - if conf.cmd == cmdCrun: conf.globalOptions.incl {optRun, optUseNimcache} - if conf.cmd notin cmdBackends + {cmdTcc}: customizeForBackend(backendC) - if conf.outDir.isEmpty: - # doc like commands can generate a lot of files (especially with --project) - # so by default should not end up in $PWD nor in $projectPath. - conf.outDir = block: - var ret = if optUseNimcache in conf.globalOptions: getNimcacheDir(conf) - else: conf.projectPath - doAssert ret.string.isAbsolute # `AbsoluteDir` is not a real guarantee - if conf.cmd in cmdDocLike + {cmdRst2html, cmdRst2tex}: ret = ret / htmldocsDir - ret + ## command prepass + if conf.cmd == cmdCrun: conf.globalOptions.incl {optRun, optUseNimcache} + if conf.cmd notin cmdBackends + {cmdTcc}: customizeForBackend(backendC) + if conf.outDir.isEmpty: + # doc like commands can generate a lot of files (especially with --project) + # so by default should not end up in $PWD nor in $projectPath. + var ret = if optUseNimcache in conf.globalOptions: getNimcacheDir(conf) + else: conf.projectPath + doAssert ret.string.isAbsolute # `AbsoluteDir` is not a real guarantee + if conf.cmd in cmdDocLike + {cmdRst2html, cmdRst2tex}: ret = ret / htmldocsDir + conf.outDir = ret ## process all commands case conf.cmd From 38b8d080f29021d2685e09d552c2a7753ef25484 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Fri, 8 Jan 2021 07:42:38 -0600 Subject: [PATCH 0014/2472] close #1550 add testcase (#16640) --- tests/iter/t1550.nim | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 tests/iter/t1550.nim diff --git a/tests/iter/t1550.nim b/tests/iter/t1550.nim new file mode 100644 index 000000000..8ad96f0da --- /dev/null +++ b/tests/iter/t1550.nim @@ -0,0 +1,20 @@ +type + A[T] = iterator(x: T): T {.gcsafe, closure.} + +iterator aimp[T](x: T): T {.gcsafe, closure.} = + var total = 0 + while (total < 100): + yield total + total += x + +iterator bimp(y: A[int], z:int): int {.gcsafe, closure.} = + for i in y(z): + yield i + +for x in aimp[int](3): + discard x + +var y = aimp[int] +var z = bimp +for x in z(y, 1): + discard x \ No newline at end of file From bfcb7c1621ba4cadb228125e924f1fe6bf91e20c Mon Sep 17 00:00:00 2001 From: Joey Date: Fri, 8 Jan 2021 11:48:23 -0700 Subject: [PATCH 0015/2472] DELETE requests should always have a content-length header (#16618) * DELETE requests should always have a content-length header Not having DELETE in this list is causing hanging when trying to close webdriver sessions in [halonium](https://github.com/halonium/halonium/issues/10) and likely any other implementation of the webdriver protocol. Both at least chromedriver and geckodriver are affected by this issue. * Change the content length calculation to match the http spec For reference: https://www.w3.org/Protocols/HTTP/1.0/draft-ietf-http-spec.html#Entity-Body --- lib/pure/httpclient.nim | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 3093f5564..70f3327b1 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -980,8 +980,11 @@ proc requestAux(client: HttpClient | AsyncHttpClient, url, httpMethod: string, var data: seq[string] if multipart != nil and multipart.content.len > 0: data = await client.format(multipart) - elif httpMethod in ["POST", "PATCH", "PUT"] or body.len != 0: - client.headers["Content-Length"] = $body.len + else: + if body.len != 0: + client.headers["Content-Length"] = $body.len + elif httpMethod notin ["GET", "HEAD"] and not client.headers.hasKey("Content-Length"): + client.headers["Content-Length"] = "0" when client is AsyncHttpClient: if not client.parseBodyFut.isNil: From 2a426ca8e2d64e4e415c20e97a6fbc8563ab1b2f Mon Sep 17 00:00:00 2001 From: alaviss Date: Fri, 8 Jan 2021 14:04:17 -0600 Subject: [PATCH 0016/2472] kochdocs: fusion needs the js backend too (#16644) --- tools/kochdocs.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 81117dd52..75bb4443f 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -15,7 +15,9 @@ const var nimExe*: string -template isJsOnly(file: string): bool = file.isRelativeTo("lib/js") +template isJsOnly(file: string): bool = + file.isRelativeTo("lib/js") or + file.isRelativeTo("lib/fusion/js") proc exe*(f: string): string = result = addFileExt(f, ExeExt) From ffb130b59c3f3d03e694b76af37563b070e28af0 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 8 Jan 2021 14:09:58 -0800 Subject: [PATCH 0017/2472] skip docs for `lib/fusion` (docs already run in fusion repo) (#16645) * run CI docs on koch.nim changes to avoid future regressions * kochdocs: skip lib/fusion --- .github/workflows/ci_docs.yml | 2 +- tools/kochdocs.nim | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci_docs.yml b/.github/workflows/ci_docs.yml index 74ca804f3..fcc89dfc1 100644 --- a/.github/workflows/ci_docs.yml +++ b/.github/workflows/ci_docs.yml @@ -26,7 +26,7 @@ on: - 'tools/dochack/dochack.nim' - 'tools/kochdocs.nim' - '.github/workflows/ci_docs.yml' - + - 'koch.nim' jobs: build: diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 75bb4443f..f258087e7 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -15,9 +15,7 @@ const var nimExe*: string -template isJsOnly(file: string): bool = - file.isRelativeTo("lib/js") or - file.isRelativeTo("lib/fusion/js") +template isJsOnly(file: string): bool = file.isRelativeTo("lib/js") proc exe*(f: string): string = result = addFileExt(f, ExeExt) @@ -187,7 +185,8 @@ lib/system/widestrs.nim """.splitWhitespace() proc follow(a: PathEntry): bool = - a.path.lastPathPart notin ["nimcache", "htmldocs", "includes", "deprecated", "genode"] + result = a.path.lastPathPart notin ["nimcache", "htmldocs", "includes", "deprecated", "genode"] and + not a.path.isRelativeTo("lib/fusion") for entry in walkDirRecFilter("lib", follow = follow): let a = entry.path if entry.kind != pcFile or a.splitFile.ext != ".nim" or From b7ff0b2a1128dc976379102e180ae883641d6c82 Mon Sep 17 00:00:00 2001 From: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Date: Sat, 9 Jan 2021 00:24:41 +0100 Subject: [PATCH 0018/2472] Use func in lenientops (#16641) --- lib/pure/lenientops.nim | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/lib/pure/lenientops.nim b/lib/pure/lenientops.nim index f73df5e5f..a8fc78e39 100644 --- a/lib/pure/lenientops.nim +++ b/lib/pure/lenientops.nim @@ -8,49 +8,49 @@ # ## This module offers implementations of common binary operations -## like ``+``, ``-``, ``*``, ``/`` and comparison operations, +## like `+`, `-`, `*`, `/` and comparison operations, ## which work for mixed float/int operands. ## All operations convert the integer operand into the ## type of the float operand. For numerical expressions, the return ## type is always the type of the float involved in the expression, ## i.e., there is no auto conversion from float32 to float64. ## -## Note: In general, auto-converting from int to float loses +## **Note:** In general, auto-converting from int to float loses ## information, which is why these operators live in a separate ## module. Use with care. ## ## Regarding binary comparison, this module only provides unequal operators. -## The equality operator ``==`` is omitted, because depending on the use case +## The equality operator `==` is omitted, because depending on the use case ## either casting to float or rounding to int might be preferred, and users ## should make an explicit choice. -proc `+`*[I: SomeInteger, F: SomeFloat](i: I, f: F): F {.noSideEffect, inline.} = +func `+`*[I: SomeInteger, F: SomeFloat](i: I, f: F): F {.inline.} = F(i) + f -proc `+`*[I: SomeInteger, F: SomeFloat](f: F, i: I): F {.noSideEffect, inline.} = +func `+`*[I: SomeInteger, F: SomeFloat](f: F, i: I): F {.inline.} = f + F(i) -proc `-`*[I: SomeInteger, F: SomeFloat](i: I, f: F): F {.noSideEffect, inline.} = +func `-`*[I: SomeInteger, F: SomeFloat](i: I, f: F): F {.inline.} = F(i) - f -proc `-`*[I: SomeInteger, F: SomeFloat](f: F, i: I): F {.noSideEffect, inline.} = +func `-`*[I: SomeInteger, F: SomeFloat](f: F, i: I): F {.inline.} = f - F(i) -proc `*`*[I: SomeInteger, F: SomeFloat](i: I, f: F): F {.noSideEffect, inline.} = +func `*`*[I: SomeInteger, F: SomeFloat](i: I, f: F): F {.inline.} = F(i) * f -proc `*`*[I: SomeInteger, F: SomeFloat](f: F, i: I): F {.noSideEffect, inline.} = +func `*`*[I: SomeInteger, F: SomeFloat](f: F, i: I): F {.inline.} = f * F(i) -proc `/`*[I: SomeInteger, F: SomeFloat](i: I, f: F): F {.noSideEffect, inline.} = +func `/`*[I: SomeInteger, F: SomeFloat](i: I, f: F): F {.inline.} = F(i) / f -proc `/`*[I: SomeInteger, F: SomeFloat](f: F, i: I): F {.noSideEffect, inline.} = +func `/`*[I: SomeInteger, F: SomeFloat](f: F, i: I): F {.inline.} = f / F(i) -proc `<`*[I: SomeInteger, F: SomeFloat](i: I, f: F): bool {.noSideEffect, inline.} = +func `<`*[I: SomeInteger, F: SomeFloat](i: I, f: F): bool {.inline.} = F(i) < f -proc `<`*[I: SomeInteger, F: SomeFloat](f: F, i: I): bool {.noSideEffect, inline.} = +func `<`*[I: SomeInteger, F: SomeFloat](f: F, i: I): bool {.inline.} = f < F(i) -proc `<=`*[I: SomeInteger, F: SomeFloat](i: I, f: F): bool {.noSideEffect, inline.} = +func `<=`*[I: SomeInteger, F: SomeFloat](i: I, f: F): bool {.inline.} = F(i) <= f -proc `<=`*[I: SomeInteger, F: SomeFloat](f: F, i: I): bool {.noSideEffect, inline.} = +func `<=`*[I: SomeInteger, F: SomeFloat](f: F, i: I): bool {.inline.} = f <= F(i) # Note that we must not defined `>=` and `>`, because system.nim already has a From dbff2cd938b326279de0f3f97b2dd8c54a90468a Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Sat, 9 Jan 2021 04:54:26 -0600 Subject: [PATCH 0019/2472] close #4834 add testcase (#16649) --- tests/cpp/t4834.nim | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 tests/cpp/t4834.nim diff --git a/tests/cpp/t4834.nim b/tests/cpp/t4834.nim new file mode 100644 index 000000000..0275b1b70 --- /dev/null +++ b/tests/cpp/t4834.nim @@ -0,0 +1,17 @@ +discard """ + targets: "cpp" +""" + +# issue #4834 +block: + defer: + let x = 0 + + +proc main() = + block: + defer: + raise newException(Exception, "foo") + +doAssertRaises(Exception): + main() From 65df5762a1f2011497350da0713a7bca05343326 Mon Sep 17 00:00:00 2001 From: vabresto <77133146+vabresto@users.noreply.github.com> Date: Sun, 10 Jan 2021 06:42:23 -0500 Subject: [PATCH 0020/2472] Add support for Transfer-Encoding: chunked (#16636) * Add support for Transfer-Encoding: chunked * Minor whitespace fixes * Use recv instead of recvLineInto * Undo changes to httpcore, inline changes --- lib/pure/asynchttpserver.nim | 44 ++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index 29a695379..86688d4b5 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -142,6 +142,17 @@ proc parseProtocol(protocol: string): tuple[orig: string, major, minor: int] = proc sendStatus(client: AsyncSocket, status: string): Future[void] = client.send("HTTP/1.1 " & status & "\c\L\c\L") +func hasChunkedEncoding(request: Request): bool = + ## Searches for a chunked transfer encoding + const transferEncoding = "Transfer-Encoding" + + if request.headers.hasKey(transferEncoding): + for encoding in seq[string](request.headers[transferEncoding]): + if "chunked" == encoding.strip: + # Returns true if it is both an HttpPost and has chunked encoding + return request.reqMethod == HttpPost + return false + proc processRequest( server: AsyncHttpServer, req: FutureVar[Request], @@ -261,6 +272,39 @@ proc processRequest( if request.body.len != contentLength: await request.respond(Http400, "Bad Request. Content-Length does not match actual.") return true + elif hasChunkedEncoding(request): + # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Transfer-Encoding + var sizeOrData = 0 + var bytesToRead = 0 + request.body = "" + + while true: + lineFut.mget.setLen(0) + lineFut.clean() + + # The encoding format alternates between specifying a number of bytes to read + # and the data to be read, of the previously specified size + if sizeOrData mod 2 == 0: + # Expect a number of chars to read + await client.recvLineInto(lineFut, maxLength = maxLine) + try: + bytesToRead = lineFut.mget.parseHexInt + except ValueError: + # Malformed request + await request.respond(Http411, ("Invalid chunked transfer encoding - " & + "chunk data size must be hex encoded")) + return true + else: + if bytesToRead == 0: + # Done reading chunked data + break + + # Read bytesToRead and add to body + # Note we add +2 because the line must be terminated by \r\n + let chunk = await client.recv(bytesToRead + 2) + request.body = request.body & chunk + + inc sizeOrData elif request.reqMethod == HttpPost: await request.respond(Http411, "Content-Length required.") return true From 7bde6aa37f52695736917a070bc25097f0cb0b34 Mon Sep 17 00:00:00 2001 From: Antonis Geralis <43617260+planetis-m@users.noreply.github.com> Date: Sun, 10 Jan 2021 15:40:53 +0200 Subject: [PATCH 0021/2472] Httpclient improvements (#15919) * Allow passing Uri instead of strings * Teach httpclient about 308 * Deprecate request proc where httpMethod is string * More use of HttpMethod enum Also fix handling of 308, I forgot to add the hunk to the previous commit. * Well behaved redirect handler * Also remove Transfer-Encoding * Removed unused proc * Secure redirection rules Strip sensitive headers for cross-domain redirects. * Allow httpMethod to be a string again This way unknown http verbs can be used without any problem. * Respect user-specified Host header * Missed multipart argument. * Try another method * add changelog * Fix hidden deprecation warning, parseEnum failing * This is wrong * Have to do it manually, parseEnum is not suitable * Review comments * update Co-authored-by: LemonBoy Co-authored-by: Dominik Picheta --- changelog.md | 3 + lib/pure/httpclient.nim | 189 ++++++++++++++++++++++------------- lib/pure/httpcore.nim | 57 ++++------- tests/stdlib/thttpclient.nim | 8 +- 4 files changed, 147 insertions(+), 110 deletions(-) diff --git a/changelog.md b/changelog.md index afe49f1f3..0705d0090 100644 --- a/changelog.md +++ b/changelog.md @@ -34,6 +34,9 @@ - Removed deprecated `iup` module from stdlib, it has already moved to [nimble](https://github.com/nim-lang/iup). +- various functions in `httpclient` now accept `url` of type `Uri`. Moreover `request` function's + `httpMethod` argument of type `string` was deprecated in favor of `HttpMethod` enum type. + - `nodejs` backend now supports osenv: `getEnv`, `putEnv`, `envPairs`, `delEnv`, `existsEnv`. - Added `cmpMem` to `system`. diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 70f3327b1..ea847ef8d 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -325,10 +325,14 @@ proc getDefaultSSL(): SslContext = result = defaultSslContext doAssert result != nil, "failure to initialize the SSL context" -proc newProxy*(url: string, auth = ""): Proxy = +proc newProxy*(url: string; auth = ""): Proxy = ## Constructs a new ``TProxy`` object. result = Proxy(url: parseUri(url), auth: auth) +proc newProxy*(url: Uri; auth = ""): Proxy = + ## Constructs a new ``TProxy`` object. + result = Proxy(url: url, auth: auth) + proc newMultipartData*: MultipartData {.inline.} = ## Constructs a new ``MultipartData`` object. MultipartData() @@ -457,29 +461,23 @@ proc sendFile(socket: Socket | AsyncSocket, await socket.send(buffer) file.close() -proc redirection(status: string): bool = - const redirectionNRs = ["301", "302", "303", "307", "308"] - for i in items(redirectionNRs): - if status.startsWith(i): - return true - -proc getNewLocation(lastURL: string, headers: HttpHeaders): string = - result = headers.getOrDefault"Location" - if result == "": httpError("location header expected") +proc getNewLocation(lastURL: Uri, headers: HttpHeaders): Uri = + let newLocation = headers.getOrDefault"Location" + if newLocation == "": httpError("location header expected") # Relative URLs. (Not part of the spec, but soon will be.) - let r = parseUri(result) - if r.hostname == "" and r.path != "": - var parsed = parseUri(lastURL) - parsed.path = r.path - parsed.query = r.query - parsed.anchor = r.anchor - result = $parsed + let parsedLocation = parseUri(newLocation) + if parsedLocation.hostname == "" and parsedLocation.path != "": + result = lastURL + result.path = parsedLocation.path + result.query = parsedLocation.query + result.anchor = parsedLocation.anchor + else: + result = parsedLocation -proc generateHeaders(requestUrl: Uri, httpMethod: string, headers: HttpHeaders, +proc generateHeaders(requestUrl: Uri, httpMethod: HttpMethod, headers: HttpHeaders, proxy: Proxy): string = # GET - let upperMethod = httpMethod.toUpperAscii() - result = upperMethod + result = $httpMethod result.add ' ' if proxy.isNil or requestUrl.scheme == "https": @@ -898,7 +896,7 @@ proc newConnection(client: HttpClient | AsyncHttpClient, connectUrl.hostname = url.hostname connectUrl.port = if url.port != "": url.port else: "443" - let proxyHeaderString = generateHeaders(connectUrl, $HttpConnect, + let proxyHeaderString = generateHeaders(connectUrl, HttpConnect, newHttpHeaders(), client.proxy) await client.socket.send(proxyHeaderString) let proxyResp = await parseResponse(client, false) @@ -967,14 +965,12 @@ proc override(fallback, override: HttpHeaders): HttpHeaders = for k, vs in override.table: result[k] = vs -proc requestAux(client: HttpClient | AsyncHttpClient, url, httpMethod: string, - body = "", headers: HttpHeaders = nil, +proc requestAux(client: HttpClient | AsyncHttpClient, url: Uri, + httpMethod: HttpMethod, body = "", headers: HttpHeaders = nil, multipart: MultipartData = nil): Future[Response | AsyncResponse] {.multisync.} = # Helper that actually makes the request. Does not handle redirects. - let requestUrl = parseUri(url) - - if requestUrl.scheme == "": + if url.scheme == "": raise newException(ValueError, "No uri scheme supplied.") var data: seq[string] @@ -992,13 +988,13 @@ proc requestAux(client: HttpClient | AsyncHttpClient, url, httpMethod: string, await client.parseBodyFut client.parseBodyFut = nil - await newConnection(client, requestUrl) + await newConnection(client, url) let newHeaders = client.headers.override(headers) if not newHeaders.hasKey("user-agent") and client.userAgent.len > 0: newHeaders["User-Agent"] = client.userAgent - let headerString = generateHeaders(requestUrl, httpMethod, newHeaders, + let headerString = generateHeaders(url, httpMethod, newHeaders, client.proxy) await client.socket.send(headerString) @@ -1020,12 +1016,13 @@ proc requestAux(client: HttpClient | AsyncHttpClient, url, httpMethod: string, elif body.len > 0: await client.socket.send(body) - let getBody = httpMethod.toLowerAscii() notin ["head", "connect"] and + let getBody = httpMethod notin {HttpHead, HttpConnect} and client.getBody result = await parseResponse(client, getBody) -proc request*(client: HttpClient | AsyncHttpClient, url: string, - httpMethod: string, body = "", headers: HttpHeaders = nil, +proc request*(client: HttpClient | AsyncHttpClient, url: Uri | string, + httpMethod: HttpMethod | string = HttpGet, body = "", + headers: HttpHeaders = nil, multipart: MultipartData = nil): Future[Response | AsyncResponse] {.multisync.} = ## Connects to the hostname specified by the URL and performs a request @@ -1040,33 +1037,87 @@ proc request*(client: HttpClient | AsyncHttpClient, url: string, ## ## You need to make sure that the ``url`` doesn't contain any newline ## characters. Failing to do so will raise ``AssertionDefect``. - doAssert(not url.contains({'\c', '\L'}), "url shouldn't contain any newline characters") + ## + ## **Deprecated since v1.5**: use HttpMethod enum instead; string parameter httpMethod is deprecated + when url is string: + doAssert(not url.contains({'\c', '\L'}), "url shouldn't contain any newline characters") + let url = parseUri(url) + + when httpMethod is string: + {.warning: + "Deprecated since v1.5; use HttpMethod enum instead; string parameter httpMethod is deprecated".} + let httpMethod = case httpMethod + of "HEAD": + HttpHead + of "GET": + HttpGet + of "POST": + HttpPost + of "PUT": + HttpPut + of "DELETE": + HttpDelete + of "TRACE": + HttpTrace + of "OPTIONS": + HttpOptions + of "CONNECT": + HttpConnect + of "PATCH": + HttpPatch + else: + raise newException(ValueError, "Invalid HTTP method name: " & httpMethod) result = await client.requestAux(url, httpMethod, body, headers, multipart) var lastURL = url for i in 1..client.maxRedirects: - if result.status.redirection(): - let redirectTo = getNewLocation(lastURL, result.headers) - # Guarantee method for HTTP 307: see https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/307 - var meth = if result.status == "307": httpMethod else: "GET" - result = await client.requestAux(redirectTo, meth, body, headers, multipart) - lastURL = redirectTo + let statusCode = result.code -proc request*(client: HttpClient | AsyncHttpClient, url: string, - httpMethod = HttpGet, body = "", headers: HttpHeaders = nil, - multipart: MultipartData = nil): Future[Response | AsyncResponse] - {.multisync.} = - ## Connects to the hostname specified by the URL and performs a request - ## using the method specified. - ## - ## Connection will be kept alive. Further requests on the same ``client`` to - ## the same hostname will not require a new connection to be made. The - ## connection can be closed by using the ``close`` procedure. - ## - ## When a request is made to a different hostname, the current connection will - ## be closed. - result = await request(client, url, $httpMethod, body, headers, multipart) + if statusCode notin {Http301, Http302, Http303, Http307, Http308}: + break + + let redirectTo = getNewLocation(lastURL, result.headers) + var redirectMethod: HttpMethod + var redirectBody: string + # For more informations about the redirect methods see: + # https://developer.mozilla.org/en-US/docs/Web/HTTP/Redirections + case statusCode + of Http301, Http302, Http303: + # The method is changed to GET unless it is GET or HEAD (RFC2616) + if httpMethod notin {HttpGet, HttpHead}: + redirectMethod = HttpGet + else: + redirectMethod = httpMethod + # The body is stripped away + redirectBody = "" + # Delete any header value associated with the body + if not headers.isNil(): + headers.del("Content-Length") + headers.del("Content-Type") + headers.del("Transfer-Encoding") + of Http307, Http308: + # The method and the body are unchanged + redirectMethod = httpMethod + redirectBody = body + else: + # Unreachable + doAssert(false) + + # Check if the redirection is to the same domain or a sub-domain (foo.com + # -> sub.foo.com) + if redirectTo.hostname != lastURL.hostname and + not redirectTo.hostname.endsWith("." & lastURL.hostname): + # Perform some cleanup of the header values + if headers != nil: + # Delete the Host header + headers.del("Host") + # Do not send any sensitive info to a unknown host + headers.del("Authorization") + + result = await client.requestAux(redirectTo, redirectMethod, redirectBody, + headers, multipart) + lastURL = redirectTo proc responseContent(resp: Response | AsyncResponse): Future[string] {.multisync.} = ## Returns the content of a response as a string. @@ -1079,79 +1130,79 @@ proc responseContent(resp: Response | AsyncResponse): Future[string] {.multisync return await resp.bodyStream.readAll() proc head*(client: HttpClient | AsyncHttpClient, - url: string): Future[Response | AsyncResponse] {.multisync.} = + url: Uri | string): Future[Response | AsyncResponse] {.multisync.} = ## Connects to the hostname specified by the URL and performs a HEAD request. ## ## This procedure uses httpClient values such as ``client.maxRedirects``. result = await client.request(url, HttpHead) proc get*(client: HttpClient | AsyncHttpClient, - url: string): Future[Response | AsyncResponse] {.multisync.} = + url: Uri | string): Future[Response | AsyncResponse] {.multisync.} = ## Connects to the hostname specified by the URL and performs a GET request. ## ## This procedure uses httpClient values such as ``client.maxRedirects``. result = await client.request(url, HttpGet) proc getContent*(client: HttpClient | AsyncHttpClient, - url: string): Future[string] {.multisync.} = + url: Uri | string): Future[string] {.multisync.} = ## Connects to the hostname specified by the URL and returns the content of a GET request. let resp = await get(client, url) return await responseContent(resp) proc delete*(client: HttpClient | AsyncHttpClient, - url: string): Future[Response | AsyncResponse] {.multisync.} = + url: Uri | string): Future[Response | AsyncResponse] {.multisync.} = ## Connects to the hostname specified by the URL and performs a DELETE request. ## This procedure uses httpClient values such as ``client.maxRedirects``. result = await client.request(url, HttpDelete) proc deleteContent*(client: HttpClient | AsyncHttpClient, - url: string): Future[string] {.multisync.} = + url: Uri | string): Future[string] {.multisync.} = ## Connects to the hostname specified by the URL and returns the content of a DELETE request. let resp = await delete(client, url) return await responseContent(resp) -proc post*(client: HttpClient | AsyncHttpClient, url: string, body = "", +proc post*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "", multipart: MultipartData = nil): Future[Response | AsyncResponse] {.multisync.} = ## Connects to the hostname specified by the URL and performs a POST request. ## This procedure uses httpClient values such as ``client.maxRedirects``. - result = await client.request(url, $HttpPost, body, multipart=multipart) + result = await client.request(url, HttpPost, body, multipart=multipart) -proc postContent*(client: HttpClient | AsyncHttpClient, url: string, body = "", +proc postContent*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "", multipart: MultipartData = nil): Future[string] {.multisync.} = ## Connects to the hostname specified by the URL and returns the content of a POST request. let resp = await post(client, url, body, multipart) return await responseContent(resp) -proc put*(client: HttpClient | AsyncHttpClient, url: string, body = "", +proc put*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "", multipart: MultipartData = nil): Future[Response | AsyncResponse] {.multisync.} = ## Connects to the hostname specified by the URL and performs a PUT request. ## This procedure uses httpClient values such as ``client.maxRedirects``. - result = await client.request(url, $HttpPut, body, multipart=multipart) + result = await client.request(url, HttpPut, body, multipart=multipart) -proc putContent*(client: HttpClient | AsyncHttpClient, url: string, body = "", +proc putContent*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "", multipart: MultipartData = nil): Future[string] {.multisync.} = ## Connects to the hostname specified by the URL andreturns the content of a PUT request. let resp = await put(client, url, body, multipart) return await responseContent(resp) -proc patch*(client: HttpClient | AsyncHttpClient, url: string, body = "", +proc patch*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "", multipart: MultipartData = nil): Future[Response | AsyncResponse] {.multisync.} = ## Connects to the hostname specified by the URL and performs a PATCH request. ## This procedure uses httpClient values such as ``client.maxRedirects``. - result = await client.request(url, $HttpPatch, body, multipart=multipart) + result = await client.request(url, HttpPatch, body, multipart=multipart) -proc patchContent*(client: HttpClient | AsyncHttpClient, url: string, body = "", +proc patchContent*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "", multipart: MultipartData = nil): Future[string] {.multisync.} = ## Connects to the hostname specified by the URL and returns the content of a PATCH request. let resp = await patch(client, url, body, multipart) return await responseContent(resp) -proc downloadFile*(client: HttpClient, url: string, filename: string) = +proc downloadFile*(client: HttpClient, url: Uri | string, filename: string) = ## Downloads ``url`` and saves it to ``filename``. client.getBody = false defer: @@ -1167,10 +1218,10 @@ proc downloadFile*(client: HttpClient, url: string, filename: string) = if resp.code.is4xx or resp.code.is5xx: raise newException(HttpRequestError, resp.status) -proc downloadFile*(client: AsyncHttpClient, url: string, +proc downloadFile*(client: AsyncHttpClient, url: Uri | string, filename: string): Future[void] = proc downloadFileEx(client: AsyncHttpClient, - url, filename: string): Future[void] {.async.} = + url: Uri | string, filename: string): Future[void] {.async.} = ## Downloads ``url`` and saves it to ``filename``. client.getBody = false let resp = await client.get(url) diff --git a/lib/pure/httpcore.nim b/lib/pure/httpcore.nim index 9287e9864..774de1260 100644 --- a/lib/pure/httpcore.nim +++ b/lib/pure/httpcore.nim @@ -29,24 +29,26 @@ type HttpVer11, HttpVer10 - HttpMethod* = enum ## the requested HttpMethod - HttpHead, ## Asks for the response identical to the one that would - ## correspond to a GET request, but without the response - ## body. - HttpGet, ## Retrieves the specified resource. - HttpPost, ## Submits data to be processed to the identified - ## resource. The data is included in the body of the - ## request. - HttpPut, ## Uploads a representation of the specified resource. - HttpDelete, ## Deletes the specified resource. - HttpTrace, ## Echoes back the received request, so that a client - ## can see what intermediate servers are adding or - ## changing in the request. - HttpOptions, ## Returns the HTTP methods that the server supports - ## for specified address. - HttpConnect, ## Converts the request connection to a transparent - ## TCP/IP tunnel, usually used for proxies. - HttpPatch ## Applies partial modifications to a resource. + HttpMethod* = enum ## the requested HttpMethod + HttpHead = "HEAD" ## Asks for the response identical to the one that + ## would correspond to a GET request, but without + ## the response body. + HttpGet = "GET" ## Retrieves the specified resource. + HttpPost = "POST" ## Submits data to be processed to the identified + ## resource. The data is included in the body of + ## the request. + HttpPut = "PUT" ## Uploads a representation of the specified + ## resource. + HttpDelete = "DELETE" ## Deletes the specified resource. + HttpTrace = "TRACE" ## Echoes back the received request, so that a + ## client + ## can see what intermediate servers are adding or + ## changing in the request. + HttpOptions = "OPTIONS" ## Returns the HTTP methods that the server + ## supports for specified address. + HttpConnect = "CONNECT" ## Converts the request connection to a transparent + ## TCP/IP tunnel, usually used for proxies. + HttpPatch = "PATCH" ## Applies partial modifications to a resource. const @@ -150,7 +152,6 @@ func newHttpHeaders*(keyValuePairs: else: result.table[key] = @[pair.val] - func `$`*(headers: HttpHeaders): string {.inline.} = $headers.table @@ -378,21 +379,3 @@ func is4xx*(code: HttpCode): bool {.inline.} = func is5xx*(code: HttpCode): bool {.inline.} = ## Determines whether ``code`` is a 5xx HTTP status code. code.int in {500 .. 599} - -func `$`*(httpMethod: HttpMethod): string {.inline.} = - runnableExamples: - doAssert $HttpHead == "HEAD" - doAssert $HttpPatch == "PATCH" - doAssert $HttpGet == "GET" - doAssert $HttpPost == "POST" - - result = case httpMethod - of HttpHead: "HEAD" - of HttpGet: "GET" - of HttpPost: "POST" - of HttpPut: "PUT" - of HttpDelete: "DELETE" - of HttpTrace: "TRACE" - of HttpOptions: "OPTIONS" - of HttpConnect: "CONNECT" - of HttpPatch: "PATCH" diff --git a/tests/stdlib/thttpclient.nim b/tests/stdlib/thttpclient.nim index 6a634d90f..4881370ee 100644 --- a/tests/stdlib/thttpclient.nim +++ b/tests/stdlib/thttpclient.nim @@ -39,7 +39,7 @@ proc makeIPv6HttpServer(hostname: string, port: Port, proc asyncTest() {.async.} = var client = newAsyncHttpClient() - var resp = await client.request("http://example.com/") + var resp = await client.request("http://example.com/", HttpGet) doAssert(resp.code.is2xx) var body = await resp.body body = await resp.body # Test caching @@ -48,7 +48,7 @@ proc asyncTest() {.async.} = resp = await client.request("http://example.com/404") doAssert(resp.code.is4xx) doAssert(resp.code == Http404) - doAssert(resp.status == Http404) + doAssert(resp.status == $Http404) resp = await client.request("https://google.com/") doAssert(resp.code.is2xx or resp.code.is3xx) @@ -102,14 +102,14 @@ proc asyncTest() {.async.} = proc syncTest() = var client = newHttpClient() - var resp = client.request("http://example.com/") + var resp = client.request("http://example.com/", HttpGet) doAssert(resp.code.is2xx) doAssert("Example Domain" in resp.body) resp = client.request("http://example.com/404") doAssert(resp.code.is4xx) doAssert(resp.code == Http404) - doAssert(resp.status == Http404) + doAssert(resp.status == $Http404) resp = client.request("https://google.com/") doAssert(resp.code.is2xx or resp.code.is3xx) From f82100ac93ddb8977d2a404d2cd85975114f50ab Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Sun, 10 Jan 2021 10:19:40 -0600 Subject: [PATCH 0022/2472] fix broken CI (#16663) --- lib/pure/httpclient.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index ea847ef8d..6acf8d9cd 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -979,7 +979,7 @@ proc requestAux(client: HttpClient | AsyncHttpClient, url: Uri, else: if body.len != 0: client.headers["Content-Length"] = $body.len - elif httpMethod notin ["GET", "HEAD"] and not client.headers.hasKey("Content-Length"): + elif httpMethod notin [HttpGet, HttpHead] and not client.headers.hasKey("Content-Length"): client.headers["Content-Length"] = "0" when client is AsyncHttpClient: From 2c6f5ae6815367dcb44e441c8f3bda2e2513b061 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Sun, 10 Jan 2021 15:51:29 -0600 Subject: [PATCH 0023/2472] fix #16650 (#16660) --- compiler/semfold.nim | 7 ++++++- tests/system/tostring.nim | 19 ++++++++++++------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/compiler/semfold.nim b/compiler/semfold.nim index 3d0a9d0ae..4cc2dcba0 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -15,6 +15,8 @@ import platform, math, msgs, idents, renderer, types, commands, magicsys, modulegraphs, strtabs, lineinfos +from system/memory import nimCStrLen + proc errorType*(g: ModuleGraph): PType = ## creates a type representing an error state result = newType(tyError, nextTypeId(g.idgen), g.owners[^1]) @@ -133,7 +135,10 @@ proc evalOp(m: TMagic, n, a, b, c: PNode; g: ModuleGraph): PNode = if a.kind == nkNilLit: result = newIntNodeT(Zero, n, g) elif a.kind in {nkStrLit..nkTripleStrLit}: - result = newIntNodeT(toInt128(a.strVal.len), n, g) + if a.typ.kind == tyString: + result = newIntNodeT(toInt128(a.strVal.len), n, g) + elif a.typ.kind == tyCString: + result = newIntNodeT(toInt128(nimCStrLen(a.strVal)), n, g) else: result = newIntNodeT(toInt128(a.len), n, g) of mUnaryPlusI, mUnaryPlusF64: result = a # throw `+` away diff --git a/tests/system/tostring.nim b/tests/system/tostring.nim index 4ff363075..fa82acc3b 100644 --- a/tests/system/tostring.nim +++ b/tests/system/tostring.nim @@ -1,7 +1,3 @@ -discard """ - output: "DONE: tostring.nim" -""" - doAssert "@[23, 45]" == $(@[23, 45]) doAssert "[32, 45]" == $([32, 45]) doAssert """@["", "foo", "bar"]""" == $(@["", "foo", "bar"]) @@ -108,7 +104,7 @@ bar(nilstring) static: stringCompare() -# bug 8847 +# issue #8847 var a2: cstring = "fo\"o2" block: @@ -116,5 +112,14 @@ block: s.addQuoted a2 doAssert s == "\"fo\\\"o2\"" - -echo "DONE: tostring.nim" +# issue #16650 +template fn() = + doAssert len(cstring"ab\0c") == 5 + doAssert len(cstring("ab\0c")) == 2 + when nimvm: + discard + else: + let c = cstring("ab\0c") + doAssert len(c) == 2 +fn() +static: fn() From 510e383d9233641a82ef25d3e3fe3e2eba3a4388 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Mon, 11 Jan 2021 00:26:52 -0600 Subject: [PATCH 0024/2472] add error messages (#16679) --- testament/testament.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/testament.nim b/testament/testament.nim index 5686da81c..314a83936 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -795,7 +795,7 @@ proc main() = var subPath = p.key.string let nimRoot = currentSourcePath / "../.." # makes sure points to this regardless of cwd or which nim is used to compile this. - doAssert dirExists(nimRoot/testsDir) # sanity check + doAssert(dirExists(nimRoot/testsDir), nimRoot/testsDir & " doesn't exist!") # sanity check if subPath.isAbsolute: subPath = subPath.relativePath(nimRoot) # at least one directory is required in the path, to use as a category name let pathParts = subPath.relativePath(testsDir).split({DirSep, AltSep}) From 0286a0879bc44e5267a5fd36e6f4aac8f78713ea Mon Sep 17 00:00:00 2001 From: cooldome Date: Mon, 11 Jan 2021 09:09:38 +0000 Subject: [PATCH 0025/2472] fix #16651 (#16658) * fix #16651 --- compiler/semtypes.nim | 7 +++++++ tests/converter/tgenericconverter.nim | 24 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 64113a4c6..45f20da9b 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -1059,6 +1059,13 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode, # disable the bindOnce behavior for the type class result = recurse(paramType.base, true) + of tyTuple: + for i in 0.. Date: Mon, 11 Jan 2021 01:16:20 -0800 Subject: [PATCH 0026/2472] fix #16555, fixes #16405: len, high honors '\0' for cstring in vm (#16610) --- compiler/vm.nim | 4 ++++ compiler/vmdef.nim | 1 + compiler/vmgen.nim | 12 +++++++----- lib/system.nim | 20 +++++++++++++------- tests/stdlib/thashes.nim | 18 ++++++++++++++++-- tests/stdlib/tstring.nim | 34 +++++++++++++++++++++++++++++++++- 6 files changed, 74 insertions(+), 15 deletions(-) diff --git a/compiler/vm.nim b/compiler/vm.nim index 0f75faee3..b8df456e0 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -872,6 +872,10 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = decodeBImm(rkInt) assert regs[rb].kind == rkNode regs[ra].intVal = regs[rb].node.strVal.len - imm + of opcLenCstring: + decodeBImm(rkInt) + assert regs[rb].kind == rkNode + regs[ra].intVal = regs[rb].node.strVal.cstring.len - imm of opcIncl: decodeB(rkNode) let b = regs[rb].regToNode diff --git a/compiler/vmdef.nim b/compiler/vmdef.nim index 6d6c55250..eab18f417 100644 --- a/compiler/vmdef.nim +++ b/compiler/vmdef.nim @@ -86,6 +86,7 @@ type opcSubImmInt, opcLenSeq, opcLenStr, + opcLenCstring, opcIncl, opcInclRange, opcExcl, opcCard, opcMulInt, opcDivInt, opcModInt, opcAddFloat, opcSubFloat, opcMulFloat, opcDivFloat, diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 46931eb54..6790276a9 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1034,7 +1034,10 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = of mLengthOpenArray, mLengthArray, mLengthSeq: genUnaryABI(c, n, dest, opcLenSeq) of mLengthStr: - genUnaryABI(c, n, dest, opcLenStr) + case n[1].typ.kind + of tyString: genUnaryABI(c, n, dest, opcLenStr) + of tyCString: genUnaryABI(c, n, dest, opcLenCstring) + else: doAssert false, $n[1].typ.kind of mIncl, mExcl: unused(c, n, dest) var d = c.genx(n[1]) @@ -1178,10 +1181,9 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = if dest < 0: dest = c.getTemp(n.typ) let tmp = c.genx(n[1]) case n[1].typ.skipTypes(abstractVar-{tyTypeDesc}).kind: - of tyString, tyCString: - c.gABI(n, opcLenStr, dest, tmp, 1) - else: - c.gABI(n, opcLenSeq, dest, tmp, 1) + of tyString: c.gABI(n, opcLenStr, dest, tmp, 1) + of tyCString: c.gABI(n, opcLenCstring, dest, tmp, 1) + else: c.gABI(n, opcLenSeq, dest, tmp, 1) c.freeTemp(tmp) of mEcho: unused(c, n, dest) diff --git a/lib/system.nim b/lib/system.nim index e220ba7a3..25184eb15 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -701,18 +701,24 @@ proc len*(x: string): int {.magic: "LengthStr", noSideEffect.} ## var str = "Hello world!" ## echo len(str) # => 12 -proc len*(x: cstring): int {.magic: "LengthStr", noSideEffect.} - ## Returns the length of a compatible string. This is sometimes - ## an O(n) operation. +proc len*(x: cstring): int {.magic: "LengthStr", noSideEffect.} = + ## Returns the length of a compatible string. This is an O(n) operation except + ## in js at runtime. ## ## **Note:** On the JS backend this currently counts UTF-16 code points ## instead of bytes at runtime (not at compile time). For now, if you ## need the byte length of the UTF-8 encoding, convert to string with ## `$` first then call `len`. - ## - ## .. code-block:: Nim - ## var str: cstring = "Hello world!" - ## len(str) # => 12 + runnableExamples: + doAssert len(cstring"abc") == 3 + doAssert len(cstring r"ab\0c") == 5 # \0 is escaped + doAssert len(cstring"ab\0c") == 5 # ditto + var a: cstring = "ab\0c" + when defined(js): doAssert a.len == 4 # len ignores \0 for js + else: doAssert a.len == 2 # \0 is a null terminator + static: + var a2: cstring = "ab\0c" + doAssert a2.len == 2 # \0 is a null terminator, even in js vm proc len*(x: (type array)|array): int {.magic: "LengthArray", noSideEffect.} ## Returns the length of an array or an array type. diff --git a/tests/stdlib/thashes.nim b/tests/stdlib/thashes.nim index 17640387a..ce7bb7d8c 100644 --- a/tests/stdlib/thashes.nim +++ b/tests/stdlib/thashes.nim @@ -86,8 +86,6 @@ block largeSize: # longer than 4 characters doAssert hash(xx, 0, 3) == hash(ssl, 0, 3) proc main() = - - doAssert hash(0.0) == hash(0) doAssert hash(cstring"abracadabra") == 97309975 doAssert hash(cstring"abracadabra") == hash("abracadabra") @@ -115,6 +113,22 @@ proc main() = doAssert hash(-9999.283456) != 0 doAssert hash(84375674.0) != 0 + block: # bug #16555 + proc fn(): auto = + # avoids hardcoding values + var a = "abc\0def" + var b = a.cstring + result = (hash(a), hash(b)) + doAssert result[0] != result[1] + when not defined(js): + doAssert fn() == static(fn()) + else: + # xxx this is a tricky case; consistency of hashes for cstring's containing + # '\0\' matters for c backend but less for js backend since such strings + # are much less common in js backend; we make vm for js backend consistent + # with c backend instead of js backend because FFI code (or other) could + # run at CT, expecting c semantics. + discard static: main() main() diff --git a/tests/stdlib/tstring.nim b/tests/stdlib/tstring.nim index 4d5a15940..fcbacc533 100644 --- a/tests/stdlib/tstring.nim +++ b/tests/stdlib/tstring.nim @@ -96,11 +96,43 @@ func reverse*(a: string): string = proc main() = + # xxx put all tests here to test in VM and RT test_string_slice() test_string_cmp() tester(1) - doAssert reverse("hello") == "olleh" + + block: # reverse + doAssert reverse("hello") == "olleh" + + block: # len, high + var a = "ab\0cd" + var b = a.cstring + doAssert a.len == 5 + block: # bug #16405 + when defined(js): + when nimvm: doAssert b.len == 2 + else: doAssert b.len == 5 + else: doAssert b.len == 2 + + doAssert a.high == a.len - 1 + doAssert b.high == b.len - 1 + + doAssert "".len == 0 + doAssert "".high == -1 + doAssert "".cstring.len == 0 + doAssert "".cstring.high == -1 + + var c: cstring = nil + template impl() = + doAssert c.len == 0 + doAssert c.high == -1 + when defined js: + when nimvm: impl() + else: + # xxx pending bug #16674 + discard + else: impl() static: main() main() From bbc96f974d643b5ab4e2b9d2855e94ef30ed3ee4 Mon Sep 17 00:00:00 2001 From: Saem Ghani Date: Mon, 11 Jan 2021 01:18:01 -0800 Subject: [PATCH 0027/2472] fixed nim-lang/nimsuggest#82 pure enum field sug (#16676) - previous code wasn't account for tyEnum being wrapped in tyTypeDesc - now pure enum fields are suggested --- compiler/suggest.nim | 21 +++++++++++---------- nimsuggest/tests/tsug_enum.nim | 18 ++++++++++++++++++ 2 files changed, 29 insertions(+), 10 deletions(-) create mode 100644 nimsuggest/tests/tsug_enum.nim diff --git a/compiler/suggest.nim b/compiler/suggest.nim index 560d20b3f..283690080 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -386,17 +386,17 @@ proc suggestFieldAccess(c: PContext, n, field: PNode, outputs: var Suggestions) else: # fallback: suggestEverything(c, n, field, outputs) - elif typ.kind == tyEnum and n.kind == nkSym and n.sym.kind == skType: - # look up if the identifier belongs to the enum: - var t = typ - while t != nil: - suggestSymList(c, t.n, field, n.info, outputs) - t = t[0] - suggestOperations(c, n, field, typ, outputs) else: - let orig = typ # skipTypes(typ, {tyGenericInst, tyAlias, tySink}) - typ = skipTypes(typ, {tyGenericInst, tyVar, tyLent, tyPtr, tyRef, tyAlias, tySink, tyOwned}) - if typ.kind == tyObject: + let orig = typ + typ = skipTypes(orig, {tyTypeDesc, tyGenericInst, tyVar, tyLent, tyPtr, tyRef, tyAlias, tySink, tyOwned}) + + if typ.kind == tyEnum and n.kind == nkSym and n.sym.kind == skType: + # look up if the identifier belongs to the enum: + var t = typ + while t != nil: + suggestSymList(c, t.n, field, n.info, outputs) + t = t[0] + elif typ.kind == tyObject: var t = typ while true: suggestObject(c, t.n, field, n.info, outputs) @@ -404,6 +404,7 @@ proc suggestFieldAccess(c: PContext, n, field: PNode, outputs: var Suggestions) t = skipTypes(t[0], skipPtrs) elif typ.kind == tyTuple and typ.n != nil: suggestSymList(c, typ.n, field, n.info, outputs) + suggestOperations(c, n, field, orig, outputs) if typ != orig: suggestOperations(c, n, field, typ, outputs) diff --git a/nimsuggest/tests/tsug_enum.nim b/nimsuggest/tests/tsug_enum.nim new file mode 100644 index 000000000..97a225f16 --- /dev/null +++ b/nimsuggest/tests/tsug_enum.nim @@ -0,0 +1,18 @@ +## suggestions for enums + +type + LogLevel {.pure.} = enum + debug, log, warn, error + + FooBar = enum + fbFoo, fbBar + +echo fbFoo, fbBar + +echo LogLevel.deb#[!]# + +discard """ +$nimsuggest --tester $file +>sug $1 +sug;;skEnumField;;debug;;LogLevel;;*nimsuggest/tests/tsug_enum.nim;;5;;4;;"";;100;;Prefix +""" \ No newline at end of file From 5897ed9d3d5ca5f84423e87a70addc8c6764923e Mon Sep 17 00:00:00 2001 From: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Date: Mon, 11 Jan 2021 10:53:15 +0100 Subject: [PATCH 0028/2472] Improve documentation of strmisc (#16665) Simplify examples --- lib/pure/strmisc.nim | 60 ++++++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/lib/pure/strmisc.nim b/lib/pure/strmisc.nim index 5060deb78..c8cd839be 100644 --- a/lib/pure/strmisc.nim +++ b/lib/pure/strmisc.nim @@ -8,12 +8,12 @@ # ## This module contains various string utility routines that are uncommonly -## used in comparison to `strutils `_. +## used in comparison to the ones in `strutils `_. -import strutils +import std/strutils -proc expandTabs*(s: string, tabSize: int = 8): string {.noSideEffect.} = - ## Expand tab characters in `s` replacing them by spaces. +func expandTabs*(s: string, tabSize: int = 8): string = + ## Expands tab characters in `s`, replacing them by spaces. ## ## The amount of inserted spaces for each tab character is the difference ## between the current column number and the next tab position. Tab positions @@ -24,9 +24,7 @@ proc expandTabs*(s: string, tabSize: int = 8): string {.noSideEffect.} = runnableExamples: doAssert expandTabs("\t", 4) == " " doAssert expandTabs("\tfoo\t", 4) == " foo " - doAssert expandTabs("\tfoo\tbar", 4) == " foo bar" - doAssert expandTabs("\tfoo\tbar\t", 4) == " foo bar " - doAssert expandTabs("ab\tcd\n\txy\t", 3) == "ab cd\n xy " + doAssert expandTabs("a\tb\n\txy\t", 3) == "a b\n xy " result = newStringOfCap(s.len + s.len shr 2) var pos = 0 @@ -50,37 +48,39 @@ proc expandTabs*(s: string, tabSize: int = 8): string {.noSideEffect.} = if c == '\l': pos = 0 -proc partition*(s: string, sep: string, - right: bool = false): (string, string, string) - {.noSideEffect.} = - ## Split the string at the first or last occurrence of `sep` into a 3-tuple +func partition*(s: string, sep: string, + right: bool = false): (string, string, string) = + ## Splits the string at the first (if `right` is false) + ## or last (if `right` is true) occurrence of `sep` into a 3-tuple. ## - ## Returns a 3 string tuple of (beforeSep, `sep`, afterSep) or - ## (`s`, "", "") if `sep` is not found and `right` is false or - ## ("", "", `s`) if `sep` is not found and `right` is true + ## Returns a 3-tuple of strings, `(beforeSep, sep, afterSep)` or + ## `(s, "", "")` if `sep` is not found and `right` is false or + ## `("", "", s)` if `sep` is not found and `right` is true. + ## + ## **See also:** + ## * `rpartition proc <#rpartition,string,string>`_ runnableExamples: - doAssert partition("foo:bar", ":") == ("foo", ":", "bar") - doAssert partition("foobarbar", "bar") == ("foo", "bar", "bar") - doAssert partition("foobarbar", "bank") == ("foobarbar", "", "") - doAssert partition("foobarbar", "foo") == ("", "foo", "barbar") - doAssert partition("foofoobar", "bar") == ("foofoo", "bar", "") + doAssert partition("foo:bar:baz", ":") == ("foo", ":", "bar:baz") + doAssert partition("foo:bar:baz", ":", right = true) == ("foo:bar", ":", "baz") + doAssert partition("foobar", ":") == ("foobar", "", "") + doAssert partition("foobar", ":", right = true) == ("", "", "foobar") let position = if right: s.rfind(sep) else: s.find(sep) if position != -1: return (s[0 ..< position], sep, s[position + sep.len ..< s.len]) return if right: ("", "", s) else: (s, "", "") -proc rpartition*(s: string, sep: string): (string, string, string) - {.noSideEffect.} = - ## Split the string at the last occurrence of `sep` into a 3-tuple +func rpartition*(s: string, sep: string): (string, string, string) = + ## Splits the string at the last occurrence of `sep` into a 3-tuple. ## - ## Returns a 3 string tuple of (beforeSep, `sep`, afterSep) or - ## ("", "", `s`) if `sep` is not found + ## Returns a 3-tuple of strings, `(beforeSep, sep, afterSep)` or + ## `("", "", s)` if `sep` is not found. This is the same as + ## `partition(s, sep, right = true)`. + ## + ## **See also:** + ## * `partition proc <#partition,string,string,bool>`_ runnableExamples: - doAssert rpartition("foo:bar", ":") == ("foo", ":", "bar") - doAssert rpartition("foobarbar", "bar") == ("foobar", "bar", "") - doAssert rpartition("foobarbar", "bank") == ("", "", "foobarbar") - doAssert rpartition("foobarbar", "foo") == ("", "foo", "barbar") - doAssert rpartition("foofoobar", "bar") == ("foofoo", "bar", "") + doAssert rpartition("foo:bar:baz", ":") == ("foo:bar", ":", "baz") + doAssert rpartition("foobar", ":") == ("", "", "foobar") - return partition(s, sep, right = true) + partition(s, sep, right = true) From be6e8916faa51d227d51e1291d1f24751385b010 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Mon, 11 Jan 2021 04:39:38 -0600 Subject: [PATCH 0029/2472] fix negative nan (#16628) --- compiler/jsgen.nim | 5 ++++- compiler/rodutils.nim | 13 +++++++++++-- tests/stdlib/tmath.nim | 16 ++++++++++++++-- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index f588f9555..10a33423e 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -2485,7 +2485,10 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) = let f = n.floatVal case classify(f) of fcNan: - r.res = rope"NaN" + if signbit(f): + r.res = rope"-NaN" + else: + r.res = rope"NaN" of fcNegZero: r.res = rope"-0.0" of fcZero: diff --git a/compiler/rodutils.nim b/compiler/rodutils.nim index 7070e6c3f..353992fca 100644 --- a/compiler/rodutils.nim +++ b/compiler/rodutils.nim @@ -8,7 +8,7 @@ # ## Serialization utilities for the compiler. -import strutils, math +import std/[strutils, math] # bcc on windows doesn't have C99 functions when defined(windows) and defined(bcc): @@ -33,10 +33,19 @@ when defined(windows) and defined(bcc): proc c_snprintf(s: cstring; n:uint; frmt: cstring): cint {.importc: "snprintf", header: "", nodecl, varargs.} + +when not declared(signbit): + proc c_signbit(x: SomeFloat): cint {.importc: "signbit", header: "".} + proc signbit*(x: SomeFloat): bool {.inline.} = + result = c_signbit(x) != 0 + proc toStrMaxPrecision*(f: BiggestFloat, literalPostfix = ""): string = case classify(f) of fcNan: - result = "NAN" + if signbit(f): + result = "-NAN" + else: + result = "NAN" of fcNegZero: result = "-0.0" & literalPostfix of fcZero: diff --git a/tests/stdlib/tmath.nim b/tests/stdlib/tmath.nim index 62fdcd19f..edab62a66 100644 --- a/tests/stdlib/tmath.nim +++ b/tests/stdlib/tmath.nim @@ -319,6 +319,18 @@ template main = doAssert not Inf.isNaN doAssert isNaN(Inf - Inf) + block: # signbit + let x1 = NaN + let x2 = -NaN + let x3 = -x1 + + doAssert isNaN(x1) + doAssert isNaN(x2) + doAssert isNaN(x3) + doAssert not signbit(x1) + doAssert signbit(x2) + doAssert signbit(x3) + block: # copySign doAssert copySign(10.0, -1.0) == -10.0 doAssert copySign(-10.0, -1.0) == -10.0 @@ -385,8 +397,8 @@ template main = discard else: when not defined(js): - doAssert copySign(-1.0, -NaN) == 1.0 - doAssert copySign(10.0, -NaN) == 10.0 + doAssert copySign(-1.0, -NaN) == -1.0 + doAssert copySign(10.0, -NaN) == -10.0 doAssert copySign(1.0, copySign(NaN, -1.0)) == -1.0 # fails in VM block: From aa185c0e9b4904b305496bd1f83cbbadf2fbfa36 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Mon, 11 Jan 2021 08:07:48 -0600 Subject: [PATCH 0030/2472] fix #13517 (#16681) --- compiler/vm.nim | 6 +++--- tests/distinct/tdistinct_issues.nim | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/compiler/vm.nim b/compiler/vm.nim index b8df456e0..00f0aca7c 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -429,13 +429,13 @@ proc opConv(c: PCtx; dest: var TFullReg, src: TFullReg, desttyp, srctyp: PType): return true of tyUInt..tyUInt64: dest.ensureKind(rkInt) - case skipTypes(srctyp, abstractRange).kind + let styp = srctyp.skipTypes(abstractRange) # skip distinct types(dest type could do this too if needed) + case styp.kind of tyFloat..tyFloat64: dest.intVal = int(src.floatVal) else: - let srcDist = (sizeof(src.intVal) - srctyp.size) * 8 + let srcDist = (sizeof(src.intVal) - styp.size) * 8 let destDist = (sizeof(dest.intVal) - desttyp.size) * 8 - var value = cast[BiggestUInt](src.intVal) value = (value shl srcDist) shr srcDist value = (value shl destDist) shr destDist diff --git a/tests/distinct/tdistinct_issues.nim b/tests/distinct/tdistinct_issues.nim index ce71344d0..747cf0b8d 100644 --- a/tests/distinct/tdistinct_issues.nim +++ b/tests/distinct/tdistinct_issues.nim @@ -65,3 +65,17 @@ block t9322: proc mystr(s: string) = echo s mystr($Fix("apr")) + + +block: # bug #13517 + type MyUint64 = distinct uint64 + + proc `==`(a: MyUint64, b: uint64): bool = uint64(a) == b + + block: + doAssert MyUint64.high is MyUint64 + doAssert MyUint64.high == 18446744073709551615'u64 + + static: + doAssert MyUint64.high is MyUint64 + doAssert MyUint64.high == 18446744073709551615'u64 From 0c128259bb01cde8ed27430929b0a01c5ee7b88c Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Mon, 11 Jan 2021 08:13:13 -0600 Subject: [PATCH 0031/2472] close #7097 add testcase (#16682) --- tests/converter/t7097.nim | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 tests/converter/t7097.nim diff --git a/tests/converter/t7097.nim b/tests/converter/t7097.nim new file mode 100644 index 000000000..fdb573588 --- /dev/null +++ b/tests/converter/t7097.nim @@ -0,0 +1,38 @@ +type + Byte* = uint8 + Bytes* = seq[Byte] + + BytesRange* = object + bytes: Bytes + ibegin, iend: int + +proc initBytesRange*(s: var Bytes, ibegin = 0, iend = -1): BytesRange = + let e = if iend < 0: s.len + iend + 1 + else: iend + assert ibegin > 0 and e <= s.len + + shallow(s) + result.bytes = s + result.ibegin = ibegin + result.iend = e + +template `[]=`*(r: var BytesRange, i: int, v: Byte) = + r.bytes[r.ibegin + i] = v + +converter fromSeq*(s: Bytes): BytesRange = + var seqCopy = s + return initBytesRange(seqCopy) + +type + Reader* = object + data: BytesRange + position: int + +proc readerFromHex*(input: string): Reader = + let totalBytes = input.len div 2 + var backingStore = newSeq[Byte](totalBytes) + result.data = initBytesRange(backingStore) + + for i in 0 ..< totalBytes: + var nextByte = 0 + result.data[i] = Byte(nextByte) # <-------- instantiated from here From 5af13c5aceaedf211ff59113384bcb6a85c35c2b Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Mon, 11 Jan 2021 11:02:32 -0600 Subject: [PATCH 0032/2472] close #9655 add testcase (#16683) --- tests/ccgbugs/t9655.nim | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 tests/ccgbugs/t9655.nim diff --git a/tests/ccgbugs/t9655.nim b/tests/ccgbugs/t9655.nim new file mode 100644 index 000000000..29fb903a4 --- /dev/null +++ b/tests/ccgbugs/t9655.nim @@ -0,0 +1,30 @@ +discard """ + action: "compile" +""" + +import std/[asynchttpserver, asyncdispatch] +import std/[strformat] + +proc main() = + let local = "123" + + proc serveIndex(req: Request) {.async, gcsafe.} = + await req.respond(Http200, &"{local}") + + proc serve404(req: Request) {.async, gcsafe.} = + echo req.url.path + await req.respond(Http404, "not found") + + proc serve(req: Request) {.async, gcsafe.} = + let handler = case req.url.path: + of "/": + serveIndex + else: + serve404 + await handler(req) + + let server = newAsyncHttpServer() + waitFor server.serve(Port(8080), serve, address = "127.0.0.1") + +when isMainModule: + main() From da28df6113cdc552ad2ef3a029638efd9bfdadf2 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Mon, 11 Jan 2021 11:02:53 -0600 Subject: [PATCH 0033/2472] remove deprecated specs (#16684) --- testament/specs.nim | 4 ---- testament/tests/shouldfail/tmsg.nim | 6 ------ 2 files changed, 10 deletions(-) delete mode 100644 testament/tests/shouldfail/tmsg.nim diff --git a/testament/specs.nim b/testament/specs.nim index 58fe7bf4f..48fe51658 100644 --- a/testament/specs.nim +++ b/testament/specs.nim @@ -303,10 +303,6 @@ proc parseSpec*(filename: string): TSpec = of "exitcode": discard parseInt(e.value, result.exitCode) result.action = actionRun - of "msg": - result.msg = e.value - if result.action != actionRun: - result.action = actionCompile of "errormsg": result.msg = e.value result.action = actionReject diff --git a/testament/tests/shouldfail/tmsg.nim b/testament/tests/shouldfail/tmsg.nim deleted file mode 100644 index 4ad17fa95..000000000 --- a/testament/tests/shouldfail/tmsg.nim +++ /dev/null @@ -1,6 +0,0 @@ -discard """ -msg: "Hello World" -""" - -static: - echo "something else" From 335f849c36c4d7e618de277f6da64d8c1fdcc1c9 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Mon, 11 Jan 2021 12:00:57 -0600 Subject: [PATCH 0034/2472] close #9901 add testcase (#16662) * close #9901 add testcase * follow advice --- tests/misc/tproveinit.nim | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 tests/misc/tproveinit.nim diff --git a/tests/misc/tproveinit.nim b/tests/misc/tproveinit.nim new file mode 100644 index 000000000..c9f688309 --- /dev/null +++ b/tests/misc/tproveinit.nim @@ -0,0 +1,18 @@ +discard """ + joinable: false +""" + +{.warningAsError[ProveInit]:on.} +template main() = + proc fn(): var int = + discard + discard fn() +doAssert not compiles(main()) + +# bug #9901 +import std/[sequtils, times] +proc parseMyDates(line: string): DateTime = + result = parse(line, "yyyy-MM-dd") +var dateStrings = @["2018-12-01", "2018-12-02", "2018-12-03"] +var parsed = dateStrings.map(parseMyDates) +discard parsed From fd5c8ef20845a511ad4e0af8dd8ad4331bf46ffc Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Mon, 11 Jan 2021 21:51:04 +0300 Subject: [PATCH 0035/2472] RST: implement internal targets (#16614) --- compiler/docgen.nim | 2 +- config/nimdoc.cfg | 3 +- lib/packages/docutils/rst.nim | 168 +++++++++++++++++++++++-------- lib/packages/docutils/rstast.nim | 14 ++- lib/packages/docutils/rstgen.nim | 135 +++++++++++++++---------- tests/stdlib/trstgen.nim | 158 +++++++++++++++++++++++++++++ 6 files changed, 379 insertions(+), 101 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index d21db7634..f07adc428 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -560,7 +560,7 @@ proc getAllRunnableExamplesImpl(d: PDoc; n: PNode, dest: var Rope, state: Runnab "\n\\textbf{$1}\n", [msg.rope]) inc d.listingCounter let id = $d.listingCounter - dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim"]) + dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim", ""]) var dest2 = "" renderNimCode(dest2, code, isLatex = d.conf.cmd == cmdRst2tex) dest.add dest2 diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index 585b3cc49..ff4dac781 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -206,7 +206,8 @@ $moduledesc $content """ -doc.listing_start = "
"
+# $1 - number of listing in document, $2 - language (e.g. langNim), $3 - anchor
+doc.listing_start = ""
 doc.listing_end = "
" # * $analytics: Google analytics location, includes -Create a ``calculator.nim`` file with the following content (or reuse the one +Create a `calculator.nim` file with the following content (or reuse the one from the previous section): .. code-block:: nim @@ -209,9 +211,9 @@ from the previous section): when isMainModule: echo addTwoIntegers(3, 7) -Compile the Nim code to JavaScript with ``nim js -o:calculator.js -calculator.nim`` and open ``host.html`` in a browser. If the browser supports -javascript, you should see the value ``10`` in the browser's console. Use the +Compile the Nim code to JavaScript with `nim js -o:calculator.js +calculator.nim` and open `host.html` in a browser. If the browser supports +javascript, you should see the value `10` in the browser's console. Use the `dom module `_ for specific DOM querying and modification procs or take a look at `karax `_ for how to develop browser-based applications. @@ -222,29 +224,29 @@ Backend code calling Nim Backend code can interface with Nim code exposed through the `exportc pragma `_. The -``exportc`` pragma is the *generic* way of making Nim symbols available to +`exportc` pragma is the *generic* way of making Nim symbols available to the backends. By default, the Nim compiler will mangle all the Nim symbols to -avoid any name collision, so the most significant thing the ``exportc`` pragma +avoid any name collision, so the most significant thing the `exportc` pragma does is maintain the Nim symbol name, or if specified, use an alternative symbol for the backend in case the symbol rules don't match. The JavaScript target doesn't have any further interfacing considerations since it also has garbage collection, but the C targets require you to -initialize Nim's internals, which is done calling a ``NimMain`` function. +initialize Nim's internals, which is done calling a `NimMain` function. Also, C code requires you to specify a forward declaration for functions or the compiler will assume certain types for the return value and parameters which will likely make your program crash at runtime. -The Nim compiler can generate a C interface header through the ``--header`` +The Nim compiler can generate a C interface header through the `--header` command-line switch. The generated header will contain all the exported -symbols and the ``NimMain`` proc which you need to call before any other +symbols and the `NimMain` proc which you need to call before any other Nim code. Nim invocation example from C ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Create a ``fib.nim`` file with the following content: +Create a `fib.nim` file with the following content: .. code-block:: nim @@ -254,7 +256,7 @@ Create a ``fib.nim`` file with the following content: else: result = fib(a - 1) + fib(a - 2) -Create a ``maths.c`` file with the following content: +Create a `maths.c` file with the following content: .. code-block:: c @@ -277,30 +279,30 @@ program:: $ gcc -o m -I$HOME/.cache/nim/fib_d -Ipath/to/nim/lib $HOME/.cache/nim/fib_d/*.c maths.c The first command runs the Nim compiler with three special options to avoid -generating a ``main()`` function in the generated files, avoid linking the +generating a `main()` function in the generated files, avoid linking the object files into a final binary, and explicitly generate a header file for C -integration. All the generated files are placed into the ``nimcache`` -directory. That's why the next command compiles the ``maths.c`` source plus -all the ``.c`` files from ``nimcache``. In addition to this path, you also -have to tell the C compiler where to find Nim's ``nimbase.h`` header file. +integration. All the generated files are placed into the `nimcache` +directory. That's why the next command compiles the `maths.c` source plus +all the `.c` files from `nimcache`. In addition to this path, you also +have to tell the C compiler where to find Nim's `nimbase.h` header file. -Instead of depending on the generation of the individual ``.c`` files you can +Instead of depending on the generation of the individual `.c` files you can also ask the Nim compiler to generate a statically linked library:: $ nim c --app:staticLib --noMain --header fib.nim $ gcc -o m -Inimcache -Ipath/to/nim/lib libfib.nim.a maths.c The Nim compiler will handle linking the source files generated in the -``nimcache`` directory into the ``libfib.nim.a`` static library, which you can +`nimcache` directory into the `libfib.nim.a` static library, which you can then link into your C program. Note that these commands are generic and will vary for each system. For instance, on Linux systems you will likely need to -use ``-ldl`` too to link in required dlopen functionality. +use `-ldl` too to link in required dlopen functionality. Nim invocation example from JavaScript ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Create a ``mhost.html`` file with the following content: +Create a `mhost.html` file with the following content: .. code-block:: @@ -311,7 +313,7 @@ Create a ``mhost.html`` file with the following content: -Create a ``fib.nim`` file with the following content (or reuse the one +Create a `fib.nim` file with the following content (or reuse the one from the previous section): .. code-block:: nim @@ -322,10 +324,10 @@ from the previous section): else: result = fib(a - 1) + fib(a - 2) -Compile the Nim code to JavaScript with ``nim js -o:fib.js fib.nim`` and -open ``mhost.html`` in a browser. If the browser supports javascript, you -should see an alert box displaying the text ``Fib for 9 is 34``. As mentioned -earlier, JavaScript doesn't require an initialization call to ``NimMain`` or +Compile the Nim code to JavaScript with `nim js -o:fib.js fib.nim` and +open `mhost.html` in a browser. If the browser supports javascript, you +should see an alert box displaying the text `Fib for 9 is 34`. As mentioned +earlier, JavaScript doesn't require an initialization call to `NimMain` or a similar function and you can call the exported Nim proc directly. @@ -335,14 +337,14 @@ Nimcache naming logic The `nimcache`:idx: directory is generated during compilation and will hold either temporary or final files depending on your backend target. The default name for the directory depends on the used backend and on your OS but you can -use the ``--nimcache`` `compiler switch +use the `--nimcache` `compiler switch `_ to change it. Memory management ================= -In the previous sections, the ``NimMain()`` function reared its head. Since +In the previous sections, the `NimMain()` function reared its head. Since JavaScript already provides automatic memory management, you can freely pass objects between the two languages without problems. In C and derivate languages you need to be careful about what you do and how you share memory. The @@ -357,15 +359,15 @@ Strings and C strings The manual mentions that `Nim strings are implicitly convertible to cstrings `_ which makes interaction usually painless. Most C functions accepting a Nim string converted to a -``cstring`` will likely not need to keep this string around and by the time +`cstring` will likely not need to keep this string around and by the time they return the string won't be needed anymore. However, for the rare cases where a Nim string has to be preserved and made available to the C backend -as a ``cstring``, you will need to manually prevent the string data from being +as a `cstring`, you will need to manually prevent the string data from being freed with `GC_ref `_ and `GC_unref `_. A similar thing happens with C code invoking Nim code which returns a -``cstring``. Consider the following proc: +`cstring`. Consider the following proc: .. code-block:: nim @@ -373,8 +375,8 @@ A similar thing happens with C code invoking Nim code which returns a result = "Hey there C code! " & $rand(100) Since Nim's garbage collector is not aware of the C code, once the -``gimme`` proc has finished it can reclaim the memory of the ``cstring``. -However, from a practical standpoint, the C code invoking the ``gimme`` +`gimme` proc has finished it can reclaim the memory of the `cstring`. +However, from a practical standpoint, the C code invoking the `gimme` function directly will be able to use it since Nim's garbage collector has not had a chance to run *yet*. This gives you enough time to make a copy for the C side of the program, as calling any further Nim procs *might* trigger @@ -397,14 +399,14 @@ Again, if you are wrapping a library which *mallocs* and *frees* data structures, you need to expose the appropriate *free* function to Nim so you can clean it up. And of course, once cleaned you should avoid accessing it from Nim (or C for that matter). Typically C data structures have their own -``malloc_structure`` and ``free_structure`` specific functions, so wrapping +`malloc_structure` and `free_structure` specific functions, so wrapping these for the Nim side should be enough. Thread coordination ------------------- -When the ``NimMain()`` function is called Nim initializes the garbage +When the `NimMain()` function is called Nim initializes the garbage collector to the current thread, which is usually the main thread of your application. If your C code later spawns a different thread and calls Nim code, the garbage collector will fail to work properly and you will crash. diff --git a/doc/destructors.rst b/doc/destructors.rst index 4f06de455..01e2d2ee9 100644 --- a/doc/destructors.rst +++ b/doc/destructors.rst @@ -1,3 +1,5 @@ +.. default-role:: code + ================================== Nim Destructors and Move Semantics ================================== @@ -16,7 +18,7 @@ not use classical GC algorithms anymore but is based on destructors and move semantics. The new runtime's advantages are that Nim programs become oblivious to the involved heap sizes and programs are easier to write to make effective use of multi-core machines. As a nice bonus, files and sockets and -the like will not require manual ``close`` calls anymore. +the like will not require manual `close` calls anymore. This document aims to be a precise specification about how move semantics and destructors work in Nim. @@ -89,12 +91,12 @@ written as: Lifetime-tracking hooks ======================= -The memory management for Nim's standard ``string`` and ``seq`` types as +The memory management for Nim's standard `string` and `seq` types as well as other standard collections is performed via so-called "Lifetime-tracking hooks", which are particular `type bound operators `_. -There are 3 different hooks for each (generic or concrete) object type ``T`` (``T`` can also be a -``distinct`` type) that are called implicitly by the compiler. +There are 3 different hooks for each (generic or concrete) object type `T` (`T` can also be a +`distinct` type) that are called implicitly by the compiler. (Note: The word "hook" here does not imply any kind of dynamic binding or runtime indirections, the implicit calls are statically bound and @@ -109,14 +111,14 @@ other associated resources. Variables are destroyed via this hook when they go out of scope or when the routine they were declared in is about to return. -The prototype of this hook for a type ``T`` needs to be: +The prototype of this hook for a type `T` needs to be: .. code-block:: nim proc `=destroy`(x: var T) -The general pattern in ``=destroy`` looks like: +The general pattern in `=destroy` looks like: .. code-block:: nim @@ -133,20 +135,20 @@ The general pattern in ``=destroy`` looks like: A `=sink` hook moves an object around, the resources are stolen from the source and passed to the destination. It is ensured that the source's destructor does not free the resources afterward by setting the object to its default value -(the value the object's state started in). Setting an object ``x`` back to its -default value is written as ``wasMoved(x)``. When not provided the compiler +(the value the object's state started in). Setting an object `x` back to its +default value is written as `wasMoved(x)`. When not provided the compiler is using a combination of `=destroy` and `copyMem` instead. This is efficient hence users rarely need to implement their own `=sink` operator, it is enough to provide `=destroy` and `=copy`, compiler will take care of the rest. -The prototype of this hook for a type ``T`` needs to be: +The prototype of this hook for a type `T` needs to be: .. code-block:: nim proc `=sink`(dest: var T; source: T) -The general pattern in ``=sink`` looks like: +The general pattern in `=sink` looks like: .. code-block:: nim @@ -156,25 +158,25 @@ The general pattern in ``=sink`` looks like: dest.field = source.field -**Note**: ``=sink`` does not need to check for self-assignments. +**Note**: `=sink` does not need to check for self-assignments. How self-assignments are handled is explained later in this document. `=copy` hook --------------- -The ordinary assignment in Nim conceptually copies the values. The ``=copy`` hook -is called for assignments that couldn't be transformed into ``=sink`` +The ordinary assignment in Nim conceptually copies the values. The `=copy` hook +is called for assignments that couldn't be transformed into `=sink` operations. -The prototype of this hook for a type ``T`` needs to be: +The prototype of this hook for a type `T` needs to be: .. code-block:: nim proc `=copy`(dest: var T; source: T) -The general pattern in ``=copy`` looks like: +The general pattern in `=copy` looks like: .. code-block:: nim @@ -186,48 +188,48 @@ The general pattern in ``=copy`` looks like: dest.field = duplicateResource(source.field) -The ``=copy`` proc can be marked with the ``{.error.}`` pragma. Then any assignment +The `=copy` proc can be marked with the `{.error.}` pragma. Then any assignment that otherwise would lead to a copy is prevented at compile-time. This looks like: .. code-block:: nim proc `=copy`(dest: var T; source: T) {.error.} -but a custom error message (e.g., ``{.error: "custom error".}``) will not be emitted -by the compiler. Notice that there is no ``=`` before the ``{.error.}`` pragma. +but a custom error message (e.g., `{.error: "custom error".}`) will not be emitted +by the compiler. Notice that there is no `=` before the `{.error.}` pragma. Move semantics ============== A "move" can be regarded as an optimized copy operation. If the source of the copy operation is not used afterward, the copy can be replaced by a move. This -document uses the notation ``lastReadOf(x)`` to describe that ``x`` is not +document uses the notation `lastReadOf(x)` to describe that `x` is not used afterwards. This property is computed by a static control flow analysis -but can also be enforced by using ``system.move`` explicitly. +but can also be enforced by using `system.move` explicitly. Swap ==== The need to check for self-assignments and also the need to destroy previous -objects inside ``=copy`` and ``=sink`` is a strong indicator to treat -``system.swap`` as a builtin primitive of its own that simply swaps every -field in the involved objects via ``copyMem`` or a comparable mechanism. -In other words, ``swap(a, b)`` is **not** implemented -as ``let tmp = move(b); b = move(a); a = move(tmp)``. +objects inside `=copy` and `=sink` is a strong indicator to treat +`system.swap` as a builtin primitive of its own that simply swaps every +field in the involved objects via `copyMem` or a comparable mechanism. +In other words, `swap(a, b)` is **not** implemented +as `let tmp = move(b); b = move(a); a = move(tmp)`. This has further consequences: * Objects that contain pointers that point to the same object are not supported by Nim's model. Otherwise swapped objects would end up in an inconsistent state. -* Seqs can use ``realloc`` in the implementation. +* Seqs can use `realloc` in the implementation. Sink parameters =============== -To move a variable into a collection usually ``sink`` parameters are involved. -A location that is passed to a ``sink`` parameter should not be used afterward. +To move a variable into a collection usually `sink` parameters are involved. +A location that is passed to a `sink` parameter should not be used afterward. This is ensured by a static analysis over a control flow graph. If it cannot be proven to be the last usage of the location, a copy is done instead and this copy is then passed to the sink parameter. @@ -235,9 +237,9 @@ copy is then passed to the sink parameter. A sink parameter *may* be consumed once in the proc's body but doesn't have to be consumed at all. The reason for this is that signatures -like ``proc put(t: var Table; k: sink Key, v: sink Value)`` should be possible -without any further overloads and ``put`` might not take ownership of ``k`` if -``k`` already exists in the table. Sink parameters enable an affine type system, +like `proc put(t: var Table; k: sink Key, v: sink Value)` should be possible +without any further overloads and `put` might not take ownership of `k` if +`k` already exists in the table. Sink parameters enable an affine type system, not a linear type system. The employed static analysis is limited and only concerned with local variables; @@ -254,7 +256,7 @@ however, object and tuple fields are treated as separate entities: echo tup[1] -Sometimes it is required to explicitly ``move`` a value into its final position: +Sometimes it is required to explicitly `move` a value into its final position: .. code-block:: nim @@ -294,9 +296,9 @@ Rewrite rules **Note**: There are two different allowed implementation strategies: -1. The produced ``finally`` section can be a single section that is wrapped +1. The produced `finally` section can be a single section that is wrapped around the complete routine body. -2. The produced ``finally`` section is wrapped around the enclosing scope. +2. The produced `finally` section is wrapped around the enclosing scope. The current implementation follows strategy (2). This means that resources are destroyed at the scope exit. @@ -359,13 +361,13 @@ Object and array construction ============================= Object and array construction is treated as a function call where the -function has ``sink`` parameters. +function has `sink` parameters. Destructor removal ================== -``wasMoved(x);`` followed by a `=destroy(x)` operation cancel each other +`wasMoved(x);` followed by a `=destroy(x)` operation cancel each other out. An implementation is encouraged to exploit this in order to improve efficiency and code sizes. The current implementation does perform this optimization. @@ -374,22 +376,22 @@ optimization. Self assignments ================ -``=sink`` in combination with ``wasMoved`` can handle self-assignments but +`=sink` in combination with `wasMoved` can handle self-assignments but it's subtle. -The simple case of ``x = x`` cannot be turned -into ``=sink(x, x); wasMoved(x)`` because that would lose ``x``'s value. +The simple case of `x = x` cannot be turned +into `=sink(x, x); wasMoved(x)` because that would lose `x`'s value. The solution is that simple self-assignments that consist of -- Symbols: ``x = x`` -- Field access: ``x.f = x.f`` -- Array, sequence or string access with indices known at compile-time: ``x[0] = x[0]`` +- Symbols: `x = x` +- Field access: `x.f = x.f` +- Array, sequence or string access with indices known at compile-time: `x[0] = x[0]` are transformed into an empty statement that does nothing. The compiler is free to optimize further cases. -The complex case looks like a variant of ``x = f(x)``, we consider -``x = select(rand() < 0.5, x, y)`` here: +The complex case looks like a variant of `x = f(x)`, we consider +`x = select(rand() < 0.5, x, y)` here: .. code-block:: nim @@ -450,17 +452,17 @@ self-assignments. Lent type ========= -``proc p(x: sink T)`` means that the proc ``p`` takes ownership of ``x``. +`proc p(x: sink T)` means that the proc `p` takes ownership of `x`. To eliminate even more creation/copy <-> destruction pairs, a proc's return -type can be annotated as ``lent T``. This is useful for "getter" accessors +type can be annotated as `lent T`. This is useful for "getter" accessors that seek to allow an immutable view into a container. -The ``sink`` and ``lent`` annotations allow us to remove most (if not all) +The `sink` and `lent` annotations allow us to remove most (if not all) superfluous copies and destructions. -``lent T`` is like ``var T`` a hidden pointer. It is proven by the compiler +`lent T` is like `var T` a hidden pointer. It is proven by the compiler that the pointer does not outlive its origin. No destructor call is injected -for expressions of type ``lent T`` or of type ``var T``. +for expressions of type `lent T` or of type `var T`. .. code-block:: nim @@ -494,9 +496,9 @@ for expressions of type ``lent T`` or of type ``var T``. The .cursor annotation ====================== -Under the ``--gc:arc|orc`` modes Nim's `ref` type is implemented via the same runtime +Under the `--gc:arc|orc` modes Nim's `ref` type is implemented via the same runtime "hooks" and thus via reference counting. This means that cyclic structures cannot be freed -immediately (``--gc:orc`` ships with a cycle collector). With the ``.cursor`` annotation +immediately (`--gc:orc` ships with a cycle collector). With the `.cursor` annotation one can break up cycles declaratively: .. code-block:: nim @@ -510,7 +512,7 @@ But please notice that this is not C++'s weak_ptr, it means the right field is n involved in the reference counting, it is a raw pointer without runtime checks. Automatic reference counting also has the disadvantage that it introduces overhead -when iterating over linked structures. The ``.cursor`` annotation can also be used +when iterating over linked structures. The `.cursor` annotation can also be used to avoid this overhead: .. code-block:: nim @@ -521,11 +523,11 @@ to avoid this overhead: it = it.next -In fact, ``.cursor`` more generally prevents object construction/destruction pairs +In fact, `.cursor` more generally prevents object construction/destruction pairs and so can also be useful in other contexts. The alternative solution would be to -use raw pointers (``ptr``) instead which is more cumbersome and also more dangerous -for Nim's evolution: Later on, the compiler can try to prove ``.cursor`` annotations -to be safe, but for ``ptr`` the compiler has to remain silent about possible +use raw pointers (`ptr`) instead which is more cumbersome and also more dangerous +for Nim's evolution: Later on, the compiler can try to prove `.cursor` annotations +to be safe, but for `ptr` the compiler has to remain silent about possible problems. @@ -556,13 +558,13 @@ indirections: Hook lifting ============ -The hooks of a tuple type ``(A, B, ...)`` are generated by lifting the -hooks of the involved types ``A``, ``B``, ... to the tuple type. In -other words, a copy ``x = y`` is implemented -as ``x[0] = y[0]; x[1] = y[1]; ...``, likewise for ``=sink`` and ``=destroy``. +The hooks of a tuple type `(A, B, ...)` are generated by lifting the +hooks of the involved types `A`, `B`, ... to the tuple type. In +other words, a copy `x = y` is implemented +as `x[0] = y[0]; x[1] = y[1]; ...`, likewise for `=sink` and `=destroy`. -Other value-based compound types like ``object`` and ``array`` are handled -correspondingly. For ``object`` however, the compiler-generated hooks +Other value-based compound types like `object` and `array` are handled +correspondingly. For `object` however, the compiler-generated hooks can be overridden. This can also be important to use an alternative traversal of the involved data structure that is more efficient or in order to avoid deep recursions. @@ -588,18 +590,18 @@ The ability to override a hook leads to a phase ordering problem: discard -The solution is to define ``proc `=destroy`[T](f: var Foo[T])`` before +The solution is to define `proc `=destroy`[T](f: var Foo[T])` before it is used. The compiler generates implicit hooks for all types in *strategic places* so that an explicitly provided hook that comes too "late" can be detected reliably. These *strategic places* have been derived from the rewrite rules and are as follows: -- In the construct ``let/var x = ...`` (var/let binding) - hooks are generated for ``typeof(x)``. -- In ``x = ...`` (assignment) hooks are generated for ``typeof(x)``. -- In ``f(...)`` (function call) hooks are generated for ``typeof(f(...))``. -- For every sink parameter ``x: sink T`` the hooks are generated - for ``typeof(x)``. +- In the construct `let/var x = ...` (var/let binding) + hooks are generated for `typeof(x)`. +- In `x = ...` (assignment) hooks are generated for `typeof(x)`. +- In `f(...)` (function call) hooks are generated for `typeof(f(...))`. +- For every sink parameter `x: sink T` the hooks are generated + for `typeof(x)`. nodestroy pragma @@ -645,20 +647,18 @@ Instead the variable simply points to the literal. The literal is shared between different variables which are pointing to it. The copy operation is deferred until the first write. -```nim -var x = "abc" # no copy -var y = x # no copy -``` +.. code-block:: nim + var x = "abc" # no copy + var y = x # no copy The string literal "abc" is stored in static memory and not allocated on the heap. The variable `x` points to the literal and the variable `y` points to the literal too. There is no copy during assigning operations. -```nim -var x = "abc" # no copy -var y = x # no copy -y[0] = 'h' # copy -``` +.. code-block:: nim + var x = "abc" # no copy + var y = x # no copy + y[0] = 'h' # copy The program above shows when the copy operations happen. When mutating the variable `y`, the Nim compiler creates a fresh copy of `x`, @@ -670,37 +670,34 @@ and the variable `y` becomes a mutable string. Let's look at a silly example demonstrating this behaviour: -```nim -var x = "abc" -var y = x +.. code-block:: nim + var x = "abc" + var y = x -moveMem(addr y[0], addr x[0], 3) -``` + moveMem(addr y[0], addr x[0], 3) The program fails because we need to prepare a fresh copy for the variable `y`. `prepareMutation` should be called before the address operation. -```nim -var x = "abc" -var y = x +.. code-block:: nim + var x = "abc" + var y = x -prepareMutation(y) -moveMem(addr y[0], addr x[0], 3) -assert y == "abc" -``` + prepareMutation(y) + moveMem(addr y[0], addr x[0], 3) + assert y == "abc" Now `prepareMutation` solves the problem. It manually creates a fresh copy and makes the variable `y` mutable. -```nim -var x = "abc" -var y = x +.. code-block:: nim + var x = "abc" + var y = x -prepareMutation(y) -moveMem(addr y[0], addr x[0], 3) -moveMem(addr y[0], addr x[0], 3) -moveMem(addr y[0], addr x[0], 3) -assert y == "abc" -``` + prepareMutation(y) + moveMem(addr y[0], addr x[0], 3) + moveMem(addr y[0], addr x[0], 3) + moveMem(addr y[0], addr x[0], 3) + assert y == "abc" No matter how many times `moveMem` is called, the program compiles and runs. diff --git a/doc/docgen.rst b/doc/docgen.rst index e383bd8d0..c002ddb83 100644 --- a/doc/docgen.rst +++ b/doc/docgen.rst @@ -1,3 +1,5 @@ +.. default-role:: code + =================================== Nim DocGen Tools Guide =================================== @@ -15,7 +17,7 @@ This document describes the `documentation generation tools`:idx: built into the `Nim compiler `_, which can generate HTML and JSON output from input .nim files and projects, as well as HTML and LaTeX from input RST (reStructuredText) files. The output documentation will include the module -dependencies (``import``), any top-level documentation comments (##), and +dependencies (`import`), any top-level documentation comments (##), and exported symbols (*), including procedures, types, and variables. Quick start @@ -125,12 +127,12 @@ Document Types HTML ---- -The generation of HTML documents is done via the ``doc`` command. This command +The generation of HTML documents is done via the `doc` command. This command takes either a single .nim file, outputting a single .html file with the same base filename, or multiple .nim files, outputting multiple .html files and, optionally, an index file. -The ``doc`` command:: +The `doc` command:: nim doc sample Partial Output:: @@ -146,12 +148,12 @@ compiler. JSON ---- -The generation of JSON documents is done via the ``jsondoc`` command. This command +The generation of JSON documents is done via the `jsondoc` command. This command takes in a .nim file and outputs a .json file with the same base filename. Note -that this tool is built off of the ``doc`` command (previously ``doc2``), and +that this tool is built off of the `doc` command (previously `doc2`), and contains the same information. -The ``jsondoc`` command:: +The `jsondoc` command:: nim jsondoc sample Output:: @@ -171,10 +173,10 @@ Output:: ] } -Similarly to the old ``doc`` command, the old ``jsondoc`` command has been -renamed to ``jsondoc0``. +Similarly to the old `doc` command, the old `jsondoc` command has been +renamed to `jsondoc0`. -The ``jsondoc0`` command:: +The `jsondoc0` command:: nim jsondoc0 sample Output:: @@ -190,8 +192,8 @@ Output:: } ] -Note that the ``jsondoc`` command outputs it's JSON without pretty-printing it, -while ``jsondoc0`` outputs pretty-printed JSON. +Note that the `jsondoc` command outputs it's JSON without pretty-printing it, +while `jsondoc0` outputs pretty-printed JSON. Related Options =============== @@ -203,7 +205,7 @@ Project switch nim doc --project filename.nim This will recursively generate documentation of all nim modules imported -into the input module that belong to the Nimble package that ``filename.nim`` +into the input module that belong to the Nimble package that `filename.nim` belongs to. @@ -214,13 +216,13 @@ Index switch nim doc --index:on filename.nim This will generate an index of all the exported symbols in the input Nim -module, and put it into a neighboring file with the extension of ``.idx``. The +module, and put it into a neighboring file with the extension of `.idx`. The index file is line-oriented (newlines have to be escaped). Each line represents a tab-separated record of several columns, the first two mandatory, the rest optional. See the `Index (idx) file format`_ section for details. Once index files have been generated for one or more modules, the Nim -compiler command ``buildIndex directory`` can be run to go over all the index +compiler command `buildIndex directory` can be run to go over all the index files in the specified directory to generate a `theindex.html `_ file. @@ -230,28 +232,28 @@ See source switch :: nim doc --git.url: filename.nim -With the ``git.url`` switch the *See source* hyperlink will appear below each +With the `git.url` switch the *See source* hyperlink will appear below each documented item in your source code pointing to the implementation of that item on a GitHub repository. You can click the link to see the implementation of the item. -The ``git.commit`` switch overrides the hardcoded `devel` branch in config/nimdoc.cfg. +The `git.commit` switch overrides the hardcoded `devel` branch in config/nimdoc.cfg. This is useful to link to a different branch e.g. `--git.commit:master`, or to a tag e.g. `--git.commit:1.2.3` or a commit. Source URLs are generated as `href="${url}/tree/${commit}/${path}#L${line}"` by default and this compatible with GitHub but not with GitLab. -Similarly, ``git.devel`` switch overrides the hardcoded `devel` branch for the `Edit` link which is also useful if you have a different working branch than `devel` e.g. `--git.devel:master`. +Similarly, `git.devel` switch overrides the hardcoded `devel` branch for the `Edit` link which is also useful if you have a different working branch than `devel` e.g. `--git.devel:master`. Edit URLs are generated as `href="${url}/tree/${devel}/${path}#L${line}"` by default. -You can edit ``config/nimdoc.cfg`` and modify the ``doc.item.seesrc`` value with a hyperlink to your own code repository. +You can edit `config/nimdoc.cfg` and modify the `doc.item.seesrc` value with a hyperlink to your own code repository. -In the case of Nim's own documentation, the ``commit`` value is just a commit +In the case of Nim's own documentation, the `commit` value is just a commit hash to append to a formatted URL to https://github.com/nim-lang/Nim. The -``tools/nimweb.nim`` helper queries the current git commit hash during the doc +`tools/nimweb.nim` helper queries the current git commit hash during the doc generation, but since you might be working on an unpublished repository, it -also allows specifying a ``githash`` value in ``web/website.ini`` to force a +also allows specifying a `githash` value in `web/website.ini` to force a specific commit in the output. @@ -259,9 +261,9 @@ Other Input Formats =================== The *Nim compiler* also has support for RST (reStructuredText) files with -the ``rst2html`` and ``rst2tex`` commands. Documents like this one are +the `rst2html` and `rst2tex` commands. Documents like this one are initially written in a dialect of RST which adds support for nim source code -highlighting with the ``.. code-block:: nim`` prefix. ``code-block`` also +highlighting with the `.. code-block:: nim` prefix. `code-block` also supports highlighting of C++ and some other c-like languages. Usage:: @@ -270,17 +272,17 @@ Usage:: Output:: You're reading it! -The ``rst2tex`` command is invoked identically to ``rst2html``, but outputs +The `rst2tex` command is invoked identically to `rst2html`, but outputs a .tex file instead of .html. HTML anchor generation ====================== -When you run the ``rst2html`` command, all sections in the RST document will +When you run the `rst2html` command, all sections in the RST document will get an anchor you can hyperlink to. Usually, you can guess the anchor lower casing the section title and replacing spaces with dashes, and in any case, you -can get it from the table of contents. But when you run the ``doc`` +can get it from the table of contents. But when you run the `doc` command to generate API documentation, some symbol get one or two anchors at the same time: a numerical identifier, or a plain name plus a complex name. @@ -312,31 +314,31 @@ suffix may be added depending on the type of the callable: Callable type Suffix ------------- -------------- proc *empty string* -macro ``.m`` -method ``.e`` -iterator ``.i`` -template ``.t`` -converter ``.c`` +macro `.m` +method `.e` +iterator `.i` +template `.t` +converter `.c` ------------- -------------- -The relationship of type to suffix is made by the proc ``complexName`` in the -``compiler/docgen.nim`` file. Here are some examples of complex names for +The relationship of type to suffix is made by the proc `complexName` in the +`compiler/docgen.nim` file. Here are some examples of complex names for symbols in the `system module `_. -* ``type SomeSignedInt = int | int8 | int16 | int32 | int64`` **=>** +* `type SomeSignedInt = int | int8 | int16 | int32 | int64` **=>** `#SomeSignedInt `_ -* ``var globalRaiseHook: proc (e: ref E_Base): bool {.nimcall.}`` **=>** +* `var globalRaiseHook: proc (e: ref E_Base): bool {.nimcall.}` **=>** `#globalRaiseHook `_ -* ``const NimVersion = "0.0.0"`` **=>** +* `const NimVersion = "0.0.0"` **=>** `#NimVersion `_ -* ``proc getTotalMem(): int {.rtl, raises: [], tags: [].}`` **=>** +* `proc getTotalMem(): int {.rtl, raises: [], tags: [].}` **=>** `#getTotalMem, `_ -* ``proc len[T](x: seq[T]): int {.magic: "LengthSeq", noSideEffect.}`` **=>** +* `proc len[T](x: seq[T]): int {.magic: "LengthSeq", noSideEffect.}` **=>** `#len,seq[T] `_ -* ``iterator pairs[T](a: seq[T]): tuple[key: int, val: T] {.inline.}`` **=>** +* `iterator pairs[T](a: seq[T]): tuple[key: int, val: T] {.inline.}` **=>** `#pairs.i,seq[T] `_ -* ``template newException[](exceptn: typedesc; message: string; - parentException: ref Exception = nil): untyped`` **=>** +* `template newException[](exceptn: typedesc; message: string; + parentException: ref Exception = nil): untyped` **=>** `#newException.t,typedesc,string,ref.Exception `_ @@ -344,13 +346,13 @@ symbols in the `system module `_. Index (idx) file format ======================= -Files with the ``.idx`` extension are generated when you use the `Index +Files with the `.idx` extension are generated when you use the `Index switch <#related-options-index-switch>`_ along with commands to generate documentation from source or text files. You can programmatically generate indices with the `setIndexTerm() `_ and `writeIndexFile() `_ procs. -The purpose of ``idx`` files is to hold the interesting symbols and their HTML +The purpose of `idx` files is to hold the interesting symbols and their HTML references so they can be later concatenated into a big index file with `mergeIndexes() `_. This section documents the file format in detail. @@ -362,7 +364,7 @@ columns is: 1. Mandatory term being indexed. Terms can include quoting according to Nim's rules (e.g. \`^\`). -2. Base filename plus anchor hyperlink (e.g. ``algorithm.html#*,int,SortOrder``). +2. Base filename plus anchor hyperlink (e.g. `algorithm.html#*,int,SortOrder`). 3. Optional human-readable string to display as a hyperlink. If the value is not present or is the empty string, the hyperlink will be rendered using the term. Prefix whitespace indicates that this entry is @@ -371,8 +373,8 @@ columns is: this as a tooltip after hovering a moment over the hyperlink. The index generation tools try to differentiate between documentation -generated from ``.nim`` files and documentation generated from ``.txt`` or -``.rst`` files. The former are always closely related to source code and +generated from `.nim` files and documentation generated from `.txt` or +`.rst` files. The former are always closely related to source code and consist mainly of API entries. The latter are generic documents meant for human reading. @@ -391,7 +393,7 @@ the index file with their third column having as much prefix spaces as their level is in the TOC (at least 1 character). The prefix whitespace helps to filter TOC entries from API or text symbols. This is important because the amount of spaces is used to replicate the hierarchy for document TOCs in the -final index, and TOC entries found in ``.nim`` files are discarded. +final index, and TOC entries found in `.nim` files are discarded. Additional resources @@ -402,8 +404,8 @@ Additional resources `RST Quick Reference `_ -The output for HTML and LaTeX comes from the ``config/nimdoc.cfg`` and -``config/nimdoc.tex.cfg`` configuration files. You can add and modify these +The output for HTML and LaTeX comes from the `config/nimdoc.cfg` and +`config/nimdoc.tex.cfg` configuration files. You can add and modify these files to your project to change the look of the docgen output. You can import the `packages/docutils/rstgen module `_ in your diff --git a/doc/docstyle.rst b/doc/docstyle.rst index 090032b98..7f3fa8cf2 100644 --- a/doc/docstyle.rst +++ b/doc/docstyle.rst @@ -6,7 +6,7 @@ General Guidelines * See also `nep1`_ which should probably be merged here. * Authors should document anything that is exported; documentation for private - procs can be useful too (visible via ``nim doc --docInternal foo.nim``). + procs can be useful too (visible via `nim doc --docInternal foo.nim`). * Within documentation, a period (`.`) should follow each sentence (or sentence fragment) in a comment block. The documentation may be limited to one sentence fragment, but if multiple sentences are within the documentation, each sentence after the first should be complete and in present tense. @@ -15,7 +15,7 @@ General Guidelines and `nim doc` supports it. Likewise with rst files: `nim rst2html` will render those as monospace, and adding `.. default-role:: code` to an rst file will also make those render as monospace when rendered directly in tools such as github. -* In nim sources, for links, prefer `[link text](link.html)` to ``` `link text`_ ``` +* In nim sources, for links, prefer `[link text](link.html)` to `` `link text`_ `` since the syntax is simpler and markdown is more common (likewise, `nim rst2html` also supports it in rst files). .. code-block:: nim @@ -28,8 +28,8 @@ General Guidelines Module-level documentation -------------------------- -Documentation of a module is placed at the top of the module itself. Each line of documentation begins with double hashes (``##``). -Sometimes ``##[ multiline docs containing code ]##`` is preferable, see ``lib/pure/times.nim``. +Documentation of a module is placed at the top of the module itself. Each line of documentation begins with double hashes (`##`). +Sometimes `##[ multiline docs containing code ]##` is preferable, see `lib/pure/times.nim`. Code samples are encouraged, and should follow the general RST syntax: .. code-block:: Nim @@ -80,7 +80,7 @@ Whenever an example of usage would be helpful to the user, you should include on doAssert addThree(3, 125, 6) == -122 result = x +% y +% z -The command ``nim doc`` will then correctly syntax highlight the Nim code within the documentation. +The command `nim doc` will then correctly syntax highlight the Nim code within the documentation. Types ----- @@ -116,8 +116,8 @@ Make sure to place the documentation beside or within the object. .. code-block:: Nim type - ## Bad: this documentation disappears because it annotates the ``type`` keyword - ## above, not ``NamedQueue``. + ## Bad: this documentation disappears because it annotates the `type` keyword + ## above, not `NamedQueue`. NamedQueue*[T] = object name*: string ## This becomes the main documentation for the object, which ## is not what we want. @@ -127,7 +127,7 @@ Make sure to place the documentation beside or within the object. Var, Let, and Const ------------------- -When declaring module-wide constants and values, documentation is encouraged. The placement of doc comments is similar to the ``type`` sections. +When declaring module-wide constants and values, documentation is encouraged. The placement of doc comments is similar to the `type` sections. .. code-block:: Nim @@ -137,9 +137,9 @@ When declaring module-wide constants and values, documentation is encouraged. Th [1,2,3], [2,3,1], [3,1,2], - ] ## Doc comment for ``SpreadArray``. + ] ## Doc comment for `SpreadArray`. -Placement of comments in other areas is usually allowed, but will not become part of the documentation output and should therefore be prefaced by a single hash (``#``). +Placement of comments in other areas is usually allowed, but will not become part of the documentation output and should therefore be prefaced by a single hash (`#`). .. code-block:: Nim diff --git a/doc/drnim.rst b/doc/drnim.rst index ee6e0ea17..d33a6066e 100644 --- a/doc/drnim.rst +++ b/doc/drnim.rst @@ -1,3 +1,5 @@ +.. default-role:: code + =================================== DrNim User Guide =================================== @@ -18,7 +20,7 @@ DrNim's command-line options are the same as the Nim compiler's. DrNim currently only checks the sections of your code that are marked -via ``staticBoundChecks: on``: +via `staticBoundChecks: on`: .. code-block:: nim @@ -31,9 +33,9 @@ overflow errors are *not* prevented. Overflows will be checked for in the future. Later versions of the **Nim compiler** will **assume** that the checks inside -the ``staticBoundChecks: on`` environment have been proven correct and so +the `staticBoundChecks: on` environment have been proven correct and so it will **omit** the runtime checks. If you do not want this behavior, use -instead ``{.push staticBoundChecks: defined(nimDrNim).}``. This way the +instead `{.push staticBoundChecks: defined(nimDrNim).}`. This way the Nim compiler remains unaware of the performed proofs but DrNim will prove your code. @@ -41,7 +43,7 @@ your code. Installation ============ -Run ``koch drnim``, the executable will afterwards be in ``$nim/bin/drnim``. +Run `koch drnim`, the executable will afterwards be in `$nim/bin/drnim`. Motivating Example @@ -67,7 +69,7 @@ detects it and produces the following error message:: cannot prove: i <= len(a) + -1; counter example: i -> 0 a.len -> 0 [IndexCheck] -In other words for ``i == 0`` and ``a.len == 0`` (for example!) there would be +In other words for `i == 0` and `a.len == 0` (for example!) there would be an index out of bounds error. @@ -82,37 +84,37 @@ DrNim adds 4 additional annotations (pragmas) to Nim: - `assume`:idx: These pragmas are ignored by the Nim compiler so that they don't have to -be disabled via ``when defined(nimDrNim)``. +be disabled via `when defined(nimDrNim)`. Invariant --------- -An ``invariant`` is a proposition that must be true after every loop +An `invariant` is a proposition that must be true after every loop iteration, it's tied to the loop body it's part of. Requires -------- -A ``requires`` annotation describes what the function expects to be true -before it's called so that it can perform its operation. A ``requires`` +A `requires` annotation describes what the function expects to be true +before it's called so that it can perform its operation. A `requires` annotation is also called a `precondition`:idx:. Ensures ------- -An ``ensures`` annotation describes what will be true after the function -call. An ``ensures`` annotation is also called a `postcondition`:idx:. +An `ensures` annotation describes what will be true after the function +call. An `ensures` annotation is also called a `postcondition`:idx:. Assume ------ -An ``assume`` annotation describes what DrNim should **assume** to be true +An `assume` annotation describes what DrNim should **assume** to be true in this section of the program. It is an unsafe escape mechanism comparable -to Nim's ``cast`` statement. Use it only when you really know better +to Nim's `cast` statement. Use it only when you really know better than DrNim. You should add a comment to a paper that proves the proposition you assume. @@ -143,12 +145,12 @@ Example: insertionSort Unfortunately, the invariants required to prove that this code is correct take more code than the imperative instructions. However, this effort can be compensated by the fact that the result needs very little testing. Be aware though that -DrNim only proves that after ``insertionSort`` this condition holds:: +DrNim only proves that after `insertionSort` this condition holds:: forall(i in 1..``. -A ``prop`` is either a comparison or a compound:: +The basic syntax is `ensures|requires|invariant: `. +A `prop` is either a comparison or a compound:: prop = nim_bool_expression | prop 'and' prop @@ -186,17 +188,17 @@ A ``prop`` is either a comparison or a compound:: quantifier = 'in' nim_iteration_expression -``nim_iteration_expression`` here is an ordinary expression of Nim code -that describes an iteration space, for example ``1..4`` or ``1.. a.len``. +`nim_bool_expression` here is an ordinary expression of Nim code of +type `bool` like `a == 3` or `23 > a.len`. The supported subset of Nim code that can be used in these expressions -is currently underspecified but ``let`` variables, function parameters -and ``result`` (which represents the function's final result) are amenable +is currently underspecified but `let` variables, function parameters +and `result` (which represents the function's final result) are amenable for verification. The expressions must not have any side-effects and must terminate. -The operators ``forall``, ``exists``, ``->``, ``<->`` have to imported -from ``std / logic``. +The operators `forall`, `exists`, `->`, `<->` have to imported +from `std / logic`. diff --git a/doc/estp.rst b/doc/estp.rst index 805a84eb7..8146562b6 100644 --- a/doc/estp.rst +++ b/doc/estp.rst @@ -1,3 +1,5 @@ +.. default-role:: code + =================================================== Embedded Stack Trace Profiler (ESTP) User Guide =================================================== @@ -10,21 +12,21 @@ Nim comes with a platform independent profiler - the Embedded Stack Trace Profiler (ESTP). The profiler is *embedded* into your executable. To activate the profiler you need to do: -* compile your program with the ``--profiler:on --stackTrace:on`` command +* compile your program with the `--profiler:on --stackTrace:on` command line options -* import the ``nimprof`` module +* import the `nimprof` module * run your program as usual. -You can in fact look at ``nimprof``'s source code to see how to implement +You can in fact look at `nimprof`'s source code to see how to implement your own profiler. -The setting ``--profiler:on`` defines the conditional symbol ``profiler``. +The setting `--profiler:on` defines the conditional symbol `profiler`. After your program has finished the profiler will create a -file ``profile_results.txt`` containing the profiling results. +file `profile_results.txt` containing the profiling results. Since the profiler works by examining stack traces, it's essential that -the option ``--stackTrace:on`` is active! Unfortunately this means that a +the option `--stackTrace:on` is active! Unfortunately this means that a profiling build is much slower than a release build. @@ -35,12 +37,12 @@ You can also use ESTP as a memory profiler to see which stack traces allocate the most memory and thus create the most GC pressure. It may also help to find memory leaks. To activate the memory profiler you need to do: -* compile your program with the ``--profiler:off --stackTrace:on -d:memProfiler`` - command line options. Yes it's ``--profiler:off``. -* import the ``nimprof`` module +* compile your program with the `--profiler:off --stackTrace:on -d:memProfiler` + command line options. Yes it's `--profiler:off`. +* import the `nimprof` module * run your program as usual. -Define the symbol ``ignoreAllocationSize`` so that only the number of +Define the symbol `ignoreAllocationSize` so that only the number of allocations is counted and the sizes of the memory allocations do not matter. @@ -51,7 +53,7 @@ The results file lists stack traces ordered by significance. The following example file has been generated by profiling the Nim compiler itself: It shows that in total 5.4% of the runtime has been spent -in ``crcFromRope`` or its children. +in `crcFromRope` or its children. In general the stack traces show you immediately where the problem is because the trace acts like an explanation; in traditional profilers you can only find diff --git a/doc/filters.rst b/doc/filters.rst index 40346ecaf..52704411f 100644 --- a/doc/filters.rst +++ b/doc/filters.rst @@ -1,3 +1,5 @@ +.. default-role:: code + =================== Source Code Filters =================== @@ -8,7 +10,7 @@ A `Source Code Filter (SCF)` transforms the input character stream to an in-mem output stream before parsing. A filter can be used to provide templating systems or preprocessors. -To use a filter for a source file the ``#?`` notation is used:: +To use a filter for a source file the `#?` notation is used:: #? stdtmpl(subsChar = '$', metaChar = '#') #proc generateXML(name, age: string): string = @@ -21,9 +23,9 @@ To use a filter for a source file the ``#?`` notation is used:: As the example shows, passing arguments to a filter can be done just like an ordinary procedure call with named or positional arguments. The available parameters depend on the invoked filter. Before version 0.12.0 of -the language ``#!`` was used instead of ``#?``. +the language `#!` was used instead of `#?`. -**Hint:** With ``--hint[codeBegin]:on`` or ``--verbosity:2`` +**Hint:** With `--hint[codeBegin]:on` or `--verbosity:2` (or higher) while compiling or `nim check`, Nim lists the processed code after each filter application. @@ -32,8 +34,8 @@ Usage First, put your SCF code in a separate file with filters specified in the first line. **Note:** You can name your SCF file with any file extension you want, but the -conventional extension is ``.nimf`` -(it used to be ``.tmpl`` but that was too generic, for example preventing github to +conventional extension is `.nimf` +(it used to be `.tmpl` but that was too generic, for example preventing github to recognize it as Nim source file). If we use `generateXML` code shown above and call the SCF file `xmlGen.nimf` @@ -47,7 +49,7 @@ In your `main.nim`: Pipe operator ============= -Filters can be combined with the ``|`` pipe operator:: +Filters can be combined with the `|` pipe operator:: #? strip(startswith="<") | stdtmpl #proc generateXML(name, age: string): string = @@ -68,10 +70,10 @@ The replace filter replaces substrings in each line. Parameters and their defaults: - ``sub: string = ""`` + `sub: string = ""` the substring that is searched for - ``by: string = ""`` + `by: string = ""` the string the substring is replaced with @@ -83,14 +85,14 @@ each line. Parameters and their defaults: - ``startswith: string = ""`` + `startswith: string = ""` strip only the lines that start with *startswith* (ignoring leading whitespace). If empty every line is stripped. - ``leading: bool = true`` + `leading: bool = true` strip leading whitespace - ``trailing: bool = true`` + `trailing: bool = true` strip trailing whitespace @@ -99,25 +101,25 @@ StdTmpl filter The stdtmpl filter provides a simple templating engine for Nim. The filter uses a line based parser: Lines prefixed with a *meta character* -(default: ``#``) contain Nim code, other lines are verbatim. Because +(default: `#`) contain Nim code, other lines are verbatim. Because indentation-based parsing is not suited for a templating engine, control flow -statements need ``end X`` delimiters. +statements need `end X` delimiters. Parameters and their defaults: - ``metaChar: char = '#'`` + `metaChar: char = '#'` prefix for a line that contains Nim code - ``subsChar: char = '$'`` + `subsChar: char = '$'` prefix for a Nim expression within a template line - ``conc: string = " & "`` + `conc: string = " & "` the operation for concatenation - ``emit: string = "result.add"`` + `emit: string = "result.add"` the operation to emit a string literal - ``toString: string = "$"`` + `toString: string = "$"` the operation that is applied to each expression Example:: @@ -174,18 +176,18 @@ The filter transforms this into: Each line that does not start with the meta character (ignoring leading -whitespace) is converted to a string literal that is added to ``result``. +whitespace) is converted to a string literal that is added to `result`. The substitution character introduces a Nim expression *e* within the string literal. *e* is converted to a string with the *toString* operation -which defaults to ``$``. For strong type checking, set ``toString`` to the +which defaults to `$`. For strong type checking, set `toString` to the empty string. *e* must match this PEG pattern:: e <- [a-zA-Z\128-\255][a-zA-Z0-9\128-\255_.]* / '{' x '}' x <- '{' x+ '}' / [^}]* -To produce a single substitution character it has to be doubled: ``$$`` -produces ``$``. +To produce a single substitution character it has to be doubled: `$$` +produces `$`. The template engine is quite flexible. It is easy to produce a procedure that writes the template code directly to a file:: diff --git a/doc/gc.rst b/doc/gc.rst index 29b9a4131..4455afcbe 100644 --- a/doc/gc.rst +++ b/doc/gc.rst @@ -1,3 +1,5 @@ +.. default-role:: code + ======================= Nim's Memory Management ======================= @@ -27,37 +29,37 @@ and how the memory management strategies other than garbage collectors work. Multi-paradigm Memory Management Strategies =========================================== -To choose the memory management strategy use the ``--gc:`` switch. +To choose the memory management strategy use the `--gc:` switch. -- ``--gc:refc``. This is the default GC. It's a +- `--gc:refc`. This is the default GC. It's a deferred reference counting based garbage collector with a simple Mark&Sweep backup GC in order to collect cycles. Heaps are thread-local. -- ``--gc:markAndSweep``. Simple Mark-And-Sweep based garbage collector. Heaps are thread-local. -- ``--gc:boehm``. Boehm based garbage collector, it offers a shared heap. -- ``--gc:go``. Go's garbage collector, useful for interoperability with Go. Offers a shared heap. -- ``--gc:arc``. Plain reference counting with +- `--gc:markAndSweep`. Simple Mark-And-Sweep based garbage collector. Heaps are thread-local. +- `--gc:boehm`. Boehm based garbage collector, it offers a shared heap. +- `--gc:go`. Go's garbage collector, useful for interoperability with Go. Offers a shared heap. +- `--gc:arc`. Plain reference counting with `move semantic optimizations `_, offers a shared heap. It offers deterministic performance for `hard realtime`:idx: systems. Reference cycles cause memory leaks, beware. -- ``--gc:orc``. Same as ``--gc:arc`` but adds a cycle collector based on "trial deletion". +- `--gc:orc`. Same as `--gc:arc` but adds a cycle collector based on "trial deletion". Unfortunately, that makes its performance profile hard to reason about so it is less useful for hard real-time systems. -- ``--gc:none``. No memory management strategy nor a garbage collector. Allocated memory is - simply never freed. You should use ``--gc:arc`` instead. +- `--gc:none`. No memory management strategy nor a garbage collector. Allocated memory is + simply never freed. You should use `--gc:arc` instead. ================== ======== ================= ============== =================== Memory Management Heap Reference Cycles Stop-The-World Command line switch ================== ======== ================= ============== =================== -RefC Local Cycle Collector No ``--gc:refc`` -Mark & Sweep Local Cycle Collector No ``--gc:markAndSweep`` -ARC Shared Leak No ``--gc:arc`` -ORC Shared Cycle Collector No ``--gc:orc`` -Boehm Shared Cycle Collector Yes ``--gc:boehm`` -Go Shared Cycle Collector Yes ``--gc:go`` -None Manual Manual Manual ``--gc:none`` +RefC Local Cycle Collector No `--gc:refc` +Mark & Sweep Local Cycle Collector No `--gc:markAndSweep` +ARC Shared Leak No `--gc:arc` +ORC Shared Cycle Collector No `--gc:orc` +Boehm Shared Cycle Collector Yes `--gc:boehm` +Go Shared Cycle Collector Yes `--gc:go` +None Manual Manual Manual `--gc:none` ================== ======== ================= ============== =================== JavaScript's garbage collector is used for the `JavaScript and NodeJS @@ -73,14 +75,14 @@ Cycle collector --------------- The cycle collector can be en-/disabled independently from the other parts of -the garbage collector with ``GC_enableMarkAndSweep`` and ``GC_disableMarkAndSweep``. +the garbage collector with `GC_enableMarkAndSweep` and `GC_disableMarkAndSweep`. Soft real-time support ---------------------- To enable real-time support, the symbol `useRealtimeGC`:idx: needs to be -defined via ``--define:useRealtimeGC`` (you can put this into your config +defined via `--define:useRealtimeGC` (you can put this into your config file as well). With this switch the garbage collector supports the following operations: @@ -88,29 +90,29 @@ With this switch the garbage collector supports the following operations: proc GC_setMaxPause*(maxPauseInUs: int) proc GC_step*(us: int, strongAdvice = false, stackSize = -1) -The unit of the parameters ``maxPauseInUs`` and ``us`` is microseconds. +The unit of the parameters `maxPauseInUs` and `us` is microseconds. These two procs are the two modus operandi of the real-time garbage collector: (1) GC_SetMaxPause Mode - You can call ``GC_SetMaxPause`` at program startup and then each triggered - garbage collector run tries to not take longer than ``maxPause`` time. However, it is + You can call `GC_SetMaxPause` at program startup and then each triggered + garbage collector run tries to not take longer than `maxPause` time. However, it is possible (and common) that the work is nevertheless not evenly distributed - as each call to ``new`` can trigger the garbage collector and thus take ``maxPause`` + as each call to `new` can trigger the garbage collector and thus take `maxPause` time. (2) GC_step Mode - This allows the garbage collector to perform some work for up to ``us`` time. + This allows the garbage collector to perform some work for up to `us` time. This is useful to call in the main loop to ensure the garbage collector can do its work. - To bind all garbage collector activity to a ``GC_step`` call, - deactivate the garbage collector with ``GC_disable`` at program startup. - If ``strongAdvice`` is set to ``true``, + To bind all garbage collector activity to a `GC_step` call, + deactivate the garbage collector with `GC_disable` at program startup. + If `strongAdvice` is set to `true`, then the garbage collector will be forced to perform the collection cycle. Otherwise, the garbage collector may decide not to do anything, if there is not much garbage to collect. - You may also specify the current stack size via ``stackSize`` parameter. + You may also specify the current stack size via `stackSize` parameter. It can improve performance when you know that there are no unique Nim references below a certain point on the stack. Make sure the size you specify is greater than the potential worst-case size. @@ -130,16 +132,16 @@ Time measurement with garbage collectors ---------------------------------------- The garbage collectors' way of measuring time uses -(see ``lib/system/timers.nim`` for the implementation): +(see `lib/system/timers.nim` for the implementation): -1) ``QueryPerformanceCounter`` and ``QueryPerformanceFrequency`` on Windows. -2) ``mach_absolute_time`` on Mac OS X. -3) ``gettimeofday`` on Posix systems. +1) `QueryPerformanceCounter` and `QueryPerformanceFrequency` on Windows. +2) `mach_absolute_time` on Mac OS X. +3) `gettimeofday` on Posix systems. As such it supports a resolution of nanoseconds internally; however, the API uses microseconds for convenience. -Define the symbol ``reportMissedDeadlines`` to make the +Define the symbol `reportMissedDeadlines` to make the garbage collector output whenever it missed a deadline. The reporting will be enhanced and supported by the API in later versions of the collector. @@ -148,9 +150,9 @@ Tweaking the garbage collector ------------------------------ The collector checks whether there is still time left for its work after -every ``workPackage``'th iteration. This is currently set to 100 which means +every `workPackage`'th iteration. This is currently set to 100 which means that up to 100 objects are traversed and freed before it checks again. Thus -``workPackage`` affects the timing granularity and may need to be tweaked in +`workPackage` affects the timing granularity and may need to be tweaked in highly specialized environments or for older hardware. @@ -158,22 +160,22 @@ Keeping track of memory ======================= If you need to pass around memory allocated by Nim to C, you can use the -procs ``GC_ref`` and ``GC_unref`` to mark objects as referenced to avoid them +procs `GC_ref` and `GC_unref` to mark objects as referenced to avoid them being freed by the garbage collector. Other useful procs from `system `_ you can use to keep track of memory are: -* ``getTotalMem()`` Returns the amount of total memory managed by the garbage collector. -* ``getOccupiedMem()`` Bytes reserved by the garbage collector and used by objects. -* ``getFreeMem()`` Bytes reserved by the garbage collector and not in use. -* ``GC_getStatistics()`` Garbage collector statistics as a human-readable string. +* `getTotalMem()` Returns the amount of total memory managed by the garbage collector. +* `getOccupiedMem()` Bytes reserved by the garbage collector and used by objects. +* `getFreeMem()` Bytes reserved by the garbage collector and not in use. +* `GC_getStatistics()` Garbage collector statistics as a human-readable string. These numbers are usually only for the running thread, not for the whole heap, -with the exception of ``--gc:boehm`` and ``--gc:go``. +with the exception of `--gc:boehm` and `--gc:go`. -In addition to ``GC_ref`` and ``GC_unref`` you can avoid the garbage collector by manually -allocating memory with procs like ``alloc``, ``alloc0``, ``allocShared``, ``allocShared0`` or ``allocCStringArray``. +In addition to `GC_ref` and `GC_unref` you can avoid the garbage collector by manually +allocating memory with procs like `alloc`, `alloc0`, `allocShared`, `allocShared0` or `allocCStringArray`. The garbage collector won't try to free them, you need to call their respective *dealloc* pairs -(``dealloc``, ``deallocShared``, ``deallocCStringArray``, etc) +(`dealloc`, `deallocShared`, `deallocCStringArray`, etc) when you are done with them or they will leak. @@ -182,12 +184,12 @@ Heap dump The heap dump feature is still in its infancy, but it already proved useful for us, so it might be useful for you. To get a heap dump, compile -with ``-d:nimTypeNames`` and call ``dumpNumberOfInstances`` at a strategic place in your program. +with `-d:nimTypeNames` and call `dumpNumberOfInstances` at a strategic place in your program. This produces a list of the used types in your program and for every type the total amount of object instances for this type as well as the total amount of bytes these instances take up. The numbers count the number of objects in all garbage collector heaps, they refer to all running threads, not only to the current thread. (The current thread -would be the thread that calls ``dumpNumberOfInstances``.) This might +would be the thread that calls `dumpNumberOfInstances`.) This might change in later versions. diff --git a/doc/hcr.rst b/doc/hcr.rst index 6dc65b1cd..7e9d71da3 100644 --- a/doc/hcr.rst +++ b/doc/hcr.rst @@ -1,3 +1,5 @@ +.. default-role:: code + =================================== Hot code reloading =================================== @@ -19,8 +21,8 @@ so we have to use a helper module where the major logic we want to change during development resides. In this example, we use SDL2 to create a window and we reload the logic -code when ``F9`` is pressed. The important lines are marked with ``#***``. -To install SDL2 you can use ``nimble install sdl2``. +code when `F9` is pressed. The important lines are marked with `#***`. +To install SDL2 you can use `nimble install sdl2`. .. code-block:: nim @@ -125,7 +127,7 @@ Then recompile the project, but do not restart or quit the mymain.exe program! nim c --hotcodereloading:on mymain.nim -Now give the ``mymain`` SDL window the focus, press F9, and watch the +Now give the `mymain` SDL window the focus, press F9, and watch the updated version of the program. @@ -133,8 +135,8 @@ updated version of the program. Reloading API ============= -One can use the special event handlers ``beforeCodeReload`` and -``afterCodeReload`` to reset the state of a particular variable or to force +One can use the special event handlers `beforeCodeReload` and +`afterCodeReload` to reset the state of a particular variable or to force the execution of certain statements: .. code-block:: Nim @@ -178,8 +180,8 @@ It's expected that most projects will implement the reloading with a suitable build-system triggered IPC notification mechanism, but a polling solution is also possible through the provided `hasAnyModuleChanged()`:idx: API. -In order to access ``beforeCodeReload``, ``afterCodeReload``, ``hasModuleChanged`` -or ``hasAnyModuleChanged`` one must import the `hotcodereloading`:idx: module. +In order to access `beforeCodeReload`, `afterCodeReload`, `hasModuleChanged` +or `hasAnyModuleChanged` one must import the `hotcodereloading`:idx: module. Native code targets @@ -187,11 +189,11 @@ Native code targets Native projects using the hot code reloading option will be implicitly compiled with the `-d:useNimRtl` option and they will depend on both -the ``nimrtl`` library and the ``nimhcr`` library which implements the -hot code reloading run-time. Both libraries can be found in the ``lib`` +the `nimrtl` library and the `nimhcr` library which implements the +hot code reloading run-time. Both libraries can be found in the `lib` folder of Nim and can be compiled into dynamic libraries to satisfy runtime demands of the example code above. An example of compiling -``nimhcr.nim`` and ``nimrtl.nim`` when the source dir of Nim is installed +`nimhcr.nim` and `nimrtl.nim` when the source dir of Nim is installed with choosenim follows. :: @@ -208,16 +210,16 @@ with choosenim follows. # source directory (.dll for Windows, .so for Unix, .dylib for MacOS) All modules of the project will be compiled to separate dynamic link -libraries placed in the ``nimcache`` directory. Please note that during +libraries placed in the `nimcache` directory. Please note that during the execution of the program, the hot code reloading run-time will load only copies of these libraries in order to not interfere with any newly issued build commands. The main module of the program is considered non-reloadable. Please note that procs from reloadable modules should not appear in the call stack of -program while ``performCodeReload`` is being called. Thus, the main module +program while `performCodeReload` is being called. Thus, the main module is a suitable place for implementing a program loop capable of calling -``performCodeReload``. +`performCodeReload`. Please note that reloading won't be possible when any of the type definitions in the program has been changed. When closure iterators are used (directly or diff --git a/doc/idetools.rst b/doc/idetools.rst index 27926b14f..dcafaf45f 100644 --- a/doc/idetools.rst +++ b/doc/idetools.rst @@ -1,3 +1,5 @@ +.. default-role:: code + ================================ Nim IDE Integration Guide ================================ @@ -18,8 +20,8 @@ Note: this is mostly outdated, see instead `nimsuggest `_ Nim differs from many other compilers in that it is really fast, and being so fast makes it suited to provide external queries for text editors about the source code being written. Through the -``idetools`` command of `the compiler `_, any IDE -can query a ``.nim`` source file and obtain useful information like +`idetools` command of `the compiler `_, any IDE +can query a `.nim` source file and obtain useful information like definition of symbols or suggestions for completion. This document will guide you through the available options. If you @@ -36,7 +38,7 @@ Specifying the location of the query ------------------------------------ All of the available idetools commands require you to specify a -query location through the ``--track`` or ``--trackDirty`` switches. +query location through the `--track` or `--trackDirty` switches. The general idetools invocations are:: nim idetools --track:FILE,LINE,COL proj.nim @@ -45,35 +47,35 @@ Or:: nim idetools --trackDirty:DIRTY_FILE,FILE,LINE,COL proj.nim -``proj.nim`` +`proj.nim` This is the main *project* filename. Most of the time you will pass in the same as **FILE**, but for bigger projects this is the file which is used as main entry point for the program, the one which users compile to generate a final binary. -```` +`` This would be any of the other idetools available options, like - ``--def`` or ``--suggest`` explained in the following sections. + `--def` or `--suggest` explained in the following sections. -``COL`` +`COL` An integer with the column you are going to query. For the compiler columns start at zero, so the first column will be **0** and the last in an 80 column terminal will be **79**. -``LINE`` +`LINE` An integer with the line you are going to query. For the compiler lines start at **1**. -``FILE`` +`FILE` The file you want to perform the query on. Usually you will pass in the same value as **proj.nim**. -``DIRTY_FILE`` +`DIRTY_FILE` The **FILE** parameter is enough for static analysis, but IDEs tend to have *unsaved buffers* where the user may still be in the middle of typing a line. In such situations the IDE can save the current contents to a temporary file and then use the - ``--trackDirty`` switch. + `--trackDirty` switch. Dirty files are likely to contain errors and they are usually compiled partially only to the point needed to service the @@ -91,7 +93,7 @@ Or:: Definitions ----------- -The ``--def`` idetools switch performs a query about the definition +The `--def` idetools switch performs a query about the definition of a specific symbol. If available, idetools will answer with the type, source file, line/column information and other accessory data if available like a docstring. With this information an IDE can @@ -112,7 +114,7 @@ can't find any valid symbol matching the position of the query. Suggestions ----------- -The ``--suggest`` idetools switch performs a query about possible +The `--suggest` idetools switch performs a query about possible completion symbols at some point in the file. IDEs can easily provide an autocompletion feature where the IDE scans the current file (and related ones, if it knows about the language being edited and follows @@ -134,7 +136,7 @@ Idetools will try to return the suggestions sorted first by scope Invocation context ------------------ -The ``--context`` idetools switch is very similar to the suggestions +The `--context` idetools switch is very similar to the suggestions switch, but instead of being used after the user has typed a dot character, this one is meant to be used after the user has typed an opening brace to start typing parameters. @@ -143,7 +145,7 @@ an opening brace to start typing parameters. Symbol usages ------------- -The ``--usages`` idetools switch lists all usages of the symbol at +The `--usages` idetools switch lists all usages of the symbol at a position. IDEs can use this to find all the places in the file where the symbol is used and offer the user to rename it in all places at the same time. Again, a pure string based search and @@ -207,15 +209,15 @@ Idetools outputs is always returned on single lines separated by tab characters (``\t``). The values of each column are: 1. Three characters indicating the type of returned answer (e.g. - def for definition, ``sug`` for suggestion, etc). -2. Type of the symbol. This can be ``skProc``, ``skLet``, and just - about any of the enums defined in the module ``compiler/ast.nim``. + def for definition, `sug` for suggestion, etc). +2. Type of the symbol. This can be `skProc`, `skLet`, and just + about any of the enums defined in the module `compiler/ast.nim`. 3. Full qualified path of the symbol. If you are querying a symbol - defined in the ``proj.nim`` file, this would have the form - ``proj.symbolName``. + defined in the `proj.nim` file, this would have the form + `proj.symbolName`. 4. Type/signature. For variables and enums this will contain the type of the symbol, for procs, methods and templates this will - contain the full unique signature (e.g. ``proc (File)``). + contain the full unique signature (e.g. `proc (File)`). 5. Full path to the file containing the symbol. 6. Line where the symbol is located in the file. Lines start to count at **1**. @@ -374,8 +376,8 @@ While at the language level a method is differentiated from others by the parameters and return value, the signature of the method returned by idetools returns also the pragmas for the method. -Note that at the moment the word ``proc`` is returned for the -signature of the found method instead of the expected ``method``. +Note that at the moment the word `proc` is returned for the +signature of the found method instead of the expected `method`. This may change in the future. | **Third column**: module + [n scope nesting] + method name. @@ -517,7 +519,7 @@ Test suite ========== To verify that idetools is working properly there are files in the -``tests/caas/`` directory which provide unit testing. If you find +`tests/caas/` directory which provide unit testing. If you find odd idetools behaviour and are able to reproduce it, you are welcome to report it as a bug and add a test to the suite to avoid future regressions. @@ -533,27 +535,27 @@ run it manually. First you have to compile the tester:: $ cd my/nim/checkout/tests $ nim c testament/caasdriver.nim -Running the ``caasdriver`` without parameters will attempt to process +Running the `caasdriver` without parameters will attempt to process all the test cases in all three operation modes. If a test succeeds nothing will be printed and the process will exit with zero. If any test fails, the specific line of the test preceding the failure and the failure itself will be dumped to stdout, along with a final indicator of the success state and operation mode. You can pass the -parameter ``verbose`` to force all output even on successful tests. +parameter `verbose` to force all output even on successful tests. -The normal operation mode is called ``ProcRun`` and it involves +The normal operation mode is called `ProcRun` and it involves starting a process for each command or query, similar to running -manually the Nim compiler from the commandline. The ``CaasRun`` -mode starts a server process to answer all queries. The ``SymbolProcRun`` +manually the Nim compiler from the commandline. The `CaasRun` +mode starts a server process to answer all queries. The `SymbolProcRun` mode is used by compiler developers. This means that running all -tests involves processing all ``*.txt`` files three times, which +tests involves processing all `*.txt` files three times, which can be quite time consuming. If you don't want to run all the test case files you can pass any -substring as a parameter to ``caasdriver``. Only files matching the +substring as a parameter to `caasdriver`. Only files matching the passed substring will be run. The filtering doesn't use any globbing metacharacters, it's a plain match. For example, to run only -``*-compile*.txt`` tests in verbose mode:: +`*-compile*.txt` tests in verbose mode:: ./caasdriver verbose -compile @@ -561,18 +563,18 @@ metacharacters, it's a plain match. For example, to run only Test case file format --------------------- -All the ``tests/caas/*.txt`` files encode a session with the compiler: +All the `tests/caas/*.txt` files encode a session with the compiler: * The first line indicates the main project file. -* Lines starting with ``>`` indicate a command to be sent to the +* Lines starting with `>` indicate a command to be sent to the compiler and the lines following a command include checks for - expected or forbidden output (``!`` for forbidden). + expected or forbidden output (`!` for forbidden). -* If a line starts with ``#`` it will be ignored completely, so you +* If a line starts with `#` it will be ignored completely, so you can use that for comments. -* Since some cases are specific to either ``ProcRun`` or ``CaasRun`` +* Since some cases are specific to either `ProcRun` or `CaasRun` modes, you can prefix a line with the mode and the line will be processed only in that mode. diff --git a/doc/intern.rst b/doc/intern.rst index 0eb70e143..2456b25fd 100644 --- a/doc/intern.rst +++ b/doc/intern.rst @@ -1,3 +1,5 @@ +.. default-role:: code + ========================================= Internals of the Nim Compiler ========================================= @@ -19,19 +21,19 @@ The Nim project's directory structure is: ============ =================================================== Path Purpose ============ =================================================== -``bin`` generated binary files -``build`` generated C code for the installation -``compiler`` the Nim compiler itself; note that this +`bin` generated binary files +`build` generated C code for the installation +`compiler` the Nim compiler itself; note that this code has been translated from a bootstrapping version written in Pascal, so the code is **not** a poster child of good Nim code -``config`` configuration files for Nim -``dist`` additional packages for the distribution -``doc`` the documentation; it is a bunch of +`config` configuration files for Nim +`dist` additional packages for the distribution +`doc` the documentation; it is a bunch of reStructuredText files -``lib`` the Nim library -``web`` website of Nim; generated by ``nimweb`` - from the ``*.txt`` and ``*.nimf`` files +`lib` the Nim library +`web` website of Nim; generated by `nimweb` + from the `*.txt` and `*.nimf` files ============ =================================================== @@ -53,7 +55,7 @@ And for a debug version compatible with GDB:: nim c koch.nim ./koch boot --debuginfo --linedir:on -The ``koch`` program is Nim's maintenance script. It is a replacement for +The `koch` program is Nim's maintenance script. It is a replacement for make and shell scripting with the advantage that it is much more portable. More information about its options can be found in the `koch `_ documentation. @@ -67,8 +69,8 @@ Coding Guidelines * Max line length is 80 characters. * Provide spaces around binary operators if that enhances readability. * Use a space after a colon, but not before it. -* [deprecated] Start types with a capital ``T``, unless they are - pointers/references which start with ``P``. +* [deprecated] Start types with a capital `T`, unless they are + pointers/references which start with `P`. See also the `API naming design `_ document. @@ -81,12 +83,12 @@ portable programming language (within certain limits) and Nim generates C code, porting the code generator is not necessary. POSIX-compliant systems on conventional hardware are usually pretty easy to -port: Add the platform to ``platform`` (if it is not already listed there), +port: Add the platform to `platform` (if it is not already listed there), check that the OS, System modules work and recompile Nim. The only case where things aren't as easy is when the garbage collector needs some assembler tweaking to work. The standard -version of the GC uses C's ``setjmp`` function to store all registers +version of the GC uses C's `setjmp` function to store all registers on the hardware stack. It may be necessary that the new platform needs to replace this generic code by some assembler code. @@ -111,7 +113,7 @@ Complex assignments We already know the type information as a graph in the compiler. Thus we need to serialize this graph as RTTI for C code generation. -Look at the file ``lib/system/hti.nim`` for more information. +Look at the file `lib/system/hti.nim` for more information. Rebuilding the compiler ======================== @@ -139,7 +141,7 @@ Debugging the compiler ====================== You can of course use GDB or Visual Studio to debug the -compiler (via ``--debuginfo --lineDir:on``). However, there +compiler (via `--debuginfo --lineDir:on`). However, there are also lots of procs that aid in debugging: @@ -170,19 +172,19 @@ These procs may not be imported by a module. You can import them directly for de from renderer import renderTree from msgs import `??` -To create a new compiler for each run, use ``koch temp``:: +To create a new compiler for each run, use `koch temp`:: ./koch temp c /tmp/test.nim -``koch temp`` creates a debug build of the compiler, which is useful +`koch temp` creates a debug build of the compiler, which is useful to create stacktraces for compiler debugging. See also `Rebuilding the compiler`_ if you need more control. Bisecting for regressions ========================= -``koch temp`` returns 125 as the exit code in case the compiler -compilation fails. This exit code tells ``git bisect`` to skip the +`koch temp` returns 125 as the exit code in case the compiler +compilation fails. This exit code tells `git bisect` to skip the current commit.:: git bisect start bad-commit good-commit @@ -219,15 +221,15 @@ examples how the AST represents each syntactic structure. How the RTL is compiled ======================= -The ``system`` module contains the part of the RTL which needs support by +The `system` module contains the part of the RTL which needs support by compiler magic (and the stuff that needs to be in it because the spec says so). The C code generator generates the C code for it, just like any other -module. However, calls to some procedures like ``addInt`` are inserted by -the CCG. Therefore the module ``magicsys`` contains a table (``compilerprocs``) -with all symbols that are marked as ``compilerproc``. ``compilerprocs`` are -needed by the code generator. A ``magic`` proc is not the same as a -``compilerproc``: A ``magic`` is a proc that needs compiler magic for its -semantic checking, a ``compilerproc`` is a proc that is used by the code +module. However, calls to some procedures like `addInt` are inserted by +the CCG. Therefore the module `magicsys` contains a table (`compilerprocs`) +with all symbols that are marked as `compilerproc`. `compilerprocs` are +needed by the code generator. A `magic` proc is not the same as a +`compilerproc`: A `magic` is a proc that needs compiler magic for its +semantic checking, a `compilerproc` is a proc that is used by the code generator. @@ -254,7 +256,7 @@ This solves the problem without having to special case the logic that fills the internal seqs which are affected by the pragmas. In fact, this describes how the AST should be stored in the database, -as a "shallow" tree. Let's assume we compile module ``m`` with the +as a "shallow" tree. Let's assume we compile module `m` with the following contents: .. code-block:: nim @@ -279,21 +281,21 @@ Conceptually this is the AST we store for the module: static: echo "static" -The symbol's ``ast`` field is loaded lazily, on demand. This is where most +The symbol's `ast` field is loaded lazily, on demand. This is where most savings come from, only the shallow outer AST is reconstructed immediately. -It is also important that the replay involves the ``import`` statement so +It is also important that the replay involves the `import` statement so that dependencies are resolved properly. Shared global compiletime state ------------------------------- -Nim allows ``.global, compiletime`` variables that can be filled by macro +Nim allows `.global, compiletime` variables that can be filled by macro invocations across different modules. This feature breaks modularity in a severe way. Plenty of different solutions have been proposed: -- Restrict the types of global compiletime variables to ``Set[T]`` or +- Restrict the types of global compiletime variables to `Set[T]` or similar unordered, only-growable collections so that we can track the module's write effects to these variables and reapply the changes in a different order. @@ -306,7 +308,7 @@ severe way. Plenty of different solutions have been proposed: Since we adopt the "replay the top level statements" idea, the natural solution to this problem is to emit pseudo top level statements that reflect the mutations done to the global variable. However, this is -MUCH harder than it sounds, for example ``squeaknim`` uses this +MUCH harder than it sounds, for example `squeaknim` uses this snippet: .. code-block:: nim @@ -314,12 +316,12 @@ snippet: "\t^self externalCallFailed\C!\C\C") stCode.add(st & "\C\t\"Generated by NimSqueak\"\C\t" & apicall) -We can "replay" ``stCode.add`` only if the values of ``st`` -and ``apicall`` are known. And even then a hash table's ``add`` with its +We can "replay" `stCode.add` only if the values of `st` +and `apicall` are known. And even then a hash table's `add` with its hashing mechanism is too hard to replay. -In practice, things are worse still, consider ``someGlobal[i][j].add arg``. -We only know the root is ``someGlobal`` but the concrete path to the data +In practice, things are worse still, consider `someGlobal[i][j].add arg`. +We only know the root is `someGlobal` but the concrete path to the data is unknown as is the value that is added. We could compute a "diff" between the global states and use that to compute a symbol patchset, but this is quite some work, expensive to do at runtime (it would need to run after @@ -342,7 +344,7 @@ an alien API and works with some existing Nimble packages, at least. On the other hand, in Nim's future I would like to replace the VM by native code. A diff algorithm wouldn't work for that. -Instead the native code would work with an API like ``put``, ``get``: +Instead the native code would work with an API like `put`, `get`: .. code-block:: nim @@ -350,7 +352,7 @@ Instead the native code would work with an API like ``put``, ``get``: proc cacheGet*(key: string): NimNode The API should embrace the AST diffing notion: See the -module ``macrocache`` for the final details. +module `macrocache` for the final details. @@ -382,9 +384,9 @@ too. Type converters fall into this category: if 1: echo "ugly, but should work" -If in the above example module ``B`` is re-compiled, but ``A`` is not then -``B`` needs to be aware of ``toBool`` even though ``toBool`` is not referenced -in ``B`` *explicitly*. +If in the above example module `B` is re-compiled, but `A` is not then +`B` needs to be aware of `toBool` even though `toBool` is not referenced +in `B` *explicitly*. Both the multi method and the type converter problems are solved by the AST replay implementation. @@ -395,7 +397,7 @@ Generics We cache generic instantiations and need to ensure this caching works well with the incremental compilation feature. Since the cache is -attached to the ``PSym`` datastructure, it should work without any +attached to the `PSym` datastructure, it should work without any special logic. @@ -405,22 +407,22 @@ Backend issues - Init procs must not be "forgotten" to be called. - Files must not be "forgotten" to be linked. - Method dispatchers are global. -- DLL loading via ``dlsym`` is global. +- DLL loading via `dlsym` is global. - Emulated thread vars are global. However the biggest problem is that dead code elimination breaks modularity! -To see why, consider this scenario: The module ``G`` (for example the huge +To see why, consider this scenario: The module `G` (for example the huge Gtk2 module...) is compiled with dead code elimination turned on. So none -of ``G``'s procs is generated at all. +of `G`'s procs is generated at all. -Then module ``B`` is compiled that requires ``G.P1``. Ok, no problem, -``G.P1`` is loaded from the symbol file and ``G.c`` now contains ``G.P1``. +Then module `B` is compiled that requires `G.P1`. Ok, no problem, +`G.P1` is loaded from the symbol file and `G.c` now contains `G.P1`. -Then module ``A`` (that depends on ``B`` and ``G``) is compiled and ``B`` -and ``G`` are left unchanged. ``A`` requires ``G.P2``. +Then module `A` (that depends on `B` and `G`) is compiled and `B` +and `G` are left unchanged. `A` requires `G.P2`. -So now ``G.c`` MUST contain both ``P1`` and ``P2``, but we haven't even -loaded ``P1`` from the symbol file, nor do we want to because we then quickly +So now `G.c` MUST contain both `P1` and `P2`, but we haven't even +loaded `P1` from the symbol file, nor do we want to because we then quickly would restore large parts of the whole program. @@ -428,7 +430,7 @@ Solution ~~~~~~~~ The backend must have some logic so that if the currently processed module -is from the compilation cache, the ``ast`` field is not accessed. Instead +is from the compilation cache, the `ast` field is not accessed. Instead the generated C(++) for the symbol's body needs to be cached too and inserted back into the produced C file. This approach seems to deal with all the outlined problems above. @@ -444,8 +446,8 @@ in mind: keeps allocating memory! Thus a stack overflow may happen, hiding the real issue. * What seem to be C code generation problems is often a bug resulting from - not producing prototypes, so that some types default to ``cint``. Testing - without the ``-w`` option helps! + not producing prototypes, so that some types default to `cint`. Testing + without the `-w` option helps! The Garbage Collector @@ -464,9 +466,9 @@ code generation. Each cell has a header consisting of a RC and a pointer to its type descriptor. However the program does not know about these, so they are placed at -negative offsets. In the GC code the type ``PCell`` denotes a pointer +negative offsets. In the GC code the type `PCell` denotes a pointer decremented by the right offset, so that the header can be accessed easily. It -is extremely important that ``pointer`` is not confused with a ``PCell`` +is extremely important that `pointer` is not confused with a `PCell` as this would lead to a memory corruption. @@ -474,9 +476,9 @@ The CellSet data structure -------------------------- The GC depends on an extremely efficient datastructure for storing a -set of pointers - this is called a ``TCellSet`` in the source code. +set of pointers - this is called a `TCellSet` in the source code. Inserting, deleting and searching are done in constant time. However, -modifying a ``TCellSet`` during traversal leads to undefined behaviour. +modifying a `TCellSet` during traversal leads to undefined behaviour. .. code-block:: Nim type @@ -559,11 +561,11 @@ Code generation for closures is implemented by `lambda lifting`:idx:. Design ------ -A ``closure`` proc var can call ordinary procs of the default Nim calling +A `closure` proc var can call ordinary procs of the default Nim calling convention. But not the other way round! A closure is implemented as a -``tuple[prc, env]``. ``env`` can be nil implying a call without a closure. -This means that a call through a closure generates an ``if`` but the -interoperability is worth the cost of the ``if``. Thunk generation would be +`tuple[prc, env]`. `env` can be nil implying a call without a closure. +This means that a call through a closure generates an `if` but the +interoperability is worth the cost of the `if`. Thunk generation would be possible too, but it's slightly more effort to implement. Tests with GCC on Amd64 showed that it's really beneficial if the @@ -579,7 +581,7 @@ A thunk would need to call 'returnsDefaultCC[i]' somehow and that would require an *additional* closure generation... Ok, not really, but it requires to pass the function to call. So we'd end up with 2 indirect calls instead of one. Another much more severe problem which this solution is that it's not GC-safe -to pass a proc pointer around via a generic ``ref`` type. +to pass a proc pointer around via a generic `ref` type. Example code: @@ -695,15 +697,15 @@ Accumulator Internals --------- -Lambda lifting is implemented as part of the ``transf`` pass. The ``transf`` +Lambda lifting is implemented as part of the `transf` pass. The `transf` pass generates code to setup the environment and to pass it around. However, this pass does not change the types! So we have some kind of mismatch here; on the one hand the proc expression becomes an explicit tuple, on the other hand the tyProc(ccClosure) type is not changed. For C code generation it's also -important the hidden formal param is ``void*`` and not something more +important the hidden formal param is `void*` and not something more specialized. However the more specialized env type needs to passed to the -backend somehow. We deal with this by modifying ``s.ast[paramPos]`` to contain -the formal hidden parameter, but not ``s.typ``! +backend somehow. We deal with this by modifying `s.ast[paramPos]` to contain +the formal hidden parameter, but not `s.typ`! Integer literals: diff --git a/doc/koch.rst b/doc/koch.rst index c92e29812..1eb02d785 100644 --- a/doc/koch.rst +++ b/doc/koch.rst @@ -1,3 +1,5 @@ +.. default-role:: code + =============================== Nim maintenance script =============================== @@ -17,7 +19,7 @@ Introduction The `koch`:idx: program is Nim's maintenance script. It is a replacement for make and shell scripting with the advantage that it is much more portable. -The word *koch* means *cook* in German. ``koch`` is used mainly to build the +The word *koch* means *cook* in German. `koch` is used mainly to build the Nim compiler, but it can also be used for other tasks. This document describes the supported commands and their options. @@ -39,8 +41,8 @@ options: Use the linenoise library for interactive mode (not needed on Windows). After compilation is finished you will hopefully end up with the nim -compiler in the ``bin`` directory. You can add Nim's ``bin`` directory to -your ``$PATH`` or use the install command to place it where it will be +compiler in the `bin` directory. You can add Nim's `bin` directory to +your `$PATH` or use the install command to place it where it will be found. csource command @@ -54,33 +56,33 @@ temp command ------------ The temp command builds the Nim compiler but with a different final name -(``nim_temp``), so it doesn't overwrite your normal compiler. You can use +(`nim_temp`), so it doesn't overwrite your normal compiler. You can use this command to test different options, the same you would issue for the `boot command <#commands-boot-command>`_. test command ------------ -The `test`:idx: command can also be invoked with the alias ``tests``. This -command will compile and run ``testament/tester.nim``, which is the main -driver of Nim's test suite. You can pass options to the ``test`` command, +The `test`:idx: command can also be invoked with the alias `tests`. This +command will compile and run `testament/tester.nim`, which is the main +driver of Nim's test suite. You can pass options to the `test` command, they will be forwarded to the tester. See its source code for available options. web command ----------- -The `web`:idx: command converts the documentation in the ``doc`` directory +The `web`:idx: command converts the documentation in the `doc` directory from rst to HTML. It also repeats the same operation but places the result in -the ``web/upload`` which can be used to update the website at +the `web/upload` which can be used to update the website at https://nim-lang.org. By default, the documentation will be built in parallel using the number of available CPU cores. If any documentation build sub-commands fail, they will be rerun in serial fashion so that meaningful error output can be gathered for -inspection. The ``--parallelBuild:n`` switch or configuration option can be +inspection. The `--parallelBuild:n` switch or configuration option can be used to force a specific number of parallel jobs or run everything serially -from the start (``n == 1``). +from the start (`n == 1`). pdf command ----------- diff --git a/doc/lib.rst b/doc/lib.rst index 2a4c2c03c..3202c5a53 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -1,3 +1,5 @@ +.. default-role:: code + ==================== Nim Standard Library ==================== @@ -9,7 +11,7 @@ Nim Standard Library Nim's library is divided into *pure libraries*, *impure libraries*, and *wrappers*. -Pure libraries do not depend on any external ``*.dll`` or ``lib*.so`` binary +Pure libraries do not depend on any external `*.dll` or `lib*.so` binary while impure libraries do. A wrapper is an impure library that is a very low-level interface to a C library. @@ -37,11 +39,11 @@ Automatic imports * `threads `_ Basic Nim thread support. **Note:** This is part of the system module. Do not - import it explicitly. Enabled with ``--threads:on``. + import it explicitly. Enabled with `--threads:on`. * `channels `_ Nim message passing support for threads. **Note:** This is part of the - system module. Do not import it explicitly. Enabled with ``--threads:on``. + system module. Do not import it explicitly. Enabled with `--threads:on`. Core @@ -86,14 +88,14 @@ Algorithms This module implements some common generic algorithms like sort or binary search. * `std/enumutils `_ - This module adds functionality for the built-in ``enum`` type. + This module adds functionality for the built-in `enum` type. * `sequtils `_ - This module implements operations for the built-in ``seq`` type + This module implements operations for the built-in `seq` type which were inspired by functional programming languages. * `std/setutils `_ - This module adds functionality for the built-in ``set`` type. + This module adds functionality for the built-in `set` type. Collections @@ -105,7 +107,7 @@ Collections * `deques `_ Implementation of a double-ended queue. - The underlying implementation uses a ``seq``. + The underlying implementation uses a `seq`. * `heapqueue `_ Implementation of a heap data structure that can be used as a priority queue. @@ -140,7 +142,7 @@ String handling --------------- * `cstrutils `_ - Utilities for ``cstring`` handling. + Utilities for `cstring` handling. * `std/editdistance `_ This module contains an algorithm to compute the edit distance between two @@ -148,7 +150,7 @@ String handling * `encodings `_ Converts between different character encodings. On UNIX, this uses - the ``iconv`` library, on Windows the Windows API. + the `iconv` library, on Windows the Windows API. * `parseutils `_ This module contains helpers for parsing tokens, numbers, identifiers, etc. @@ -166,17 +168,17 @@ String handling * `strformat `_ Macro based standard string interpolation/formatting. Inspired by - Python's ``f``-strings. + Python's `f`-strings. * `strmisc `_ This module contains uncommon string handling operations that do not fit with the commonly used operations in strutils. * `strscans `_ - This module contains a ``scanf`` macro for convenient parsing of mini languages. + This module contains a `scanf` macro for convenient parsing of mini languages. * `strtabs `_ - The ``strtabs`` module implements an efficient hash table that is a mapping + The `strtabs` module implements an efficient hash table that is a mapping from strings to strings. Supports a case-sensitive, case-insensitive and style-insensitive modes. @@ -200,10 +202,10 @@ Time handling ------------- * `std/monotimes `_ - The ``monotimes`` module implements monotonic timestamps. + The `monotimes` module implements monotonic timestamps. * `times `_ - The ``times`` module contains support for working with time. + The `times` module contains support for working with time. Generic Operating System Services @@ -225,7 +227,7 @@ Generic Operating System Services data structures. * `memfiles `_ - This module provides support for memory-mapped files (Posix's ``mmap``) + This module provides support for memory-mapped files (Posix's `mmap`) on the different operating systems. * `os `_ @@ -234,12 +236,12 @@ Generic Operating System Services commands, etc. * `osproc `_ - Module for process communication beyond ``os.execShellCmd``. + Module for process communication beyond `os.execShellCmd`. * `streams `_ This module provides a stream interface and two implementations thereof: - the ``FileStream`` and the ``StringStream`` which implement the stream - interface for Nim file objects (``File``) and strings. Other modules + the `FileStream` and the `StringStream` which implement the stream + interface for Nim file objects (`File`) and strings. Other modules may provide other implementations for this standard stream interface. * `terminal `_ @@ -288,22 +290,22 @@ Internet Protocols and Support * `asyncfile `_ This module implements asynchronous file reading and writing using - ``asyncdispatch``. + `asyncdispatch`. * `asyncftpclient `_ - This module implements an asynchronous FTP client using the ``asyncnet`` + This module implements an asynchronous FTP client using the `asyncnet` module. * `asynchttpserver `_ - This module implements an asynchronous HTTP server using the ``asyncnet`` + This module implements an asynchronous HTTP server using the `asyncnet` module. * `asyncnet `_ - This module implements asynchronous sockets based on the ``asyncdispatch`` + This module implements asynchronous sockets based on the `asyncdispatch` module. * `asyncstreams `_ - This module provides ``FutureStream`` - a future that acts as a queue. + This module provides `FutureStream` - a future that acts as a queue. * `cgi `_ This module implements helpers for CGI applications. @@ -323,7 +325,7 @@ Internet Protocols and Support * `net `_ This module implements a high-level sockets API. It replaces the - ``sockets`` module. + `sockets` module. * `selectors `_ This module implements a selector API with backends specific to each OS. @@ -357,23 +359,23 @@ Parsers scheme for lexers and parsers. This is used by the diverse parsing modules. * `parsecfg `_ - The ``parsecfg`` module implements a high-performance configuration file - parser. The configuration file's syntax is similar to the Windows ``.ini`` + The `parsecfg` module implements a high-performance configuration file + parser. The configuration file's syntax is similar to the Windows `.ini` format, but much more powerful, as it is not a line based parser. String literals, raw string literals, and triple quote string literals are supported as in the Nim programming language. * `parsecsv `_ - The ``parsecsv`` module implements a simple high-performance CSV parser. + The `parsecsv` module implements a simple high-performance CSV parser. * `parseopt `_ - The ``parseopt`` module implements a command line option parser. + The `parseopt` module implements a command line option parser. * `parsesql `_ - The ``parsesql`` module implements a simple high-performance SQL parser. + The `parsesql` module implements a simple high-performance SQL parser. * `parsexml `_ - The ``parsexml`` module implements a simple high performance XML/HTML parser. + The `parsexml` module implements a simple high performance XML/HTML parser. The only encoding that is supported is UTF-8. The parser has been designed to be somewhat error-correcting, so that even some "wild HTML" found on the web can be parsed with it. @@ -458,7 +460,7 @@ Miscellaneous This module implements a simple logger. * `segfaults `_ - Turns access violations or segfaults into a ``NilAccessDefect`` exception. + Turns access violations or segfaults into a `NilAccessDefect` exception. * `sugar `_ This module implements nice syntactic sugar based on Nim's macro system. @@ -480,11 +482,11 @@ Modules for JS backend Declaration of the Document Object Model for the JS backend. * `jsconsole `_ - Wrapper for the ``console`` object. + Wrapper for the `console` object. * `jscore `_ The wrapper of core JavaScript functions. For most purposes, you should be using - the ``math``, ``json``, and ``times`` stdlib modules instead of this module. + the `math`, `json`, and `times` stdlib modules instead of this module. * `jsffi `_ Types and macros for easier interaction with JavaScript. diff --git a/doc/manual/var_t_return.rst b/doc/manual/var_t_return.rst index c6de8cf7c..e34993e3e 100644 --- a/doc/manual/var_t_return.rst +++ b/doc/manual/var_t_return.rst @@ -1,6 +1,8 @@ -Memory safety for returning by ``var T`` is ensured by a simple borrowing -rule: If ``result`` does not refer to a location pointing to the heap -(that is in ``result = X`` the ``X`` involves a ``ptr`` or ``ref`` access) +.. default-role:: code + +Memory safety for returning by `var T` is ensured by a simple borrowing +rule: If `result` does not refer to a location pointing to the heap +(that is in `result = X` the `X` involves a `ptr` or `ref` access) then it has to be derived from the routine's first parameter: .. code-block:: nim @@ -11,10 +13,10 @@ then it has to be derived from the routine's first parameter: var x: int # we know 'forward' provides a view into the location derived from # its first argument 'x'. - result = forward(x) # Error: location is derived from ``x`` + result = forward(x) # Error: location is derived from `x` # which is not p's first parameter and lives # on the stack. -In other words, the lifetime of what ``result`` points to is attached to the +In other words, the lifetime of what `result` points to is attached to the lifetime of the first parameter and that is enough knowledge to verify memory safety at the call site. diff --git a/doc/manual_experimental.rst b/doc/manual_experimental.rst index b76839842..cf2e0c247 100644 --- a/doc/manual_experimental.rst +++ b/doc/manual_experimental.rst @@ -1,3 +1,5 @@ +.. default-role:: code + ========================= Nim Experimental Features ========================= @@ -12,8 +14,8 @@ About this document =================== This document describes features of Nim that are to be considered experimental. -Some of these are not covered by the ``.experimental`` pragma or -``--experimental`` switch because they are already behind a special syntax and +Some of these are not covered by the `.experimental` pragma or +`--experimental` switch because they are already behind a special syntax and one may want to use Nim libraries using these features without using them oneself. @@ -28,13 +30,13 @@ Every Nim module resides in a (nimble) package. An object type can be attached to the package it resides in. If that is done, the type can be referenced from other modules as an `incomplete`:idx: object type. This feature allows to break up recursive type dependencies across module boundaries. Incomplete -object types are always passed ``byref`` and can only be used in pointer like -contexts (``var/ref/ptr IncompleteObject``) in general since the compiler does +object types are always passed `byref` and can only be used in pointer like +contexts (`var/ref/ptr IncompleteObject`) in general since the compiler does not yet know the size of the object. To complete an incomplete object -the ``package`` pragma has to be used. ``package`` implies ``byref``. +the `package` pragma has to be used. `package` implies `byref`. -As long as a type ``T`` is incomplete, neither ``sizeof(T)`` nor runtime -type information for ``T`` is available. +As long as a type `T` is incomplete, neither `sizeof(T)` nor runtime +type information for `T` is available. Example: @@ -63,8 +65,8 @@ Example: Void type ========= -The ``void`` type denotes the absence of any type. Parameters of -type ``void`` are treated as non-existent, ``void`` as a return type means that +The `void` type denotes the absence of any type. Parameters of +type `void` are treated as non-existent, `void` as a return type means that the procedure does not return a value: .. code-block:: nim @@ -73,7 +75,7 @@ the procedure does not return a value: nothing() # writes "ha" to stdout -The ``void`` type is particularly useful for generic code: +The `void` type is particularly useful for generic code: .. code-block:: nim proc callProc[T](p: proc (x: T), x: T) = @@ -88,7 +90,7 @@ The ``void`` type is particularly useful for generic code: callProc[int](intProc, 12) callProc[void](emptyProc) -However, a ``void`` type cannot be inferred in generic code: +However, a `void` type cannot be inferred in generic code: .. code-block:: nim callProc(emptyProc) @@ -96,8 +98,8 @@ However, a ``void`` type cannot be inferred in generic code: # but expected one of: # callProc(p: proc (T), x: T) -The ``void`` type is only valid for parameters and return types; other symbols -cannot have the type ``void``. +The `void` type is only valid for parameters and return types; other symbols +cannot have the type `void`. @@ -105,19 +107,19 @@ Covariance ========== Covariance in Nim can be introduced only through pointer-like types such -as ``ptr`` and ``ref``. Sequence, Array and OpenArray types, instantiated +as `ptr` and `ref`. Sequence, Array and OpenArray types, instantiated with pointer-like types will be considered covariant if and only if they -are also immutable. The introduction of a ``var`` modifier or additional -``ptr`` or ``ref`` indirections would result in invariant treatment of +are also immutable. The introduction of a `var` modifier or additional +`ptr` or `ref` indirections would result in invariant treatment of these types. -``proc`` types are currently always invariant, but future versions of Nim +`proc` types are currently always invariant, but future versions of Nim may relax this rule. User-defined generic types may also be covariant with respect to some of their parameters. By default, all generic params are considered invariant, -but you may choose the apply the prefix modifier ``in`` to a parameter to -make it contravariant or ``out`` to make it covariant: +but you may choose the apply the prefix modifier `in` to a parameter to +make it contravariant or `out` to make it covariant: .. code-block:: nim type @@ -166,10 +168,10 @@ values: # to point to a ComboBox On the other hand, in the `RingBuffer` example above, the designated generic -param is used to instantiate the non-pointer ``seq`` type, which means that +param is used to instantiate the non-pointer `seq` type, which means that the resulting generic type will have covariance that mimics an array or -sequence (i.e. it will be covariant only when instantiated with ``ptr`` and -``ref`` types): +sequence (i.e. it will be covariant only when instantiated with `ptr` and +`ref` types): .. code-block:: nim @@ -196,7 +198,7 @@ as `seq[AnnotatedPtr[T]]` or `RingBuffer[AnnotatedPtr[T]]` will also be considered covariant and you can create new pointer-like types by instantiating other user-defined pointer-like types. -The contravariant parameters introduced with the ``in`` modifier are currently +The contravariant parameters introduced with the `in` modifier are currently useful only when interfacing with imported types having such semantics. @@ -204,7 +206,7 @@ Automatic dereferencing ======================= Automatic dereferencing is performed for the first argument of a routine call. -This feature has to be enabled via ``{.experimental: "implicitDeref".}``: +This feature has to be enabled via `{.experimental: "implicitDeref".}`: .. code-block:: nim {.experimental: "implicitDeref".} @@ -257,7 +259,7 @@ preface definitions inside a module. Please note that if a callable symbol is never used in this scenario, its body will never be compiled. This is the default behavior leading to best compilation times, but if exhaustive compilation of all definitions is - required, using ``nim check`` provides this option as well. + required, using `nim check` provides this option as well. Example: @@ -290,10 +292,10 @@ what code is executed at the top level: .. TODO: Let's table this for now. This is an *experimental feature* and so the - specific manner in which ``declared`` operates with it can be decided in + specific manner in which `declared` operates with it can be decided in eventuality, because right now it works a bit weirdly. - The values of expressions involving ``declared`` are decided *before* the + The values of expressions involving `declared` are decided *before* the code reordering process, and not after. As an example, the output of this code is the same as it would be with code reordering disabled. @@ -325,7 +327,7 @@ Named argument overloading ========================== Routines with the same type signature can be called differently if a parameter -has different names. This does not need an ``experimental`` switch, but is an +has different names. This does not need an `experimental` switch, but is an unstable feature. .. code-block::nim @@ -344,7 +346,7 @@ Do notation =========== As a special more convenient notation, proc expressions involved in procedure -calls can use the ``do`` keyword: +calls can use the `do` keyword: .. code-block:: nim sort(cities) do (x,y: string) -> int: @@ -359,13 +361,13 @@ calls can use the ``do`` keyword: if not `ex`: echo `info`, ": Check failed: ", `expString` -``do`` is written after the parentheses enclosing the regular proc params. +`do` is written after the parentheses enclosing the regular proc params. The proc expression represented by the do block is appended to them. In calls using the command syntax, the do block will bind to the immediately preceding expression, transforming it in a call. -``do`` with parentheses is an anonymous ``proc``; however a ``do`` without -parentheses is just a block of code. The ``do`` notation can be used to +`do` with parentheses is an anonymous `proc`; however a `do` without +parentheses is just a block of code. The `do` notation can be used to pass multiple blocks to a macro: .. code-block:: nim @@ -385,7 +387,7 @@ dot operators ------------- **Note**: Dot operators are still experimental and so need to be enabled -via ``{.experimental: "dotOperators".}``. +via `{.experimental: "dotOperators".}`. Nim offers a special family of dot operators that can be used to intercept and rewrite proc call and field access attempts, referring @@ -398,7 +400,7 @@ When Nim encounters an expression that cannot be resolved by the standard overload resolution rules, the current scope will be searched for a dot operator that can be matched against a re-written form of the expression, where the unknown field or proc name is passed to -an ``untyped`` parameter: +an `untyped` parameter: .. code-block:: nim a.b # becomes `.`(a, b) @@ -471,10 +473,10 @@ Not nil annotation ================== **Note:** This is an experimental feature. It can be enabled with -``{.experimental: "notnil"}``. +`{.experimental: "notnil"}`. -All types for which ``nil`` is a valid value can be annotated with the ``not -nil`` annotation to exclude ``nil`` as a valid value: +All types for which `nil` is a valid value can be annotated with the `not +nil` annotation to exclude `nil` as a valid value: .. code-block:: nim {.experimental: "notnil"} @@ -526,9 +528,9 @@ The concept is a match if: a) all of the expressions within the body can be compiled for the tested type b) all statically evaluable boolean expressions in the body must be true -The identifiers following the ``concept`` keyword represent instances of the +The identifiers following the `concept` keyword represent instances of the currently matched type. You can apply any of the standard type modifiers such -as ``var``, ``ref``, ``ptr`` and ``static`` to denote a more specific type of +as `var`, `ref`, `ptr` and `static` to denote a more specific type of instance. You can also apply the `type` modifier to create a named instance of the type itself: @@ -546,9 +548,9 @@ the presence of callable symbols with specific signatures: OutputStream = concept var s s.write(string) -In order to check for symbols accepting ``type`` params, you must prefix -the type with the explicit ``type`` modifier. The named instance of the -type, following the ``concept`` keyword is also considered to have the +In order to check for symbols accepting `type` params, you must prefix +the type with the explicit `type` modifier. The named instance of the +type, following the `concept` keyword is also considered to have the explicit modifier and will be matched only as a type. .. code-block:: nim @@ -570,7 +572,7 @@ explicit modifier and will be matched only as a type. -x is T x - y is T -Please note that the ``is`` operator allows one to easily verify the precise +Please note that the `is` operator allows one to easily verify the precise type signatures of the required operations, but since type inference and default parameters are still applied in the concept body, it's also possible to describe usage protocols that do not reveal implementation details. @@ -586,7 +588,7 @@ By default, the compiler will report the matching errors in concepts only when no other overload can be selected and a normal compilation error is produced. When you need to understand why the compiler is not matching a particular concept and, as a result, a wrong overload is selected, you can apply the -``explain`` pragma to either the concept body or a particular call-site. +`explain` pragma to either the concept body or a particular call-site. .. code-block:: nim type @@ -670,7 +672,7 @@ resembles the way generic parameters of callable symbols are inferred on call sites. Unbound types can appear both as params to calls such as `s.push(T)` and -on the right-hand side of the ``is`` operator in cases such as `x.pop is T` +on the right-hand side of the `is` operator in cases such as `x.pop is T` and `x.data is seq[T]`. Unbound static params will be inferred from expressions involving the `==` @@ -684,8 +686,8 @@ operator and also when types dependent on them are being matched: The Nim compiler includes a simple linear equation solver, allowing it to infer static params in some situations where integer arithmetic is involved. -Just like in regular type classes, Nim discriminates between ``bind once`` -and ``bind many`` types when matching the concept. You can add the ``distinct`` +Just like in regular type classes, Nim discriminates between `bind once` +and `bind many` types when matching the concept. You can add the `distinct` modifier to any of the otherwise inferable types to get a type that will be matched without permanently inferring it. This may be useful when you need to match several procs accepting the same wide class of types: @@ -708,7 +710,7 @@ to match several procs accepting the same wide class of types: type Enum = distinct Enumerable o.baz is Enum -On the other hand, using ``bind once`` types allows you to test for equivalent +On the other hand, using `bind once` types allows you to test for equivalent types used in multiple signatures, without actually requiring any concrete types, thus allowing you to encode implementation-defined types: @@ -804,7 +806,7 @@ concept, we say that the outer concept is a refinement of the inner concept and thus it is more-specific. When both concepts are matched in a call during overload resolution, Nim will assign a higher precedence to the most specific one. As an alternative way of defining concept refinements, you can use the -object inheritance syntax involving the ``of`` keyword: +object inheritance syntax involving the `of` keyword: .. code-block:: nim type @@ -895,8 +897,8 @@ object inheritance syntax involving the ``of`` keyword: any type can implement an unlimited number of protocols or interfaces not originally envisioned by the type's author. - Any concept type can be turned into a VTable type by using the ``vtref`` - or the ``vtptr`` compiler magics. Under the hood, these magics generate + Any concept type can be turned into a VTable type by using the `vtref` + or the `vtptr` compiler magics. Under the hood, these magics generate a converter type class, which converts the regular instances of the matching types to the corresponding VTable type. @@ -928,8 +930,8 @@ object inheritance syntax involving the ``of`` keyword: but it will include a smaller number of captured procs. A completely empty vtable will be reported as an error. - The ``vtref`` magic produces types which can be bound to ``ref`` types and - the ``vtptr`` magic produced types bound to ``ptr`` types. + The `vtref` magic produces types which can be bound to `ref` types and + the `vtptr` magic produced types bound to `ptr` types. Type bound operations @@ -944,12 +946,12 @@ There are 4 operations that are bound to a type: These operations can be *overridden* instead of *overloaded*. This means the implementation is automatically lifted to structured types. For instance if type -``T`` has an overridden assignment operator ``=`` this operator is also used -for assignments of the type ``seq[T]``. Since these operations are bound to a +`T` has an overridden assignment operator `=` this operator is also used +for assignments of the type `seq[T]`. Since these operations are bound to a type they have to be bound to a nominal type for reasons of simplicity of -implementation: This means an overridden ``deepCopy`` for ``ref T`` is really -bound to ``T`` and not to ``ref T``. This also means that one cannot override -``deepCopy`` for both ``ptr T`` and ``ref T`` at the same time; instead a +implementation: This means an overridden `deepCopy` for `ref T` is really +bound to `T` and not to `ref T`. This also means that one cannot override +`deepCopy` for both `ptr T` and `ref T` at the same time; instead a helper distinct or object type has to be used for one pointer type. Assignments, moves and destruction are specified in @@ -959,9 +961,9 @@ the `destructors `_ document. deepCopy -------- -``=deepCopy`` is a builtin that is invoked whenever data is passed to -a ``spawn``'ed proc to ensure memory safety. The programmer can override its -behaviour for a specific ``ref`` or ``ptr`` type ``T``. (Later versions of the +`=deepCopy` is a builtin that is invoked whenever data is passed to +a `spawn`'ed proc to ensure memory safety. The programmer can override its +behaviour for a specific `ref` or `ptr` type `T`. (Later versions of the language may weaken this restriction.) The signature has to be: @@ -972,7 +974,7 @@ The signature has to be: This mechanism will be used by most data structures that support shared memory like channels to implement thread safe automatic memory management. -The builtin ``deepCopy`` can even clone closures and their environments. See +The builtin `deepCopy` can even clone closures and their environments. See the documentation of `spawn <#parallel-amp-spawn-spawn-statement>`_ for details. @@ -982,7 +984,7 @@ Case statement macros Macros named `case` can rewrite `case` statements for certain types in order to implement `pattern matching`:idx:. The following example implements a simplistic form of pattern matching for tuples, leveraging the existing -equality operator for tuples (as provided in ``system.==``): +equality operator for tuples (as provided in `system.==`): .. code-block:: nim :test: "nim c $1" @@ -1013,7 +1015,7 @@ equality operator for tuples (as provided in ``system.==``): Currently case statement macros must be enabled explicitly -via ``{.experimental: "caseStmtMacros".}``. +via `{.experimental: "caseStmtMacros".}`. `case` macros are subject to overload resolution. The type of the `case` statement's selector expression is matched against the type @@ -1039,10 +1041,10 @@ compilation pipeline with user defined optimizations: let x = 3 echo x * 2 -The compiler now rewrites ``x * 2`` as ``x + x``. The code inside the -curlies is the pattern to match against. The operators ``*``, ``**``, -``|``, ``~`` have a special meaning in patterns if they are written in infix -notation, so to match verbatim against ``*`` the ordinary function call syntax +The compiler now rewrites `x * 2` as `x + x`. The code inside the +curlies is the pattern to match against. The operators `*`, `**`, +`|`, `~` have a special meaning in patterns if they are written in infix +notation, so to match verbatim against `*` the ordinary function call syntax needs to be used. Term rewriting macro are applied recursively, up to a limit. This means that @@ -1080,7 +1082,7 @@ You can make one overload matching with a constraint and one without, and the one with a constraint will have precedence, and so you can handle both cases differently. -So what about ``2 * a``? We should tell the compiler ``*`` is commutative. We +So what about `2 * a`? We should tell the compiler `*` is commutative. We cannot really do that however as the following code only swaps arguments blindly: @@ -1092,59 +1094,59 @@ What optimizers really need to do is a *canonicalization*: .. code-block:: nim template canonMul{`*`(a, b)}(a: int{lit}, b: int): int = b*a -The ``int{lit}`` parameter pattern matches against an expression of -type ``int``, but only if it's a literal. +The `int{lit}` parameter pattern matches against an expression of +type `int`, but only if it's a literal. Parameter constraints --------------------- -The `parameter constraint`:idx: expression can use the operators ``|`` (or), -``&`` (and) and ``~`` (not) and the following predicates: +The `parameter constraint`:idx: expression can use the operators `|` (or), +`&` (and) and `~` (not) and the following predicates: =================== ===================================================== Predicate Meaning =================== ===================================================== -``atom`` The matching node has no children. -``lit`` The matching node is a literal like "abc", 12. -``sym`` The matching node must be a symbol (a bound +`atom` The matching node has no children. +`lit` The matching node is a literal like "abc", 12. +`sym` The matching node must be a symbol (a bound identifier). -``ident`` The matching node must be an identifier (an unbound +`ident` The matching node must be an identifier (an unbound identifier). -``call`` The matching AST must be a call/apply expression. -``lvalue`` The matching AST must be an lvalue. -``sideeffect`` The matching AST must have a side effect. -``nosideeffect`` The matching AST must have no side effect. -``param`` A symbol which is a parameter. -``genericparam`` A symbol which is a generic parameter. -``module`` A symbol which is a module. -``type`` A symbol which is a type. -``var`` A symbol which is a variable. -``let`` A symbol which is a ``let`` variable. -``const`` A symbol which is a constant. -``result`` The special ``result`` variable. -``proc`` A symbol which is a proc. -``method`` A symbol which is a method. -``iterator`` A symbol which is an iterator. -``converter`` A symbol which is a converter. -``macro`` A symbol which is a macro. -``template`` A symbol which is a template. -``field`` A symbol which is a field in a tuple or an object. -``enumfield`` A symbol which is a field in an enumeration. -``forvar`` A for loop variable. -``label`` A label (used in ``block`` statements). -``nk*`` The matching AST must have the specified kind. - (Example: ``nkIfStmt`` denotes an ``if`` statement.) -``alias`` States that the marked parameter needs to alias +`call` The matching AST must be a call/apply expression. +`lvalue` The matching AST must be an lvalue. +`sideeffect` The matching AST must have a side effect. +`nosideeffect` The matching AST must have no side effect. +`param` A symbol which is a parameter. +`genericparam` A symbol which is a generic parameter. +`module` A symbol which is a module. +`type` A symbol which is a type. +`var` A symbol which is a variable. +`let` A symbol which is a `let` variable. +`const` A symbol which is a constant. +`result` The special `result` variable. +`proc` A symbol which is a proc. +`method` A symbol which is a method. +`iterator` A symbol which is an iterator. +`converter` A symbol which is a converter. +`macro` A symbol which is a macro. +`template` A symbol which is a template. +`field` A symbol which is a field in a tuple or an object. +`enumfield` A symbol which is a field in an enumeration. +`forvar` A for loop variable. +`label` A label (used in `block` statements). +`nk*` The matching AST must have the specified kind. + (Example: `nkIfStmt` denotes an `if` statement.) +`alias` States that the marked parameter needs to alias with *some* other parameter. -``noalias`` States that *every* other parameter must not alias +`noalias` States that *every* other parameter must not alias with the marked parameter. =================== ===================================================== Predicates that share their name with a keyword have to be escaped with backticks. -The ``alias`` and ``noalias`` predicates refer not only to the matching AST, +The `alias` and `noalias` predicates refer not only to the matching AST, but also to every other bound parameter; syntactically they need to occur after the ordinary AST predicates: @@ -1158,14 +1160,14 @@ the ordinary AST predicates: Pattern operators ----------------- -The operators ``*``, ``**``, ``|``, ``~`` have a special meaning in patterns +The operators `*`, `**`, `|`, `~` have a special meaning in patterns if they are written in infix notation. -The ``|`` operator +The `|` operator ~~~~~~~~~~~~~~~~~~ -The ``|`` operator if used as infix operator creates an ordered choice: +The `|` operator if used as infix operator creates an ordered choice: .. code-block:: nim template t{0|1}(): untyped = 3 @@ -1182,15 +1184,15 @@ constant folding, so the following does not work: echo 1 The reason is that the compiler already transformed the 1 into "1" for -the ``echo`` statement. However, a term rewriting macro should not change the -semantics anyway. In fact they can be deactivated with the ``--patterns:off`` -command line option or temporarily with the ``patterns`` pragma. +the `echo` statement. However, a term rewriting macro should not change the +semantics anyway. In fact they can be deactivated with the `--patterns:off` +command line option or temporarily with the `patterns` pragma. -The ``{}`` operator +The `{}` operator ~~~~~~~~~~~~~~~~~~~ -A pattern expression can be bound to a pattern parameter via the ``expr{param}`` +A pattern expression can be bound to a pattern parameter via the `expr{param}` notation: .. code-block:: nim @@ -1200,10 +1202,10 @@ notation: echo a -The ``~`` operator +The `~` operator ~~~~~~~~~~~~~~~~~~ -The ``~`` operator is the **not** operator in patterns: +The `~` operator is the **not** operator in patterns: .. code-block:: nim template t{x = (~x){y} and (~x){z}}(x, y, z: bool) = @@ -1218,11 +1220,11 @@ The ``~`` operator is the **not** operator in patterns: echo a -The ``*`` operator +The `*` operator ~~~~~~~~~~~~~~~~~~ -The ``*`` operator can *flatten* a nested binary expression like ``a & b & c`` -to ``&(a, b, c)``: +The `*` operator can *flatten* a nested binary expression like `a & b & c` +to `&(a, b, c)`: .. code-block:: nim var @@ -1243,19 +1245,19 @@ to ``&(a, b, c)``: The second operator of `*` must be a parameter; it is used to gather all the -arguments. The expression ``"my" && (space & "awe" && "some " ) && "concat"`` -is passed to ``optConc`` in ``a`` as a special list (of kind ``nkArgList``) -which is flattened into a call expression; thus the invocation of ``optConc`` +arguments. The expression `"my" && (space & "awe" && "some " ) && "concat"` +is passed to `optConc` in `a` as a special list (of kind `nkArgList`) +which is flattened into a call expression; thus the invocation of `optConc` produces: .. code-block:: nim `&&`("my", space & "awe", "some ", "concat") -The ``**`` operator +The `**` operator ~~~~~~~~~~~~~~~~~~~ -The ``**`` is much like the ``*`` operator, except that it gathers not only +The `**` is much like the `*` operator, except that it gathers not only all the arguments, but also the matched operators in reverse polish notation: .. code-block:: nim @@ -1280,8 +1282,8 @@ all the arguments, but also the matched operators in reverse polish notation: echo x + y * z - x -This passes the expression ``x + y * z - x`` to the ``optM`` macro as -an ``nnkArglist`` node containing:: +This passes the expression `x + y * z - x` to the `optM` macro as +an `nnkArglist` node containing:: Arglist Sym "x" @@ -1292,14 +1294,14 @@ an ``nnkArglist`` node containing:: Sym "x" Sym "-" -(Which is the reverse polish notation of ``x + y * z - x``.) +(Which is the reverse polish notation of `x + y * z - x`.) Parameters ---------- Parameters in a pattern are type checked in the matching process. If a -parameter is of the type ``varargs`` it is treated specially and it can match +parameter is of the type `varargs` it is treated specially and it can match 0 or more arguments in the AST to be matched against: .. code-block:: nim @@ -1341,9 +1343,9 @@ The following example shows how some form of hoisting can be implemented: echo match("(a b c)", peg"'(' @ ')'") echo match("W_HI_Le", peg"\y 'while'") -The ``optPeg`` template optimizes the case of a peg constructor with a string +The `optPeg` template optimizes the case of a peg constructor with a string literal, so that the pattern will only be parsed once at program startup and -stored in a global ``gl`` which is then re-used. This optimization is called +stored in a global `gl` which is then re-used. This optimization is called hoisting because it is comparable to classical loop hoisting. @@ -1369,7 +1371,7 @@ constraints affect ordinary overloading resolution then: optLit(constant) optLit(variable) -However, the constraints ``alias`` and ``noalias`` are not available in +However, the constraints `alias` and `noalias` are not available in ordinary routines. @@ -1377,26 +1379,26 @@ Parallel & Spawn ================ Nim has two flavors of parallelism: -1) `Structured`:idx: parallelism via the ``parallel`` statement. -2) `Unstructured`:idx: parallelism via the standalone ``spawn`` statement. +1) `Structured`:idx: parallelism via the `parallel` statement. +2) `Unstructured`:idx: parallelism via the standalone `spawn` statement. Nim has a builtin thread pool that can be used for CPU intensive tasks. For -IO intensive tasks the ``async`` and ``await`` features should be +IO intensive tasks the `async` and `await` features should be used instead. Both parallel and spawn need the `threadpool `_ module to work. -Somewhat confusingly, ``spawn`` is also used in the ``parallel`` statement -with slightly different semantics. ``spawn`` always takes a call expression of -the form ``f(a, ...)``. Let ``T`` be ``f``'s return type. If ``T`` is ``void`` -then ``spawn``'s return type is also ``void`` otherwise it is ``FlowVar[T]``. +Somewhat confusingly, `spawn` is also used in the `parallel` statement +with slightly different semantics. `spawn` always takes a call expression of +the form `f(a, ...)`. Let `T` be `f`'s return type. If `T` is `void` +then `spawn`'s return type is also `void` otherwise it is `FlowVar[T]`. -Within a ``parallel`` section sometimes the ``FlowVar[T]`` is eliminated -to ``T``. This happens when ``T`` does not contain any GC'ed memory. -The compiler can ensure the location in ``location = spawn f(...)`` is not -read prematurely within a ``parallel`` section and so there is no need for -the overhead of an indirection via ``FlowVar[T]`` to ensure correctness. +Within a `parallel` section sometimes the `FlowVar[T]` is eliminated +to `T`. This happens when `T` does not contain any GC'ed memory. +The compiler can ensure the location in `location = spawn f(...)` is not +read prematurely within a `parallel` section and so there is no need for +the overhead of an indirection via `FlowVar[T]` to ensure correctness. -**Note**: Currently exceptions are not propagated between ``spawn``'ed tasks! +**Note**: Currently exceptions are not propagated between `spawn`'ed tasks! Spawn statement @@ -1415,25 +1417,25 @@ Spawn statement sync() For reasons of type safety and implementation simplicity the expression -that ``spawn`` takes is restricted: +that `spawn` takes is restricted: -* It must be a call expression ``f(a, ...)``. -* ``f`` must be ``gcsafe``. -* ``f`` must not have the calling convention ``closure``. -* ``f``'s parameters may not be of type ``var``. - This means one has to use raw ``ptr``'s for data passing reminding the +* It must be a call expression `f(a, ...)`. +* `f` must be `gcsafe`. +* `f` must not have the calling convention `closure`. +* `f`'s parameters may not be of type `var`. + This means one has to use raw `ptr`'s for data passing reminding the programmer to be careful. -* ``ref`` parameters are deeply copied which is a subtle semantic change and +* `ref` parameters are deeply copied which is a subtle semantic change and can cause performance problems but ensures memory safety. This deep copy - is performed via ``system.deepCopy`` and so can be overridden. -* For *safe* data exchange between ``f`` and the caller a global ``TChannel`` + is performed via `system.deepCopy` and so can be overridden. +* For *safe* data exchange between `f` and the caller a global `TChannel` needs to be used. However, since spawn can return a result, often no further communication is required. -``spawn`` executes the passed expression on the thread pool and returns -a `data flow variable`:idx: ``FlowVar[T]`` that can be read from. The reading -with the ``^`` operator is **blocking**. However, one can use ``blockUntilAny`` to +`spawn` executes the passed expression on the thread pool and returns +a `data flow variable`:idx: `FlowVar[T]` that can be read from. The reading +with the `^` operator is **blocking**. However, one can use `blockUntilAny` to wait on multiple flow variables at the same time: .. code-block:: nim @@ -1450,8 +1452,8 @@ wait on multiple flow variables at the same time: discard blockUntilAny(responses) Data flow variables ensure that no data races -are possible. Due to technical limitations not every type ``T`` is possible in -a data flow variable: ``T`` has to be of the type ``ref``, ``string``, ``seq`` +are possible. Due to technical limitations not every type `T` is possible in +a data flow variable: `T` has to be of the type `ref`, `string`, `seq` or of a type that doesn't contain a type that is garbage collected. This restriction is not hard to work-around in practice. @@ -1483,7 +1485,7 @@ Example: The parallel statement is the preferred mechanism to introduce parallelism in a -Nim program. A subset of the Nim language is valid within a ``parallel`` +Nim program. A subset of the Nim language is valid within a `parallel` section. This subset is checked during semantic analysis to be free of data races. A sophisticated `disjoint checker`:idx: ensures that no data races are possible even though shared memory is extensively supported! @@ -1491,25 +1493,25 @@ possible even though shared memory is extensively supported! The subset is in fact the full language with the following restrictions / changes: -* ``spawn`` within a ``parallel`` section has special semantics. -* Every location of the form ``a[i]`` and ``a[i..j]`` and ``dest`` where - ``dest`` is part of the pattern ``dest = spawn f(...)`` has to be +* `spawn` within a `parallel` section has special semantics. +* Every location of the form `a[i]` and `a[i..j]` and `dest` where + `dest` is part of the pattern `dest = spawn f(...)` has to be provably disjoint. This is called the *disjoint check*. -* Every other complex location ``loc`` that is used in a spawned - proc (``spawn f(loc)``) has to be immutable for the duration of - the ``parallel`` section. This is called the *immutability check*. Currently +* Every other complex location `loc` that is used in a spawned + proc (`spawn f(loc)`) has to be immutable for the duration of + the `parallel` section. This is called the *immutability check*. Currently it is not specified what exactly "complex location" means. We need to make this an optimization! * Every array access has to be provably within bounds. This is called the *bounds check*. * Slices are optimized so that no copy is performed. This optimization is not - yet performed for ordinary slices outside of a ``parallel`` section. + yet performed for ordinary slices outside of a `parallel` section. Guards and locks ================ -Apart from ``spawn`` and ``parallel`` Nim also provides all the common low level +Apart from `spawn` and `parallel` Nim also provides all the common low level concurrency mechanisms like locks, atomic intrinsics or condition variables. Nim significantly improves on the safety of these features via additional @@ -1528,13 +1530,13 @@ Guards and the locks section Protecting global variables ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Object fields and global variables can be annotated via a ``guard`` pragma: +Object fields and global variables can be annotated via a `guard` pragma: .. code-block:: nim var glock: TLock var gdata {.guard: glock.}: int -The compiler then ensures that every access of ``gdata`` is within a ``locks`` +The compiler then ensures that every access of `gdata` is within a `locks` section: .. code-block:: nim @@ -1547,11 +1549,11 @@ section: {.locks: [glock].}: echo gdata -Top level accesses to ``gdata`` are always allowed so that it can be initialized +Top level accesses to `gdata` are always allowed so that it can be initialized conveniently. It is *assumed* (but not enforced) that every top level statement is executed before any concurrent action happens. -The ``locks`` section deliberately looks ugly because it has no runtime +The `locks` section deliberately looks ugly because it has no runtime semantics and should not be used directly! It should only be used in templates that also implement some form of locking at runtime: @@ -1580,7 +1582,7 @@ model low level lockfree mechanisms: echo atomicRead(atomicCounter) -The ``locks`` pragma takes a list of lock expressions ``locks: [a, b, ...]`` +The `locks` pragma takes a list of lock expressions `locks: [a, b, ...]` in order to support *multi lock* statements. Why these are essential is explained in the `lock levels <#guards-and-locks-lock-levels>`_ section. @@ -1588,7 +1590,7 @@ explained in the `lock levels <#guards-and-locks-lock-levels>`_ section. Protecting general locations ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The ``guard`` annotation can also be used to protect fields within an object. +The `guard` annotation can also be used to protect fields within an object. The guard then needs to be another field within the same object or a global variable. @@ -1606,7 +1608,7 @@ expressivity of the language: lock counters[i].L: inc counters[i].v -The access to field ``x.v`` is allowed since its guard ``x.L`` is active. +The access to field `x.v` is allowed since its guard `x.L` is active. After template expansion, this amounts to: .. code-block:: nim @@ -1619,10 +1621,10 @@ After template expansion, this amounts to: finally: pthread_mutex_unlock(counters[i].L) -There is an analysis that checks that ``counters[i].L`` is the lock that -corresponds to the protected location ``counters[i].v``. This analysis is called +There is an analysis that checks that `counters[i].L` is the lock that +corresponds to the protected location `counters[i].v`. This analysis is called `path analysis`:idx: because it deals with paths to locations -like ``obj.field[i].fieldB[j]``. +like `obj.field[i].fieldB[j]`. The path analysis is **currently unsound**, but that doesn't make it useless. Two paths are considered equivalent if they are syntactically the same. @@ -1644,10 +1646,10 @@ potential deadlocks during semantic analysis. A lock level is an constant integer in the range 0..1_000. Lock level 0 means that no lock is acquired at all. -If a section of code holds a lock of level ``M`` than it can also acquire any -lock of level ``N < M``. Another lock of level ``M`` cannot be acquired. Locks +If a section of code holds a lock of level `M` than it can also acquire any +lock of level `N < M`. Another lock of level `M` cannot be acquired. Locks of the same level can only be acquired *at the same time* within a -single ``locks`` section: +single `locks` section: .. code-block:: nim var a, b: TLock[2] @@ -1672,8 +1674,8 @@ single ``locks`` section: Here is how a typical multilock statement can be implemented in Nim. Note how -the runtime check is required to ensure a global ordering for two locks ``a`` -and ``b`` of the same lock level: +the runtime check is required to ensure a global ordering for two locks `a` +and `b` of the same lock level: .. code-block:: nim template multilock(a, b: ptr TLock; body: untyped) = @@ -1691,9 +1693,9 @@ and ``b`` of the same lock level: pthread_mutex_unlock(b) -Whole routines can also be annotated with a ``locks`` pragma that takes a lock +Whole routines can also be annotated with a `locks` pragma that takes a lock level. This then means that the routine may acquire locks of up to this level. -This is essential so that procs can be called within a ``locks`` section: +This is essential so that procs can be called within a `locks` section: .. code-block:: nim proc p() {.locks: 3.} = discard @@ -1704,17 +1706,17 @@ This is essential so that procs can be called within a ``locks`` section: p() -As usual ``locks`` is an inferred effect and there is a subtype -relation: ``proc () {.locks: N.}`` is a subtype of ``proc () {.locks: M.}`` +As usual `locks` is an inferred effect and there is a subtype +relation: `proc () {.locks: N.}` is a subtype of `proc () {.locks: M.}` iff (M <= N). -The ``locks`` pragma can also take the special value ``"unknown"``. This +The `locks` pragma can also take the special value `"unknown"`. This is useful in the context of dynamic method dispatching. In the following -example, the compiler can infer a lock level of 0 for the ``base`` case. +example, the compiler can infer a lock level of 0 for the `base` case. However, one of the overloaded methods calls a procvar which is -potentially locking. Thus, the lock level of calling ``g.testMethod`` +potentially locking. Thus, the lock level of calling `g.testMethod` cannot be inferred statically, leading to compiler warnings. By using -``{.locks: "unknown".}``, the base method can be marked explicitly as +`{.locks: "unknown".}`, the base method can be marked explicitly as having unknown lock level as well: .. code-block:: nim @@ -1736,8 +1738,8 @@ they will rewrite as long as there is a match. There was no way to ensure some rewrite happens only once, e.g. when rewriting term to same term plus extra content. -``noRewrite`` pragma can actually prevent further rewriting on marked code, -e.g. with given example ``echo("ab")`` will be rewritten just once: +`noRewrite` pragma can actually prevent further rewriting on marked code, +e.g. with given example `echo("ab")` will be rewritten just once: .. code-block:: nim template pwnEcho{echo(x)}(x: untyped) = @@ -1745,7 +1747,7 @@ e.g. with given example ``echo("ab")`` will be rewritten just once: echo "ab" -``noRewrite`` pragma can be useful to control term-rewriting macros recursion. +`noRewrite` pragma can be useful to control term-rewriting macros recursion. Aliasing restrictions in parameter passing @@ -1755,8 +1757,8 @@ Aliasing restrictions in parameter passing implementation and need to be fleshed out further. "Aliasing" here means that the underlying storage locations overlap in memory -at runtime. An "output parameter" is a parameter of type ``var T``, -an input parameter is any parameter that is not of type ``var``. +at runtime. An "output parameter" is a parameter of type `var T`, +an input parameter is any parameter that is not of type `var`. 1. Two output parameters should never be aliased. 2. An input and an output parameter should not be aliased. @@ -1767,25 +1769,25 @@ an input parameter is any parameter that is not of type ``var``. One problem with rules 3 and 4 is that they affect specific global or thread local variables, but Nim's effect tracking only tracks "uses no global variable" -via ``.noSideEffect``. The rules 3 and 4 can also be approximated by a different rule: +via `.noSideEffect`. The rules 3 and 4 can also be approximated by a different rule: 5. A global or thread local variable (or a location derived from such a location) - can only passed to a parameter of a ``.noSideEffect`` proc. + can only passed to a parameter of a `.noSideEffect` proc. Noalias annotation ================== -Since version 1.4 of the Nim compiler, there is a ``.noalias`` annotation for variables -and parameters. It is mapped directly to C/C++'s ``restrict`` keyword and means that +Since version 1.4 of the Nim compiler, there is a `.noalias` annotation for variables +and parameters. It is mapped directly to C/C++'s `restrict` keyword and means that the underlying pointer is pointing to a unique location in memory, no other aliases to this location exist. It is *unchecked* that this alias restriction is followed, if the restriction is violated, the backend optimizer is free to miscompile the code. This is an **unsafe** language feature. Ideally in later versions of the language, the restriction will be enforced at -compile time. (Which is also why the name ``noalias`` was choosen instead of a more -verbose name like ``unsafeAssumeNoAlias``.) +compile time. (Which is also why the name `noalias` was choosen instead of a more +verbose name like `unsafeAssumeNoAlias`.) Strict funcs @@ -1796,7 +1798,7 @@ to the existing rule that a side effect is calling a function with side effects the following rule is also enforced: Any mutation to an object does count as a side effect if that object is reachable -via a parameter that is not declared as a ``var`` parameter. +via a parameter that is not declared as a `var` parameter. For example: @@ -1830,14 +1832,14 @@ the `view types section <#view-types-algorithm>`_. View types ========== -**Note**: ``--experimental:views`` is more effective -with ``--experimental:strictFuncs``. +**Note**: `--experimental:views` is more effective +with `--experimental:strictFuncs`. A view type is a type that is or contains one of the following types: -- ``var T`` (mutable view into ``T``) -- ``lent T`` (immutable view into ``T``) -- ``openArray[T]`` (pair of (pointer to array of ``T``, size)) +- `var T` (mutable view into `T`) +- `lent T` (immutable view into `T`) +- `openArray[T]` (pair of (pointer to array of `T`, size)) For example: @@ -1850,7 +1852,7 @@ For example: View4 = Table[openArray[char], int] -Exceptions to this rule are types constructed via ``ptr`` or ``proc``. +Exceptions to this rule are types constructed via `ptr` or `proc`. For example, the following types are **not** view types: .. code-block:: nim @@ -1861,13 +1863,13 @@ For example, the following types are **not** view types: NotView3 = ptr array[4, var int] -A *mutable* view type is a type that is or contains a ``var T`` type. +A *mutable* view type is a type that is or contains a `var T` type. An *immutable* view type is a view type that is not a mutable view type. A *view* is a symbol (a let, var, const, etc.) that has a view type. Since version 1.4 Nim allows view types to be used as local variables. -This feature needs to be enabled via ``{.experimental: "views".}``. +This feature needs to be enabled via `{.experimental: "views".}`. A local variable of a view type *borrows* from the locations and it is statically enforced that the view does not outlive the location @@ -1901,47 +1903,47 @@ For example: A local variable of a view type can borrow from a location -derived from a parameter, another local variable, a global ``const`` or ``let`` -symbol or a thread-local ``var`` or ``let``. +derived from a parameter, another local variable, a global `const` or `let` +symbol or a thread-local `var` or `let`. -Let ``p`` the proc that is analysed for the correctness of the borrow operation. +Let `p` the proc that is analysed for the correctness of the borrow operation. -Let ``source`` be one of: +Let `source` be one of: -- A formal parameter of ``p``. Note that this does not cover parameters of +- A formal parameter of `p`. Note that this does not cover parameters of inner procs. -- The ``result`` symbol of ``p``. -- A local ``var`` or ``let`` or ``const`` of ``p``. Note that this does +- The `result` symbol of `p`. +- A local `var` or `let` or `const` of `p`. Note that this does not cover locals of inner procs. -- A thread-local ``var`` or ``let``. -- A global ``let`` or ``const``. +- A thread-local `var` or `let`. +- A global `let` or `const`. - A constant array/seq/object/tuple constructor. Path expressions ---------------- -A location derived from ``source`` is then defined as a path expression that -has ``source`` as the owner. A path expression ``e`` is defined recursively: +A location derived from `source` is then defined as a path expression that +has `source` as the owner. A path expression `e` is defined recursively: -- ``source`` itself is a path expression. -- Container access like ``e[i]`` is a path expression. -- Tuple access ``e[0]`` is a path expression. -- Object field access ``e.field`` is a path expression. -- ``system.toOpenArray(e, ...)`` is a path expression. -- Pointer dereference ``e[]`` is a path expression. -- An address ``addr e``, ``unsafeAddr e`` is a path expression. -- A type conversion ``T(e)`` is a path expression. -- A cast expression ``cast[T](e)`` is a path expression. -- ``f(e, ...)`` is a path expression if ``f``'s return type is a view type. - Because the view can only have been borrowed from ``e``, we then know - that owner of ``f(e, ...)`` is ``e``. +- `source` itself is a path expression. +- Container access like `e[i]` is a path expression. +- Tuple access `e[0]` is a path expression. +- Object field access `e.field` is a path expression. +- `system.toOpenArray(e, ...)` is a path expression. +- Pointer dereference `e[]` is a path expression. +- An address `addr e`, `unsafeAddr e` is a path expression. +- A type conversion `T(e)` is a path expression. +- A cast expression `cast[T](e)` is a path expression. +- `f(e, ...)` is a path expression if `f`'s return type is a view type. + Because the view can only have been borrowed from `e`, we then know + that owner of `f(e, ...)` is `e`. If a view type is used as a return type, the location must borrow from a location that is derived from the first parameter that is passed to the proc. See https://nim-lang.org/docs/manual.html#procedures-var-return-type for -details about how this is done for ``var T``. +details about how this is done for `var T`. A mutable view can borrow from a mutable location, an immutable view can borrow from both a mutable or an immutable location. @@ -1979,8 +1981,8 @@ The scope of the view does not matter: The analysis requires as much precision about mutations as is reasonably obtainable, so it is more effective with the experimental `strict funcs <#strict-funcs>`_ -feature. In other words ``--experimental:views`` works better -with ``--experimental:strictFuncs``. +feature. In other words `--experimental:views` works better +with `--experimental:strictFuncs`. The analysis is currently control flow insensitive: @@ -1992,8 +1994,8 @@ The analysis is currently control flow insensitive: s.setLen 0 echo v.field -In this example, the compiler assumes that ``s.setLen 0`` invalidates the -borrow operation of ``v`` even though a human being can easily see that it +In this example, the compiler assumes that `s.setLen 0` invalidates the +borrow operation of `v` even though a human being can easily see that it will never do that at runtime. @@ -2016,9 +2018,9 @@ A borrow operation ends with the last usage of the view variable. Reborrows --------- -A view ``v`` can borrow from multiple different locations. However, the borrow -is always the full span of ``v``'s lifetime and every location that is borrowed -from is sealed during ``v``'s lifetime. +A view `v` can borrow from multiple different locations. However, the borrow +is always the full span of `v`'s lifetime and every location that is borrowed +from is sealed during `v`'s lifetime. Algorithm @@ -2034,41 +2036,41 @@ a notion of an "abstract time", in the implementation it's a simple integer that incremented for every visited node. In the second pass information about the underlying object "graphs" is computed. -Let ``v`` be a parameter or a local variable. Let ``G(v)`` be the graph -that ``v`` belongs to. A graph is defined by the set of variables that belong -to the graph. Initially for all ``v``: ``G(v) = {v}``. Every variable can only +Let `v` be a parameter or a local variable. Let `G(v)` be the graph +that `v` belongs to. A graph is defined by the set of variables that belong +to the graph. Initially for all `v`: `G(v) = {v}`. Every variable can only be part of a single graph. -Assignments like ``a = b`` "connect" two variables, both variables end up in the -same graph ``{a, b} = G(a) = G(b)``. Unfortunately, the pattern to look for is +Assignments like `a = b` "connect" two variables, both variables end up in the +same graph `{a, b} = G(a) = G(b)`. Unfortunately, the pattern to look for is much more complex than that and can involve multiple assignment targets and sources:: f(x, y) = g(a, b) -connects ``x`` and ``y`` to ``a`` and ``b``: ``G(x) = G(y) = G(a) = G(b) = {x, y, a, b}``. +connects `x` and `y` to `a` and `b`: `G(x) = G(y) = G(a) = G(b) = {x, y, a, b}`. A type based alias analysis rules out some of these combinations, for example -a ``string`` value cannot possibly be connected to a ``seq[int]``. +a `string` value cannot possibly be connected to a `seq[int]`. -A pattern like ``v[] = value`` or ``v.field = value`` marks ``G(v)`` as mutated. +A pattern like `v[] = value` or `v.field = value` marks `G(v)` as mutated. After the second pass a set of disjoint graphs was computed. For strict functions it is then enforced that there is no graph that is both mutated and has an element that is an immutable parameter (that is a parameter that is not -of type ``var T``). +of type `var T`). -For borrow checking a different set of checks is performed. Let ``v`` be the view -and ``b`` the location that is borrowed from. +For borrow checking a different set of checks is performed. Let `v` be the view +and `b` the location that is borrowed from. -- The lifetime of ``v`` must not exceed ``b``'s lifetime. Note: The lifetime of +- The lifetime of `v` must not exceed `b`'s lifetime. Note: The lifetime of a parameter is the complete proc body. -- If ``v`` is a mutable view and ``v`` is used to actually mutate the - borrowed location, then ``b`` has to be a mutable location. +- If `v` is a mutable view and `v` is used to actually mutate the + borrowed location, then `b` has to be a mutable location. Note: If it is not actually used for mutation, borrowing a mutable view from an immutable location is allowed! This allows for many important idioms and will be justified in an upcoming RFC. -- During ``v``'s lifetime, ``G(b)`` can only be modified by ``v`` (and only if - ``v`` is a mutable view). -- If ``v`` is ``result`` then ``b`` has to be a location derived from the first +- During `v`'s lifetime, `G(b)` can only be modified by `v` (and only if + `v` is a mutable view). +- If `v` is `result` then `b` has to be a location derived from the first formal parameter or from a constant location. - A view cannot be used for a read or a write access before it was assigned to. diff --git a/doc/manual_experimental_strictnotnil.rst b/doc/manual_experimental_strictnotnil.rst index 10b59a74e..c7c045683 100644 --- a/doc/manual_experimental_strictnotnil.rst +++ b/doc/manual_experimental_strictnotnil.rst @@ -1,3 +1,4 @@ +.. default-role:: code Strict not nil checking ========================= @@ -14,9 +15,9 @@ or In the second case it would check builtin and imported modules as well. -It checks the nilability of ref-like types and makes dereferencing safer based on flow typing and ``not nil`` annotations. +It checks the nilability of ref-like types and makes dereferencing safer based on flow typing and `not nil` annotations. -Its implementation is different than the ``notnil`` one: defined under ``strictNotNil``. Keep in mind the difference in option names, be careful with distinguishing them. +Its implementation is different than the `notnil` one: defined under `strictNotNil`. Keep in mind the difference in option names, be careful with distinguishing them. We check several kinds of types for nilability: @@ -28,14 +29,14 @@ We check several kinds of types for nilability: nil ------- -The default kind of nilability types is the nilable kind: they can have the value ``nil``. -If you have a non-nilable type ``T``, you can use ``T nil`` to get a nilable type for it. +The default kind of nilability types is the nilable kind: they can have the value `nil`. +If you have a non-nilable type `T`, you can use `T nil` to get a nilable type for it. not nil -------- -You can annotate a type where nil isn't a valid value with ``not nil``. +You can annotate a type where nil isn't a valid value with `not nil`. .. code-block:: nim type @@ -58,40 +59,40 @@ You can annotate a type where nil isn't a valid value with ``not nil``. -If a type can include ``nil`` as a valid value, dereferencing values of the type -is checked by the compiler: if a value which might be nil is derefenced, this produces a warning by default, you can turn this into an error using the compiler options ``--warningAsError:strictNotNil`` +If a type can include `nil` as a valid value, dereferencing values of the type +is checked by the compiler: if a value which might be nil is derefenced, this produces a warning by default, you can turn this into an error using the compiler options `--warningAsError:strictNotNil` -If a type is nilable, you should dereference its values only after a ``isNil`` or equivalent check. +If a type is nilable, you should dereference its values only after a `isNil` or equivalent check. local turn on/off --------------------- -You can still turn off nil checking on function/module level by using a ``{.strictNotNil: off}.`` pragma. +You can still turn off nil checking on function/module level by using a `{.strictNotNil: off}.` pragma. Note: test that/TODO for code/manual. nilability state ----------------- -Currently a nilable value can be ``Safe``, ``MaybeNil`` or ``Nil`` : we use internally ``Parent`` and ``Unreachable`` but this is an implementation detail(a parent layer has the actual nilability). +Currently a nilable value can be `Safe`, `MaybeNil` or `Nil` : we use internally `Parent` and `Unreachable` but this is an implementation detail(a parent layer has the actual nilability). -``Safe`` means it shouldn't be nil at that point: e.g. after assignment to a non-nil value or ``not a.isNil`` check -``MaybeNil`` means it might be nil, but it might not be nil: e.g. an argument, a call argument or a value after an ``if`` and ``else``. -``Nil`` means it should be nil at that point; e.g. after an assignment to ``nil`` or a ``.isNil`` check. +`Safe` means it shouldn't be nil at that point: e.g. after assignment to a non-nil value or `not a.isNil` check +`MaybeNil` means it might be nil, but it might not be nil: e.g. an argument, a call argument or a value after an `if` and `else`. +`Nil` means it should be nil at that point; e.g. after an assignment to `nil` or a `.isNil` check. -``Unreachable`` means it shouldn't be possible to access this in this branch: so we do generate a warning as well. +`Unreachable` means it shouldn't be possible to access this in this branch: so we do generate a warning as well. -We show an error for each dereference (``[]``, ``.field``, ``[index]`` ``()`` etc) which is of a tracked expression which is -in ``MaybeNil`` or ``Nil`` state. +We show an error for each dereference (`[]`, `.field`, `[index]` `()` etc) which is of a tracked expression which is +in `MaybeNil` or `Nil` state. type nilability ---------------- Types are either nilable or non-nilable. -When you pass a param or a default value, we use the type : for nilable types we return ``MaybeNil`` -and for non-nilable ``Safe``. +When you pass a param or a default value, we use the type : for nilable types we return `MaybeNil` +and for non-nilable `Safe`. -TODO: fix the manual here. (This is not great, as default values for non-nilables and nilables are usually actually ``nil`` , so we should think a bit more about this section.) +TODO: fix the manual here. (This is not great, as default values for non-nilables and nilables are usually actually `nil` , so we should think a bit more about this section.) params rules ------------ @@ -102,9 +103,9 @@ Param's nilability is detected based on type nilability. We use the type of the assignment rules ----------------- -Let's say we have ``left = right``. +Let's say we have `left = right`. -When we assign, we pass the right's nilability to the left's expression. There should be special handling of aliasing and compound expressions which we specify in their sections. (Assignment is a possible alias ``move`` or ``move out``). +When we assign, we pass the right's nilability to the left's expression. There should be special handling of aliasing and compound expressions which we specify in their sections. (Assignment is a possible alias `move` or `move out`). call args rules ----------------- @@ -114,20 +115,20 @@ When we call with arguments, we have two cases when we might change the nilabili .. code-block:: nim callByVar(a) -Here ``callByVar`` can re-assign ``a``, so this might change ``a``'s nilability, so we change it to ``MaybeNil``. -This is also a possible aliasing ``move out`` (moving out of a current alias set). +Here `callByVar` can re-assign `a`, so this might change `a`'s nilability, so we change it to `MaybeNil`. +This is also a possible aliasing `move out` (moving out of a current alias set). .. code-block:: nim call(a) -Here ``call`` can change a field or element of ``a``, so if we have a dependant expression of ``a`` : e.g. ``a.field``. Dependats become ``MaybeNil``. +Here `call` can change a field or element of `a`, so if we have a dependant expression of `a` : e.g. `a.field`. Dependats become `MaybeNil`. branches rules --------------- Branches are the reason we do nil checking like this: with flow checking. -Sources of brancing are ``if``, ``while``, ``for``, ``and``, ``or``, ``case``, ``try`` and combinations with ``return``, ``break``, ``continue`` and ``raise`` +Sources of brancing are `if`, `while`, `for`, `and`, `or`, `case`, `try` and combinations with `return`, `break`, `continue` and `raise` We create a new layer/"scope" for each branch where we map expressions to nilability. This happens when we "fork": usually on the beginning of a construct. When branches "join" we usually unify their expression maps or/and nilabilities. @@ -142,33 +143,33 @@ Merging usually merges maps and alias sets: nilabilities are merged like this: else: MaybeNil -Special handling is for ``.isNil`` and `` == nil``, also for ``not``, ``and`` and ``or``. +Special handling is for `.isNil` and ` == nil`, also for `not`, `and` and `or`. -``not`` reverses the nilability, ``and`` is similar to "forking" : the right expression is checked in the layer resulting from the left one and ``or`` is similar to "merging": the right and left expression should be both checked in the original layer. +`not` reverses the nilability, `and` is similar to "forking" : the right expression is checked in the layer resulting from the left one and `or` is similar to "merging": the right and left expression should be both checked in the original layer. -``isNil``, ``== nil`` make expressions ``Nil``. If there is a ``not`` or ``!= nil``, they make them ``Safe``. -We also reverse the nilability in the opposite branch: e.g. ``else``. +`isNil`, `== nil` make expressions `Nil`. If there is a `not` or `!= nil`, they make them `Safe`. +We also reverse the nilability in the opposite branch: e.g. `else`. compound expressions: field, index expressions ----------------------------------------------- We want to track also field(dot) and index(bracket) expressions. -We track some of those compound expressions which might be nilable as dependants of their bases: ``a.field`` is changed if ``a`` is moved (re-assigned), -similarly ``a[index]`` is dependent on ``a`` and ``a.field.field`` on ``a.field``. +We track some of those compound expressions which might be nilable as dependants of their bases: `a.field` is changed if `a` is moved (re-assigned), +similarly `a[index]` is dependent on `a` and `a.field.field` on `a.field`. -When we move the base, we update dependants to ``MaybeNil``. Otherwise we usually start with type nilability. +When we move the base, we update dependants to `MaybeNil`. Otherwise we usually start with type nilability. -When we call args, we update the nilability of their dependants to ``MaybeNil`` as the calls usually can change them. -We might need to check for ``strictFuncs`` pure funcs and not do that then. +When we call args, we update the nilability of their dependants to `MaybeNil` as the calls usually can change them. +We might need to check for `strictFuncs` pure funcs and not do that then. -For field expressions ``a.field``, we calculate an integer value based on a hash of the tree and just accept equivalent trees as equivalent expressions. +For field expressions `a.field`, we calculate an integer value based on a hash of the tree and just accept equivalent trees as equivalent expressions. -For item expression ``a[index]``, we also calculate an integer value based on a hash of the tree and accept equivalent trees as equivalent expressions: for static values only. -For now we support only constant indices: we dont track expression with no-const indices. For those we just report a warning even if they are safe for now: one can use a local variable to workaround. For loops this might be annoying: so one should be able to turn off locally the warning using the ``{.warning[StrictCheckNotNil]:off}.``. +For item expression `a[index]`, we also calculate an integer value based on a hash of the tree and accept equivalent trees as equivalent expressions: for static values only. +For now we support only constant indices: we dont track expression with no-const indices. For those we just report a warning even if they are safe for now: one can use a local variable to workaround. For loops this might be annoying: so one should be able to turn off locally the warning using the `{.warning[StrictCheckNotNil]:off}.`. -For bracket expressions, in the future we might count ``a[]`` as the same general expression. -This means we should should the index but otherwise handle it the same for assign (maybe "aliasing" all the non-static elements) and differentiate only for static: e.g. ``a[0]`` and ``a[1]``. +For bracket expressions, in the future we might count `a[]` as the same general expression. +This means we should should the index but otherwise handle it the same for assign (maybe "aliasing" all the non-static elements) and differentiate only for static: e.g. `a[0]` and `a[1]`. element tracking ----------------- @@ -185,8 +186,8 @@ Also related to tracking initialization of expressions/fields. unstructured control flow rules ------------------------------- -Unstructured control flow keywords as ``return``, ``break``, ``continue``, ``raise`` mean that we jump from a branch out. -This means that if there is code after the finishing of the branch, it would be ran if one hasn't hit the direct parent branch of those: so it is similar to an ``else``. In those cases we should use the reverse nilabilities for the local to the condition expressions. E.g. +Unstructured control flow keywords as `return`, `break`, `continue`, `raise` mean that we jump from a branch out. +This means that if there is code after the finishing of the branch, it would be ran if one hasn't hit the direct parent branch of those: so it is similar to an `else`. In those cases we should use the reverse nilabilities for the local to the condition expressions. E.g. .. code-block:: nim for a in c: @@ -204,14 +205,14 @@ We support alias detection for local expressions. We track sets of aliased expressions. We start with all nilable local expressions in separate sets. Assignments and other changes to nilability can move / move out expressions of sets. -``move``: Moving ``left`` to ``right`` means we remove ``left`` from its current set and unify it with the ``right``'s set. +`move`: Moving `left` to `right` means we remove `left` from its current set and unify it with the `right`'s set. This means it stops being aliased with its previous aliases. .. code-block:: nim var left = b left = right # moving left to right -``move out``: Moving out ``left`` might remove it from the current set and ensure that it's in its own set as a single element. +`move out`: Moving out `left` might remove it from the current set and ensure that it's in its own set as a single element. e.g. @@ -229,7 +230,7 @@ warnings and errors --------------------- We show an error for each dereference (`[]`, `.field`, `[index]` `()` etc) which is of a tracked expression which is -in ``MaybeNil`` or ``Nil`` state. +in `MaybeNil` or `Nil` state. We might also show a history of the transitions and the reasons for them that might change the nilability of the expression. diff --git a/doc/nep1.rst b/doc/nep1.rst index 4a31524f6..3bae6a00b 100644 --- a/doc/nep1.rst +++ b/doc/nep1.rst @@ -1,3 +1,5 @@ +.. default-role:: code + ========================================================== Nim Enhancement Proposal #1 - Standard Library Style Guide ========================================================== @@ -124,11 +126,11 @@ Naming Conventions - In the age of HTTP, HTML, FTP, TCP, IP, UTF, WWW it is foolish to pretend these are somewhat special words requiring all uppercase. Instead treat them - as what they are: Real words. So it's ``parseUrl`` rather than - ``parseURL``, ``checkHttpHeader`` instead of ``checkHTTPHeader`` etc. + as what they are: Real words. So it's `parseUrl` rather than + `parseURL`, `checkHttpHeader` instead of `checkHTTPHeader` etc. -- Operations like ``mitems`` or ``mpairs`` (or the now deprecated ``mget``) - that allow a *mutating view* into some data structure should start with an ``m``. +- Operations like `mitems` or `mpairs` (or the now deprecated `mget`) + that allow a *mutating view* into some data structure should start with an `m`. - When both in-place mutation and 'returns transformed copy' are available the latter is a past participle of the former: @@ -136,8 +138,8 @@ Naming Conventions - sort and sorted - rotate and rotated -- When the 'returns transformed copy' version already exists like ``strutils.replace`` - an in-place version should get an ``-In`` suffix (``replaceIn`` for this example). +- When the 'returns transformed copy' version already exists like `strutils.replace` + an in-place version should get an `-In` suffix (`replaceIn` for this example). - Use `subjectVerb`, not `verbSubject`, e.g.: `fileExists`, not `existsFile`. @@ -153,25 +155,25 @@ to keep the names short but meaningful. ------------------- ------------ -------------------------------------- English word To use Notes ------------------- ------------ -------------------------------------- -initialize initFoo initializes a value type ``Foo`` -new newFoo initializes a reference type ``Foo`` - via ``new`` +initialize initFoo initializes a value type `Foo` +new newFoo initializes a reference type `Foo` + via `new` this or self self for method like procs, e.g.: `proc fun(self: Foo, a: int)` rationale: `self` is more unique in English than `this`, and `foo` would not be DRY. find find should return the position where something was found; for a bool result - use ``contains`` -contains contains often short for ``find() >= 0`` -append add use ``add`` instead of ``append`` + use `contains` +contains contains often short for `find() >= 0` +append add use `add` instead of `append` compare cmp should return an int with the - ``< 0`` ``== 0`` or ``> 0`` semantics; - for a bool result use ``sameXYZ`` -put put, ``[]=`` consider overloading ``[]=`` for put -get get, ``[]`` consider overloading ``[]`` for get; - consider to not use ``get`` as a - prefix: ``len`` instead of ``getLen`` + `< 0` `== 0` or `> 0` semantics; + for a bool result use `sameXYZ` +put put, `[]=` consider overloading `[]=` for put +get get, `[]` consider overloading `[]` for get; + consider to not use `get` as a + prefix: `len` instead of `getLen` length len also used for *number of elements* size size, len size should refer to a byte size capacity cap @@ -236,8 +238,8 @@ Coding Conventions - Use a proc when possible, only using the more powerful facilities of macros, templates, iterators, and converters when necessary. -- Use the ``let`` statement (not the ``var`` statement) when declaring variables that - do not change within their scope. Using the ``let`` statement ensures that +- Use the `let` statement (not the `var` statement) when declaring variables that + do not change within their scope. Using the `let` statement ensures that variables remain immutable, and gives those who read the code a better idea of the code's purpose. diff --git a/doc/nimc.rst b/doc/nimc.rst index ea173691f..aad889591 100644 --- a/doc/nimc.rst +++ b/doc/nimc.rst @@ -1,3 +1,5 @@ +.. default-role:: code + =================================== Nim Compiler User Guide =================================== @@ -45,8 +47,8 @@ Advanced command-line switches are: List of warnings ---------------- -Each warning can be activated individually with ``--warning[NAME]:on|off`` or -in a ``push`` pragma. +Each warning can be activated individually with `--warning[NAME]:on|off` or +in a `push` pragma. ========================== ============================================ Name Description @@ -60,7 +62,7 @@ ConfigDeprecated The project makes use of a deprecated config file. SmallLshouldNotBeUsed The letter 'l' should not be used as an identifier. -EachIdentIsTuple The code contains a confusing ``var`` +EachIdentIsTuple The code contains a confusing `var` declaration. User Some user-defined warning. ========================== ============================================ @@ -69,8 +71,8 @@ User Some user-defined warning. List of hints ------------- -Each hint can be activated individually with ``--hint[NAME]:on|off`` or in a -``push`` pragma. +Each hint can be activated individually with `--hint[NAME]:on|off` or in a +`push` pragma. ========================== ============================================ Name Description @@ -129,52 +131,52 @@ Level Description Compile-time symbols -------------------- -Through the ``-d:x`` or ``--define:x`` switch you can define compile-time +Through the `-d:x` or `--define:x` switch you can define compile-time symbols for conditional compilation. The defined switches can be checked in source code with the `when statement `_ and `defined proc `_. The typical use of this switch is -to enable builds in release mode (``-d:release``) where optimizations are -enabled for better performance. Another common use is the ``-d:ssl`` switch to +to enable builds in release mode (`-d:release`) where optimizations are +enabled for better performance. Another common use is the `-d:ssl` switch to activate SSL sockets. -Additionally, you may pass a value along with the symbol: ``-d:x=y`` +Additionally, you may pass a value along with the symbol: `-d:x=y` which may be used in conjunction with the `compile-time define pragmas`_ to override symbols during build time. Compile-time symbols are completely **case insensitive** and underscores are -ignored too. ``--define:FOO`` and ``--define:foo`` are identical. +ignored too. `--define:FOO` and `--define:foo` are identical. -Compile-time symbols starting with the ``nim`` prefix are reserved for the +Compile-time symbols starting with the `nim` prefix are reserved for the implementation and should not be used elsewhere. Configuration files ------------------- -**Note:** The *project file name* is the name of the ``.nim`` file that is +**Note:** The *project file name* is the name of the `.nim` file that is passed as a command-line argument to the compiler. -The ``nim`` executable processes configuration files in the following +The `nim` executable processes configuration files in the following directories (in this order; later files overwrite previous settings): -1) ``$nim/config/nim.cfg``, ``/etc/nim/nim.cfg`` (UNIX) or ``\config\nim.cfg`` (Windows). This file can be skipped with the ``--skipCfg`` command line option. -2) If environment variable ``XDG_CONFIG_HOME`` is defined, ``$XDG_CONFIG_HOME/nim/nim.cfg`` or ``~/.config/nim/nim.cfg`` (POSIX) or ``%APPDATA%/nim/nim.cfg`` (Windows). This file can be skipped with the ``--skipUserCfg`` command line option. -3) ``$parentDir/nim.cfg`` where ``$parentDir`` stands for any parent directory of the project file's path. These files can be skipped with the ``--skipParentCfg`` command-line option. -4) ``$projectDir/nim.cfg`` where ``$projectDir`` stands for the project file's path. This file can be skipped with the ``--skipProjCfg`` command-line option. -5) A project can also have a project-specific configuration file named ``$project.nim.cfg`` that resides in the same directory as ``$project.nim``. This file can be skipped with the ``--skipProjCfg`` command-line option. +1) `$nim/config/nim.cfg`, `/etc/nim/nim.cfg` (UNIX) or ``\config\nim.cfg`` (Windows). This file can be skipped with the `--skipCfg` command line option. +2) If environment variable `XDG_CONFIG_HOME` is defined, `$XDG_CONFIG_HOME/nim/nim.cfg` or `~/.config/nim/nim.cfg` (POSIX) or `%APPDATA%/nim/nim.cfg` (Windows). This file can be skipped with the `--skipUserCfg` command line option. +3) `$parentDir/nim.cfg` where `$parentDir` stands for any parent directory of the project file's path. These files can be skipped with the `--skipParentCfg` command-line option. +4) `$projectDir/nim.cfg` where `$projectDir` stands for the project file's path. This file can be skipped with the `--skipProjCfg` command-line option. +5) A project can also have a project-specific configuration file named `$project.nim.cfg` that resides in the same directory as `$project.nim`. This file can be skipped with the `--skipProjCfg` command-line option. Command-line settings have priority over configuration file settings. The default build of a project is a `debug build`:idx:. To compile a -`release build`:idx: define the ``release`` symbol:: +`release build`:idx: define the `release` symbol:: nim c -d:release myproject.nim - To compile a `dangerous release build`:idx: define the ``danger`` symbol:: + To compile a `dangerous release build`:idx: define the `danger` symbol:: nim c -d:danger myproject.nim @@ -186,10 +188,10 @@ Nim has the concept of a global search path (PATH) that is queried to determine where to find imported modules or include files. If multiple files are found an ambiguity error is produced. -``nim dump`` shows the contents of the PATH. +`nim dump` shows the contents of the PATH. However before the PATH is used the current directory is checked for the -file's existence. So if PATH contains ``$lib`` and ``$lib/bar`` and the +file's existence. So if PATH contains `$lib` and `$lib/bar` and the directory structure looks like this:: $lib/x.nim @@ -198,27 +200,27 @@ directory structure looks like this:: foo/main.nim other.nim -And ``main`` imports ``x``, ``foo/x`` is imported. If ``other`` imports ``x`` -then both ``$lib/x.nim`` and ``$lib/bar/x.nim`` match but ``$lib/x.nim`` is used +And `main` imports `x`, `foo/x` is imported. If `other` imports `x` +then both `$lib/x.nim` and `$lib/bar/x.nim` match but `$lib/x.nim` is used as it is the first match. Generated C code directory -------------------------- The generated files that Nim produces all go into a subdirectory called -``nimcache``. Its full path is +`nimcache`. Its full path is -- ``$XDG_CACHE_HOME/nim/$projectname(_r|_d)`` or ``~/.cache/nim/$projectname(_r|_d)`` +- `$XDG_CACHE_HOME/nim/$projectname(_r|_d)` or `~/.cache/nim/$projectname(_r|_d)` on Posix -- ``$HOME/nimcache/$projectname(_r|_d)`` on Windows. +- `$HOME/nimcache/$projectname(_r|_d)` on Windows. -The ``_r`` suffix is used for release builds, ``_d`` is for debug builds. +The `_r` suffix is used for release builds, `_d` is for debug builds. This makes it easy to delete all generated files. -The ``--nimcache`` +The `--nimcache` `compiler switch <#compiler-usage-commandminusline-switches>`_ can be used to -to change the ``nimcache`` directory. +to change the `nimcache` directory. However, the generated C code is not platform-independent. C code generated for Linux does not compile on Windows, for instance. The comment on top of the @@ -232,16 +234,16 @@ To change the compiler from the default compiler (at the command line):: nim c --cc:llvm_gcc --compile_only myfile.nim -This uses the configuration defined in ``config\nim.cfg`` for ``lvm_gcc``. +This uses the configuration defined in ``config\nim.cfg`` for `lvm_gcc`. If nimcache already contains compiled code from a different compiler for the same project, -add the ``-f`` flag to force all files to be recompiled. +add the `-f` flag to force all files to be recompiled. The default compiler is defined at the top of ``config\nim.cfg``. -Changing this setting affects the compiler used by ``koch`` to (re)build Nim. +Changing this setting affects the compiler used by `koch` to (re)build Nim. -To use the ``CC`` environment variable, use ``nim c --cc:env myfile.nim``. To use the -``CXX`` environment variable, use ``nim cpp --cc:env myfile.nim``. ``--cc:env`` is available +To use the `CC` environment variable, use `nim c --cc:env myfile.nim`. To use the +`CXX` environment variable, use `nim cpp --cc:env myfile.nim`. `--cc:env` is available since Nim version 1.4. @@ -252,7 +254,7 @@ To cross compile, use for example:: nim c --cpu:i386 --os:linux --compileOnly --genScript myproject.nim -Then move the C code and the compile script ``compile_myproject.sh`` to your +Then move the C code and the compile script `compile_myproject.sh` to your Linux i386 machine and run the script. Another way is to make Nim invoke a cross compiler toolchain:: @@ -260,8 +262,8 @@ Another way is to make Nim invoke a cross compiler toolchain:: nim c --cpu:arm --os:linux myproject.nim For cross compilation, the compiler invokes a C compiler named -like ``$cpu.$os.$cc`` (for example arm.linux.gcc) and the configuration -system is used to provide meaningful defaults. For example for ``ARM`` your +like `$cpu.$os.$cc` (for example arm.linux.gcc) and the configuration +system is used to provide meaningful defaults. For example for `ARM` your configuration file should contain something like:: arm.linux.gcc.path = "/usr/bin" @@ -275,7 +277,7 @@ To cross-compile for Windows from Linux or macOS using the MinGW-w64 toolchain:: nim c -d:mingw myproject.nim -Use ``--cpu:i386`` or ``--cpu:amd64`` to switch the CPU architecture. +Use `--cpu:i386` or `--cpu:amd64` to switch the CPU architecture. The MinGW-w64 toolchain can be installed as follows:: @@ -295,7 +297,7 @@ The first one is to treat Android as a simple Linux and use directly on android as if it was Linux. These programs are console-only programs that can't be distributed in the Play Store. -Use regular ``nim c`` inside termux to make Android terminal programs. +Use regular `nim c` inside termux to make Android terminal programs. Normal Android apps are written in Java, to use Nim inside an Android app you need a small Java stub that calls out to a native library written in @@ -303,16 +305,16 @@ Nim using the `NDK `_. You can also use `native-activity `_ to have the Java stub be auto-generated for you. -Use ``nim c -c --cpu:arm --os:android -d:androidNDK --noMain:on`` to +Use `nim c -c --cpu:arm --os:android -d:androidNDK --noMain:on` to generate the C source files you need to include in your Android Studio project. Add the generated C files to CMake build script in your Android project. Then do the final compile with Android Studio which uses Gradle to call CMake to compile the project. -Because Nim is part of a library it can't have its own c style ``main()`` -so you would need to define your own ``android_main`` and init the Java +Because Nim is part of a library it can't have its own c style `main()` +so you would need to define your own `android_main` and init the Java environment, or use a library like SDL2 or GLFM to do it. After the Android -stuff is done, it's very important to call ``NimMain()`` in order to +stuff is done, it's very important to call `NimMain()` in order to initialize Nim's garbage collector and to run the top level statements of your program. @@ -331,14 +333,14 @@ Normal languages for iOS development are Swift and Objective C. Both of these use LLVM and can be compiled into object files linked together with C, C++ or Objective C code produced by Nim. -Use ``nim c -c --os:ios --noMain:on`` to generate C files and include them in +Use `nim c -c --os:ios --noMain:on` to generate C files and include them in your XCode project. Then you can use XCode to compile, link, package and sign everything. -Because Nim is part of a library it can't have its own c style ``main()`` so you -would need to define `main` that calls ``autoreleasepool`` and -``UIApplicationMain`` to do it, or use a library like SDL2 or GLFM. After -the iOS setup is done, it's very important to call ``NimMain()`` to +Because Nim is part of a library it can't have its own c style `main()` so you +would need to define `main` that calls `autoreleasepool` and +`UIApplicationMain` to do it, or use a library like SDL2 or GLFM. After +the iOS setup is done, it's very important to call `NimMain()` to initialize Nim's garbage collector and to run the top-level statements of your program. @@ -356,8 +358,8 @@ Cross-compilation for Nintendo Switch ===================================== Simply add --os:nintendoswitch -to your usual ``nim c`` or ``nim cpp`` command and set the ``passC`` -and ``passL`` command line switches to something like: +to your usual `nim c` or `nim cpp` command and set the `passC` +and `passL` command line switches to something like: .. code-block:: console nim c ... --passC="-I$DEVKITPRO/libnx/include" ... @@ -378,8 +380,8 @@ For example, with the above-mentioned config:: nim c --os:nintendoswitch switchhomebrew.nim -This will generate a file called ``switchhomebrew.elf`` which can then be turned into -an nro file with the ``elf2nro`` tool in the DevkitPro release. Examples can be found at +This will generate a file called `switchhomebrew.elf` which can then be turned into +an nro file with the `elf2nro` tool in the DevkitPro release. Examples can be found at `the nim-libnx github repo `_. There are a few things that don't work because the DevkitPro libraries don't support them. @@ -399,62 +401,62 @@ DLL generation Nim supports the generation of DLLs. However, there must be only one instance of the GC per process/address space. This instance is contained in -``nimrtl.dll``. This means that every generated Nim DLL depends -on ``nimrtl.dll``. To generate the "nimrtl.dll" file, use the command:: +`nimrtl.dll`. This means that every generated Nim DLL depends +on `nimrtl.dll`. To generate the "nimrtl.dll" file, use the command:: nim c -d:release lib/nimrtl.nim -To link against ``nimrtl.dll`` use the command:: +To link against `nimrtl.dll` use the command:: nim c -d:useNimRtl myprog.nim -**Note**: Currently the creation of ``nimrtl.dll`` with thread support has +**Note**: Currently the creation of `nimrtl.dll` with thread support has never been tested and is unlikely to work! Additional compilation switches =============================== -The standard library supports a growing number of ``useX`` conditional defines +The standard library supports a growing number of `useX` conditional defines affecting how some features are implemented. This section tries to give a complete list. ====================== ========================================================= Define Effect ====================== ========================================================= -``release`` Turns on the optimizer. +`release` Turns on the optimizer. More aggressive optimizations are possible, e.g.: - ``--passC:-ffast-math`` (but see issue #10305) -``danger`` Turns off all runtime checks and turns on the optimizer. -``useFork`` Makes ``osproc`` use ``fork`` instead of ``posix_spawn``. -``useNimRtl`` Compile and link against ``nimrtl.dll``. -``useMalloc`` Makes Nim use C's `malloc`:idx: instead of Nim's + `--passC:-ffast-math` (but see issue #10305) +`danger` Turns off all runtime checks and turns on the optimizer. +`useFork` Makes `osproc` use `fork` instead of `posix_spawn`. +`useNimRtl` Compile and link against `nimrtl.dll`. +`useMalloc` Makes Nim use C's `malloc`:idx: instead of Nim's own memory manager, albeit prefixing each allocation with its size to support clearing memory on reallocation. - This only works with ``gc:none``, ``gc:arc`` and - ``--gc:orc``. -``useRealtimeGC`` Enables support of Nim's GC for *soft* realtime + This only works with `gc:none`, `gc:arc` and + `--gc:orc`. +`useRealtimeGC` Enables support of Nim's GC for *soft* realtime systems. See the documentation of the `gc `_ for further information. -``logGC`` Enable GC logging to stdout. -``nodejs`` The JS target is actually ``node.js``. -``ssl`` Enables OpenSSL support for the sockets module. -``memProfiler`` Enables memory profiling for the native GC. -``uClibc`` Use uClibc instead of libc. (Relevant for Unix-like OSes) -``checkAbi`` When using types from C headers, add checks that compare +`logGC` Enable GC logging to stdout. +`nodejs` The JS target is actually `node.js`. +`ssl` Enables OpenSSL support for the sockets module. +`memProfiler` Enables memory profiling for the native GC. +`uClibc` Use uClibc instead of libc. (Relevant for Unix-like OSes) +`checkAbi` When using types from C headers, add checks that compare what's in the Nim file with what's in the C header. This may become enabled by default in the future. -``tempDir`` This symbol takes a string as its value, like - ``--define:tempDir:/some/temp/path`` to override the - temporary directory returned by ``os.getTempDir()``. +`tempDir` This symbol takes a string as its value, like + `--define:tempDir:/some/temp/path` to override the + temporary directory returned by `os.getTempDir()`. The value **should** end with a directory separator character. (Relevant for the Android platform) -``useShPath`` This symbol takes a string as its value, like - ``--define:useShPath:/opt/sh/bin/sh`` to override the - path for the ``sh`` binary, in cases where it is not - located in the default location ``/bin/sh``. -``noSignalHandler`` Disable the crash handler from ``system.nim``. -``globalSymbols`` Load all ``{.dynlib.}`` libraries with the ``RTLD_GLOBAL`` +`useShPath` This symbol takes a string as its value, like + `--define:useShPath:/opt/sh/bin/sh` to override the + path for the `sh` binary, in cases where it is not + located in the default location `/bin/sh`. +`noSignalHandler` Disable the crash handler from `system.nim`. +`globalSymbols` Load all `{.dynlib.}` libraries with the `RTLD_GLOBAL` flag on Posix systems to resolve symbols in subsequently loaded libraries. ====================== ========================================================= @@ -471,20 +473,20 @@ generator and are subject to change. LineDir option -------------- -The ``lineDir`` option can be turned on or off. If turned on the -generated C code contains ``#line`` directives. This may be helpful for +The `lineDir` option can be turned on or off. If turned on the +generated C code contains `#line` directives. This may be helpful for debugging with GDB. StackTrace option ----------------- -If the ``stackTrace`` option is turned on, the generated C contains code to +If the `stackTrace` option is turned on, the generated C contains code to ensure that proper stack traces are given if the program crashes or some uncaught exception is raised. LineTrace option ---------------- -The ``lineTrace`` option implies the ``stackTrace`` option. If turned on, +The `lineTrace` option implies the `stackTrace` option. If turned on, the generated C contains code to ensure that proper stack traces with line number information are given if the program crashes or an uncaught exception is raised. @@ -493,10 +495,10 @@ is raised. DynlibOverride ============== -By default Nim's ``dynlib`` pragma causes the compiler to generate -``GetProcAddress`` (or their Unix counterparts) -calls to bind to a DLL. With the ``dynlibOverride`` command line switch this -can be prevented and then via ``--passL`` the static library can be linked +By default Nim's `dynlib` pragma causes the compiler to generate +`GetProcAddress` (or their Unix counterparts) +calls to bind to a DLL. With the `dynlibOverride` command line switch this +can be prevented and then via `--passL` the static library can be linked against. For instance, to link statically against Lua this command might work on Linux:: @@ -506,8 +508,8 @@ on Linux:: Backend language options ======================== -The typical compiler usage involves using the ``compile`` or ``c`` command to -transform a ``.nim`` file into one or more ``.c`` files which are then +The typical compiler usage involves using the `compile` or `c` command to +transform a `.nim` file into one or more `.c` files which are then compiled with the platform's C compiler into a static binary. However, there are other commands to compile to C++, Objective-C, or JavaScript. More details can be read in the `Nim Backend Integration document `_. @@ -517,7 +519,7 @@ Nim documentation tools ======================= Nim provides the `doc`:idx: command to generate HTML -documentation from ``.nim`` source files. Only exported symbols will appear in +documentation from `.nim` source files. Only exported symbols will appear in the output. For more details `see the docgen documentation `_. Nim idetools integration @@ -533,15 +535,15 @@ for further information. The Nim compiler supports an interactive mode. This is also known as a `REPL`:idx: (*read eval print loop*). If Nim has been built with the - ``-d:nimUseLinenoise`` switch, it uses the GNU readline library for terminal + `-d:nimUseLinenoise` switch, it uses the GNU readline library for terminal input management. To start Nim in interactive mode use the command - ``nim secret``. To quit use the ``quit()`` command. To determine whether an input + `nim secret`. To quit use the `quit()` command. To determine whether an input line is an incomplete statement to be continued these rules are used: 1. The line ends with ``[-+*/\\<>!\?\|%&$@~,;:=#^]\s*$`` (operator symbol followed by optional whitespace). 2. The line starts with a space (indentation). 3. The line is within a triple quoted string literal. However, the detection - does not work if the line contains more than one ``"""``. + does not work if the line contains more than one `"""`. Nim for embedded systems @@ -552,22 +554,22 @@ modern PC hardware and operating systems with ample memory, it is very well possible to run Nim code and a good part of the Nim standard libraries on small embedded microprocessors with only a few kilobytes of memory. -A good start is to use the ``any`` operating target together with the -``malloc`` memory allocator and the ``arc`` garbage collector. For example: +A good start is to use the `any` operating target together with the +`malloc` memory allocator and the `arc` garbage collector. For example: -``nim c --os:any --gc:arc -d:useMalloc [...] x.nim`` +`nim c --os:any --gc:arc -d:useMalloc [...] x.nim` -- ``--gc:arc`` will enable the reference counting memory management instead +- `--gc:arc` will enable the reference counting memory management instead of the default garbage collector. This enables Nim to use heap memory which is required for strings and seqs, for example. -- The ``--os:any`` target makes sure Nim does not depend on any specific +- The `--os:any` target makes sure Nim does not depend on any specific operating system primitives. Your platform should support only some basic - ANSI C library ``stdlib`` and ``stdio`` functions which should be available + ANSI C library `stdlib` and `stdio` functions which should be available on almost any platform. -- The ``-d:useMalloc`` option configures Nim to use only the standard C memory - manage primitives ``malloc()``, ``free()``, ``realloc()``. +- The `-d:useMalloc` option configures Nim to use only the standard C memory + manage primitives `malloc()`, `free()`, `realloc()`. If your platform does not provide these functions it should be trivial to provide an implementation for them and link these to your program. @@ -577,10 +579,10 @@ additional flags to both the Nim compiler and the C compiler and/or linker to optimize the build for size. For example, the following flags can be used when targeting a gcc compiler: -``--opt:size --passC:-flto --passL:-flto`` +`--opt:size --passC:-flto --passL:-flto` -The ``--opt:size`` flag instructs Nim to optimize code generation for small -size (with the help of the C compiler), the ``flto`` flags enable link-time +The `--opt:size` flag instructs Nim to optimize code generation for small +size (with the help of the C compiler), the `flto` flags enable link-time optimization in the compiler and linker. Check the `Cross-compilation` section for instructions on how to compile the @@ -600,7 +602,7 @@ The Nim programming language has no concept of Posix's signal handling mechanisms. However, the standard library offers some rudimentary support for signal handling, in particular, segmentation faults are turned into fatal errors that produce a stack trace. This can be disabled with the -``-d:noSignalHandler`` switch. +`-d:noSignalHandler` switch. Optimizing for Nim @@ -642,7 +644,7 @@ However, it is not efficient to do: .. code-block:: Nim var s = varA # assignment has to copy the whole string into a new buffer! -For ``let`` symbols a copy is not always necessary: +For `let` symbols a copy is not always necessary: .. code-block:: Nim let s = varA # may only copy a pointer if it safe to do so @@ -656,7 +658,7 @@ objects as `shallow`:idx:\: shallow(s) # mark 's' as a shallow string var x = s # now might not copy the string! -Usage of ``shallow`` is always safe once you know the string won't be modified +Usage of `shallow` is always safe once you know the string won't be modified anymore, similar to Ruby's `freeze`:idx:. diff --git a/doc/nimfix.rst b/doc/nimfix.rst index 62064fe69..d105346da 100644 --- a/doc/nimfix.rst +++ b/doc/nimfix.rst @@ -1,3 +1,5 @@ +.. default-role:: code + ===================== Nimfix User Guide ===================== @@ -14,12 +16,12 @@ It performs 3 different actions: 1. It makes your code case consistent. 2. It renames every symbol that has a deprecation rule. So if a module has a - rule ``{.deprecated: [TFoo: Foo].}`` then ``TFoo`` is replaced by ``Foo``. + rule `{.deprecated: [TFoo: Foo].}` then `TFoo` is replaced by `Foo`. 3. It can also check that your identifiers adhere to the official style guide - and optionally modify them to do so (via ``--styleCheck:auto``). + and optionally modify them to do so (via `--styleCheck:auto`). -Note that ``nimfix`` defaults to **overwrite** your code unless you -use ``--overwriteFiles:off``! But hey, if you do not use a version control +Note that `nimfix` defaults to **overwrite** your code unless you +use `--overwriteFiles:off`! But hey, if you do not use a version control system by this day and age, your project is already in big trouble. diff --git a/doc/nimgrep.rst b/doc/nimgrep.rst index 791ead162..5b0fe0dbb 100644 --- a/doc/nimgrep.rst +++ b/doc/nimgrep.rst @@ -1,3 +1,5 @@ +.. default-role:: code + ========================= nimgrep User's manual ========================= @@ -22,7 +24,7 @@ Compile nimgrep with the command:: nim c -d:release tools/nimgrep.nim -And copy the executable somewhere in your ``$PATH``. +And copy the executable somewhere in your `$PATH`. Command line switches diff --git a/doc/niminst.rst b/doc/niminst.rst index adb5e6f1f..3ccb47cc8 100644 --- a/doc/niminst.rst +++ b/doc/niminst.rst @@ -1,3 +1,5 @@ +.. default-role:: code + ========================= niminst User's manual ========================= @@ -31,8 +33,8 @@ configuration file. Here's an example of how the syntax looks like: :literal: The value of a key-value pair can reference user-defined variables via -the ``$variable`` notation: They can be defined in the command line with the -``--var:name=value`` switch. This is useful to not hard-coding the +the `$variable` notation: They can be defined in the command line with the +`--var:name=value` switch. This is useful to not hard-coding the program's version number into the configuration file, for instance. It follows a description of each possible section and how it affects the @@ -47,28 +49,28 @@ contain the following key-value pairs: ==================== ======================================================= Key description ==================== ======================================================= -``Name`` the project's name; this needs to be a single word -``DisplayName`` the project's long name; this can contain spaces. If - not specified, this is the same as ``Name``. -``Version`` the project's version -``OS`` the OSes to generate C code for; for example: - ``"windows;linux;macosx"`` -``CPU`` the CPUs to generate C code for; for example: - ``"i386;amd64;powerpc"`` -``Authors`` the project's authors -``Description`` the project's description -``App`` the application's type: "Console" or "GUI". If +`Name` the project's name; this needs to be a single word +`DisplayName` the project's long name; this can contain spaces. If + not specified, this is the same as `Name`. +`Version` the project's version +`OS` the OSes to generate C code for; for example: + `"windows;linux;macosx"` +`CPU` the CPUs to generate C code for; for example: + `"i386;amd64;powerpc"` +`Authors` the project's authors +`Description` the project's description +`App` the application's type: "Console" or "GUI". If "Console", niminst generates a special batch file for Windows to open up the command-line shell. -``License`` the filename of the application's license +`License` the filename of the application's license ==================== ======================================================= -``files`` key +`files` key ------------- -Many sections support the ``files`` key. Listed filenames -can be separated by semicolon or the ``files`` key can be repeated. Wildcards +Many sections support the `files` key. Listed filenames +can be separated by semicolon or the `files` key can be repeated. Wildcards in filenames are supported. If it is a directory name, all files in the directory are used:: @@ -80,63 +82,63 @@ directory are used:: Config section -------------- -The ``config`` section currently only supports the ``files`` key. Listed files +The `config` section currently only supports the `files` key. Listed files will be installed into the OS's configuration directory. Documentation section --------------------- -The ``documentation`` section supports the ``files`` key. +The `documentation` section supports the `files` key. Listed files will be installed into the OS's native documentation directory -(which might be ``$appdir/doc``). +(which might be `$appdir/doc`). -There is a ``start`` key which determines whether the Windows installer -generates a link to e.g. the ``index.html`` of your documentation. +There is a `start` key which determines whether the Windows installer +generates a link to e.g. the `index.html` of your documentation. Other section ------------- -The ``other`` section currently only supports the ``files`` key. +The `other` section currently only supports the `files` key. Listed files will be installed into the application installation directory -(``$appdir``). +(`$appdir`). Lib section ----------- -The ``lib`` section currently only supports the ``files`` key. +The `lib` section currently only supports the `files` key. Listed files will be installed into the OS's native library directory -(which might be ``$appdir/lib``). +(which might be `$appdir/lib`). Windows section --------------- -The ``windows`` section supports the ``files`` key for Windows-specific files. +The `windows` section supports the `files` key for Windows-specific files. Listed files will be installed into the application installation directory -(``$appdir``). +(`$appdir`). Other possible options are: ==================== ======================================================= Key description ==================== ======================================================= -``BinPath`` paths to add to the Windows ``%PATH%`` environment +`BinPath` paths to add to the Windows `%PATH%` environment variable. Example: ``BinPath: r"bin;dist\mingw\bin"`` -``InnoSetup`` boolean flag whether an Inno Setup installer should be - generated for Windows. Example: ``InnoSetup: "Yes"`` +`InnoSetup` boolean flag whether an Inno Setup installer should be + generated for Windows. Example: `InnoSetup: "Yes"` ==================== ======================================================= UnixBin section --------------- -The ``UnixBin`` section currently only supports the ``files`` key. +The `UnixBin` section currently only supports the `files` key. Listed files will be installed into the OS's native bin directory -(e.g. ``/usr/local/bin``). The exact location depends on the -installation path the user specifies when running the ``install.sh`` script. +(e.g. `/usr/local/bin`). The exact location depends on the +installation path the user specifies when running the `install.sh` script. Unix section @@ -147,11 +149,11 @@ Possible options are: ==================== ======================================================= Key description ==================== ======================================================= -``InstallScript`` boolean flag whether an installation shell script - should be generated. Example: ``InstallScript: "Yes"`` -``UninstallScript`` boolean flag whether a de-installation shell script +`InstallScript` boolean flag whether an installation shell script + should be generated. Example: `InstallScript: "Yes"` +`UninstallScript` boolean flag whether a de-installation shell script should be generated. - Example: ``UninstallScript: "Yes"`` + Example: `UninstallScript: "Yes"` ==================== ======================================================= @@ -163,10 +165,10 @@ Possible options are: ==================== ======================================================= Key description ==================== ======================================================= -``path`` Path to Inno Setup. +`path` Path to Inno Setup. Example: ``path = r"c:\inno setup 5\iscc.exe"`` -``flags`` Flags to pass to Inno Setup. - Example: ``flags = "/Q"`` +`flags` Flags to pass to Inno Setup. + Example: `flags = "/Q"` ==================== ======================================================= @@ -178,9 +180,9 @@ Possible options are: ==================== ======================================================= Key description ==================== ======================================================= -``path`` Path to the C compiler. -``flags`` Flags to pass to the C Compiler. - Example: ``flags = "-w"`` +`path` Path to the C compiler. +`flags` Flags to pass to the C Compiler. + Example: `flags = "-w"` ==================== ======================================================= diff --git a/doc/nims.rst b/doc/nims.rst index aa13d134a..f81637d73 100644 --- a/doc/nims.rst +++ b/doc/nims.rst @@ -1,30 +1,32 @@ +.. default-role:: code + ================================ NimScript ================================ -Strictly speaking, ``NimScript`` is the subset of Nim that can be evaluated +Strictly speaking, `NimScript` is the subset of Nim that can be evaluated by Nim's builtin virtual machine (VM). This VM is used for Nim's compiletime function evaluation features. -The ``nim`` executable processes the ``.nims`` configuration files in +The `nim` executable processes the `.nims` configuration files in the following directories (in this order; later files overwrite previous settings): -1) If environment variable ``XDG_CONFIG_HOME`` is defined, - ``$XDG_CONFIG_HOME/nim/config.nims`` or - ``~/.config/nim/config.nims`` (POSIX) or - ``%APPDATA%/nim/config.nims`` (Windows). This file can be skipped - with the ``--skipUserCfg`` command line option. -2) ``$parentDir/config.nims`` where ``$parentDir`` stands for any +1) If environment variable `XDG_CONFIG_HOME` is defined, + `$XDG_CONFIG_HOME/nim/config.nims` or + `~/.config/nim/config.nims` (POSIX) or + `%APPDATA%/nim/config.nims` (Windows). This file can be skipped + with the `--skipUserCfg` command line option. +2) `$parentDir/config.nims` where `$parentDir` stands for any parent directory of the project file's path. These files can be - skipped with the ``--skipParentCfg`` command line option. -3) ``$projectDir/config.nims`` where ``$projectDir`` stands for the - project's path. This file can be skipped with the ``--skipProjCfg`` + skipped with the `--skipParentCfg` command line option. +3) `$projectDir/config.nims` where `$projectDir` stands for the + project's path. This file can be skipped with the `--skipProjCfg` command line option. 4) A project can also have a project specific configuration file named - ``$project.nims`` that resides in the same directory as - ``$project.nim``. This file can be skipped with the same - ``--skipProjCfg`` command line option. + `$project.nims` that resides in the same directory as + `$project.nim`. This file can be skipped with the same + `--skipProjCfg` command line option. For available procs and implementation details see `nimscript `_. @@ -36,13 +38,13 @@ NimScript is subject to some limitations caused by the implementation of the VM (virtual machine): * Nim's FFI (foreign function interface) is not available in NimScript. This - means that any stdlib module which relies on ``importc`` can not be used in + means that any stdlib module which relies on `importc` can not be used in the VM. -* ``ptr`` operations are are hard to emulate with the symbolic representation +* `ptr` operations are are hard to emulate with the symbolic representation the VM uses. They are available and tested extensively but there are bugs left. -* ``var T`` function arguments rely on ``ptr`` operations internally and might +* `var T` function arguments rely on `ptr` operations internally and might also be problematic in some cases. * More than one level of `ref` is generally not supported (for example, the type @@ -50,7 +52,7 @@ NimScript is subject to some limitations caused by the implementation of the VM * Multimethods are not available. -* ``random.randomize()`` requires an ``int64`` explicitly passed as argument, you *must* pass a Seed integer. +* `random.randomize()` requires an `int64` explicitly passed as argument, you *must* pass a Seed integer. Standard library modules @@ -109,11 +111,11 @@ See also: NimScript as a configuration file ================================= -A command-line switch ``--FOO`` is written as ``switch("FOO")`` in -NimScript. Similarly, command-line ``--FOO:VAL`` translates to -``switch("FOO", "VAL")``. +A command-line switch `--FOO` is written as `switch("FOO")` in +NimScript. Similarly, command-line `--FOO:VAL` translates to +`switch("FOO", "VAL")`. -Here are few examples of using the ``switch`` proc: +Here are few examples of using the `switch` proc: .. code-block:: nim # command-line: --opt:size @@ -123,7 +125,7 @@ Here are few examples of using the ``switch`` proc: # command-line: --forceBuild switch("forceBuild") -NimScripts also support ``--`` templates for convenience, which look +NimScripts also support `--` templates for convenience, which look like command-line switches written as-is in the NimScript file. So the above example can be rewritten as: @@ -133,17 +135,17 @@ above example can be rewritten as: --forceBuild **Note**: In general, the *define* switches can also be set in -NimScripts using ``switch`` or ``--``, as shown in above -examples. Only the ``release`` define (``-d:release``) cannot be set +NimScripts using `switch` or `--`, as shown in above +examples. Only the `release` define (`-d:release`) cannot be set in NimScripts. NimScript as a build tool ========================= -The ``task`` template that the ``system`` module defines allows a NimScript +The `task` template that the `system` module defines allows a NimScript file to be used as a build tool. The following example defines a -task ``build`` that is an alias for the ``c`` command: +task `build` that is an alias for the `c` command: .. code-block:: nim task build, "builds an example": @@ -155,11 +157,11 @@ In fact, as a convention the following tasks should be available: ========= =================================================== Task Description ========= =================================================== -``help`` List all the available NimScript tasks along with their docstrings. -``build`` Build the project with the required - backend (``c``, ``cpp`` or ``js``). -``tests`` Runs the tests belonging to the project. -``bench`` Runs benchmarks belonging to the project. +`help` List all the available NimScript tasks along with their docstrings. +`build` Build the project with the required + backend (`c`, `cpp` or `js`). +`tests` Runs the tests belonging to the project. +`bench` Runs benchmarks belonging to the project. ========= =================================================== @@ -178,7 +180,7 @@ Standalone NimScript ==================== NimScript can also be used directly as a portable replacement for Bash and -Batch files. Use ``nim myscript.nims`` to run ``myscript.nims``. For example, +Batch files. Use `nim myscript.nims` to run `myscript.nims`. For example, installation of Nimble could be accomplished with this simple script: .. code-block:: nim @@ -196,8 +198,8 @@ installation of Nimble could be accomplished with this simple script: mvFile "nimble" & $id & "/src/nimble".toExe, "bin/nimble".toExe -On Unix, you can also use the shebang ``#!/usr/bin/env nim``, as long as your filename -ends with ``.nims``: +On Unix, you can also use the shebang `#!/usr/bin/env nim`, as long as your filename +ends with `.nims`: .. code-block:: nim @@ -206,7 +208,7 @@ ends with ``.nims``: echo "hello world" -Use ``#!/usr/bin/env -S nim --hints:off`` to disable hints. +Use `#!/usr/bin/env -S nim --hints:off` to disable hints. Benefits @@ -268,7 +270,7 @@ Powerful Metaprogramming NimScript can use Nim's templates, macros, types, concepts, effect tracking system, and more, you can create modules that work on compiled Nim and also on interpreted NimScript. -``func`` will still check for side effects, ``debugEcho`` also works as expected, +`func` will still check for side effects, `debugEcho` also works as expected, making it ideal for functional scripting metaprogramming. This is an example of a third party module that uses macros and templates to @@ -315,7 +317,7 @@ See the following NimScript: echo CompileDate -``likely()``, ``unlikely()``, ``static:`` and ``{.compiletime.}`` +`likely()`, `unlikely()`, `static:` and `{.compiletime.}` will produce no code at all when run on NimScript, but still no error nor warning is produced and the code just works. diff --git a/doc/nimsuggest.rst b/doc/nimsuggest.rst index 8db7c233b..509f72e8a 100644 --- a/doc/nimsuggest.rst +++ b/doc/nimsuggest.rst @@ -1,3 +1,5 @@ +.. default-role:: code + ================================ Nim IDE Integration Guide ================================ @@ -11,8 +13,8 @@ Nim differs from many other compilers in that it is really fast, and being so fast makes it suited to provide external queries for text editors about the source code being written. Through the -``nimsuggest`` tool, any IDE -can query a ``.nim`` source file and obtain useful information like +`nimsuggest` tool, any IDE +can query a `.nim` source file and obtain useful information like definition of symbols or suggestions for completion. This document will guide you through the available options. If you @@ -33,50 +35,50 @@ Nimsuggest is part of Nim's core. Build it via:: Nimsuggest invocation ===================== -Run it via ``nimsuggest --stdin --debug myproject.nim``. Nimsuggest is a -server that takes queries that are related to ``myproject``. There is some -support so that you can throw random ``.nim`` files which are not part -of ``myproject`` at Nimsuggest too, but usually the query refer to modules/files -that are part of ``myproject``. +Run it via `nimsuggest --stdin --debug myproject.nim`. Nimsuggest is a +server that takes queries that are related to `myproject`. There is some +support so that you can throw random `.nim` files which are not part +of `myproject` at Nimsuggest too, but usually the query refer to modules/files +that are part of `myproject`. -``--stdin`` means that Nimsuggest reads the query from ``stdin``. This is great +`--stdin` means that Nimsuggest reads the query from `stdin`. This is great for testing things out and playing with it but for an editor communication via sockets is more reasonable so that is the default. It listens to port 6000 by default. -Nimsuggest is basically a frontend for the nim compiler so ``--path`` flags and +Nimsuggest is basically a frontend for the nim compiler so `--path` flags and `config files `_ can be used to specify additional dependencies like -``nimsuggest --stdin --debug --path:"dependencies" myproject.nim``. +`nimsuggest --stdin --debug --path:"dependencies" myproject.nim`. Specifying the location of the query ------------------------------------ Nimsuggest then waits for queries to process. A query consists of a -cryptic 3 letter "command" ``def`` or ``con`` or ``sug`` or ``use`` followed by +cryptic 3 letter "command" `def` or `con` or `sug` or `use` followed by a location. A query location consists of: -``file.nim`` +`file.nim` This is the name of the module or include file the query refers to. -``dirtyfile.nim`` +`dirtyfile.nim` This is optional. - The ``file`` parameter is enough for static analysis, but IDEs + The `file` parameter is enough for static analysis, but IDEs tend to have *unsaved buffers* where the user may still be in the middle of typing a line. In such situations the IDE can save the current contents to a temporary file and then use the - ``dirtyfile.nim`` option to tell Nimsuggest that ``foobar.nim`` should - be taken from ``temporary/foobar.nim``. + `dirtyfile.nim` option to tell Nimsuggest that `foobar.nim` should + be taken from `temporary/foobar.nim`. -``line`` +`line` An integer with the line you are going to query. For the compiler lines start at **1**. -``col`` +`col` An integer with the column you are going to query. For the compiler columns start at **0**. @@ -84,7 +86,7 @@ a location. A query location consists of: Definitions ----------- -The ``def`` Nimsuggest command performs a query about the definition +The `def` Nimsuggest command performs a query about the definition of a specific symbol. If available, Nimsuggest will answer with the type, source file, line/column information and other accessory data if available like a docstring. With this information an IDE can @@ -105,7 +107,7 @@ can't find any valid symbol matching the position of the query. Suggestions ----------- -The ``sug`` Nimsuggest command performs a query about possible +The `sug` Nimsuggest command performs a query about possible completion symbols at some point in the file. The typical usage scenario for this option is to call it after the @@ -118,7 +120,7 @@ Nimsuggest will try to return the suggestions sorted first by scope Invocation context ------------------ -The ``con`` Nimsuggest command is very similar to the suggestions +The `con` Nimsuggest command is very similar to the suggestions command, but instead of being used after the user has typed a dot character, this one is meant to be used after the user has typed an opening brace to start typing parameters. @@ -127,7 +129,7 @@ an opening brace to start typing parameters. Symbol usages ------------- -The ``use`` Nimsuggest command lists all usages of the symbol at +The `use` Nimsuggest command lists all usages of the symbol at a position. IDEs can use this to find all the places in the file where the symbol is used and offer the user to rename it in all places at the same time. @@ -145,15 +147,15 @@ Nimsuggest output is always returned on single lines separated by tab characters (``\t``). The values of each column are: 1. Three characters indicating the type of returned answer (e.g. - ``def`` for definition, ``sug`` for suggestion, etc). -2. Type of the symbol. This can be ``skProc``, ``skLet``, and just - about any of the enums defined in the module ``compiler/ast.nim``. + `def` for definition, `sug` for suggestion, etc). +2. Type of the symbol. This can be `skProc`, `skLet`, and just + about any of the enums defined in the module `compiler/ast.nim`. 3. Fully qualified path of the symbol. If you are querying a symbol - defined in the ``proj.nim`` file, this would have the form - ``proj.symbolName``. + defined in the `proj.nim` file, this would have the form + `proj.symbolName`. 4. Type/signature. For variables and enums this will contain the type of the symbol, for procs, methods and templates this will - contain the full unique signature (e.g. ``proc (File)``). + contain the full unique signature (e.g. `proc (File)`). 5. Full path to the file containing the symbol. 6. Line where the symbol is located in the file. Lines start to count at **1**. diff --git a/doc/testament.rst b/doc/testament.rst index 5db68bfa7..04c966ffe 100644 --- a/doc/testament.rst +++ b/doc/testament.rst @@ -1,3 +1,5 @@ +.. default-role:: code + Testament is an advanced automatic unittests runner for Nim tests, is used for the development of Nim itself, offers process isolation for your tests, it can generate statistics about test cases, supports multiple targets (C, C++, ObjectiveC, JavaScript, etc), @@ -9,29 +11,29 @@ so can be useful to run your tests, even the most complex ones. Test files location =================== -By default Testament looks for test files on ``"./tests/*.nim"``. -You can overwrite this pattern glob using ``pattern ``. +By default Testament looks for test files on `"./tests/*.nim"`. +You can overwrite this pattern glob using `pattern `. The default working directory path can be changed using -``--directory:"folder/subfolder/"``. +`--directory:"folder/subfolder/"`. -Testament uses the ``nim`` compiler on ``PATH``. -You can change that using ``--nim:"folder/subfolder/nim"``. -Running JavaScript tests with ``--targets:"js"`` requires a working NodeJS on -``PATH``. +Testament uses the `nim` compiler on `PATH`. +You can change that using `--nim:"folder/subfolder/nim"`. +Running JavaScript tests with `--targets:"js"` requires a working NodeJS on +`PATH`. Options ======= -* ``--print`` Also print results to the console -* ``--simulate`` See what tests would be run but don't run them (for debugging) -* ``--failing`` Only show failing/ignored tests -* ``--targets:"c cpp js objc"`` Run tests for specified targets (default: all) -* ``--nim:path`` Use a particular nim executable (default: ``$PATH/nim``) -* ``--directory:dir`` Change to directory dir before reading the tests or doing anything else. -* ``--colors:on|off`` Turn messages coloring on|off. -* ``--backendLogging:on|off`` Disable or enable backend logging. By default turned on. -* ``--skipFrom:file`` Read tests to skip from ``file`` - one test per line, # comments ignored +* `--print` Also print results to the console +* `--simulate` See what tests would be run but don't run them (for debugging) +* `--failing` Only show failing/ignored tests +* `--targets:"c cpp js objc"` Run tests for specified targets (default: all) +* `--nim:path` Use a particular nim executable (default: `$PATH/nim`) +* `--directory:dir` Change to directory dir before reading the tests or doing anything else. +* `--colors:on|off` Turn messages coloring on|off. +* `--backendLogging:on|off` Disable or enable backend logging. By default turned on. +* `--skipFrom:file` Read tests to skip from `file` - one test per line, # comments ignored Running a single test @@ -68,7 +70,7 @@ To search for tests deeper in a directory, use HTML Reports ============ -Generate HTML Reports ``testresults.html`` from unittests, +Generate HTML Reports `testresults.html` from unittests, you have to run at least 1 test *before* generating a report: .. code:: @@ -172,7 +174,7 @@ Example "template" **to edit** and write a Testament unittest: assert 42 == 42, "Assert error message" -* As you can see the "Spec" is just a ``discard """ """``. +* As you can see the "Spec" is just a `discard """ """`. * Spec has sane defaults, so you don't need to provide them all, any simple assert will work just fine. * `This is not the full spec of Testament, check the Testament Spec on GitHub, see parseSpec(). `_ * `Nim itself uses Testament, so there are plenty of test examples. `_ diff --git a/doc/tools.rst b/doc/tools.rst index f231cdc7d..e0044e1ca 100644 --- a/doc/tools.rst +++ b/doc/tools.rst @@ -1,3 +1,5 @@ +.. default-role:: code + ======================== Tools available with Nim ======================== @@ -9,11 +11,11 @@ The standard distribution ships with the following tools: document explaining how it works. - | `Documentation generator `_ - | The builtin document generator ``nim doc`` generates HTML documentation - from ``.nim`` source files. + | The builtin document generator `nim doc` generates HTML documentation + from `.nim` source files. - | `Nimsuggest for IDE support `_ - | Through the ``nimsuggest`` tool, any IDE can query a ``.nim`` source file + | Through the `nimsuggest` tool, any IDE can query a `.nim` source file and obtain useful information like the definition of symbols or suggestions for completion. @@ -27,11 +29,11 @@ The standard distribution ships with the following tools: | Nim search and replace utility. - | nimpretty - | ``nimpretty`` is a Nim source code beautifier, + | `nimpretty` is a Nim source code beautifier, to format code according to the official style guide. - | `testament `_ - | ``testament`` is an advanced automatic *unittests runner* for Nim tests, + | `testament` is an advanced automatic *unittests runner* for Nim tests, is used for the development of Nim itself, offers process isolation for your tests, it can generate statistics about test cases, supports multiple targets (C, JS, etc), `simulated Dry-Runs `_, diff --git a/doc/tut1.rst b/doc/tut1.rst index 171b2f918..d2d6f8fe7 100644 --- a/doc/tut1.rst +++ b/doc/tut1.rst @@ -1,3 +1,5 @@ +.. default-role:: code + ===================== Nim Tutorial (Part I) ===================== @@ -46,7 +48,7 @@ Save this code to the file "greetings.nim". Now compile and run it:: nim compile --run greetings.nim -With the ``--run`` `switch `_ Nim +With the `--run` `switch `_ Nim executes the file automatically after compilation. You can give your program command-line arguments by appending them after the filename:: @@ -61,7 +63,7 @@ To compile a release version use:: nim c -d:release greetings.nim By default, the Nim compiler generates a large number of runtime checks -aiming for your debugging pleasure. With ``-d:release`` some checks are +aiming for your debugging pleasure. With `-d:release` some checks are `turned off and optimizations are turned on `_. @@ -70,8 +72,8 @@ syntax: statements which are not indented are executed when the program starts. Indentation is Nim's way of grouping statements. Indentation is done with spaces only, tabulators are not allowed. -String literals are enclosed in double-quotes. The ``var`` statement declares -a new variable named ``name`` of type ``string`` with the value that is +String literals are enclosed in double-quotes. The `var` statement declares +a new variable named `name` of type `string` with the value that is returned by the `readLine `_ procedure. Since the compiler knows that `readLine `_ returns a string, you can leave out the type in the declaration (this is called `local type @@ -85,7 +87,7 @@ Note that this is basically the only form of type inference that exists in Nim: it is a good compromise between brevity and readability. The "hello world" program contains several identifiers that are already known -to the compiler: ``echo``, `readLine `_, etc. +to the compiler: `echo`, `readLine `_, etc. These built-ins are declared in the system_ module which is implicitly imported by any other module. @@ -102,7 +104,7 @@ String and character literals ----------------------------- String literals are enclosed in double-quotes; character literals in single -quotes. Special characters are escaped with ``\``: ``\n`` means newline, ``\t`` +quotes. Special characters are escaped with ``\\``: ``\n`` means newline, ``\t`` means tabulator, etc. There are also *raw* string literals: .. code-block:: Nim @@ -111,8 +113,8 @@ means tabulator, etc. There are also *raw* string literals: In raw literals, the backslash is not an escape character. The third and last way to write string literals is *long-string literals*. -They are written with three quotes: ``""" ... """``; they can span over -multiple lines and the ``\`` is not an escape character either. They are very +They are written with three quotes: `""" ... """`; they can span over +multiple lines and the ``\\`` is not an escape character either. They are very useful for embedding HTML code templates for example. @@ -120,7 +122,7 @@ Comments -------- Comments start anywhere outside a string or character literal with the -hash character ``#``. Documentation comments start with ``##``: +hash character `#`. Documentation comments start with `##`: .. code-block:: nim :test: "nim c $1" @@ -133,7 +135,7 @@ Documentation comments are tokens; they are only allowed at certain places in the input file as they belong to the syntax tree! This feature enables simpler documentation generators. -Multiline comments are started with ``#[`` and terminated with ``]#``. Multiline +Multiline comments are started with `#[` and terminated with `]#`. Multiline comments can also be nested. .. code-block:: nim @@ -152,10 +154,10 @@ Numbers ------- Numerical literals are written as in most other languages. As a special twist, -underscores are allowed for better readability: ``1_000_000`` (one million). +underscores are allowed for better readability: `1_000_000` (one million). A number that contains a dot (or 'e' or 'E') is a floating-point literal: -``1.0e9`` (one billion). Hexadecimal literals are prefixed with ``0x``, -binary literals with ``0b`` and octal literals with ``0o``. A leading zero +`1.0e9` (one billion). Hexadecimal literals are prefixed with `0x`, +binary literals with `0b` and octal literals with `0o`. A leading zero alone does not produce an octal. @@ -164,9 +166,9 @@ The var statement The var statement declares a new local or global variable: .. code-block:: - var x, y: int # declares x and y to have the type ``int`` + var x, y: int # declares x and y to have the type `int` -Indentation can be used after the ``var`` keyword to list a whole section of +Indentation can be used after the `var` keyword to list a whole section of variables: .. code-block:: @@ -187,7 +189,7 @@ to a storage location: var x = "abc" # introduces a new variable `x` and assigns a value to it x = "xyz" # assigns a new value to `x` -``=`` is the *assignment operator*. The assignment operator can be +`=` is the *assignment operator*. The assignment operator can be overloaded. You can declare multiple variables with a single assignment statement and all the variables will have the same value: @@ -219,7 +221,7 @@ constant declaration at compile time: :test: "nim c $1" const x = "abc" # the constant x contains the string "abc" -Indentation can be used after the ``const`` keyword to list a whole section of +Indentation can be used after the `const` keyword to list a whole section of constants: .. code-block:: @@ -233,7 +235,7 @@ constants: The let statement ================= -The ``let`` statement works like the ``var`` statement but the declared +The `let` statement works like the `var` statement but the declared symbols are *single assignment* variables: After the initialization their value cannot change: @@ -241,8 +243,8 @@ value cannot change: let x = "abc" # introduces a new variable `x` and binds a value to it x = "xyz" # Illegal: assignment to `x` -The difference between ``let`` and ``const`` is: ``let`` introduces a variable -that can not be re-assigned, ``const`` means "enforce compile time evaluation +The difference between `let` and `const` is: `let` introduces a variable +that can not be re-assigned, `const` means "enforce compile time evaluation and put it into a data section": .. code-block:: @@ -276,9 +278,9 @@ The if statement is one way to branch the control flow: else: echo "Hi, ", name, "!" -There can be zero or more ``elif`` parts, and the ``else`` part is optional. -The keyword ``elif`` is short for ``else if``, and is useful to avoid -excessive indentation. (The ``""`` is the empty string. It contains no +There can be zero or more `elif` parts, and the `else` part is optional. +The keyword `elif` is short for `else if`, and is useful to avoid +excessive indentation. (The `""` is the empty string. It contains no characters.) @@ -301,7 +303,7 @@ a multi-branch: else: echo "Hi, ", name, "!" -As it can be seen, for an ``of`` branch a comma-separated list of values is also +As it can be seen, for an `of` branch a comma-separated list of values is also allowed. The case statement can deal with integers, other ordinal types, and strings. @@ -319,8 +321,8 @@ For integers or other ordinal types value ranges are also possible: of 3, 8: echo "The number is 3 or 8" However, the above code does not compile: the reason is that you have to cover -every value that ``n`` may contain, but the code only handles the values -``0..8``. Since it is not very practical to list every other possible integer +every value that `n` may contain, but the code only handles the values +`0..8`. Since it is not very practical to list every other possible integer (though it is possible thanks to the range notation), we fix this by telling the compiler that for every other value nothing should be done: @@ -334,7 +336,7 @@ the compiler that for every other value nothing should be done: The empty `discard statement <#procedures-discard-statement>`_ is a *do nothing* statement. The compiler knows that a case statement with an else part cannot fail and thus the error disappears. Note that it is impossible to cover -all possible string values: that is why string cases always need an ``else`` +all possible string values: that is why string cases always need an `else` branch. In general, the case statement is used for subrange types or enumerations where @@ -355,7 +357,7 @@ The while statement is a simple looping construct: while name == "": echo "Please tell me your name: " name = readLine(stdin) - # no ``var``, because we do not declare a new variable here + # no `var`, because we do not declare a new variable here The example uses a while loop to keep asking the users for their name, as long as the user types in nothing (only presses RETURN). @@ -364,7 +366,7 @@ as the user types in nothing (only presses RETURN). For statement ------------- -The ``for`` statement is a construct to loop over any element an *iterator* +The `for` statement is a construct to loop over any element an *iterator* provides. The example uses the built-in `countup `_ iterator: @@ -375,10 +377,10 @@ provides. The example uses the built-in `countup echo i # --> Outputs 1 2 3 4 5 6 7 8 9 10 on different lines -The variable ``i`` is implicitly declared by the -``for`` loop and has the type ``int``, because that is what `countup -`_ returns. ``i`` runs through the values -1, 2, .., 10. Each value is ``echo``-ed. This code does the same: +The variable `i` is implicitly declared by the +`for` loop and has the type `int`, because that is what `countup +`_ returns. `i` runs through the values +1, 2, .., 10. Each value is `echo`-ed. This code does the same: .. code-block:: nim echo "Counting to 10: " @@ -403,7 +405,7 @@ Since counting up occurs so often in programs, Nim also has a `.. for i in 1 .. 10: ... -Zero-indexed counting has two shortcuts ``..<`` and ``.. ^1`` +Zero-indexed counting has two shortcuts `..<` and `.. ^1` (`backward index operator `_) to simplify counting to one less than the higher index: @@ -426,8 +428,8 @@ or ... Other useful iterators for collections (like arrays and sequences) are -* ``items`` and ``mitems``, which provides immutable and mutable elements respectively, and -* ``pairs`` and ``mpairs`` which provides the element and an index number (immutable and mutable respectively) +* `items` and `mitems`, which provides immutable and mutable elements respectively, and +* `pairs` and `mpairs` which provides the element and an index number (immutable and mutable respectively) .. code-block:: nim :test: "nim c $1" @@ -439,7 +441,7 @@ Other useful iterators for collections (like arrays and sequences) are Scopes and the block statement ------------------------------ Control flow statements have a feature not covered yet: they open a -new scope. This means that in the following example, ``x`` is not accessible +new scope. This means that in the following example, `x` is not accessible outside the loop: .. code-block:: nim @@ -450,7 +452,7 @@ outside the loop: echo x # does not work A while (for) statement introduces an implicit block. Identifiers -are only visible within the block they have been declared. The ``block`` +are only visible within the block they have been declared. The `block` statement can be used to open a new block explicitly: .. code-block:: nim @@ -460,13 +462,13 @@ statement can be used to open a new block explicitly: var x = "hi" echo x # does not work either -The block's *label* (``myblock`` in the example) is optional. +The block's *label* (`myblock` in the example) is optional. Break statement --------------- -A block can be left prematurely with a ``break`` statement. The break statement -can leave a ``while``, ``for``, or a ``block`` statement. It leaves the +A block can be left prematurely with a `break` statement. The break statement +can leave a `while`, `for`, or a `block` statement. It leaves the innermost construct, unless a label of a block is given: .. code-block:: nim @@ -490,7 +492,7 @@ innermost construct, unless a label of a block is given: Continue statement ------------------ -Like in many other programming languages, a ``continue`` statement starts +Like in many other programming languages, a `continue` statement starts the next iteration immediately: .. code-block:: nim @@ -518,17 +520,17 @@ Example: else: echo "unknown operating system" -The ``when`` statement is almost identical to the ``if`` statement, but with these +The `when` statement is almost identical to the `if` statement, but with these differences: * Each condition must be a constant expression since it is evaluated by the compiler. * The statements within a branch do not open a new scope. * The compiler checks the semantics and produces code *only* for the statements - that belong to the first condition that evaluates to ``true``. + that belong to the first condition that evaluates to `true`. -The ``when`` statement is useful for writing platform-specific code, similar to -the ``#ifdef`` construct in the C programming language. +The `when` statement is useful for writing platform-specific code, similar to +the `#ifdef` construct in the C programming language. Statements and indentation @@ -539,8 +541,8 @@ indentation rules. In Nim, there is a distinction between *simple statements* and *complex statements*. *Simple statements* cannot contain other statements: -Assignment, procedure calls, or the ``return`` statement are all simple -statements. *Complex statements* like ``if``, ``when``, ``for``, ``while`` can +Assignment, procedure calls, or the `return` statement are all simple +statements. *Complex statements* like `if`, `when`, `for`, `while` can contain other statements. To avoid ambiguities, complex statements must always be indented, but single simple statements do not: @@ -575,7 +577,7 @@ contain indentation at certain places for better readability: As a rule of thumb, indentation within expressions is allowed after operators, an open parenthesis and after commas. -With parenthesis and semicolons ``(;)`` you can use statements where only +With parenthesis and semicolons `(;)` you can use statements where only an expression is allowed: .. code-block:: nim @@ -590,7 +592,7 @@ Procedures To define new commands like `echo `_ and `readLine `_ in the examples, the concept of a `procedure` is needed. (Some languages call them *methods* or *functions*.) -In Nim new procedures are defined with the ``proc`` keyword: +In Nim new procedures are defined with the `proc` keyword: .. code-block:: nim :test: "nim c $1" @@ -607,26 +609,26 @@ In Nim new procedures are defined with the ``proc`` keyword: else: echo "I think you know what the problem is just as well as I do." -This example shows a procedure named ``yes`` that asks the user a ``question`` +This example shows a procedure named `yes` that asks the user a `question` and returns true if they answered "yes" (or something similar) and returns -false if they answered "no" (or something similar). A ``return`` statement +false if they answered "no" (or something similar). A `return` statement leaves the procedure (and therefore the while loop) immediately. The -``(question: string): bool`` syntax describes that the procedure expects a -parameter named ``question`` of type ``string`` and returns a value of type -``bool``. The ``bool`` type is built-in: the only valid values for ``bool`` are -``true`` and ``false``. -The conditions in if or while statements must be of type ``bool``. +`(question: string): bool` syntax describes that the procedure expects a +parameter named `question` of type `string` and returns a value of type +`bool`. The `bool` type is built-in: the only valid values for `bool` are +`true` and `false`. +The conditions in if or while statements must be of type `bool`. -Some terminology: in the example ``question`` is called a (formal) *parameter*, -``"Should I..."`` is called an *argument* that is passed to this parameter. +Some terminology: in the example `question` is called a (formal) *parameter*, +`"Should I..."` is called an *argument* that is passed to this parameter. Result variable --------------- -A procedure that returns a value has an implicit ``result`` variable declared -that represents the return value. A ``return`` statement with no expression is -shorthand for ``return result``. The ``result`` value is always returned -automatically at the end of a procedure if there is no ``return`` statement at +A procedure that returns a value has an implicit `result` variable declared +that represents the return value. A `return` statement with no expression is +shorthand for `return result`. The `result` value is always returned +automatically at the end of a procedure if there is no `return` statement at the exit. .. code-block:: nim @@ -641,15 +643,15 @@ the exit. echo sumTillNegative(3, 4, 5) # echos 12 echo sumTillNegative(3, 4 , -1 , 6) # echos 7 -The ``result`` variable is already implicitly declared at the start of the +The `result` variable is already implicitly declared at the start of the function, so declaring it again with 'var result', for example, would shadow it with a normal variable of the same name. The result variable is also already initialized with the type's default value. Note that referential data types will -be ``nil`` at the start of the procedure, and thus may require manual +be `nil` at the start of the procedure, and thus may require manual initialization. -A procedure that does not have any ``return`` statement and does not use the -special ``result`` variable returns the value of its last expression. For example, +A procedure that does not have any `return` statement and does not use the +special `result` variable returns the value of its last expression. For example, this procedure .. code-block:: nim @@ -664,7 +666,7 @@ Parameters Parameters are immutable in the procedure body. By default, their value cannot be changed because this allows the compiler to implement parameter passing in the most efficient way. If a mutable variable is needed inside the procedure, it has -to be declared with ``var`` in the procedure body. Shadowing the parameter name +to be declared with `var` in the procedure body. Shadowing the parameter name is possible, and actually an idiom: .. code-block:: nim @@ -675,7 +677,7 @@ is possible, and actually an idiom: echo s[i] If the procedure needs to modify the argument for the -caller, a ``var`` parameter can be used: +caller, a `var` parameter can be used: .. code-block:: nim :test: "nim c $1" @@ -689,7 +691,7 @@ caller, a ``var`` parameter can be used: echo x echo y -In the example, ``res`` and ``remainder`` are `var parameters`. +In the example, `res` and `remainder` are `var parameters`. Var parameters can be modified by the procedure and the changes are visible to the caller. Note that the above example would better make use of a tuple as a return value instead of using var parameters. @@ -698,7 +700,7 @@ a tuple as a return value instead of using var parameters. Discard statement ----------------- To call a procedure that returns a value just for its side effects and ignoring -its return value, a ``discard`` statement **must** be used. Nim does not +its return value, a `discard` statement **must** be used. Nim does not allow silently throwing away a return value: .. code-block:: nim @@ -706,7 +708,7 @@ allow silently throwing away a return value: The return value can be ignored implicitly if the called proc/iterator has -been declared with the ``discardable`` pragma: +been declared with the `discardable` pragma: .. code-block:: nim :test: "nim c $1" @@ -732,7 +734,7 @@ that it is clear which argument belongs to which parameter: var w = createWindow(show = true, title = "My Application", x = 0, y = 0, height = 600, width = 800) -Now that we use named arguments to call ``createWindow`` the argument order +Now that we use named arguments to call `createWindow` the argument order does not matter anymore. Mixing named arguments with ordered arguments is also possible, but not very readable: @@ -745,7 +747,7 @@ The compiler checks that each parameter receives exactly one argument. Default values -------------- -To make the ``createWindow`` proc easier to use it should provide `default +To make the `createWindow` proc easier to use it should provide `default values`; these are values that are used as arguments if the caller does not specify them: @@ -757,11 +759,11 @@ specify them: var w = createWindow(title = "My Application", height = 600, width = 800) -Now the call to ``createWindow`` only needs to set the values that differ +Now the call to `createWindow` only needs to set the values that differ from the defaults. Note that type inference works for parameters with default values; there is -no need to write ``title: string = "unknown"``, for example. +no need to write `title: string = "unknown"`, for example. Overloaded procedures @@ -783,8 +785,8 @@ Nim provides the ability to overload procedures similar to C++: assert toString(13) == "positive" # calls the toString(x: int) proc assert toString(true) == "yep" # calls the toString(x: bool) proc -(Note that ``toString`` is usually the `$ `_ operator in -Nim.) The compiler chooses the most appropriate proc for the ``toString`` +(Note that `toString` is usually the `$ `_ operator in +Nim.) The compiler chooses the most appropriate proc for the `toString` calls. How this overloading resolution algorithm works exactly is not discussed here (it will be specified in the manual soon). However, it does not lead to nasty surprises and is based on a quite simple unification @@ -794,31 +796,31 @@ algorithm. Ambiguous calls are reported as errors. Operators --------- The Nim library makes heavy use of overloading - one reason for this is that -each operator like ``+`` is just an overloaded proc. The parser lets you -use operators in `infix notation` (``a + b``) or `prefix notation` (``+ a``). +each operator like `+` is just an overloaded proc. The parser lets you +use operators in `infix notation` (`a + b`) or `prefix notation` (`+ a`). An infix operator always receives two arguments, a prefix operator always one. (Postfix operators are not possible, because this would be ambiguous: does -``a @ @ b`` mean ``(a) @ (@b)`` or ``(a@) @ (b)``? It always means -``(a) @ (@b)``, because there are no postfix operators in Nim.) +`a @ @ b` mean `(a) @ (@b)` or `(a@) @ (b)`? It always means +`(a) @ (@b)`, because there are no postfix operators in Nim.) -Apart from a few built-in keyword operators such as ``and``, ``or``, ``not``, +Apart from a few built-in keyword operators such as `and`, `or`, `not`, operators always consist of these characters: ``+ - * \ / < > = @ $ ~ & % ! ? ^ . |`` User-defined operators are allowed. Nothing stops you from defining your own -``@!?+~`` operator, but doing so may reduce readability. +`@!?+~` operator, but doing so may reduce readability. The operator's precedence is determined by its first character. The details can be found in the manual. -To define a new operator enclose the operator in backticks "``": +To define a new operator enclose the operator in backticks "`": .. code-block:: nim proc `$` (x: myDataType): string = ... # now the $ operator also works with myDataType, overloading resolution # ensures that $ works for built-in types just like before -The "``" notation can also be used to call an operator just like any other +The "`" notation can also be used to call an operator just like any other procedure: .. code-block:: nim @@ -851,10 +853,10 @@ However, this cannot be done for mutually recursive procedures: else: n == 0 or odd(n-1) -Here ``odd`` depends on ``even`` and vice versa. Thus ``even`` needs to be +Here `odd` depends on `even` and vice versa. Thus `even` needs to be introduced to the compiler before it is completely defined. The syntax for -such a forward declaration is simple: just omit the ``=`` and the -procedure's body. The ``assert`` just adds border conditions, and will be +such a forward declaration is simple: just omit the `=` and the +procedure's body. The `assert` just adds border conditions, and will be covered later in `Modules`_ section. Later versions of the language will weaken the requirements for forward @@ -886,9 +888,9 @@ supports this loop? Lets try: inc(res) However, this does not work. The problem is that the procedure should not -only ``return``, but return and **continue** after an iteration has +only `return`, but return and **continue** after an iteration has finished. This *return and continue* is called a `yield` statement. Now -the only thing left to do is to replace the ``proc`` keyword by ``iterator`` +the only thing left to do is to replace the `proc` keyword by `iterator` and here it is - our first iterator: .. code-block:: nim @@ -903,19 +905,19 @@ Iterators look very similar to procedures, but there are several important differences: * Iterators can only be called from for loops. -* Iterators cannot contain a ``return`` statement (and procs cannot contain a - ``yield`` statement). -* Iterators have no implicit ``result`` variable. +* Iterators cannot contain a `return` statement (and procs cannot contain a + `yield` statement). +* Iterators have no implicit `result` variable. * Iterators do not support recursion. * Iterators cannot be forward declared, because the compiler must be able to inline an iterator. (This restriction will be gone in a future version of the compiler.) -However, you can also use a ``closure`` iterator to get a different set of +However, you can also use a `closure` iterator to get a different set of restrictions. See `first-class iterators `_ for details. Iterators can have the same name and parameters as a proc since essentially they have their own namespaces. Therefore it is common practice to wrap iterators in procs of the same name which accumulate the result of the -iterator and return it as a sequence, like ``split`` from the `strutils module +iterator and return it as a sequence, like `split` from the `strutils module `_. @@ -928,12 +930,12 @@ that are available for them in detail. Booleans -------- -Nim's boolean type is called ``bool`` and consists of the two -pre-defined values ``true`` and ``false``. Conditions in while, +Nim's boolean type is called `bool` and consists of the two +pre-defined values `true` and `false`. Conditions in while, if, elif, and when statements must be of type bool. -The operators ``not, and, or, xor, <, <=, >, >=, !=, ==`` are defined -for the bool type. The ``and`` and ``or`` operators perform short-circuit +The operators `not, and, or, xor, <, <=, >, >=, !=, ==` are defined +for the bool type. The `and` and `or` operators perform short-circuit evaluation. For example: .. code-block:: nim @@ -945,7 +947,7 @@ evaluation. For example: Characters ---------- -The `character type` is called ``char``. Its size is always one byte, so +The `character type` is called `char`. Its size is always one byte, so it cannot represent most UTF-8 characters, but it *can* represent one of the bytes that makes up a multi-byte UTF-8 character. The reason for this is efficiency: for the overwhelming majority of use-cases, @@ -953,57 +955,57 @@ the resulting programs will still handle UTF-8 properly as UTF-8 was especially designed for this. Character literals are enclosed in single quotes. -Chars can be compared with the ``==``, ``<``, ``<=``, ``>``, ``>=`` operators. -The ``$`` operator converts a ``char`` to a ``string``. Chars cannot be mixed -with integers; to get the ordinal value of a ``char`` use the ``ord`` proc. -Converting from an integer to a ``char`` is done with the ``chr`` proc. +Chars can be compared with the `==`, `<`, `<=`, `>`, `>=` operators. +The `$` operator converts a `char` to a `string`. Chars cannot be mixed +with integers; to get the ordinal value of a `char` use the `ord` proc. +Converting from an integer to a `char` is done with the `chr` proc. Strings ------- String variables are **mutable**, so appending to a string is possible, and quite efficient. Strings in Nim are both zero-terminated and have a -length field. A string's length can be retrieved with the builtin ``len`` +length field. A string's length can be retrieved with the builtin `len` procedure; the length never counts the terminating zero. Accessing the terminating zero is an error, it only exists so that a Nim string can be converted -to a ``cstring`` without doing a copy. +to a `cstring` without doing a copy. -The assignment operator for strings copies the string. You can use the ``&`` -operator to concatenate strings and ``add`` to append to a string. +The assignment operator for strings copies the string. You can use the `&` +operator to concatenate strings and `add` to append to a string. Strings are compared using their lexicographical order. All the comparison operators are supported. By convention, all strings are UTF-8 encoded, but this is not enforced. For example, when reading strings from binary files, they are merely -a sequence of bytes. The index operation ``s[i]`` means the i-th *char* of -``s``, not the i-th *unichar*. +a sequence of bytes. The index operation `s[i]` means the i-th *char* of +`s`, not the i-th *unichar*. -A string variable is initialized with the empty string ``""``. +A string variable is initialized with the empty string `""`. Integers -------- Nim has these integer types built-in: -``int int8 int16 int32 int64 uint uint8 uint16 uint32 uint64``. +`int int8 int16 int32 int64 uint uint8 uint16 uint32 uint64`. -The default integer type is ``int``. Integer literals can have a *type suffix* +The default integer type is `int`. Integer literals can have a *type suffix* to specify a non-default integer type: .. code-block:: nim :test: "nim c $1" let - x = 0 # x is of type ``int`` - y = 0'i8 # y is of type ``int8`` - z = 0'i64 # z is of type ``int64`` - u = 0'u # u is of type ``uint`` + x = 0 # x is of type `int` + y = 0'i8 # y is of type `int8` + z = 0'i64 # z is of type `int64` + u = 0'u # u is of type `uint` Most often integers are used for counting objects that reside in memory, so -``int`` has the same size as a pointer. +`int` has the same size as a pointer. -The common operators ``+ - * div mod < <= == != > >=`` are defined for -integers. The ``and or xor not`` operators are also defined for integers and -provide *bitwise* operations. Left bit shifting is done with the ``shl``, right -shifting with the ``shr`` operator. Bit shifting operators always treat their +The common operators `+ - * div mod < <= == != > >=` are defined for +integers. The `and or xor not` operators are also defined for integers and +provide *bitwise* operations. Left bit shifting is done with the `shl`, right +shifting with the `shr` operator. Bit shifting operators always treat their arguments as *unsigned*. For `arithmetic bit shifts`:idx: ordinary multiplication or division can be used. @@ -1018,10 +1020,10 @@ cannot be detected at compile time). Floats ------ -Nim has these floating-point types built-in: ``float float32 float64``. +Nim has these floating-point types built-in: `float float32 float64`. -The default float type is ``float``. In the current implementation, -``float`` is always 64-bits. +The default float type is `float`. In the current implementation, +`float` is always 64-bits. Float literals can have a *type suffix* to specify a non-default float type: @@ -1029,11 +1031,11 @@ type: .. code-block:: nim :test: "nim c $1" var - x = 0.0 # x is of type ``float`` - y = 0.0'f32 # y is of type ``float32`` - z = 0.0'f64 # z is of type ``float64`` + x = 0.0 # x is of type `float` + y = 0.0'f32 # y is of type `float32` + z = 0.0'f64 # z is of type `float64` -The common operators ``+ - * / < <= == != > >=`` are defined for +The common operators `+ - * / < <= == != > >=` are defined for floats and follow the IEEE-754 standard. Automatic type conversion in expressions with different kinds of floating-point types is performed: the smaller type is converted to the larger. Integer @@ -1061,13 +1063,13 @@ Internal type representation As mentioned earlier, the built-in `$ `_ (stringify) operator turns any basic type into a string, which you can then print to the console -using the ``echo`` proc. However, advanced types, and your own custom types, -won't work with the ``$`` operator until you define it for them. +using the `echo` proc. However, advanced types, and your own custom types, +won't work with the `$` operator until you define it for them. Sometimes you just want to debug the current value of a complex type without -having to write its ``$`` operator. You can use then the `repr +having to write its `$` operator. You can use then the `repr `_ proc which works with any type and even complex data graphs with cycles. The following example shows that even for basic types -there is a difference between the ``$`` and ``repr`` outputs: +there is a difference between the `$` and `repr` outputs: .. code-block:: nim :test: "nim c $1" @@ -1092,7 +1094,7 @@ there is a difference between the ``$`` and ``repr`` outputs: Advanced types ============== -In Nim new types can be defined within a ``type`` statement: +In Nim new types can be defined within a `type` statement: .. code-block:: nim :test: "nim c $1" @@ -1101,7 +1103,7 @@ In Nim new types can be defined within a ``type`` statement: biggestFloat = float64 # biggest float type that is available Enumeration and object types may only be defined within a -``type`` statement. +`type` statement. Enumerations @@ -1124,9 +1126,9 @@ at runtime by 0, the second by 1, and so on. For example: All the comparison operators can be used with enumeration types. An enumeration's symbol can be qualified to avoid ambiguities: -``Direction.south``. +`Direction.south`. -The ``$`` operator can convert any enumeration value to its name, and the ``ord`` +The `$` operator can convert any enumeration value to its name, and the `ord` proc can convert it to its underlying integer value. For better interfacing to other programming languages, the symbols of enum @@ -1136,7 +1138,7 @@ must be in ascending order. Ordinal types ------------- -Enumerations, integer types, ``char`` and ``bool`` (and +Enumerations, integer types, `char` and `bool` (and subranges) are called ordinal types. Ordinal types have quite a few special operations: @@ -1144,16 +1146,16 @@ a few special operations: ----------------- -------------------------------------------------------- Operation Comment ----------------- -------------------------------------------------------- -``ord(x)`` returns the integer value that is used to +`ord(x)` returns the integer value that is used to represent `x`'s value -``inc(x)`` increments `x` by one -``inc(x, n)`` increments `x` by `n`; `n` is an integer -``dec(x)`` decrements `x` by one -``dec(x, n)`` decrements `x` by `n`; `n` is an integer -``succ(x)`` returns the successor of `x` -``succ(x, n)`` returns the `n`'th successor of `x` -``pred(x)`` returns the predecessor of `x` -``pred(x, n)`` returns the `n`'th predecessor of `x` +`inc(x)` increments `x` by one +`inc(x, n)` increments `x` by `n`; `n` is an integer +`dec(x)` decrements `x` by one +`dec(x, n)` decrements `x` by `n`; `n` is an integer +`succ(x)` returns the successor of `x` +`succ(x, n)` returns the `n`'th successor of `x` +`pred(x)` returns the predecessor of `x` +`pred(x, n)` returns the `n`'th predecessor of `x` ----------------- -------------------------------------------------------- @@ -1174,17 +1176,17 @@ A subrange type is a range of values from an integer or enumeration type MySubrange = range[0..5] -``MySubrange`` is a subrange of ``int`` which can only hold the values 0 -to 5. Assigning any other value to a variable of type ``MySubrange`` is a +`MySubrange` is a subrange of `int` which can only hold the values 0 +to 5. Assigning any other value to a variable of type `MySubrange` is a compile-time or runtime error. Assignments from the base type to one of its subrange types (and vice versa) are allowed. -The ``system`` module defines the important `Natural `_ -type as ``range[0..high(int)]`` (`high `_ returns +The `system` module defines the important `Natural `_ +type as `range[0..high(int)]` (`high `_ returns the maximal value). Other programming languages may suggest the use of unsigned integers for natural numbers. This is often **unwise**: you don't want unsigned arithmetic (which wraps around) just because the numbers cannot be negative. -Nim's ``Natural`` type helps to avoid this common programming error. +Nim's `Natural` type helps to avoid this common programming error. Sets @@ -1197,7 +1199,7 @@ Arrays An array is a simple fixed-length container. Each element in an array has the same type. The array's index type can be any ordinal type. -Arrays can be constructed using ``[]``: +Arrays can be constructed using `[]`: .. code-block:: nim :test: "nim c $1" @@ -1210,10 +1212,10 @@ Arrays can be constructed using ``[]``: for i in low(x)..high(x): echo x[i] -The notation ``x[i]`` is used to access the i-th element of ``x``. +The notation `x[i]` is used to access the i-th element of `x`. Array access is always bounds checked (at compile-time or at runtime). These checks can be disabled via pragmas or invoking the compiler with the -``--bound_checks:off`` command line switch. +`--bound_checks:off` command line switch. Arrays are value types, like any other Nim type. The assignment operator copies the whole array contents. @@ -1263,9 +1265,9 @@ subdivided into height levels accessed through their integer index: #tower[north][east] = on #tower[0][1] = on -Note how the built-in ``len`` proc returns only the array's first dimension -length. Another way of defining the ``LightTower`` to better illustrate its -nested nature would be to omit the previous definition of the ``LevelSetting`` +Note how the built-in `len` proc returns only the array's first dimension +length. Another way of defining the `LightTower` to better illustrate its +nested nature would be to omit the previous definition of the `LevelSetting` type and instead write it embedded directly as the type of the first dimension: .. code-block:: nim @@ -1295,13 +1297,13 @@ Sequences are similar to arrays but of dynamic length which may change during runtime (like strings). Since sequences are resizable they are always allocated on the heap and garbage collected. -Sequences are always indexed with an ``int`` starting at position 0. The `len +Sequences are always indexed with an `int` starting at position 0. The `len `_, `low `_ and `high `_ operations are available for sequences too. -The notation ``x[i]`` can be used to access the i-th element of ``x``. +The notation `x[i]` can be used to access the i-th element of `x`. -Sequences can be constructed by the array constructor ``[]`` in conjunction -with the array to sequence operator ``@``. Another way to allocate space for +Sequences can be constructed by the array constructor `[]` in conjunction +with the array to sequence operator `@`. Another way to allocate space for a sequence is to call the built-in `newSeq `_ procedure. A sequence may be passed to an openarray parameter. @@ -1315,15 +1317,15 @@ Example: x: seq[int] # a reference to a sequence of integers x = @[1, 2, 3, 4, 5, 6] # the @ turns the array into a sequence allocated on the heap -Sequence variables are initialized with ``@[]``. +Sequence variables are initialized with `@[]`. -The ``for`` statement can be used with one or two variables when used with a +The `for` statement can be used with one or two variables when used with a sequence. When you use the one variable form, the variable will hold the value -provided by the sequence. The ``for`` statement is looping over the results +provided by the sequence. The `for` statement is looping over the results from the `items() `_ iterator from the `system `_ module. But if you use the two-variable form, the first variable will hold the index position and the second variable will hold the -value. Here the ``for`` statement is looping over the results from the +value. Here the `for` statement is looping over the results from the `pairs() `_ iterator from the `system `_ module. Examples: @@ -1348,7 +1350,7 @@ Open arrays Often fixed-size arrays turn out to be too inflexible; procedures should be able to deal with arrays of different sizes. The `openarray`:idx: type allows -this. Openarrays are always indexed with an ``int`` starting at position 0. +this. Openarrays are always indexed with an `int` starting at position 0. The `len `_, `low `_ and `high `_ operations are available for open arrays too. Any array with a compatible base type can be passed to an @@ -1377,7 +1379,7 @@ supported because this is seldom needed and cannot be done efficiently. Varargs ------- -A ``varargs`` parameter is like an openarray parameter. However, it is +A `varargs` parameter is like an openarray parameter. However, it is also a means to implement passing a variable number of arguments to a procedure. The compiler converts the list of arguments to an array automatically: @@ -1409,7 +1411,7 @@ type conversions in this context: myWriteln(stdout, [$123, $"abc", $4.0]) In this example `$ `_ is applied to any argument that is passed -to the parameter ``a``. Note that `$ `_ applied to strings is a +to the parameter `a`. Note that `$ `_ applied to strings is a nop. @@ -1441,7 +1443,7 @@ To understand some of the different ways of specifying the indices of strings, arrays, sequences, etc., it must be remembered that Nim uses zero-based indices. -So the string ``b`` is of length 19, and two different ways of specifying the +So the string `b` is of length 19, and two different ways of specifying the indices are .. code-block:: nim @@ -1451,22 +1453,22 @@ indices are 0 11 17 using indices ^19 ^8 ^2 using ^ syntax -where ``b[0 .. ^1]`` is equivalent to ``b[0 .. b.len-1]`` and ``b[0 ..< b.len]``, and it -can be seen that the ``^1`` provides a short-hand way of specifying the ``b.len-1``. See +where `b[0 .. ^1]` is equivalent to `b[0 .. b.len-1]` and `b[0 ..< b.len]`, and it +can be seen that the `^1` provides a short-hand way of specifying the `b.len-1`. See the `backwards index operator `_. In the above example, because the string ends in a period, to get the portion of the string that is "useless" and replace it with "useful". -``b[11 .. ^2]`` is the portion "useless", and ``b[11 .. ^2] = "useful"`` replaces the +`b[11 .. ^2]` is the portion "useless", and `b[11 .. ^2] = "useful"` replaces the "useless" portion with "useful", giving the result "Slices are useful." -Note 1: alternate ways of writing this are ``b[^8 .. ^2] = "useful"`` or -as ``b[11 .. b.len-2] = "useful"`` or as ``b[11 ..< b.len-1] = "useful"``. +Note 1: alternate ways of writing this are `b[^8 .. ^2] = "useful"` or +as `b[11 .. b.len-2] = "useful"` or as `b[11 ..< b.len-1] = "useful"`. -Note 2: As the ``^`` template returns a `distinct int `_ -of type ``BackwardsIndex``, we can have a ``lastIndex`` constant defined as ``const lastIndex = ^1``, -and later used as ``b[0 .. lastIndex]``. +Note 2: As the `^` template returns a `distinct int `_ +of type `BackwardsIndex`, we can have a `lastIndex` constant defined as `const lastIndex = ^1`, +and later used as `b[0 .. lastIndex]`. Objects ------- @@ -1476,7 +1478,7 @@ structure with a name is the object type. An object is a value type, which means that when an object is assigned to a new variable all its components are copied as well. -Each object type ``Foo`` has a constructor ``Foo(field: value, ...)`` +Each object type `Foo` has a constructor `Foo(field: value, ...)` where all of its fields can be initialized. Unspecified fields will get their default value. @@ -1510,7 +1512,7 @@ get their default value. Object fields that should be visible from outside the defining module have to -be marked with ``*``. +be marked with `*`. .. code-block:: nim :test: "nim c $1" @@ -1529,15 +1531,15 @@ Unlike object types though, tuple types are structurally typed, meaning different tuple-types are *equivalent* if they specify fields of the same type and of the same name in the same order. -The constructor ``()`` can be used to construct tuples. The order of the +The constructor `()` can be used to construct tuples. The order of the fields in the constructor must match the order in the tuple's definition. But unlike objects, a name for the tuple type may not be used here. -Like the object type the notation ``t.field`` is used to access a +Like the object type the notation `t.field` is used to access a tuple's field. Another notation that is not available for objects is -``t[i]`` to access the ``i``'th field. Here ``i`` must be a constant +`t[i]` to access the `i`'th field. Here `i` must be a constant integer. .. code-block:: nim @@ -1641,9 +1643,9 @@ untraced references are *unsafe*. However, for certain low-level operations Traced references are declared with the **ref** keyword; untraced references are declared with the **ptr** keyword. -The empty ``[]`` subscript notation can be used to *de-refer* a reference, -meaning to retrieve the item the reference points to. The ``.`` (access a -tuple/object field operator) and ``[]`` (array/string/sequence index operator) +The empty `[]` subscript notation can be used to *de-refer* a reference, +meaning to retrieve the item the reference points to. The `.` (access a +tuple/object field operator) and `[]` (array/string/sequence index operator) operators perform implicit dereferencing operations for reference types: .. code-block:: nim @@ -1659,18 +1661,18 @@ operators perform implicit dereferencing operations for reference types: n.data = 9 # no need to write n[].data; in fact n[].data is highly discouraged! -To allocate a new traced object, the built-in procedure ``new`` must be used. -To deal with untraced memory, the procedures ``alloc``, ``dealloc`` and -``realloc`` can be used. The `system `_ +To allocate a new traced object, the built-in procedure `new` must be used. +To deal with untraced memory, the procedures `alloc`, `dealloc` and +`realloc` can be used. The `system `_ module's documentation contains further details. -If a reference points to *nothing*, it has the value ``nil``. +If a reference points to *nothing*, it has the value `nil`. Procedural type --------------- A procedural type is a (somewhat abstract) pointer to a procedure. -``nil`` is an allowed value for a variable of a procedural type. +`nil` is an allowed value for a variable of a procedural type. Nim uses procedural types to achieve `functional`:idx: programming techniques. @@ -1708,7 +1710,7 @@ Nim supports splitting a program into pieces with a module concept. Each module is in its own file. Modules enable `information hiding`:idx: and `separate compilation`:idx:. A module may gain access to the symbols of another module by using the `import`:idx: statement. Only top-level symbols that are marked -with an asterisk (``*``) are exported: +with an asterisk (`*`) are exported: .. code-block:: nim # Module A @@ -1722,19 +1724,19 @@ with an asterisk (``*``) are exported: for i in 0..len(a)-1: result[i] = a[i] * b[i] when isMainModule: - # test the new ``*`` operator for sequences: + # test the new `*` operator for sequences: assert(@[1, 2, 3] * @[1, 2, 3] == @[1, 4, 9]) -The above module exports ``x`` and ``*``, but not ``y``. +The above module exports `x` and `*`, but not `y`. A module's top-level statements are executed at the start of the program. This can be used to initialize complex data structures for example. -Each module has a special magic constant ``isMainModule`` that is true if the +Each module has a special magic constant `isMainModule` that is true if the module is compiled as the main file. This is very useful to embed tests within the module as shown by the above example. -A symbol of a module *can* be *qualified* with the ``module.symbol`` syntax. And if +A symbol of a module *can* be *qualified* with the `module.symbol` syntax. And if a symbol is ambiguous, it *must* be qualified. A symbol is ambiguous if it is defined in two (or more) different modules and both modules are imported by a third one: @@ -1781,9 +1783,9 @@ rules apply: Excluding symbols ----------------- -The normal ``import`` statement will bring in all exported symbols. +The normal `import` statement will bring in all exported symbols. These can be limited by naming symbols that should be excluded using -the ``except`` qualifier. +the `except` qualifier. .. code-block:: nim import mymodule except y @@ -1792,14 +1794,14 @@ the ``except`` qualifier. From statement -------------- -We have already seen the simple ``import`` statement that just imports all +We have already seen the simple `import` statement that just imports all exported symbols. An alternative that only imports listed symbols is the -``from import`` statement: +`from import` statement: .. code-block:: nim from mymodule import x, y, z -The ``from`` statement can also force namespace qualification on +The `from` statement can also force namespace qualification on symbols, thereby making symbols available, but needing to be qualified in order to be used. @@ -1826,8 +1828,8 @@ define a shorter alias to use when qualifying symbols. Include statement ----------------- -The ``include`` statement does something fundamentally different than -importing a module: it merely includes the contents of a file. The ``include`` +The `include` statement does something fundamentally different than +importing a module: it merely includes the contents of a file. The `include` statement is useful to split up a large module into several files: .. code-block:: nim diff --git a/doc/tut2.rst b/doc/tut2.rst index e0d1bdb32..3aef6bb0f 100644 --- a/doc/tut2.rst +++ b/doc/tut2.rst @@ -1,3 +1,5 @@ +.. default-role:: code + ====================== Nim Tutorial (Part II) ====================== @@ -24,7 +26,7 @@ Pragmas Pragmas are Nim's method to give the compiler additional information/ commands without introducing a massive number of new keywords. Pragmas are -enclosed in the special ``{.`` and ``.}`` curly dot brackets. This tutorial +enclosed in the special `{.` and `.}` curly dot brackets. This tutorial does not cover pragmas. See the `manual `_ or `user guide `_ for a description of the available pragmas. @@ -45,11 +47,11 @@ Inheritance Inheritance in Nim is entirely optional. To enable inheritance with runtime type information the object needs to inherit from -``RootObj``. This can be done directly, or indirectly by -inheriting from an object that inherits from ``RootObj``. Usually -types with inheritance are also marked as ``ref`` types even though +`RootObj`. This can be done directly, or indirectly by +inheriting from an object that inherits from `RootObj`. Usually +types with inheritance are also marked as `ref` types even though this isn't strictly enforced. To check at runtime if an object is of a certain -type, the ``of`` operator can be used. +type, the `of` operator can be used. .. code-block:: nim :test: "nim c $1" @@ -69,16 +71,16 @@ type, the ``of`` operator can be used. student = Student(name: "Anton", age: 5, id: 2) echo student[] -Inheritance is done with the ``object of`` syntax. Multiple inheritance is -currently not supported. If an object type has no suitable ancestor, ``RootObj`` +Inheritance is done with the `object of` syntax. Multiple inheritance is +currently not supported. If an object type has no suitable ancestor, `RootObj` can be used as its ancestor, but this is only a convention. Objects that have -no ancestor are implicitly ``final``. You can use the ``inheritable`` pragma -to introduce new object roots apart from ``system.RootObj``. (This is used +no ancestor are implicitly `final`. You can use the `inheritable` pragma +to introduce new object roots apart from `system.RootObj`. (This is used in the GTK wrapper for instance.) Ref objects should be used whenever inheritance is used. It isn't strictly -necessary, but with non-ref objects assignments such as ``let person: Person = -Student(id: 123)`` will truncate subclass fields. +necessary, but with non-ref objects assignments such as `let person: Person = +Student(id: 123)` will truncate subclass fields. **Note**: Composition (*has-a* relation) is often preferable to inheritance (*is-a* relation) for simple code reuse. Since objects are value types in @@ -111,7 +113,7 @@ Example: Type conversions ---------------- Nim distinguishes between `type casts`:idx: and `type conversions`:idx:. -Casts are done with the ``cast`` operator and force the compiler to +Casts are done with the `cast` operator and force the compiler to interpret a bit pattern to be of another type. Type conversions are a much more polite way to convert a type into another: @@ -119,15 +121,15 @@ They preserve the abstract *value*, not necessarily the *bit-pattern*. If a type conversion is not possible, the compiler complains or an exception is raised. -The syntax for type conversions is ``destination_type(expression_to_convert)`` +The syntax for type conversions is `destination_type(expression_to_convert)` (like an ordinary call): .. code-block:: nim proc getID(x: Person): int = Student(x).id -The ``InvalidObjectConversionDefect`` exception is raised if ``x`` is not a -``Student``. +The `InvalidObjectConversionDefect` exception is raised if `x` is not a +`Student`. Object variants @@ -150,7 +152,7 @@ An example: nkSub, # a subtraction nkIf # an if statement Node = ref object - case kind: NodeKind # the ``kind`` field is the discriminator + case kind: NodeKind # the `kind` field is the discriminator of nkInt: intVal: int of nkFloat: floatVal: float of nkString: strVal: string @@ -173,9 +175,9 @@ Method call syntax ------------------ There is a syntactic sugar for calling routines: -The syntax ``obj.method(args)`` can be used instead of ``method(obj, args)``. +The syntax `obj.method(args)` can be used instead of `method(obj, args)`. If there are no remaining arguments, the parentheses can be omitted: -``obj.len`` (instead of ``len(obj)``). +`obj.len` (instead of `len(obj)`). This method call syntax is not restricted to objects, it can be used for any type: @@ -229,10 +231,10 @@ is needed: new s s.host = 34 # same as `host=`(s, 34) -(The example also shows ``inline`` procedures.) +(The example also shows `inline` procedures.) -The ``[]`` array access operator can be overloaded to provide +The `[]` array access operator can be overloaded to provide `array properties`:idx:\ : .. code-block:: nim @@ -258,14 +260,14 @@ The ``[]`` array access operator can be overloaded to provide else: assert(false) The example is silly, since a vector is better modelled by a tuple which -already provides ``v[]`` access. +already provides `v[]` access. Dynamic dispatch ---------------- Procedures always use static dispatch. For dynamic dispatch replace the -``proc`` keyword by ``method``: +`proc` keyword by `method`: .. code-block:: nim :test: "nim c $1" @@ -289,12 +291,12 @@ Procedures always use static dispatch. For dynamic dispatch replace the echo eval(newPlus(newPlus(newLit(1), newLit(2)), newLit(4))) -Note that in the example the constructors ``newLit`` and ``newPlus`` are procs -because it makes more sense for them to use static binding, but ``eval`` is a +Note that in the example the constructors `newLit` and `newPlus` are procs +because it makes more sense for them to use static binding, but `eval` is a method because it requires dynamic binding. **Note:** Starting from Nim 0.20, to use multi-methods one must explicitly pass -``--multimethods:on`` when compiling. +`--multimethods:on` when compiling. In a multi-method all parameters that have an object type are used for the dispatching: @@ -324,7 +326,7 @@ dispatching: As the example demonstrates, invocation of a multi-method cannot be ambiguous: Collide 2 is preferred over collide 1 because the resolution works from left to -right. Thus ``Unit, Thing`` is preferred over ``Thing, Unit``. +right. Thus `Unit, Thing` is preferred over `Thing, Unit`. **Performance note**: Nim does not produce a virtual method table, but generates dispatch trees. This avoids the expensive indirect branch for method @@ -338,19 +340,19 @@ Exceptions In Nim exceptions are objects. By convention, exception types are suffixed with 'Error'. The `system `_ module defines an exception hierarchy that you might want to stick to. Exceptions derive from -``system.Exception``, which provides the common interface. +`system.Exception`, which provides the common interface. Exceptions have to be allocated on the heap because their lifetime is unknown. The compiler will prevent you from raising an exception created on the stack. All raised exceptions should at least specify the reason for being raised in -the ``msg`` field. +the `msg` field. A convention is that exceptions should be raised in *exceptional* cases, they should not be used as an alternative method of control flow. Raise statement --------------- -Raising an exception is done with the ``raise`` statement: +Raising an exception is done with the `raise` statement: .. code-block:: nim :test: "nim c $1" @@ -360,9 +362,9 @@ Raising an exception is done with the ``raise`` statement: e.msg = "the request to the OS failed" raise e -If the ``raise`` keyword is not followed by an expression, the last exception +If the `raise` keyword is not followed by an expression, the last exception is *re-raised*. For the purpose of avoiding repeating this common code pattern, -the template ``newException`` in the ``system`` module can be used: +the template `newException` in the `system` module can be used: .. code-block:: nim raise newException(OSError, "the request to the OS failed") @@ -371,7 +373,7 @@ the template ``newException`` in the ``system`` module can be used: Try statement ------------- -The ``try`` statement handles exceptions: +The `try` statement handles exceptions: .. code-block:: nim :test: "nim c $1" @@ -399,23 +401,23 @@ The ``try`` statement handles exceptions: finally: close(f) -The statements after the ``try`` are executed unless an exception is -raised. Then the appropriate ``except`` part is executed. +The statements after the `try` are executed unless an exception is +raised. Then the appropriate `except` part is executed. -The empty ``except`` part is executed if there is an exception that is -not explicitly listed. It is similar to an ``else`` part in ``if`` +The empty `except` part is executed if there is an exception that is +not explicitly listed. It is similar to an `else` part in `if` statements. -If there is a ``finally`` part, it is always executed after the +If there is a `finally` part, it is always executed after the exception handlers. -The exception is *consumed* in an ``except`` part. If an exception is not +The exception is *consumed* in an `except` part. If an exception is not handled, it is propagated through the call stack. This means that often -the rest of the procedure - that is not within a ``finally`` clause - +the rest of the procedure - that is not within a `finally` clause - is not executed (if an exception occurs). If you need to *access* the actual exception object or message inside an -``except`` branch you can use the `getCurrentException() +`except` branch you can use the `getCurrentException() `_ and `getCurrentExceptionMsg() `_ procs from the `system `_ module. Example: @@ -433,10 +435,10 @@ module. Example: Annotating procs with raised exceptions --------------------------------------- -Through the use of the optional ``{.raises.}`` pragma you can specify that a +Through the use of the optional `{.raises.}` pragma you can specify that a proc is meant to raise a specific set of exceptions, or none at all. If the -``{.raises.}`` pragma is used, the compiler will verify that this is true. For -instance, if you specify that a proc raises ``IOError``, and at some point it +`{.raises.}` pragma is used, the compiler will verify that this is true. For +instance, if you specify that a proc raises `IOError`, and at some point it (or one of the procs it calls) starts raising a new exception the compiler will prevent that proc from compiling. Usage example: @@ -453,11 +455,11 @@ stopped validating the pragma and the raised exception not being caught, along with the file and line where the uncaught exception is being raised, which may help you locate the offending code which has changed. -If you want to add the ``{.raises.}`` pragma to existing code, the compiler can -also help you. You can add the ``{.effects.}`` pragma statement to your proc and +If you want to add the `{.raises.}` pragma to existing code, the compiler can +also help you. You can add the `{.effects.}` pragma statement to your proc and the compiler will output all inferred effects up to that point (exception tracking is part of Nim's effect system). Another more roundabout way to -find out the list of exceptions raised by a proc is to use the Nim ``doc`` +find out the list of exceptions raised by a proc is to use the Nim `doc` command which generates documentation for a whole module and decorates all procs with the list of raised exceptions. You can read more about Nim's `effect system and related pragmas in the manual `_. @@ -468,14 +470,14 @@ Generics Generics are Nim's means to parametrize procs, iterators or types with `type parameters`:idx:. Generic parameters are written within square -brackets, for example ``Foo[T]``. They are most useful for efficient type safe +brackets, for example `Foo[T]`. They are most useful for efficient type safe containers: .. code-block:: nim :test: "nim c $1" type BinaryTree*[T] = ref object # BinaryTree is a generic type with - # generic param ``T`` + # generic param `T` le, ri: BinaryTree[T] # left and right subtrees; may be nil data: T # the data stored in a node @@ -491,8 +493,8 @@ containers: else: var it = root while it != nil: - # compare the data items; uses the generic ``cmp`` proc - # that works for any type that has a ``==`` and ``<`` operator + # compare the data items; uses the generic `cmp` proc + # that works for any type that has a `==` and `<` operator var c = cmp(it.data, n.data) if c < 0: if it.le == nil: @@ -522,19 +524,19 @@ containers: n = n.le # and follow the left pointer var - root: BinaryTree[string] # instantiate a BinaryTree with ``string`` - add(root, newNode("hello")) # instantiates ``newNode`` and ``add`` - add(root, "world") # instantiates the second ``add`` proc + root: BinaryTree[string] # instantiate a BinaryTree with `string` + add(root, newNode("hello")) # instantiates `newNode` and `add` + add(root, "world") # instantiates the second `add` proc for str in preorder(root): stdout.writeLine(str) The example shows a generic binary tree. Depending on context, the brackets are used either to introduce type parameters or to instantiate a generic proc, iterator or type. As the example shows, generics work with overloading: the -best match of ``add`` is used. The built-in ``add`` procedure for sequences -is not hidden and is used in the ``preorder`` iterator. +best match of `add` is used. The built-in `add` procedure for sequences +is not hidden and is used in the `preorder` iterator. -There is a special ``[:T]`` syntax when using generics with the method call syntax: +There is a special `[:T]` syntax when using generics with the method call syntax: .. code-block:: nim :test: "nim c $1" @@ -567,14 +569,14 @@ Example: assert(5 != 6) # the compiler rewrites that to: assert(not (5 == 6)) -The ``!=``, ``>``, ``>=``, ``in``, ``notin``, ``isnot`` operators are in fact -templates: this has the benefit that if you overload the ``==`` operator, -the ``!=`` operator is available automatically and does the right thing. (Except +The `!=`, `>`, `>=`, `in`, `notin`, `isnot` operators are in fact +templates: this has the benefit that if you overload the `==` operator, +the `!=` operator is available automatically and does the right thing. (Except for IEEE floating point numbers - NaN breaks basic boolean logic.) -``a > b`` is transformed into ``b < a``. -``a in b`` is transformed into ``contains(b, a)``. -``notin`` and ``isnot`` have the obvious meanings. +`a > b` is transformed into `b < a`. +`a in b` is transformed into `contains(b, a)`. +`notin` and `isnot` have the obvious meanings. Templates are especially useful for lazy evaluation purposes. Consider a simple proc for logging: @@ -591,11 +593,11 @@ simple proc for logging: x = 4 log("x has the value: " & $x) -This code has a shortcoming: if ``debug`` is set to false someday, the quite -expensive ``$`` and ``&`` operations are still performed! (The argument +This code has a shortcoming: if `debug` is set to false someday, the quite +expensive `$` and `&` operations are still performed! (The argument evaluation for procedures is *eager*). -Turning the ``log`` proc into a template solves this problem: +Turning the `log` proc into a template solves this problem: .. code-block:: nim :test: "nim c $1" @@ -609,15 +611,15 @@ Turning the ``log`` proc into a template solves this problem: x = 4 log("x has the value: " & $x) -The parameters' types can be ordinary types or the meta types ``untyped``, -``typed``, or ``type``. ``type`` suggests that only a type symbol may be given -as an argument, and ``untyped`` means symbol lookups and type resolution is not +The parameters' types can be ordinary types or the meta types `untyped`, +`typed`, or `type`. `type` suggests that only a type symbol may be given +as an argument, and `untyped` means symbol lookups and type resolution is not performed before the expression is passed to the template. If the template has no explicit return type, -``void`` is used for consistency with procs and methods. +`void` is used for consistency with procs and methods. -To pass a block of statements to a template, use ``untyped`` for the last parameter: +To pass a block of statements to a template, use `untyped` for the last parameter: .. code-block:: nim :test: "nim c $1" @@ -638,10 +640,10 @@ To pass a block of statements to a template, use ``untyped`` for the last parame txt.writeLine("line 1") txt.writeLine("line 2") -In the example the two ``writeLine`` statements are bound to the ``body`` -parameter. The ``withFile`` template contains boilerplate code and helps to +In the example the two `writeLine` statements are bound to the `body` +parameter. The `withFile` template contains boilerplate code and helps to avoid a common bug: to forget to close the file. Note how the -``let fn = filename`` statement ensures that ``filename`` is evaluated only +`let fn = filename` statement ensures that `filename` is evaluated only once. Example: Lifting Procs @@ -653,7 +655,7 @@ Example: Lifting Procs template liftScalarProc(fname) = ## Lift a proc taking one scalar parameter and returning a - ## scalar value (eg ``proc sssss[T](x: T): float``), + ## scalar value (eg `proc sssss[T](x: T): float`), ## to provide templated procs that can handle a single ## parameter of seq[T] or nested seq[seq[]] or the same type ## @@ -675,15 +677,15 @@ Compilation to JavaScript Nim code can be compiled to JavaScript. However in order to write JavaScript-compatible code you should remember the following: -- ``addr`` and ``ptr`` have slightly different semantic meaning in JavaScript. +- `addr` and `ptr` have slightly different semantic meaning in JavaScript. It is recommended to avoid those if you're not sure how they are translated to JavaScript. -- ``cast[T](x)`` in JavaScript is translated to ``(x)``, except for casting +- `cast[T](x)` in JavaScript is translated to `(x)`, except for casting between signed/unsigned ints, in which case it behaves as static cast in C language. -- ``cstring`` in JavaScript means JavaScript string. It is a good practice to - use ``cstring`` only when it is semantically appropriate. E.g. don't use - ``cstring`` as a binary data buffer. +- `cstring` in JavaScript means JavaScript string. It is a good practice to + use `cstring` only when it is semantically appropriate. E.g. don't use + `cstring` as a binary data buffer. Part 3 diff --git a/doc/tut3.rst b/doc/tut3.rst index b0a3d8232..358a9b45e 100644 --- a/doc/tut3.rst +++ b/doc/tut3.rst @@ -1,3 +1,5 @@ +.. default-role:: code + ======================= Nim Tutorial (Part III) ======================= diff --git a/lib/system_overview.rst b/lib/system_overview.rst index 5e4b171fe..d6cbe1a35 100644 --- a/lib/system_overview.rst +++ b/lib/system_overview.rst @@ -1,3 +1,5 @@ +.. default-role:: code + The System module imports several separate modules, and their documentation is in separate files: @@ -36,7 +38,7 @@ Proc Usage * `strutils module `_ for common string functions * `strformat module `_ for string interpolation and formatting * `unicode module `_ for Unicode UTF-8 handling -* `strscans `_ for ``scanf`` and ``scanp`` macros, which offer +* `strscans `_ for `scanf` and `scanp` macros, which offer easier substring extraction than regular expressions * `strtabs module `_ for efficient hash tables (dictionaries, in some programming languages) mapping from strings to strings From 9e1a2e9b63fff3d3445a04600b431a42c800a0c8 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Fri, 19 Mar 2021 03:15:37 -0300 Subject: [PATCH 0444/2472] improve jsconsole (#17414) * Add jsconsole.dir * Update lib/js/jsconsole.nim Co-authored-by: Timothee Cour Co-authored-by: Timothee Cour --- changelog.md | 2 ++ lib/js/jsconsole.nim | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/changelog.md b/changelog.md index 4080f789c..219a66ae7 100644 --- a/changelog.md +++ b/changelog.md @@ -237,6 +237,8 @@ - In `std/os`, `getHomeDir`, `expandTilde`, `getTempDir`, `getConfigDir` now do not include trailing `DirSep`, unless `-d:nimLegacyHomeDir` is specified (for a transition period). +- Added `jsconsole.dir`, `jsconsole.dirxml`, `jsconsole.timeStamp`. + ## Language changes diff --git a/lib/js/jsconsole.nim b/lib/js/jsconsole.nim index b8a361af1..bf43adddd 100644 --- a/lib/js/jsconsole.nim +++ b/lib/js/jsconsole.nim @@ -110,5 +110,15 @@ since (1, 5): {.line: loc.}: {.emit: ["console.assert(", assertion, ", ", msg, ");"].} + func dir*(console: Console; obj: auto) {.importcpp.} + ## https://developer.mozilla.org/en-US/docs/Web/API/Console/dir + + func dirxml*(console: Console; obj: auto) {.importcpp.} + ## https://developer.mozilla.org/en-US/docs/Web/API/Console/dirxml + + func timeStamp*(console: Console; label: cstring) {.importcpp.} + ## https://developer.mozilla.org/en-US/docs/Web/API/Console/timeStamp + ## ..warning:: non-standard + var console* {.importc, nodecl.}: Console From 452366982d8ef7d8ebed5cd95c034a14ee2f56c0 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 18 Mar 2021 23:17:09 -0700 Subject: [PATCH 0445/2472] fix #16901: sidebar groups now works with all routines, not just proc,func (#17416) * fix #16901: sidebar groups now works with all routines, not just proc,func * fix tests --- compiler/docgen.nim | 2 +- .../expected/subdir/subdir_b/utils.html | 19 +++- nimdoc/testproject/expected/testproject.html | 104 +++++++++++++----- 3 files changed, 91 insertions(+), 34 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 37831c72c..57d5e7b01 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -1245,7 +1245,7 @@ proc genOutFile(d: PDoc, groupedToc = false): Rope = renderTocEntries(d[], j, 1, tmp) var toc = tmp.rope for i in TSymKind: - var shouldSort = i in {skProc, skFunc} and groupedToc + var shouldSort = i in routineKinds and groupedToc genSection(d, i, shouldSort) toc.add(d.toc[i]) if toc != nil: diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html index 53088dfda..cb80f2873 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html @@ -121,12 +121,21 @@ window.addEventListener('DOMContentLoaded', main);
  • Templates
    • diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html index dbdcca79e..48b3094e7 100644 --- a/nimdoc/testproject/expected/testproject.html +++ b/nimdoc/testproject/expected/testproject.html @@ -309,56 +309,104 @@ window.addEventListener('DOMContentLoaded', main);
  • Methods
  • Iterators
  • Macros
  • Templates
    • From 83e002a318dee2370b266446d5ec26ad97a65f57 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 19 Mar 2021 14:48:31 +0800 Subject: [PATCH 0446/2472] follow up #17391 add testcase (#17404) * Revert "make system random work in VM" * fix #17380 * attempt to fix bug * fix * better * fix * a bit * fix the leaks * revert * fix * better * follow up #17391 * fix * Update tchannels.nim * Update tests/stdlib/tchannels.nim * Update tchannels.nim --- tests/stdlib/tchannels.nim | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 tests/stdlib/tchannels.nim diff --git a/tests/stdlib/tchannels.nim b/tests/stdlib/tchannels.nim new file mode 100644 index 000000000..33108c50c --- /dev/null +++ b/tests/stdlib/tchannels.nim @@ -0,0 +1,33 @@ +discard """ + timeout: 5.0 # but typically < 1s + disabled: "freebsd" + matrix: "--gc:arc --threads:on; --gc:arc --threads:on -d:danger" +""" + +when true: + # bug #17380: this was either blocking (without -d:danger) or crashing with SIGSEGV (with -d:danger) + import std/[channels, isolation] + const + N1 = 10 + N2 = 100 + var + sender: array[N1, Thread[void]] + receiver: array[5, Thread[void]] + + var chan = newChannel[seq[string]](N1 * N2) # large enough to not block + proc sendHandler() = + chan.send(isolate(@["Hello, Nim"])) + proc recvHandler() = + template fn = + let x = chan.recv() + fn() + + template benchmark() = + for t in mitems(sender): + t.createThread(sendHandler) + joinThreads(sender) + for t in mitems(receiver): + t.createThread(recvHandler) + joinThreads(receiver) + for i in 0.. Date: Fri, 19 Mar 2021 21:44:13 +0800 Subject: [PATCH 0447/2472] fix a typo (#17417) * Revert "make system random work in VM" * fix #17380 * attempt to fix bug * fix * better * fix * a bit * fix the leaks * revert * fix * better * follow up #17391 * fix * Update tchannels.nim * Update tests/stdlib/tchannels.nim * Update tchannels.nim * fix a typo --- lib/std/channels.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/channels.nim b/lib/std/channels.nim index 69f6a6ffb..3bbc8a6b6 100644 --- a/lib/std/channels.nim +++ b/lib/std/channels.nim @@ -473,7 +473,7 @@ func send*[T](c: Channel[T], src: sink Isolated[T]) {.inline.} = discard sendMpmc(c.d, data.addr, sizeof(T), false) wasMoved(data) -template send*[T](c: var Channel[T]; src: T) = +template send*[T](c: Channel[T]; src: T) = ## Helper templates for `send`. send(c, isolate(src)) From 6c1c8f51b38c9bc570a70ec8d2836b823d3584cc Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 19 Mar 2021 16:53:38 +0100 Subject: [PATCH 0448/2472] IC: green tests (#17311) * IC: renamed to_packed_ast module to ic module * IC: don't store the --forceBuild flag, makes it easier to test * IC: enable hello world test * Codegen: refactorings for IC; changed the name mangling algorithm * fixed the HCR regressions * life is too short for HCR * tconvexhull is now allowed to use deepCopy * IC exposed a stdlib bug, required a refactoring * codegen: code cleanups * IC: even if a module is outdated, its dependencies might come from disk * IC: progress * IC: better name mangling, module IDs are not stable * IC: another refactoring helping with --ic:on --gc:arc * disable arraymancer on Windows for the time being * disable arraymancer altogether * IC: make basic test work with 'nim cpp' * IC: progress on --ic:on --gc:arc * wip; name mangling for type info --- compiler/ast.nim | 2 +- .../{ccgmerge.nim => ccgmerge_unused.nim} | 0 compiler/ccgtypes.nim | 60 ++++++++-- compiler/cgen.nim | 106 ++++++------------ compiler/cgendata.nim | 4 +- compiler/ic/cbackend.nim | 4 +- compiler/ic/dce.nim | 6 +- compiler/ic/{to_packed_ast.nim => ic.nim} | 21 ++-- compiler/ic/replayer.nim | 2 +- compiler/injectdestructors.nim | 8 +- compiler/liftdestructors.nim | 75 +++++++------ compiler/lineinfos.nim | 9 +- compiler/main.nim | 2 +- compiler/modulegraphs.nim | 29 ++++- compiler/modules.nim | 7 +- compiler/msgs.nim | 26 +++++ compiler/pragmas.nim | 2 +- compiler/semdata.nim | 2 +- compiler/semexprs.nim | 2 +- compiler/semparallel.nim | 14 +-- compiler/spawn.nim | 2 +- lib/pure/bitops.nim | 2 +- lib/system.nim | 1 + lib/system/countbits_impl.nim | 25 +++++ lib/system/excpt.nim | 2 +- lib/system/sets.nim | 15 --- testament/important_packages.nim | 3 +- tests/dll/nimhcr_integration.nim | 2 +- tests/ic/thallo.nim | 1 - tests/parallel/tconvexhull.nim | 2 - 30 files changed, 249 insertions(+), 187 deletions(-) rename compiler/{ccgmerge.nim => ccgmerge_unused.nim} (100%) rename compiler/ic/{to_packed_ast.nim => ic.nim} (98%) create mode 100644 lib/system/countbits_impl.nim diff --git a/compiler/ast.nim b/compiler/ast.nim index d82420519..50a2fb58c 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -1416,7 +1416,7 @@ proc newType*(kind: TTypeKind, id: ItemId; owner: PSym): PType = lockLevel: UnspecifiedLockLevel, uniqueId: id) when false: - if result.id == 76426: + if result.itemId.module == 55 and result.itemId.item == 2: echo "KNID ", kind writeStackTrace() diff --git a/compiler/ccgmerge.nim b/compiler/ccgmerge_unused.nim similarity index 100% rename from compiler/ccgmerge.nim rename to compiler/ccgmerge_unused.nim diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 9c751b1ca..73ee9cb8a 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -40,7 +40,13 @@ proc mangleName(m: BModule; s: PSym): Rope = result = s.loc.r if result == nil: result = s.name.s.mangle.rope - result.add(idOrSig(s, m.module.name.s.mangle, m.sigConflicts)) + result.add "_" + result.add m.g.graph.ifaces[s.itemId.module].uniqueName + result.add "_" + result.add rope s.itemId.item + if m.hcrOn: + result.add "_" + result.add(idOrSig(s, m.module.name.s.mangle, m.sigConflicts)) s.loc.r = result writeMangledName(m.ndi, s, m.config) @@ -1273,12 +1279,12 @@ proc genDeepCopyProc(m: BModule; s: PSym; result: Rope) = m.s[cfsTypeInit3].addf("$1.deepcopy =(void* (N_RAW_NIMCALL*)(void*))$2;$n", [result, s.loc.r]) -proc declareNimType(m: BModule, name: string; str: Rope, ownerModule: PSym) = +proc declareNimType(m: BModule, name: string; str: Rope, module: int) = let nr = rope(name) if m.hcrOn: m.s[cfsData].addf("static $2* $1;$n", [str, nr]) m.s[cfsTypeInit1].addf("\t$1 = ($3*)hcrGetGlobal($2, \"$1\");$n", - [str, getModuleDllPath(m, ownerModule), nr]) + [str, getModuleDllPath(m, module), nr]) else: m.s[cfsData].addf("extern $2 $1;$n", [str, nr]) @@ -1351,6 +1357,9 @@ proc genTypeInfoV2Impl(m: BModule, t, origType: PType, name: Rope; info: TLineIn if t.kind == tyObject and t.len > 0 and t[0] != nil and optEnableDeepCopy in m.config.globalOptions: discard genTypeInfoV1(m, t, info) +proc moduleOpenForCodegen(m: BModule; module: int32): bool {.inline.} = + result = module < m.g.modules.len and m.g.modules[module] != nil + proc genTypeInfoV2(m: BModule, t: PType; info: TLineInfo): Rope = let origType = t # distinct types can have their own destructors @@ -1374,11 +1383,11 @@ proc genTypeInfoV2(m: BModule, t: PType; info: TLineInfo): Rope = result = "NTIv2$1_" % [rope($sig)] m.typeInfoMarkerV2[sig] = result - let owner = t.skipTypes(typedescPtrs).owner.getModule - if owner != m.module: + let owner = t.skipTypes(typedescPtrs).itemId.module + if owner != m.module.position and moduleOpenForCodegen(m, owner): # make sure the type info is created in the owner module - assert m.g.modules[owner.position] != nil - discard genTypeInfoV2(m.g.modules[owner.position], origType, info) + assert m.g.modules[owner] != nil + discard genTypeInfoV2(m.g.modules[owner], origType, info) # reference the type info as extern here discard cgsym(m, "TNimTypeV2") declareNimType(m, "TNimTypeV2", result, owner) @@ -1397,6 +1406,33 @@ proc openArrayToTuple(m: BModule; t: PType): PType = result.add p result.add getSysType(m.g.graph, t.owner.info, tyInt) +proc typeToC(t: PType): string = + ## Just for more readable names, the result doesn't have + ## to be unique. + let s = typeToString(t) + result = newStringOfCap(s.len) + for i in 0.. 0: moduleInitRequired = true if addHcrGuards: prc.add("\tif (nim_hcr_do_init_) {\n\n") - prc.add(genSectionStart(section, m.config)) prc.add(m.thing.s(section)) - prc.add(genSectionEnd(section, m.config)) if addHcrGuards: prc.add("\n\t} // nim_hcr_do_init_\n") if m.preInitProc.s(cpsInit).len > 0 or m.preInitProc.s(cpsStmts).len > 0: @@ -1740,28 +1739,21 @@ proc genModule(m: BModule, cfile: Cfile): Rope = var moduleIsEmpty = true result = getFileHeader(m.config, cfile) - result.add(genMergeInfo(m)) generateThreadLocalStorage(m) generateHeaders(m) - result.add(genSectionStart(cfsHeaders, m.config)) result.add(m.s[cfsHeaders]) if m.config.cppCustomNamespace.len > 0: result.add openNamespaceNim(m.config.cppCustomNamespace) - result.add(genSectionEnd(cfsHeaders, m.config)) - result.add(genSectionStart(cfsFrameDefines, m.config)) if m.s[cfsFrameDefines].len > 0: result.add(m.s[cfsFrameDefines]) else: result.add("#define nimfr_(x, y)\n#define nimln_(x, y)\n") - result.add(genSectionEnd(cfsFrameDefines, m.config)) for i in cfsForwardTypes..cfsProcs: if m.s[i].len > 0: moduleIsEmpty = false - result.add(genSectionStart(i, m.config)) result.add(m.s[i]) - result.add(genSectionEnd(i, m.config)) if m.s[cfsInitProc].len > 0: moduleIsEmpty = false @@ -1851,9 +1843,7 @@ proc writeHeader(m: BModule) = generateThreadLocalStorage(m) for i in cfsHeaders..cfsProcs: - result.add(genSectionStart(i, m.config)) result.add(m.s[i]) - result.add(genSectionEnd(i, m.config)) if m.config.cppCustomNamespace.len > 0 and i == cfsHeaders: result.add openNamespaceNim(m.config.cppCustomNamespace) result.add(m.s[cfsInitProc]) @@ -1952,46 +1942,26 @@ proc shouldRecompile(m: BModule; code: Rope, cfile: Cfile): bool = proc writeModule(m: BModule, pending: bool) = template onExit() = close(m.ndi, m.config) let cfile = getCFile(m) - if true or optForceFullMake in m.config.globalOptions: - if moduleHasChanged(m.g.graph, m.module): - genInitCode(m) - finishTypeDescriptions(m) - if sfMainModule in m.module.flags: - # generate main file: - genMainProc(m) - m.s[cfsProcHeaders].add(m.g.mainModProcs) - generateThreadVarsSize(m) - - var cf = Cfile(nimname: m.module.name.s, cname: cfile, - obj: completeCfilePath(m.config, toObjFile(m.config, cfile)), flags: {}) - var code = genModule(m, cf) - if code != nil or m.config.symbolFiles != disabledSf: - when hasTinyCBackend: - if m.config.cmd == cmdTcc: - tccgen.compileCCode($code, m.config) - onExit() - return - - if not shouldRecompile(m, code, cf): cf.flags = {CfileFlag.Cached} - addFileToCompile(m.config, cf) - elif pending and mergeRequired(m) and sfMainModule notin m.module.flags: - let cf = Cfile(nimname: m.module.name.s, cname: cfile, - obj: completeCfilePath(m.config, toObjFile(m.config, cfile)), flags: {}) - mergeFiles(cfile, m) + if moduleHasChanged(m.g.graph, m.module): genInitCode(m) finishTypeDescriptions(m) - var code = genModule(m, cf) - if code != nil: - if not writeRope(code, cfile): - rawMessage(m.config, errCannotOpenFile, cfile.string) - addFileToCompile(m.config, cf) - else: - # Consider: first compilation compiles ``system.nim`` and produces - # ``system.c`` but then compilation fails due to an error. This means - # that ``system.o`` is missing, so we need to call the C compiler for it: - var cf = Cfile(nimname: m.module.name.s, cname: cfile, - obj: completeCfilePath(m.config, toObjFile(m.config, cfile)), flags: {}) - if fileExists(cf.obj): cf.flags = {CfileFlag.Cached} + if sfMainModule in m.module.flags: + # generate main file: + genMainProc(m) + m.s[cfsProcHeaders].add(m.g.mainModProcs) + generateThreadVarsSize(m) + + var cf = Cfile(nimname: m.module.name.s, cname: cfile, + obj: completeCfilePath(m.config, toObjFile(m.config, cfile)), flags: {}) + var code = genModule(m, cf) + if code != nil or m.config.symbolFiles != disabledSf: + when hasTinyCBackend: + if m.config.cmd == cmdTcc: + tccgen.compileCCode($code, m.config) + onExit() + return + + if not shouldRecompile(m, code, cf): cf.flags = {CfileFlag.Cached} addFileToCompile(m.config, cf) onExit() @@ -1999,21 +1969,10 @@ proc updateCachedModule(m: BModule) = let cfile = getCFile(m) var cf = Cfile(nimname: m.module.name.s, cname: cfile, obj: completeCfilePath(m.config, toObjFile(m.config, cfile)), flags: {}) - - if mergeRequired(m) and sfMainModule notin m.module.flags: - mergeFiles(cfile, m) - genInitCode(m) - finishTypeDescriptions(m) - var code = genModule(m, cf) - if code != nil: - if not writeRope(code, cfile): - rawMessage(m.config, errCannotOpenFile, cfile.string) - addFileToCompile(m.config, cf) - else: - if sfMainModule notin m.module.flags: - genMainProc(m) - cf.flags = {CfileFlag.Cached} - addFileToCompile(m.config, cf) + if sfMainModule notin m.module.flags: + genMainProc(m) + cf.flags = {CfileFlag.Cached} + addFileToCompile(m.config, cf) proc finalCodegenActions*(graph: ModuleGraph; m: BModule; n: PNode) = ## Also called from IC. @@ -2080,8 +2039,7 @@ proc genForwardedProcs(g: BModuleList) = while g.forwardedProcs.len > 0: let prc = g.forwardedProcs.pop() - ms = getModule(prc) - m = g.modules[ms.position] + m = g.modules[prc.itemId.module] if sfForward in prc.flags: internalError(m.config, prc.info, "still forwarded: " & prc.name.s) diff --git a/compiler/cgendata.nim b/compiler/cgendata.nim index 6a128466a..3678adacf 100644 --- a/compiler/cgendata.nim +++ b/compiler/cgendata.nim @@ -102,7 +102,7 @@ type TTypeSeq* = seq[PType] TypeCache* = Table[SigHash, Rope] - TypeCacheWithOwner* = Table[SigHash, tuple[str: Rope, owner: PSym]] + TypeCacheWithOwner* = Table[SigHash, tuple[str: Rope, owner: int32]] CodegenFlag* = enum preventStackTrace, # true if stack traces need to be prevented @@ -202,7 +202,7 @@ proc newProc*(prc: PSym, module: BModule): BProc = result.sigConflicts = initCountTable[string]() proc newModuleList*(g: ModuleGraph): BModuleList = - BModuleList(typeInfoMarker: initTable[SigHash, tuple[str: Rope, owner: PSym]](), + BModuleList(typeInfoMarker: initTable[SigHash, tuple[str: Rope, owner: int32]](), config: g.config, graph: g, nimtvDeclared: initIntSet()) iterator cgenModules*(g: BModuleList): BModule = diff --git a/compiler/ic/cbackend.nim b/compiler/ic/cbackend.nim index 52a4a3339..88b2a9477 100644 --- a/compiler/ic/cbackend.nim +++ b/compiler/ic/cbackend.nim @@ -23,7 +23,7 @@ import std/[packedsets, algorithm] import ".."/[ast, options, lineinfos, modulegraphs, cgendata, cgen, pathutils, extccomp, msgs] -import packed_ast, to_packed_ast, dce, rodfiles +import packed_ast, ic, dce, rodfiles proc unpackTree(g: ModuleGraph; thisModule: int; tree: PackedTree; n: NodePos): PNode = @@ -83,7 +83,7 @@ proc aliveSymsChanged(config: ConfigRef; position: int; alive: AliveSyms): bool proc generateCode*(g: ModuleGraph) = ## The single entry point, generate C(++) code for the entire ## Nim program aka `ModuleGraph`. - initStrTable(g.compilerprocs) + resetForBackend(g) var alive = computeAliveSyms(g.packed, g.config) for i in 0..high(g.packed): diff --git a/compiler/ic/dce.nim b/compiler/ic/dce.nim index c7d66465d..0918fc379 100644 --- a/compiler/ic/dce.nim +++ b/compiler/ic/dce.nim @@ -12,7 +12,7 @@ import std / [intsets, tables] import ".." / [ast, options, lineinfos, types] -import packed_ast, to_packed_ast, bitabs +import packed_ast, ic, bitabs type AliveSyms* = seq[IntSet] @@ -111,7 +111,7 @@ proc aliveCode(c: var AliveContext; g: PackedModuleGraph; tree: PackedTree; n: N let otherModule = toFileIndexCached(c.decoder, g, c.thisModule, m).int followLater(c, g, otherModule, item) of nkMacroDef, nkTemplateDef, nkTypeSection, nkTypeOfExpr, - nkCommentStmt, nkIteratorDef, nkIncludeStmt, + nkCommentStmt, nkIncludeStmt, nkImportStmt, nkImportExceptStmt, nkExportStmt, nkExportExceptStmt, nkFromStmt, nkStaticStmt: discard @@ -121,7 +121,7 @@ proc aliveCode(c: var AliveContext; g: PackedModuleGraph; tree: PackedTree; n: N aliveCode(c, g, tree, son) of nkChckRangeF, nkChckRange64, nkChckRange: rangeCheckAnalysis(c, g, tree, n) - of nkProcDef, nkConverterDef, nkMethodDef, nkLambda, nkDo, nkFuncDef: + of nkProcDef, nkConverterDef, nkMethodDef, nkFuncDef, nkIteratorDef: if n.firstSon.kind == nkSym and isNotGeneric(n): let item = n.firstSon.operand if isExportedToC(c, g, item): diff --git a/compiler/ic/to_packed_ast.nim b/compiler/ic/ic.nim similarity index 98% rename from compiler/ic/to_packed_ast.nim rename to compiler/ic/ic.nim index 44902143d..99a68e0f0 100644 --- a/compiler/ic/to_packed_ast.nim +++ b/compiler/ic/ic.nim @@ -95,6 +95,7 @@ proc rememberStartupConfig*(dest: var PackedConfig, config: ConfigRef) = template rem(x) = dest.x = config.x primConfigFields rem + dest.globalOptions.excl optForceFullMake proc hashFileCached(conf: ConfigRef; fileIdx: FileIndex): string = result = msgs.getHash(conf, fileIdx) @@ -486,8 +487,14 @@ proc storeInstantiation*(c: var PackedEncoder; m: var PackedModule; s: PSym; i: concreteTypes: t) toPackedGeneratedProcDef(i.sym, c, m) -proc loadError(err: RodFileError; filename: AbsoluteFile) = - echo "Error: ", $err, " loading file: ", filename.string +proc loadError(err: RodFileError; filename: AbsoluteFile; config: ConfigRef;) = + case err + of cannotOpen: + rawMessage(config, warnCannotOpenFile, filename.string) + of includeFileChanged: + rawMessage(config, warnFileChanged, filename.string) + else: + echo "Error: ", $err, " loading file: ", filename.string proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef; ignoreConfig = false): RodFileError = @@ -718,7 +725,7 @@ proc loadProcHeader(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: result = newNodeIT(k, translateLineInfo(c, g, thisModule, n.info), loadType(c, g, thisModule, n.typ)) result.flags = n.flags - assert k in {nkProcDef, nkMethodDef, nkIteratorDef, nkFuncDef, nkConverterDef} + assert k in {nkProcDef, nkMethodDef, nkIteratorDef, nkFuncDef, nkConverterDef, nkLambda} var i = 0 for n0 in sonsReadonly(tree, n): if i != bodyPos: @@ -932,10 +939,10 @@ proc needsRecompile(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache else: g[m] = LoadedModule(status: outdated, module: g[m].module) else: - loadError(err, rod) + loadError(err, rod, conf) g[m].status = outdated result = true - when false: loadError(err, rod) + when false: loadError(err, rod, conf) of loading, loaded: # For loading: Assume no recompile is required. result = false @@ -951,8 +958,8 @@ proc moduleFromRodFile*(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentC result = g[int fileIdx].module assert result != nil assert result.position == int(fileIdx) - for m in cachedModules: - loadToReplayNodes(g, conf, cache, m, g[int m]) + for m in cachedModules: + loadToReplayNodes(g, conf, cache, m, g[int m]) template setupDecoder() {.dirty.} = var decoder = PackedDecoder( diff --git a/compiler/ic/replayer.nim b/compiler/ic/replayer.nim index 05c473090..61aa0e697 100644 --- a/compiler/ic/replayer.nim +++ b/compiler/ic/replayer.nim @@ -16,7 +16,7 @@ import ".." / [ast, modulegraphs, trees, extccomp, btrees, import tables -import packed_ast, to_packed_ast, bitabs +import packed_ast, ic, bitabs proc replayStateChanges*(module: PSym; g: ModuleGraph) = let list = module.ast diff --git a/compiler/injectdestructors.nim b/compiler/injectdestructors.nim index d20ed8e26..b65391252 100644 --- a/compiler/injectdestructors.nim +++ b/compiler/injectdestructors.nim @@ -350,7 +350,7 @@ proc genMarkCyclic(c: var Con; result, dest: PNode) = if t.kind == tyRef: result.add callCodegenProc(c.graph, "nimMarkCyclic", dest.info, dest) else: - let xenv = genBuiltin(c.graph, mAccessEnv, "accessEnv", dest) + let xenv = genBuiltin(c.graph, c.idgen, mAccessEnv, "accessEnv", dest) xenv.typ = getSysType(c.graph, dest.info, tyPointer) result.add callCodegenProc(c.graph, "nimMarkCyclic", dest.info, xenv) @@ -395,21 +395,21 @@ It is best to factor out piece of object that needs custom destructor into separ cond.add le cond.add tmp let notExpr = newNodeIT(nkPrefix, n.info, getSysType(c.graph, unknownLineInfo, tyBool)) - notExpr.add newSymNode(createMagic(c.graph, "not", mNot)) + notExpr.add newSymNode(createMagic(c.graph, c.idgen, "not", mNot)) notExpr.add cond result.add newTree(nkIfStmt, newTree(nkElifBranch, notExpr, c.genOp(branchDestructor, le))) result.add newTree(nkFastAsgn, le, tmp) proc genWasMoved(c: var Con, n: PNode): PNode = result = newNodeI(nkCall, n.info) - result.add(newSymNode(createMagic(c.graph, "wasMoved", mWasMoved))) + result.add(newSymNode(createMagic(c.graph, c.idgen, "wasMoved", mWasMoved))) result.add copyTree(n) #mWasMoved does not take the address #if n.kind != nkSym: # message(c.graph.config, n.info, warnUser, "wasMoved(" & $n & ")") proc genDefaultCall(t: PType; c: Con; info: TLineInfo): PNode = result = newNodeI(nkCall, info) - result.add(newSymNode(createMagic(c.graph, "default", mDefault))) + result.add(newSymNode(createMagic(c.graph, c.idgen, "default", mDefault))) result.typ = t proc destructiveMoveVar(n: PNode; c: var Con; s: var Scope): PNode = diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim index 889c65cc0..980e77e4b 100644 --- a/compiler/liftdestructors.nim +++ b/compiler/liftdestructors.nim @@ -70,16 +70,19 @@ proc newAsgnStmt(le, ri: PNode): PNode = result[0] = le result[1] = ri -proc genBuiltin*(g: ModuleGraph; magic: TMagic; name: string; i: PNode): PNode = +proc genBuiltin*(g: ModuleGraph; idgen: IdGenerator; magic: TMagic; name: string; i: PNode): PNode = result = newNodeI(nkCall, i.info) - result.add createMagic(g, name, magic).newSymNode + result.add createMagic(g, idgen, name, magic).newSymNode result.add i +proc genBuiltin(c: var TLiftCtx; magic: TMagic; name: string; i: PNode): PNode = + result = genBuiltin(c.g, c.idgen, magic, name, i) + proc defaultOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = if c.kind in {attachedAsgn, attachedDeepCopy, attachedSink}: body.add newAsgnStmt(x, y) elif c.kind == attachedDestructor and c.addMemReset: - let call = genBuiltin(c.g, mDefault, "default", x) + let call = genBuiltin(c, mDefault, "default", x) call.typ = t body.add newAsgnStmt(x, call) @@ -93,7 +96,7 @@ proc genAddr(c: var TLiftCtx; x: PNode): PNode = proc genWhileLoop(c: var TLiftCtx; i, dest: PNode): PNode = result = newNodeI(nkWhileStmt, c.info, 2) - let cmp = genBuiltin(c.g, mLtI, "<", i) + let cmp = genBuiltin(c, mLtI, "<", i) cmp.add genLen(c.g, dest) cmp.typ = getSysType(c.g, c.info, tyBool) result[0] = cmp @@ -116,10 +119,10 @@ proc genContainerOf(c: var TLiftCtx; objType: PType, field, x: PSym): PNode = dotExpr.add newNodeIT(nkType, c.info, objType) dotExpr.add newSymNode(field) - let offsetOf = genBuiltin(c.g, mOffsetOf, "offsetof", dotExpr) + let offsetOf = genBuiltin(c, mOffsetOf, "offsetof", dotExpr) offsetOf.typ = intType - let minusExpr = genBuiltin(c.g, mSubI, "-", castExpr1) + let minusExpr = genBuiltin(c, mSubI, "-", castExpr1) minusExpr.typ = intType minusExpr.add offsetOf @@ -135,7 +138,7 @@ proc destructorCall(c: var TLiftCtx; op: PSym; x: PNode): PNode = if sfNeverRaises notin op.flags: c.canRaise = true if c.addMemReset: - result = newTree(nkStmtList, destroy, genBuiltin(c.g, mWasMoved, "wasMoved", x)) + result = newTree(nkStmtList, destroy, genBuiltin(c, mWasMoved, "wasMoved", x)) else: result = destroy @@ -237,7 +240,7 @@ proc fillBodyObjT(c: var TLiftCtx; t: PType, body, x, y: PNode) = #body.add newAsgnStmt(blob, x) var wasMovedCall = newNodeI(nkCall, c.info) - wasMovedCall.add(newSymNode(createMagic(c.g, "wasMoved", mWasMoved))) + wasMovedCall.add(newSymNode(createMagic(c.g, c.idgen, "wasMoved", mWasMoved))) wasMovedCall.add x # mWasMoved does not take the address body.add wasMovedCall @@ -443,25 +446,25 @@ proc declareTempOf(c: var TLiftCtx; body: PNode; value: PNode): PNode = body.add v proc addIncStmt(c: var TLiftCtx; body, i: PNode) = - let incCall = genBuiltin(c.g, mInc, "inc", i) + let incCall = genBuiltin(c, mInc, "inc", i) incCall.add lowerings.newIntLit(c.g, c.info, 1) body.add incCall -proc newSeqCall(g: ModuleGraph; x, y: PNode): PNode = +proc newSeqCall(c: var TLiftCtx; x, y: PNode): PNode = # don't call genAddr(c, x) here: - result = genBuiltin(g, mNewSeq, "newSeq", x) - let lenCall = genBuiltin(g, mLengthSeq, "len", y) - lenCall.typ = getSysType(g, x.info, tyInt) + result = genBuiltin(c, mNewSeq, "newSeq", x) + let lenCall = genBuiltin(c, mLengthSeq, "len", y) + lenCall.typ = getSysType(c.g, x.info, tyInt) result.add lenCall -proc setLenStrCall(g: ModuleGraph; x, y: PNode): PNode = - let lenCall = genBuiltin(g, mLengthStr, "len", y) - lenCall.typ = getSysType(g, x.info, tyInt) - result = genBuiltin(g, mSetLengthStr, "setLen", x) # genAddr(g, x)) +proc setLenStrCall(c: var TLiftCtx; x, y: PNode): PNode = + let lenCall = genBuiltin(c, mLengthStr, "len", y) + lenCall.typ = getSysType(c.g, x.info, tyInt) + result = genBuiltin(c, mSetLengthStr, "setLen", x) # genAddr(g, x)) result.add lenCall proc setLenSeqCall(c: var TLiftCtx; t: PType; x, y: PNode): PNode = - let lenCall = genBuiltin(c.g, mLengthSeq, "len", y) + let lenCall = genBuiltin(c, mLengthSeq, "len", y) lenCall.typ = getSysType(c.g, x.info, tyInt) var op = getSysMagic(c.g, x.info, "setLen", mSetLengthSeq) op = instantiateGeneric(c, op, t, t) @@ -487,7 +490,7 @@ proc fillSeqOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = body.add setLenSeqCall(c, t, x, y) forallElements(c, t, body, x, y) of attachedSink: - let moveCall = genBuiltin(c.g, mMove, "move", x) + let moveCall = genBuiltin(c, mMove, "move", x) moveCall.add y doAssert t.destructor != nil moveCall.add destructorCall(c, t.destructor, x) @@ -495,13 +498,13 @@ proc fillSeqOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = of attachedDestructor: # destroy all elements: forallElements(c, t, body, x, y) - body.add genBuiltin(c.g, mDestroy, "destroy", x) + body.add genBuiltin(c, mDestroy, "destroy", x) of attachedTrace: # follow all elements: forallElements(c, t, body, x, y) of attachedDispose: forallElements(c, t, body, x, y) - body.add genBuiltin(c.g, mDestroy, "destroy", x) + body.add genBuiltin(c, mDestroy, "destroy", x) proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = createTypeBoundOps(c.g, c.c, t, body.info, c.idgen) @@ -521,7 +524,7 @@ proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = body.add newHookCall(c, t.assignment, x, y) of attachedSink: # we always inline the move for better performance: - let moveCall = genBuiltin(c.g, mMove, "move", x) + let moveCall = genBuiltin(c, mMove, "move", x) moveCall.add y doAssert t.destructor != nil moveCall.add destructorCall(c, t.destructor, x) @@ -549,13 +552,13 @@ proc fillStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = of attachedAsgn, attachedDeepCopy: body.add callCodegenProc(c.g, "nimAsgnStrV2", c.info, genAddr(c, x), y) of attachedSink: - let moveCall = genBuiltin(c.g, mMove, "move", x) + let moveCall = genBuiltin(c, mMove, "move", x) moveCall.add y doAssert t.destructor != nil moveCall.add destructorCall(c, t.destructor, x) body.add moveCall of attachedDestructor, attachedDispose: - body.add genBuiltin(c.g, mDestroy, "destroy", x) + body.add genBuiltin(c, mDestroy, "destroy", x) of attachedTrace: discard "strings are atomic and have no inner elements that are to trace" @@ -599,7 +602,7 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = if isFinal(elemType): addDestructorCall(c, elemType, actions, genDeref(tmp, nkDerefExpr)) - var alignOf = genBuiltin(c.g, mAlignOf, "alignof", newNodeIT(nkType, c.info, elemType)) + var alignOf = genBuiltin(c, mAlignOf, "alignof", newNodeIT(nkType, c.info, elemType)) alignOf.typ = getSysType(c.g, c.info, tyInt) actions.add callCodegenProc(c.g, "nimRawDispose", c.info, tmp, alignOf) else: @@ -609,7 +612,7 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = var cond: PNode if isCyclic: if isFinal(elemType): - let typInfo = genBuiltin(c.g, mGetTypeInfoV2, "getTypeInfoV2", newNodeIT(nkType, x.info, elemType)) + let typInfo = genBuiltin(c, mGetTypeInfoV2, "getTypeInfoV2", newNodeIT(nkType, x.info, elemType)) typInfo.typ = getSysType(c.g, c.info, tyPointer) cond = callCodegenProc(c.g, "nimDecRefIsLastCyclicStatic", c.info, tmp, typInfo) else: @@ -641,7 +644,7 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = of attachedDeepCopy: assert(false, "cannot happen") of attachedTrace: if isFinal(elemType): - let typInfo = genBuiltin(c.g, mGetTypeInfoV2, "getTypeInfoV2", newNodeIT(nkType, x.info, elemType)) + let typInfo = genBuiltin(c, mGetTypeInfoV2, "getTypeInfoV2", newNodeIT(nkType, x.info, elemType)) typInfo.typ = getSysType(c.g, c.info, tyPointer) body.add callCodegenProc(c.g, "nimTraceRef", c.info, genAddrOf(x, c.idgen), typInfo, y) else: @@ -659,7 +662,7 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = proc atomicClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = ## Closures are really like refs except they always use a virtual destructor ## and we need to do the refcounting only on the ref field which we call 'xenv': - let xenv = genBuiltin(c.g, mAccessEnv, "accessEnv", x) + let xenv = genBuiltin(c, mAccessEnv, "accessEnv", x) xenv.typ = getSysType(c.g, c.info, tyPointer) let isCyclic = c.g.config.selectedGC == gcOrc @@ -687,7 +690,7 @@ proc atomicClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = body.add genIf(c, cond, actions) body.add newAsgnStmt(x, y) of attachedAsgn: - let yenv = genBuiltin(c.g, mAccessEnv, "accessEnv", y) + let yenv = genBuiltin(c, mAccessEnv, "accessEnv", y) yenv.typ = getSysType(c.g, c.info, tyPointer) if isCyclic: body.add genIf(c, yenv, callCodegenProc(c.g, "nimIncRefCyclic", c.info, yenv, getCycleParam(c))) @@ -741,11 +744,11 @@ proc ownedRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = let elemType = t.lastSon #fillBody(c, elemType, actions, genDeref(x), genDeref(y)) - #var disposeCall = genBuiltin(c.g, mDispose, "dispose", x) + #var disposeCall = genBuiltin(c, mDispose, "dispose", x) if isFinal(elemType): addDestructorCall(c, elemType, actions, genDeref(x, nkDerefExpr)) - var alignOf = genBuiltin(c.g, mAlignOf, "alignof", newNodeIT(nkType, c.info, elemType)) + var alignOf = genBuiltin(c, mAlignOf, "alignof", newNodeIT(nkType, c.info, elemType)) alignOf.typ = getSysType(c.g, c.info, tyInt) actions.add callCodegenProc(c.g, "nimRawDispose", c.info, x, alignOf) else: @@ -767,12 +770,12 @@ proc closureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = # have to go through some indirection; we delegate this to the codegen: let call = newNodeI(nkCall, c.info, 2) call.typ = t - call[0] = newSymNode(createMagic(c.g, "deepCopy", mDeepCopy)) + call[0] = newSymNode(createMagic(c.g, c.idgen, "deepCopy", mDeepCopy)) call[1] = y body.add newAsgnStmt(x, call) elif (optOwnedRefs in c.g.config.globalOptions and optRefCheck in c.g.config.options) or c.g.config.selectedGC in {gcArc, gcOrc}: - let xx = genBuiltin(c.g, mAccessEnv, "accessEnv", x) + let xx = genBuiltin(c, mAccessEnv, "accessEnv", x) xx.typ = getSysType(c.g, c.info, tyPointer) case c.kind of attachedSink: @@ -781,7 +784,7 @@ proc closureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = body.add genIf(c, xx, callCodegenProc(c.g, "nimDecWeakRef", c.info, xx)) body.add newAsgnStmt(x, y) of attachedAsgn: - let yy = genBuiltin(c.g, mAccessEnv, "accessEnv", y) + let yy = genBuiltin(c, mAccessEnv, "accessEnv", y) yy.typ = getSysType(c.g, c.info, tyPointer) body.add genIf(c, yy, callCodegenProc(c.g, "nimIncRef", c.info, yy)) body.add genIf(c, xx, callCodegenProc(c.g, "nimDecWeakRef", c.info, xx)) @@ -796,7 +799,7 @@ proc closureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = of attachedTrace, attachedDispose: discard proc ownedClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = - let xx = genBuiltin(c.g, mAccessEnv, "accessEnv", x) + let xx = genBuiltin(c, mAccessEnv, "accessEnv", x) xx.typ = getSysType(c.g, c.info, tyPointer) var actions = newNodeI(nkStmtList, c.info) #discard addDestructorCall(c, elemType, newNodeI(nkStmtList, c.info), genDeref(xx)) @@ -859,7 +862,7 @@ proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode) = discard considerUserDefinedOp(c, t, body, x, y) elif tfHasAsgn in t.flags: if c.kind in {attachedAsgn, attachedSink, attachedDeepCopy}: - body.add newSeqCall(c.g, x, y) + body.add newSeqCall(c, x, y) forallElements(c, t, body, x, y) else: defaultOp(c, t, body, x, y) diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index 96a382453..d8f82aea0 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -59,7 +59,10 @@ type warnLockLevel = "LockLevel", warnResultShadowed = "ResultShadowed", warnInconsistentSpacing = "Spacing", warnCaseTransition = "CaseTransition", warnCycleCreated = "CycleCreated", warnObservableStores = "ObservableStores", - warnUser = "User", warnStrictNotNil = "StrictNotNil", + warnStrictNotNil = "StrictNotNil", + warnCannotOpen = "CannotOpen", + warnFileChanged = "FileChanged", + warnUser = "User", hintSuccess = "Success", hintSuccessX = "SuccessX", hintCC = "CC", hintLineTooLong = "LineTooLong", hintXDeclaredButNotUsed = "XDeclaredButNotUsed", @@ -133,8 +136,10 @@ const warnCaseTransition: "Potential object case transition, instantiate new object instead", warnCycleCreated: "$1", warnObservableStores: "observable stores to '$1'", - warnUser: "$1", warnStrictNotNil: "$1", + warnCannotOpen: "cannot open: $1", + warnFileChanged: "file changed: $1", + warnUser: "$1", hintSuccess: "operation successful: $#", # keep in sync with `testament.isSuccess` hintSuccessX: "${loc} lines; ${sec}s; $mem; $build build; proj: $project; out: $output", diff --git a/compiler/main.nim b/compiler/main.nim index 895923606..b61cdcadb 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -22,7 +22,7 @@ import modulegraphs, tables, lineinfos, pathutils, vmprofiler import ic / cbackend -from ic / to_packed_ast import rodViewer +from ic / ic import rodViewer when not defined(leanCompiler): import jsgen, docgen, docgen2 diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index b8a8b3e2c..de3773ca5 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -12,9 +12,9 @@ ## or stored in a rod-file. import ast, astalgo, intsets, tables, options, lineinfos, hashes, idents, - btrees, md5 + btrees, md5, ropes, msgs -import ic / [packed_ast, to_packed_ast] +import ic / [packed_ast, ic] type SigHash* = distinct MD5Digest @@ -30,6 +30,7 @@ type patterns*: seq[LazySym] pureEnums*: seq[LazySym] interf: TStrTable + uniqueName*: Rope Operators* = object opNot*, opContains*, opLe*, opLt*, opAnd*, opOr*, opIsNil*, opEq*: PSym @@ -117,6 +118,15 @@ type close: TPassClose, isFrontend: bool] +proc resetForBackend*(g: ModuleGraph) = + initStrTable(g.compilerprocs) + g.typeInstCache.clear() + g.procInstCache.clear() + for a in mitems(g.attachedOps): + a.clear() + g.methodsPerType.clear() + g.enumToStringProcs.clear() + const cb64 = [ "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", @@ -169,7 +179,7 @@ proc initEncoder*(g: ModuleGraph; module: PSym) = let id = module.position if id >= g.encoders.len: setLen g.encoders, id+1 - to_packed_ast.initEncoder(g.encoders[id], + ic.initEncoder(g.encoders[id], g.packed[id].fromDisk, module, g.config, g.startupPackedConfig) type @@ -359,11 +369,14 @@ else: proc stopCompile*(g: ModuleGraph): bool {.inline.} = result = g.doStopCompile != nil and g.doStopCompile() -proc createMagic*(g: ModuleGraph; name: string, m: TMagic): PSym = - result = newSym(skProc, getIdent(g.cache, name), nextSymId(g.idgen), nil, unknownLineInfo, {}) +proc createMagic*(g: ModuleGraph; idgen: IdGenerator; name: string, m: TMagic): PSym = + result = newSym(skProc, getIdent(g.cache, name), nextSymId(idgen), nil, unknownLineInfo, {}) result.magic = m result.flags = {sfNeverRaises} +proc createMagic(g: ModuleGraph; name: string, m: TMagic): PSym = + result = createMagic(g, g.idgen, name, m) + proc registerModule*(g: ModuleGraph; m: PSym) = assert m != nil assert m.kind == skModule @@ -374,9 +387,13 @@ proc registerModule*(g: ModuleGraph; m: PSym) = if m.position >= g.packed.len: setLen(g.packed, m.position + 1) - g.ifaces[m.position] = Iface(module: m, converters: @[], patterns: @[]) + g.ifaces[m.position] = Iface(module: m, converters: @[], patterns: @[], + uniqueName: rope(uniqueModuleName(g.config, FileIndex(m.position)))) initStrTable(g.ifaces[m.position].interf) +proc registerModuleById*(g: ModuleGraph; m: FileIndex) = + registerModule(g, g.packed[int m].module) + proc initOperators(g: ModuleGraph): Operators = # These are safe for IC. result.opLe = createMagic(g, "<=", mLeI) diff --git a/compiler/modules.nim b/compiler/modules.nim index 7503d4da2..7d7a2b6f7 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -108,9 +108,10 @@ proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): P if sfSystemModule in flags: graph.systemModule = result partialInitModule(result, graph, fileIdx, filename) - for m in cachedModules: - replayStateChanges(graph.packed[m.int].module, graph) - replayGenericCacheInformation(graph, m.int) + for m in cachedModules: + registerModuleById(graph, m) + replayStateChanges(graph.packed[m.int].module, graph) + replayGenericCacheInformation(graph, m.int) elif graph.isDirty(result): result.flags.excl sfDirty # reset module fields: diff --git a/compiler/msgs.nim b/compiler/msgs.nim index b384dad24..bbe40507f 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -115,6 +115,7 @@ proc fileInfoIdx*(conf: ConfigRef; filename: AbsoluteFile; isKnownFile: var bool else: isKnownFile = false result = conf.m.fileInfos.len.FileIndex + #echo "ID ", result.int, " ", canon2 conf.m.fileInfos.add(newFileInfo(canon, if pseudoPath: RelativeFile filename else: relativeTo(canon, conf.projectPath))) conf.m.filenameToIndexTbl[canon2] = result @@ -630,3 +631,28 @@ template listMsg(title, r) = proc listWarnings*(conf: ConfigRef) = listMsg("Warnings:", warnMin..warnMax) proc listHints*(conf: ConfigRef) = listMsg("Hints:", hintMin..hintMax) + +proc uniqueModuleName*(conf: ConfigRef; fid: FileIndex): string = + ## The unique module name is guaranteed to only contain {'A'..'Z', 'a'..'z', '0'..'9', '_'} + ## so that it is useful as a C identifier snippet. + let path = AbsoluteFile toFullPath(conf, fid) + let rel = + if path.string.startsWith(conf.libpath.string): + relativeTo(path, conf.libpath).string + else: + relativeTo(path, conf.projectPath).string + let trunc = if rel.endsWith(".nim"): rel.len - len(".nim") else: rel.len + result = newStringOfCap(trunc) + for i in 0.. 0: result = shallowCopy(n) for i in 0.. 0: diff --git a/compiler/spawn.nim b/compiler/spawn.nim index 61bcc424b..54ed51dbc 100644 --- a/compiler/spawn.nim +++ b/compiler/spawn.nim @@ -245,7 +245,7 @@ proc setupArgsForParallelism(g: ModuleGraph; n: PNode; objType: PType; # important special case: we always create a zero-copy slice: let slice = newNodeI(nkCall, n.info, 4) slice.typ = n.typ - slice[0] = newSymNode(createMagic(g, "slice", mSlice)) + slice[0] = newSymNode(createMagic(g, idgen, "slice", mSlice)) slice[0].typ = getSysType(g, n.info, tyInt) # fake type var fieldB = newSym(skField, tmpName, nextSymId idgen, objType.owner, n.info, g.config.options) fieldB.typ = getSysType(g, n.info, tyInt) diff --git a/lib/pure/bitops.nim b/lib/pure/bitops.nim index b75b0cf9a..57e3c7989 100644 --- a/lib/pure/bitops.nim +++ b/lib/pure/bitops.nim @@ -439,7 +439,7 @@ func fastlog2Nim(x: uint64): int {.inline.} = # sets.nim cannot import bitops, but bitops can use include # system/sets to eliminate code duplication. sets.nim defines # countBits32 and countBits64. -include system/sets +import system/countbits_impl template countSetBitsNim(n: uint32): int = countBits32(n) template countSetBitsNim(n: uint64): int = countBits64(n) diff --git a/lib/system.nim b/lib/system.nim index f0c3da517..574043125 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2339,6 +2339,7 @@ when notJSnotNims: when hostOS != "standalone" and hostOS != "any": include "system/dyncalls" + import system/countbits_impl include "system/sets" when defined(gogc): diff --git a/lib/system/countbits_impl.nim b/lib/system/countbits_impl.nim new file mode 100644 index 000000000..6c85612e2 --- /dev/null +++ b/lib/system/countbits_impl.nim @@ -0,0 +1,25 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2012 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Contains the used algorithms for counting bits. + +proc countBits32*(n: uint32): int {.compilerproc.} = + # generic formula is from: https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel + var v = uint32(n) + v = v - ((v shr 1'u32) and 0x55555555'u32) + v = (v and 0x33333333'u32) + ((v shr 2'u32) and 0x33333333'u32) + result = (((v + (v shr 4'u32) and 0xF0F0F0F'u32) * 0x1010101'u32) shr 24'u32).int + +proc countBits64*(n: uint64): int {.compilerproc, inline.} = + # generic formula is from: https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel + var v = uint64(n) + v = v - ((v shr 1'u64) and 0x5555555555555555'u64) + v = (v and 0x3333333333333333'u64) + ((v shr 2'u64) and 0x3333333333333333'u64) + v = (v + (v shr 4'u64) and 0x0F0F0F0F0F0F0F0F'u64) + result = ((v * 0x0101010101010101'u64) shr 56'u64).int diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index 8f7e2cbc7..06fa45097 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -567,7 +567,7 @@ when defined(cpp) and appType != "lib" and not gotoBasedExceptions and type StdException {.importcpp: "std::exception", header: "".} = object - proc what(ex: StdException): cstring {.importcpp: "((char *)#.what())".} + proc what(ex: StdException): cstring {.importcpp: "((char *)#.what())", nodecl.} proc setTerminate(handler: proc() {.noconv.}) {.importc: "std::set_terminate", header: "".} diff --git a/lib/system/sets.nim b/lib/system/sets.nim index 42c448848..04e10ba04 100644 --- a/lib/system/sets.nim +++ b/lib/system/sets.nim @@ -14,21 +14,6 @@ type # bitops can't be imported here, therefore the code duplication. -proc countBits32(n: uint32): int {.compilerproc.} = - # generic formula is from: https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel - var v = uint32(n) - v = v - ((v shr 1'u32) and 0x55555555'u32) - v = (v and 0x33333333'u32) + ((v shr 2'u32) and 0x33333333'u32) - result = (((v + (v shr 4'u32) and 0xF0F0F0F'u32) * 0x1010101'u32) shr 24'u32).int - -proc countBits64(n: uint64): int {.compilerproc, inline.} = - # generic formula is from: https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel - var v = uint64(n) - v = v - ((v shr 1'u64) and 0x5555555555555555'u64) - v = (v and 0x3333333333333333'u64) + ((v shr 2'u64) and 0x3333333333333333'u64) - v = (v + (v shr 4'u64) and 0x0F0F0F0F0F0F0F0F'u64) - result = ((v * 0x0101010101010101'u64) shr 56'u64).int - proc cardSet(s: NimSet, len: int): int {.compilerproc, inline.} = var i = 0 result = 0 diff --git a/testament/important_packages.nim b/testament/important_packages.nim index a3b6db57d..1dd7c69ca 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -30,7 +30,8 @@ proc pkg(name: string; cmd = "nimble test"; url = "", useHead = true) = # pkg "alea" pkg "argparse" -pkg "arraymancer", "nim c tests/tests_cpu.nim" +when false: + pkg "arraymancer", "nim c tests/tests_cpu.nim" # pkg "ast_pattern_matching", "nim c -r --oldgensym:on tests/test1.nim" pkg "awk" pkg "bigints", url = "https://github.com/Araq/nim-bigints" diff --git a/tests/dll/nimhcr_integration.nim b/tests/dll/nimhcr_integration.nim index 58851b5c4..ac34f1f85 100644 --- a/tests/dll/nimhcr_integration.nim +++ b/tests/dll/nimhcr_integration.nim @@ -1,5 +1,5 @@ discard """ - disabled: "openbsd" + disabled: "true" output: ''' main: HELLO! main: hasAnyModuleChanged? true diff --git a/tests/ic/thallo.nim b/tests/ic/thallo.nim index c29a0820c..7ead7c8ba 100644 --- a/tests/ic/thallo.nim +++ b/tests/ic/thallo.nim @@ -1,6 +1,5 @@ discard """ output: "Hello World" - disabled: "true" """ const str = "Hello World" diff --git a/tests/parallel/tconvexhull.nim b/tests/parallel/tconvexhull.nim index ebadb874d..0a07e6b76 100644 --- a/tests/parallel/tconvexhull.nim +++ b/tests/parallel/tconvexhull.nim @@ -1,8 +1,6 @@ discard """ output: ''' ''' - -ccodeCheck: "\\i ! @'deepCopy(' .*" """ # parallel convex hull for Nim bigbreak From e332c20ba7f0fffc0ad42751a99e11232973f27f Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 19 Mar 2021 23:54:10 +0800 Subject: [PATCH 0449/2472] follow up #17276 (#17355) * improve test coverage for isolation * a bit better * rename channels to channels_builtin * follow up #17276 * fix * Update lib/std/private/jsutils.nim --- lib/std/private/jsutils.nim | 12 ++++++++++-- tools/kochdocs.nim | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/std/private/jsutils.nim b/lib/std/private/jsutils.nim index cf58b3b81..32c717c56 100644 --- a/lib/std/private/jsutils.nim +++ b/lib/std/private/jsutils.nim @@ -1,4 +1,4 @@ -when defined(js): +when defined(js) or defined(nimdoc): import std/jsbigints type @@ -40,4 +40,12 @@ when defined(js): proc isInteger*[T](x: T): bool {.importjs: "Number.isInteger(#)".} - proc isSafeInteger*[T](x: T): bool {.importjs: "Number.isSafeInteger(#)".} + proc isSafeInteger*[T](x: T): bool {.importjs: "Number.isSafeInteger(#)".} = + runnableExamples: + import std/jsffi + assert not "123".toJs.isSafeInteger + assert 123.toJs.isSafeInteger + assert 9007199254740991.toJs.isSafeInteger + assert not 9007199254740992.toJs.isSafeInteger + + let maxSafeInteger* {.importjs: "Number.MAX_SAFE_INTEGER".} : int64 diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 8f4dd87a9..621dc643f 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -14,7 +14,7 @@ const webUploadOutput = "web/upload" var nimExe*: string -const allowList = ["jsbigints.nim", "jsheaders.nim", "jsformdata.nim", "jsfetch.nim"] +const allowList = ["jsbigints.nim", "jsheaders.nim", "jsformdata.nim", "jsfetch.nim", "jsutils.nim"] template isJsOnly(file: string): bool = file.isRelativeTo("lib/js") or From 8e8bea9044f0ae1f0583cb6130f6fbac390bf26a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9ctor=20M=2E=20Monacci?= Date: Fri, 19 Mar 2021 15:19:41 -0300 Subject: [PATCH 0450/2472] Clarify behaviour of char replace (#17339) Clarify behaviour of char replace by adding ```every ocurrence of character``` --- lib/pure/strutils.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 36a211545..510f0682d 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -2079,7 +2079,7 @@ func replace*(s, sub: string, by = ""): string {.rtl, func replace*(s: string, sub, by: char): string {.rtl, extern: "nsuReplaceChar".} = - ## Replaces `sub` in `s` by the character `by`. + ## Replaces every occurence of character `sub` in `s` by the character `by`. ## ## Optimized version of `replace <#replace,string,string,string>`_ for ## characters. From 430c30299f4c1cc5632d89c37487a78d4b4b3fde Mon Sep 17 00:00:00 2001 From: haxscramper Date: Fri, 19 Mar 2021 22:16:52 +0300 Subject: [PATCH 0451/2472] [FIX] use `mixin` for strscans.scanp (#17371) --- lib/pure/strscans.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pure/strscans.nim b/lib/pure/strscans.nim index 6bedf2de2..73b53e3d6 100644 --- a/lib/pure/strscans.nim +++ b/lib/pure/strscans.nim @@ -577,6 +577,7 @@ macro scanp*(input, idx: typed; pattern: varargs[untyped]): bool = of nnkCallKinds: # *{'A'..'Z'} !! s.add(!_) template buildWhile(input, idx, init, cond, action): untyped = + mixin hasNxt while hasNxt(input, idx): init if not cond: break @@ -688,4 +689,4 @@ macro scanp*(input, idx: typed; pattern: varargs[untyped]): bool = result.add toIfChain(conds, idx, res, 0) result.add res when defined(debugScanp): - echo repr result \ No newline at end of file + echo repr result From 9997b42c3512d0422910704d6472c486efd4db19 Mon Sep 17 00:00:00 2001 From: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Date: Fri, 19 Mar 2021 22:22:48 +0100 Subject: [PATCH 0452/2472] Use importjs (#17422) --- lib/js/asyncjs.nim | 8 ++-- lib/js/dom_extensions.nim | 2 +- lib/js/jsffi.nim | 98 +++++++++++++++++++-------------------- lib/std/jsbigints.nim | 4 +- 4 files changed, 56 insertions(+), 56 deletions(-) diff --git a/lib/js/asyncjs.nim b/lib/js/asyncjs.nim index 73af232b4..c62ac633f 100644 --- a/lib/js/asyncjs.nim +++ b/lib/js/asyncjs.nim @@ -70,7 +70,7 @@ type future*: T ## Wraps the return type of an asynchronous procedure. - PromiseJs* {.importcpp: "Promise".} = ref object + PromiseJs* {.importjs: "Promise".} = ref object ## A JavaScript Promise. @@ -113,7 +113,7 @@ proc generateJsasync(arg: NimNode): NimNode = if len(code) > 0: var awaitFunction = quote: - proc await[T](f: Future[T]): T {.importcpp: "(await #)", used.} + proc await[T](f: Future[T]): T {.importjs: "(await #)", used.} result.body.add(awaitFunction) var resolve: NimNode @@ -150,11 +150,11 @@ macro async*(arg: untyped): untyped = else: result = generateJsasync(arg) -proc newPromise*[T](handler: proc(resolve: proc(response: T))): Future[T] {.importcpp: "(new Promise(#))".} +proc newPromise*[T](handler: proc(resolve: proc(response: T))): Future[T] {.importjs: "(new Promise(#))".} ## A helper for wrapping callback-based functions ## into promises and async procedures. -proc newPromise*(handler: proc(resolve: proc())): Future[void] {.importcpp: "(new Promise(#))".} +proc newPromise*(handler: proc(resolve: proc())): Future[void] {.importjs: "(new Promise(#))".} ## A helper for wrapping callback-based functions ## into promises and async procedures. diff --git a/lib/js/dom_extensions.nim b/lib/js/dom_extensions.nim index f7d37f4bf..a1ceff5b4 100644 --- a/lib/js/dom_extensions.nim +++ b/lib/js/dom_extensions.nim @@ -2,4 +2,4 @@ import std/dom {.push importcpp.} proc elementsFromPoint*(n: DocumentOrShadowRoot; x, y: float): seq[Element] -{.pop.} \ No newline at end of file +{.pop.} diff --git a/lib/js/jsffi.nim b/lib/js/jsffi.nim index 0734d891a..937e3727b 100644 --- a/lib/js/jsffi.nim +++ b/lib/js/jsffi.nim @@ -21,7 +21,7 @@ runnableExamples: var document {.importc, nodecl.}: JsObject var console {.importc, nodecl.}: JsObject # import the "$" function - proc jq(selector: JsObject): JsObject {.importcpp: "$$(#)".} + proc jq(selector: JsObject): JsObject {.importjs: "$$(#)".} # Use jQuery to make the following code run, after the document is ready. # This uses an experimental `.()` operator for `JsObject`, to emit @@ -70,7 +70,7 @@ template mangleJsName(name: cstring): cstring = # only values that can be mapped 1 to 1 with cstring should be keys: they have an injective function with cstring -proc toJsKey*[T: SomeInteger](text: cstring, t: type T): T {.importcpp: "parseInt(#)".} +proc toJsKey*[T: SomeInteger](text: cstring, t: type T): T {.importjs: "parseInt(#)".} proc toJsKey*[T: enum](text: cstring, t: type T): T = T(text.toJsKey(int)) @@ -78,7 +78,7 @@ proc toJsKey*[T: enum](text: cstring, t: type T): T = proc toJsKey*(text: cstring, t: type cstring): cstring = text -proc toJsKey*[T: SomeFloat](text: cstring, t: type T): T {.importcpp: "parseFloat(#)".} +proc toJsKey*[T: SomeFloat](text: cstring, t: type T): T {.importjs: "parseFloat(#)".} type JsKey* = concept a, type T @@ -103,10 +103,10 @@ var jsFilename* {.importc: "__filename", nodecl.}: cstring ## JavaScript's __filename pseudo-variable. -proc isNull*[T](x: T): bool {.noSideEffect, importcpp: "(# === null)".} +proc isNull*[T](x: T): bool {.noSideEffect, importjs: "(# === null)".} ## Checks if a value is exactly null. -proc isUndefined*[T](x: T): bool {.noSideEffect, importcpp: "(# === undefined)".} +proc isUndefined*[T](x: T): bool {.noSideEffect, importjs: "(# === undefined)".} ## Checks if a value is exactly undefined. # Exceptions @@ -121,35 +121,35 @@ type JsURIError* {.importc: "URIError".} = object of JsError # New -proc newJsObject*: JsObject {.importcpp: "{@}".} +proc newJsObject*: JsObject {.importjs: "{@}".} ## Creates a new empty JsObject. -proc newJsAssoc*[K: JsKey, V]: JsAssoc[K, V] {.importcpp: "{@}".} +proc newJsAssoc*[K: JsKey, V]: JsAssoc[K, V] {.importjs: "{@}".} ## Creates a new empty JsAssoc with key type `K` and value type `V`. # Checks proc hasOwnProperty*(x: JsObject, prop: cstring): bool - {.importcpp: "#.hasOwnProperty(#)".} + {.importjs: "#.hasOwnProperty(#)".} ## Checks, whether `x` has a property of name `prop`. -proc jsTypeOf*(x: JsObject): cstring {.importcpp: "typeof(#)".} +proc jsTypeOf*(x: JsObject): cstring {.importjs: "typeof(#)".} ## Returns the name of the JsObject's JavaScript type as a cstring. -proc jsNew*(x: auto): JsObject {.importcpp: "(new #)".} +proc jsNew*(x: auto): JsObject {.importjs: "(new #)".} ## Turns a regular function call into an invocation of the ## JavaScript's `new` operator. -proc jsDelete*(x: auto): JsObject {.importcpp: "(delete #)".} +proc jsDelete*(x: auto): JsObject {.importjs: "(delete #)".} ## JavaScript's `delete` operator. proc require*(module: cstring): JsObject {.importc.} ## JavaScript's `require` function. # Conversion to and from JsObject -proc to*(x: JsObject, T: typedesc): T {.importcpp: "(#)".} +proc to*(x: JsObject, T: typedesc): T {.importjs: "(#)".} ## Converts a JsObject `x` to type `T`. -proc toJs*[T](val: T): JsObject {.importcpp: "(#)".} +proc toJs*[T](val: T): JsObject {.importjs: "(#)".} ## Converts a value of any type to type JsObject. template toJs*(s: string): JsObject = cstring(s).toJs @@ -160,50 +160,50 @@ macro jsFromAst*(n: untyped): untyped = result = newProc(procType = nnkDo, body = result) return quote: toJs(`result`) -proc `&`*(a, b: cstring): cstring {.importcpp: "(# + #)".} +proc `&`*(a, b: cstring): cstring {.importjs: "(# + #)".} ## Concatenation operator for JavaScript strings. -proc `+` *(x, y: JsObject): JsObject {.importcpp: "(# + #)".} -proc `-` *(x, y: JsObject): JsObject {.importcpp: "(# - #)".} -proc `*` *(x, y: JsObject): JsObject {.importcpp: "(# * #)".} -proc `/` *(x, y: JsObject): JsObject {.importcpp: "(# / #)".} -proc `%` *(x, y: JsObject): JsObject {.importcpp: "(# % #)".} -proc `+=` *(x, y: JsObject): JsObject {.importcpp: "(# += #)", discardable.} -proc `-=` *(x, y: JsObject): JsObject {.importcpp: "(# -= #)", discardable.} -proc `*=` *(x, y: JsObject): JsObject {.importcpp: "(# *= #)", discardable.} -proc `/=` *(x, y: JsObject): JsObject {.importcpp: "(# /= #)", discardable.} -proc `%=` *(x, y: JsObject): JsObject {.importcpp: "(# %= #)", discardable.} -proc `++` *(x: JsObject): JsObject {.importcpp: "(++#)".} -proc `--` *(x: JsObject): JsObject {.importcpp: "(--#)".} -proc `>` *(x, y: JsObject): JsObject {.importcpp: "(# > #)".} -proc `<` *(x, y: JsObject): JsObject {.importcpp: "(# < #)".} -proc `>=` *(x, y: JsObject): JsObject {.importcpp: "(# >= #)".} -proc `<=` *(x, y: JsObject): JsObject {.importcpp: "(# <= #)".} -proc `**` *(x, y: JsObject): JsObject {.importcpp: "((#) ** #)".} +proc `+` *(x, y: JsObject): JsObject {.importjs: "(# + #)".} +proc `-` *(x, y: JsObject): JsObject {.importjs: "(# - #)".} +proc `*` *(x, y: JsObject): JsObject {.importjs: "(# * #)".} +proc `/` *(x, y: JsObject): JsObject {.importjs: "(# / #)".} +proc `%` *(x, y: JsObject): JsObject {.importjs: "(# % #)".} +proc `+=` *(x, y: JsObject): JsObject {.importjs: "(# += #)", discardable.} +proc `-=` *(x, y: JsObject): JsObject {.importjs: "(# -= #)", discardable.} +proc `*=` *(x, y: JsObject): JsObject {.importjs: "(# *= #)", discardable.} +proc `/=` *(x, y: JsObject): JsObject {.importjs: "(# /= #)", discardable.} +proc `%=` *(x, y: JsObject): JsObject {.importjs: "(# %= #)", discardable.} +proc `++` *(x: JsObject): JsObject {.importjs: "(++#)".} +proc `--` *(x: JsObject): JsObject {.importjs: "(--#)".} +proc `>` *(x, y: JsObject): JsObject {.importjs: "(# > #)".} +proc `<` *(x, y: JsObject): JsObject {.importjs: "(# < #)".} +proc `>=` *(x, y: JsObject): JsObject {.importjs: "(# >= #)".} +proc `<=` *(x, y: JsObject): JsObject {.importjs: "(# <= #)".} +proc `**` *(x, y: JsObject): JsObject {.importjs: "((#) ** #)".} # (#) needed, refs https://github.com/nim-lang/Nim/pull/16409#issuecomment-760550812 -proc `and`*(x, y: JsObject): JsObject {.importcpp: "(# && #)".} -proc `or` *(x, y: JsObject): JsObject {.importcpp: "(# || #)".} -proc `not`*(x: JsObject): JsObject {.importcpp: "(!#)".} -proc `in` *(x, y: JsObject): JsObject {.importcpp: "(# in #)".} +proc `and`*(x, y: JsObject): JsObject {.importjs: "(# && #)".} +proc `or` *(x, y: JsObject): JsObject {.importjs: "(# || #)".} +proc `not`*(x: JsObject): JsObject {.importjs: "(!#)".} +proc `in` *(x, y: JsObject): JsObject {.importjs: "(# in #)".} -proc `[]`*(obj: JsObject, field: cstring): JsObject {.importcpp: getImpl.} +proc `[]`*(obj: JsObject, field: cstring): JsObject {.importjs: getImpl.} ## Returns the value of a property of name `field` from a JsObject `obj`. -proc `[]`*(obj: JsObject, field: int): JsObject {.importcpp: getImpl.} +proc `[]`*(obj: JsObject, field: int): JsObject {.importjs: getImpl.} ## Returns the value of a property of name `field` from a JsObject `obj`. -proc `[]=`*[T](obj: JsObject, field: cstring, val: T) {.importcpp: setImpl.} +proc `[]=`*[T](obj: JsObject, field: cstring, val: T) {.importjs: setImpl.} ## Sets the value of a property of name `field` in a JsObject `obj` to `v`. -proc `[]=`*[T](obj: JsObject, field: int, val: T) {.importcpp: setImpl.} +proc `[]=`*[T](obj: JsObject, field: int, val: T) {.importjs: setImpl.} ## Sets the value of a property of name `field` in a JsObject `obj` to `v`. proc `[]`*[K: JsKey, V](obj: JsAssoc[K, V], field: K): V - {.importcpp: getImpl.} + {.importjs: getImpl.} ## Returns the value of a property of name `field` from a JsAssoc `obj`. proc `[]=`*[K: JsKey, V](obj: JsAssoc[K, V], field: K, val: V) - {.importcpp: setImpl.} + {.importjs: setImpl.} ## Sets the value of a property of name `field` in a JsAssoc `obj` to `v`. proc `[]`*[V](obj: JsAssoc[cstring, V], field: string): V = @@ -212,7 +212,7 @@ proc `[]`*[V](obj: JsAssoc[cstring, V], field: string): V = proc `[]=`*[V](obj: JsAssoc[cstring, V], field: string, val: V) = obj[cstring(field)] = val -proc `==`*(x, y: JsRoot): bool {.importcpp: "(# === #)".} +proc `==`*(x, y: JsRoot): bool {.importjs: "(# === #)".} ## Compares two JsObjects or JsAssocs. Be careful though, as this is comparison ## like in JavaScript, so if your JsObjects are in fact JavaScript Objects, ## and not strings or numbers, this is a *comparison of references*. @@ -229,7 +229,7 @@ macro `.`*(obj: JsObject, field: untyped): JsObject = let importString = "#." & $field result = quote do: proc helper(o: JsObject): JsObject - {.importcpp: `importString`, gensym.} + {.importjs: `importString`, gensym.} helper(`obj`) else: if not mangledNames.hasKey($field): @@ -237,7 +237,7 @@ macro `.`*(obj: JsObject, field: untyped): JsObject = let importString = "#." & mangledNames[$field] result = quote do: proc helper(o: JsObject): JsObject - {.importcpp: `importString`, gensym.} + {.importjs: `importString`, gensym.} helper(`obj`) macro `.=`*(obj: JsObject, field, value: untyped): untyped = @@ -247,7 +247,7 @@ macro `.=`*(obj: JsObject, field, value: untyped): untyped = let importString = "#." & $field & " = #" result = quote do: proc helper(o: JsObject, v: auto) - {.importcpp: `importString`, gensym.} + {.importjs: `importString`, gensym.} helper(`obj`, `value`) else: if not mangledNames.hasKey($field): @@ -255,7 +255,7 @@ macro `.=`*(obj: JsObject, field, value: untyped): untyped = let importString = "#." & mangledNames[$field] & " = #" result = quote do: proc helper(o: JsObject, v: auto) - {.importcpp: `importString`, gensym.} + {.importjs: `importString`, gensym.} helper(`obj`, `value`) macro `.()`*(obj: JsObject, @@ -286,7 +286,7 @@ macro `.()`*(obj: JsObject, importString = "#." & mangledNames[$field] & "(@)" result = quote: proc helper(o: JsObject): JsObject - {.importcpp: `importString`, gensym, discardable.} + {.importjs: `importString`, gensym, discardable.} helper(`obj`) for idx in 0 ..< args.len: let paramName = newIdentNode("param" & $idx) @@ -306,7 +306,7 @@ macro `.`*[K: cstring, V](obj: JsAssoc[K, V], importString = "#." & mangledNames[$field] result = quote do: proc helper(o: type(`obj`)): `obj`.V - {.importcpp: `importString`, gensym.} + {.importjs: `importString`, gensym.} helper(`obj`) macro `.=`*[K: cstring, V](obj: JsAssoc[K, V], @@ -323,7 +323,7 @@ macro `.=`*[K: cstring, V](obj: JsAssoc[K, V], importString = "#." & mangledNames[$field] & " = #" result = quote do: proc helper(o: type(`obj`), v: `obj`.V) - {.importcpp: `importString`, gensym.} + {.importjs: `importString`, gensym.} helper(`obj`, `value`) macro `.()`*[K: cstring, V: proc](obj: JsAssoc[K, V], diff --git a/lib/std/jsbigints.nim b/lib/std/jsbigints.nim index 21bb9e1a0..ccf14080b 100644 --- a/lib/std/jsbigints.nim +++ b/lib/std/jsbigints.nim @@ -3,8 +3,8 @@ when not defined(js): {.fatal: "Module jsbigints is designed to be used with the JavaScript backend.".} -type JsBigIntImpl {.importc: "bigint".} = int # https://github.com/nim-lang/Nim/pull/16606 -type JsBigInt* = distinct JsBigIntImpl ## Arbitrary precision integer for JavaScript target. +type JsBigIntImpl {.importjs: "bigint".} = int # https://github.com/nim-lang/Nim/pull/16606 +type JsBigInt* = distinct JsBigIntImpl ## Arbitrary precision integer for JavaScript target. func big*(integer: SomeInteger): JsBigInt {.importjs: "BigInt(#)".} = ## Constructor for `JsBigInt`. From b70e33f5bbb7b9c146c97d39aabae57504f4ee75 Mon Sep 17 00:00:00 2001 From: ee7 <45465154+ee7@users.noreply.github.com> Date: Sat, 20 Mar 2021 13:22:50 +0100 Subject: [PATCH 0453/2472] strutils: improve doc comments for `replace` funcs (#17427) This commit fixes mispellings of "occurrence" introduced by: - 76a3b350ce0f (#17337) - 8e8bea9044f0 (#17339) and adds the same "every occurrence of" in the `replaceWord` func. Other changes: - Prefer "replace with" to "replace by". - Be more consistent with "the" - prefer "of the character" given that we wrote "by the character". - Try to be more consistent with writing the types - add "the string `sub`" given that we wrote "the character `sub`". --- lib/pure/strutils.nim | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 510f0682d..4098749ce 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -2037,7 +2037,7 @@ func contains*(s: string, chars: set[char]): bool = func replace*(s, sub: string, by = ""): string {.rtl, extern: "nsuReplaceStr".} = - ## Replaces every occurence of `sub` in `s` by the string `by`. + ## Replaces every occurrence of the string `sub` in `s` with the string `by`. ## ## See also: ## * `find func<#find,string,string,Natural,int>`_ @@ -2079,7 +2079,8 @@ func replace*(s, sub: string, by = ""): string {.rtl, func replace*(s: string, sub, by: char): string {.rtl, extern: "nsuReplaceChar".} = - ## Replaces every occurence of character `sub` in `s` by the character `by`. + ## Replaces every occurrence of the character `sub` in `s` with the character + ## `by`. ## ## Optimized version of `replace <#replace,string,string,string>`_ for ## characters. @@ -2097,7 +2098,7 @@ func replace*(s: string, sub, by: char): string {.rtl, func replaceWord*(s, sub: string, by = ""): string {.rtl, extern: "nsuReplaceWord".} = - ## Replaces `sub` in `s` by the string `by`. + ## Replaces every occurrence of the string `sub` in `s` with the string `by`. ## ## Each occurrence of `sub` has to be surrounded by word boundaries ## (comparable to `\b` in regular expressions), otherwise it is not From eca0b8754458e6d57a0ebc248ae0d1d024e1723c Mon Sep 17 00:00:00 2001 From: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Date: Sun, 21 Mar 2021 02:30:57 +0100 Subject: [PATCH 0454/2472] Close #8545 by add a test case (#17432) Co-authored-by: Timothee Cour --- tests/misc/t8545.nim | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 tests/misc/t8545.nim diff --git a/tests/misc/t8545.nim b/tests/misc/t8545.nim new file mode 100644 index 000000000..89957e1d3 --- /dev/null +++ b/tests/misc/t8545.nim @@ -0,0 +1,23 @@ +discard """ + targets: "c cpp js" +""" + +# bug #8545 + +template bar(a: static[bool]): untyped = int + +proc main() = + proc foo1(a: static[bool]): auto = 1 + doAssert foo1(true) == 1 + + proc foo2(a: static[bool]): bar(a) = 1 + doAssert foo2(true) == 1 + + proc foo3(a: static[bool]): bar(cast[static[bool]](a)) = 1 + doAssert foo3(true) == 1 + + proc foo4(a: static[bool]): bar(static(a)) = 1 + doAssert foo4(true) == 1 + +static: main() +main() From c5b109233a1ffe283d460be40575bdaf5beb0104 Mon Sep 17 00:00:00 2001 From: Danil Yarantsev Date: Sun, 21 Mar 2021 12:34:10 +0300 Subject: [PATCH 0455/2472] Add documentation to the `macrocache` module (#17431) * Add docs to macrocache * use hint * Use incl in the incl example * add macrocache to lib * consistency * Update doc/lib.rst Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * apply suggestions * clarify the warning Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- doc/lib.rst | 3 + lib/core/macrocache.nim | 191 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 179 insertions(+), 15 deletions(-) diff --git a/doc/lib.rst b/doc/lib.rst index 3202c5a53..9d715b1c7 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -64,6 +64,9 @@ Core * `locks `_ Locks and condition variables for Nim. +* `macrocache `_ + Provides an API for macros to collect compile-time information across modules. + * `macros `_ Contains the AST API and documentation of Nim for writing macros. diff --git a/lib/core/macrocache.nim b/lib/core/macrocache.nim index 8fe1fa603..e376ad87f 100644 --- a/lib/core/macrocache.nim +++ b/lib/core/macrocache.nim @@ -7,38 +7,199 @@ # distribution, for details about the copyright. # -## This module provides an API for macros that need to collect compile -## time information across module boundaries in global variables. -## Starting with version 0.19 of Nim this is not directly supported anymore -## as it breaks incremental compilations. -## Instead the API here needs to be used. +## This module provides an API for macros to collect compile-time information +## across module boundaries. It should be used instead of global `{.compileTime.}` +## variables as those break incremental compilation. +## +## The main feature of this module is that if you create `CacheTable`s or +## any other `Cache` types with the same name in different modules, their +## content will be shared, meaning that you can fill a `CacheTable` in +## one module, and iterate over its contents in another. + +runnableExamples: + import std/macros + + const mcTable = CacheTable"myTable" + const mcSeq = CacheSeq"mySeq" + const mcCounter = CacheCounter"myCounter" + + static: + # add new key "val" with the value `myval` + let myval = newLit("hello ic") + mcTable["val"] = myval + assert mcTable["val"].kind == nnkStrLit + + # Can access the same cache from different static contexts + # All the information is retained + static: + # get value from `mcTable` and add it to `mcSeq` + mcSeq.add(mcTable["val"]) + assert mcSeq.len == 1 + + static: + assert mcSeq[0].strVal == "hello ic" + + # increase `mcCounter` by 3 + mcCounter.inc(3) + assert mcCounter.value == 3 + type CacheSeq* = distinct string + ## Compile-time sequence of `NimNode`s. CacheTable* = distinct string + ## Compile-time table of key-value pairs. + ## + ## Keys are `string`s and values are `NimNode`s. CacheCounter* = distinct string + ## Compile-time counter, uses `int` for storing the count. -proc value*(c: CacheCounter): int {.magic: "NccValue".} -proc inc*(c: CacheCounter; by = 1) {.magic: "NccInc".} +proc value*(c: CacheCounter): int {.magic: "NccValue".} = + ## Returns the value of a counter `c`. + runnableExamples: + static: + let counter = CacheCounter"valTest" + # default value is 0 + assert counter.value == 0 -proc add*(s: CacheSeq; value: NimNode) {.magic: "NcsAdd".} -proc incl*(s: CacheSeq; value: NimNode) {.magic: "NcsIncl".} -proc len*(s: CacheSeq): int {.magic: "NcsLen".} -proc `[]`*(s: CacheSeq; i: int): NimNode {.magic: "NcsAt".} + inc counter + assert counter.value == 1 + +proc inc*(c: CacheCounter; by = 1) {.magic: "NccInc".} = + ## Increments the counter `c` with the value `by`. + runnableExamples: + static: + let counter = CacheCounter"incTest" + inc counter + inc counter, 5 + + assert counter.value == 6 + +proc add*(s: CacheSeq; value: NimNode) {.magic: "NcsAdd".} = + ## Adds `value` to `s`. + runnableExamples: + import std/macros + const mySeq = CacheSeq"addTest" + + static: + mySeq.add(newLit(5)) + mySeq.add(newLit("hello ic")) + + assert mySeq.len == 2 + assert mySeq[1].strVal == "hello ic" + +proc incl*(s: CacheSeq; value: NimNode) {.magic: "NcsIncl".} = + ## Adds `value` to `s`. + ## + ## .. hint:: This doesn't do anything if `value` is already in `s`. + runnableExamples: + import std/macros + const mySeq = CacheSeq"inclTest" + + static: + mySeq.incl(newLit(5)) + mySeq.incl(newLit(5)) + + # still one element + assert mySeq.len == 1 + +proc len*(s: CacheSeq): int {.magic: "NcsLen".} = + ## Returns the length of `s`. + runnableExamples: + import std/macros + + const mySeq = CacheSeq"lenTest" + static: + let val = newLit("helper") + mySeq.add(val) + assert mySeq.len == 1 + + mySeq.add(val) + assert mySeq.len == 2 + +proc `[]`*(s: CacheSeq; i: int): NimNode {.magic: "NcsAt".} = + ## Returns the `i`th value from `s`. + runnableExamples: + import std/macros + + const mySeq = CacheSeq"subTest" + static: + mySeq.add(newLit(42)) + assert mySeq[0].intVal == 42 iterator items*(s: CacheSeq): NimNode = + ## Iterates over each item in `s`. + runnableExamples: + import std/macros + const myseq = CacheSeq"itemsTest" + + static: + myseq.add(newLit(5)) + myseq.add(newLit(42)) + + for val in myseq: + # check that all values in `myseq` are int literals + assert val.kind == nnkIntLit + for i in 0 ..< len(s): yield s[i] -proc `[]=`*(t: CacheTable; key: string, value: NimNode) {.magic: "NctPut".} - ## 'key' has to be unique! +proc `[]=`*(t: CacheTable; key: string, value: NimNode) {.magic: "NctPut".} = + ## Inserts a `(key, value)` pair into `t`. + ## + ## .. warning:: `key` has to be unique! Assigning `value` to a `key` that is already + ## in the table will result in a compiler error. + runnableExamples: + import std/macros -proc len*(t: CacheTable): int {.magic: "NctLen".} -proc `[]`*(t: CacheTable; key: string): NimNode {.magic: "NctGet".} + const mcTable = CacheTable"subTest" + static: + # assign newLit(5) to the key "value" + mcTable["value"] = newLit(5) + + # check that we can get the value back + assert mcTable["value"].kind == nnkIntLit + +proc len*(t: CacheTable): int {.magic: "NctLen".} = + ## Returns the number of elements in `t`. + runnableExamples: + import std/macros + + const dataTable = CacheTable"lenTest" + static: + dataTable["key"] = newLit(5) + assert dataTable.len == 1 + +proc `[]`*(t: CacheTable; key: string): NimNode {.magic: "NctGet".} = + ## Retrieves the `NimNode` value at `t[key]`. + runnableExamples: + import std/macros + + const mcTable = CacheTable"subTest" + static: + mcTable["toAdd"] = newStmtList() + + # get the NimNode back + assert mcTable["toAdd"].kind == nnkStmtList proc hasNext(t: CacheTable; iter: int): bool {.magic: "NctHasNext".} proc next(t: CacheTable; iter: int): (string, NimNode, int) {.magic: "NctNext".} iterator pairs*(t: CacheTable): (string, NimNode) = + ## Iterates over all `(key, value)` pairs in `t`. + runnableExamples: + import std/macros + const mytabl = CacheTable"values" + + static: + mytabl["intVal"] = newLit(5) + mytabl["otherVal"] = newLit(6) + for key, val in mytabl: + # make sure that we actually get the same keys + assert key in ["intVal", "otherVal"] + + # all vals are int literals + assert val.kind == nnkIntLit + var h = 0 while hasNext(t, h): let (a, b, h2) = next(t, h) From 05743bc9f72f2e3cbf4b17f5974a811637f03241 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 21 Mar 2021 02:35:00 -0700 Subject: [PATCH 0456/2472] improve jsutils docs (#17421) * improve jsutils docs * address comments --- lib/std/private/jsutils.nim | 52 ++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/lib/std/private/jsutils.nim b/lib/std/private/jsutils.nim index 32c717c56..836b3512a 100644 --- a/lib/std/private/jsutils.nim +++ b/lib/std/private/jsutils.nim @@ -1,4 +1,4 @@ -when defined(js) or defined(nimdoc): +when defined(js): import std/jsbigints type @@ -14,7 +14,6 @@ when defined(js) or defined(nimdoc): func newUint32Array*(buffer: ArrayBuffer): Uint32Array {.importjs: "new Uint32Array(#)".} func newBigUint64Array*(buffer: ArrayBuffer): BigUint64Array {.importjs: "new BigUint64Array(#)".} - func newUint8Array*(n: int): Uint8Array {.importjs: "new Uint8Array(#)".} func `[]`*(arr: Uint32Array, i: int): uint32 {.importjs: "#[#]".} @@ -22,12 +21,22 @@ when defined(js) or defined(nimdoc): func `[]`*(arr: BigUint64Array, i: int): JsBigInt {.importjs: "#[#]".} func `[]=`*(arr: Float64Array, i: int, v: float) {.importjs: "#[#] = #".} - - proc jsTypeOf*[T](x: T): cstring {.importjs: "typeof(#)".} - ## Returns the name of the JsObject's JavaScript type as a cstring. - # xxx replace jsffi.jsTypeOf with this definition and add tests + proc jsTypeOf*[T](x: T): cstring {.importjs: "typeof(#)".} = + ## Returns the name of the JsObject's JavaScript type as a cstring. + # xxx replace jsffi.jsTypeOf with this definition and add tests + runnableExamples: + import std/[jsffi, jsbigints] + assert jsTypeOf(1.toJs) == "number" + assert jsTypeOf(false.toJs) == "boolean" + assert [1].toJs.jsTypeOf == "object" # note the difference with `getProtoName` + assert big"1".toJs.jsTypeOf == "bigint" proc jsConstructorName*[T](a: T): cstring = + runnableExamples: + import std/jsffi + let a = array[2, float64].default + assert jsConstructorName(a) == "Float64Array" + assert jsConstructorName(a.toJs) == "Float64Array" asm """`result` = `a`.constructor.name""" proc hasJsBigInt*(): bool = @@ -36,16 +45,39 @@ when defined(js) or defined(nimdoc): proc hasBigUint64Array*(): bool = asm """`result` = typeof BigUint64Array != 'undefined'""" - proc getProtoName*[T](a: T): cstring {.importjs: "Object.prototype.toString.call(#)".} + proc getProtoName*[T](a: T): cstring {.importjs: "Object.prototype.toString.call(#)".} = + runnableExamples: + import std/[jsffi, jsbigints] + type A = ref object + assert 1.toJs.getProtoName == "[object Number]" + assert "a".toJs.getProtoName == "[object String]" + assert big"1".toJs.getProtoName == "[object BigInt]" + assert false.toJs.getProtoName == "[object Boolean]" + assert (a: 1).toJs.getProtoName == "[object Object]" + assert A.default.toJs.getProtoName == "[object Null]" + assert [1].toJs.getProtoName == "[object Int32Array]" # implementation defined + assert @[1].toJs.getProtoName == "[object Array]" # ditto - proc isInteger*[T](x: T): bool {.importjs: "Number.isInteger(#)".} + const maxSafeInteger* = 9007199254740991 + ## The same as `Number.MAX_SAFE_INTEGER` or `2^53 - 1`. + ## See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number/MAX_SAFE_INTEGER + runnableExamples: + let a {.importjs: "Number.MAX_SAFE_INTEGER".}: int64 + assert a == maxSafeInteger + + proc isInteger*[T](x: T): bool {.importjs: "Number.isInteger(#)".} = + runnableExamples: + import std/jsffi + assert 1.isInteger + assert not 1.5.isInteger + assert 1.toJs.isInteger + assert not 1.5.toJs.isInteger proc isSafeInteger*[T](x: T): bool {.importjs: "Number.isSafeInteger(#)".} = runnableExamples: import std/jsffi assert not "123".toJs.isSafeInteger + assert 123.isSafeInteger assert 123.toJs.isSafeInteger assert 9007199254740991.toJs.isSafeInteger assert not 9007199254740992.toJs.isSafeInteger - - let maxSafeInteger* {.importjs: "Number.MAX_SAFE_INTEGER".} : int64 From fb38d906a284a088f052e6fc842fb7f2df26a486 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Sun, 21 Mar 2021 06:35:55 -0300 Subject: [PATCH 0457/2472] Improve jsre (#17365) * Add dollar for regex * Add dollar for regex * Peer review feedbacks * Peer review feedbacks * Update lib/js/jsre.nim Co-authored-by: Timothee Cour * Update lib/js/jsre.nim Co-authored-by: Timothee Cour * Update lib/js/jsre.nim Co-authored-by: Timothee Cour * Pear review * Beer review * Beer review Co-authored-by: Timothee Cour --- changelog.md | 2 ++ lib/js/jsre.nim | 77 ++++++++++++++++++++++++++++++------------------- 2 files changed, 49 insertions(+), 30 deletions(-) diff --git a/changelog.md b/changelog.md index 219a66ae7..cc4f54fe0 100644 --- a/changelog.md +++ b/changelog.md @@ -239,6 +239,8 @@ - Added `jsconsole.dir`, `jsconsole.dirxml`, `jsconsole.timeStamp`. +- Added dollar `$` and `len` for `jsre.RegExp`. + ## Language changes diff --git a/lib/js/jsre.nim b/lib/js/jsre.nim index 7be7221bc..7d51db646 100644 --- a/lib/js/jsre.nim +++ b/lib/js/jsre.nim @@ -1,46 +1,63 @@ ## Regular Expressions for the JavaScript target. ## * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions - -runnableExamples: - let jsregex: RegExp = newRegExp(r"\s+", r"i") - jsregex.compile(r"\w+", r"i") - doAssert jsregex.test(r"nim javascript") - doAssert jsregex.exec(r"nim javascript") == @["nim".cstring] - doAssert jsregex.toString() == r"/\w+/i" - jsregex.compile(r"[0-9]", r"i") - doAssert jsregex.test(r"0123456789abcd") - - when not defined(js): {.error: "This module only works on the JavaScript platform".} -type RegExp* {.importjs.} = object ## Regular Expressions for JavaScript target. - flags* {.importjs.}: cstring ## cstring that contains the flags of the RegExp object. - dotAll* {.importjs.}: bool ## Whether `.` matches newlines or not. - global* {.importjs.}: bool ## Whether to test against all possible matches in a string, or only against the first. - ignoreCase* {.importjs.}: bool ## Whether to ignore case while attempting a match in a string. - multiline* {.importjs.}: bool ## Whether to search in strings across multiple lines. - source* {.importjs.}: cstring ## The text of the pattern. - sticky* {.importjs.}: bool ## Whether the search is sticky. - unicode* {.importjs.}: bool ## Whether Unicode features are enabled. - lastIndex* {.importjs.}: cint ## Index at which to start the next match (read/write property). - input* {.importjs.}: cstring ## Read-only and modified on successful match. - lastMatch* {.importjs.}: cstring ## Ditto. - lastParen* {.importjs.}: cstring ## Ditto. - leftContext* {.importjs.}: cstring ## Ditto. - rightContext* {.importjs.}: cstring ## Ditto. +type RegExp* = ref object of JsRoot + ## Regular Expressions for JavaScript target. + ## See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp + flags*: cstring ## cstring that contains the flags of the RegExp object. + dotAll*: bool ## Whether `.` matches newlines or not. + global*: bool ## Whether to test against all possible matches in a string, or only against the first. + ignoreCase*: bool ## Whether to ignore case while attempting a match in a string. + multiline*: bool ## Whether to search in strings across multiple lines. + source*: cstring ## The text of the pattern. + sticky*: bool ## Whether the search is sticky. + unicode*: bool ## Whether Unicode features are enabled. + lastIndex*: cint ## Index at which to start the next match (read/write property). + input*: cstring ## Read-only and modified on successful match. + lastMatch*: cstring ## Ditto. + lastParen*: cstring ## Ditto. + leftContext*: cstring ## Ditto. + rightContext*: cstring ## Ditto. + func newRegExp*(pattern: cstring; flags: cstring): RegExp {.importjs: "new RegExp(@)".} ## Creates a new RegExp object. +func newRegExp*(pattern: cstring): RegExp {.importjs: "new RegExp(@)".} + func compile*(self: RegExp; pattern: cstring; flags: cstring) {.importjs: "#.compile(@)".} ## Recompiles a regular expression during execution of a script. func exec*(self: RegExp; pattern: cstring): seq[cstring] {.importjs: "#.exec(#)".} ## Executes a search for a match in its string parameter. -func test*(self: RegExp; pattern: cstring): bool {.importjs: "#.test(#)".} - ## Tests for a match in its string parameter. - -func toString*(self: RegExp): cstring {.importjs: "#.toString()".} +func toCstring*(self: RegExp): cstring {.importjs: "#.toString()".} ## Returns a string representing the RegExp object. + +func `$`*(self: RegExp): string = $toCstring(self) + +func test*(self: RegExp; pattern: cstring): bool {.importjs: "#.test(#)", deprecated: "Use contains instead".} + +func toString*(self: RegExp): cstring {.importjs: "#.toString()", deprecated: "Use toCstring instead".} + +func contains*(pattern: cstring; self: RegExp): bool = + ## Tests for a substring match in its string parameter. + runnableExamples: + let jsregex: RegExp = newRegExp(r"bc$", r"i") + assert jsregex in r"abc" + assert jsregex notin r"abcd" + assert "xabc".contains jsregex + asm "`result` = `self`.test(`pattern`);" + + +runnableExamples: + let jsregex: RegExp = newRegExp(r"\s+", r"i") + jsregex.compile(r"\w+", r"i") + assert "nim javascript".contains jsregex + assert jsregex.exec(r"nim javascript") == @["nim".cstring] + assert jsregex.toCstring() == r"/\w+/i" + jsregex.compile(r"[0-9]", r"i") + assert "0123456789abcd".contains jsregex + assert $jsregex == "/[0-9]/i" From fd09ace55796ae6605f34dafecab24cc0d6ac9fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Derek=20=E5=91=86?= <116649+derekdai@users.noreply.github.com> Date: Sun, 21 Mar 2021 21:29:39 +0800 Subject: [PATCH 0458/2472] prevent bitmasks double included in mmdist if -d:nimArcDebug added (#17436) --- lib/system/cellsets.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/system/cellsets.nim b/lib/system/cellsets.nim index ea00176b5..779f1a91f 100644 --- a/lib/system/cellsets.nim +++ b/lib/system/cellsets.nim @@ -13,7 +13,8 @@ when defined(gcOrc) or defined(gcArc): type PCell = Cell - include bitmasks + when not declaredInScope(PageShift): + include bitmasks else: type From 5bed7d282ad043d945225fe72adb654d49b4f2ee Mon Sep 17 00:00:00 2001 From: AFaurholt Date: Sun, 21 Mar 2021 18:35:22 +0100 Subject: [PATCH 0459/2472] added more modules to docs lib (#17430) * added more modules * Update doc/lib.rst Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Update doc/lib.rst Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Update doc/lib.rst Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Update doc/lib.rst Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Update doc/lib.rst Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * listed alphabetically + link to json module * Added suggestion #17430 Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- doc/lib.rst | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/doc/lib.rst b/doc/lib.rst index 9d715b1c7..62e02815c 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -357,6 +357,9 @@ Parsers * `json `_ High-performance JSON parser. +* `std/jsonutils `_ + This module implements a hookable (de)serialization for arbitrary types. + * `lexbase `_ This is a low-level module that implements an extremely efficient buffering scheme for lexers and parsers. This is used by the diverse parsing modules. @@ -371,6 +374,9 @@ Parsers * `parsecsv `_ The `parsecsv` module implements a simple high-performance CSV parser. +* `parsejson `_ + This module implements a JSON parser. It is used and exported by the `json `_ module, but can also be used in its own right. + * `parseopt `_ The `parseopt` module implements a command line option parser. @@ -459,6 +465,9 @@ Miscellaneous * `coro `_ This module implements experimental coroutines in Nim. +* `std/enumerate `_ + This module implements `enumerate` syntactic sugar based on Nim's macro system. + * `logging `_ This module implements a simple logger. @@ -474,6 +483,10 @@ Miscellaneous * `std/varints `_ Decode variable-length integers that are compatible with SQLite. +* `std/with `_ + This module implements the `with` macro for easy function chaining. + + Modules for JS backend ---------------------- @@ -506,6 +519,7 @@ Regular expressions expressions. The current implementation uses PCRE. + Database support ---------------- @@ -522,6 +536,13 @@ Database support for other databases too. +Generic Operating System Services +--------------------------------- + +* `rdstdin `_ + This module contains code for reading from stdin. + + Wrappers ======== @@ -572,3 +593,11 @@ Network Programming and Internet Protocols * `openssl `_ Wrapper for OpenSSL. + + + +Unstable +======== + +* `atomics `_ + Types and operations for atomic operations and lockless algorithms. From 23fd0984283fe94108dea5d569450f5e0564c59d Mon Sep 17 00:00:00 2001 From: Saem Ghani Date: Sun, 21 Mar 2021 16:33:37 -0700 Subject: [PATCH 0460/2472] Fixes #17433; gensym callDef return in templ body (#17445) --- compiler/semtempl.nim | 2 +- tests/template/t17433.nim | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 tests/template/t17433.nim diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim index 568873269..4bb9f3e6b 100644 --- a/compiler/semtempl.nim +++ b/compiler/semtempl.nim @@ -435,8 +435,8 @@ proc semTemplBody(c: var TemplCtx, n: PNode): PNode = of nkLetSection: semTemplSomeDecl(c, n, skLet) of nkFormalParams: checkMinSonsLen(n, 1, c.c.config) - n[0] = semTemplBody(c, n[0]) semTemplSomeDecl(c, n, skParam, 1) + n[0] = semTemplBody(c, n[0]) of nkConstSection: for i in 0.. Date: Mon, 22 Mar 2021 07:36:48 +0800 Subject: [PATCH 0461/2472] close #11330 sets uses optimized countSetBits (#17334) * Update lib/pure/bitops.nim * Update lib/system/sets.nim * Apply suggestions from code review Co-authored-by: Andreas Rumpf --- lib/pure/bitops.nim | 88 ++++------------------------------- lib/std/private/vmutils.nim | 17 +++++++ lib/system/countbits_impl.nim | 77 ++++++++++++++++++++++++++++-- lib/system/sets.nim | 1 - 4 files changed, 98 insertions(+), 85 deletions(-) create mode 100644 lib/std/private/vmutils.nim diff --git a/lib/pure/bitops.nim b/lib/pure/bitops.nim index 57e3c7989..377602e75 100644 --- a/lib/pure/bitops.nim +++ b/lib/pure/bitops.nim @@ -27,6 +27,9 @@ import macros import std/private/since +from std/private/vmutils import forwardImpl, toUnsigned + + func bitnot*[T: SomeInteger](x: T): T {.magic: "BitnotI".} ## Computes the `bitwise complement` of the integer `x`. @@ -58,34 +61,6 @@ macro bitxor*[T: SomeInteger](x, y: T; z: varargs[T]): T = for extra in z: result = newCall(fn, result, extra) -const useBuiltins = not defined(noIntrinsicsBitOpts) -const noUndefined = defined(noUndefinedBitOpts) -const useGCC_builtins = (defined(gcc) or defined(llvm_gcc) or - defined(clang)) and useBuiltins -const useICC_builtins = defined(icc) and useBuiltins -const useVCC_builtins = defined(vcc) and useBuiltins -const arch64 = sizeof(int) == 8 -const useBuiltinsRotate = (defined(amd64) or defined(i386)) and - (defined(gcc) or defined(clang) or defined(vcc) or - (defined(icl) and not defined(cpp))) and useBuiltins - -template toUnsigned(x: int8): uint8 = cast[uint8](x) -template toUnsigned(x: int16): uint16 = cast[uint16](x) -template toUnsigned(x: int32): uint32 = cast[uint32](x) -template toUnsigned(x: int64): uint64 = cast[uint64](x) -template toUnsigned(x: int): uint = cast[uint](x) - -template forwardImpl(impl, arg) {.dirty.} = - when sizeof(x) <= 4: - when x is SomeSignedInt: - impl(cast[uint32](x.int32)) - else: - impl(x.uint32) - else: - when x is SomeSignedInt: - impl(cast[uint64](x.int64)) - else: - impl(x.uint64) type BitsRange*[T] = range[0..sizeof(T)*8-1] ## A range with all bit positions for type `T`. @@ -436,13 +411,12 @@ func fastlog2Nim(x: uint64): int {.inline.} = v = v or v shr 32 result = lookup[(v * 0x03F6EAF2CD271461'u64) shr 58].int -# sets.nim cannot import bitops, but bitops can use include -# system/sets to eliminate code duplication. sets.nim defines -# countBits32 and countBits64. import system/countbits_impl -template countSetBitsNim(n: uint32): int = countBits32(n) -template countSetBitsNim(n: uint64): int = countBits64(n) +const arch64 = sizeof(int) == 8 +const useBuiltinsRotate = (defined(amd64) or defined(i386)) and + (defined(gcc) or defined(clang) or defined(vcc) or + (defined(icl) and not defined(cpp))) and useBuiltins template parityImpl[T](value: T): int = # formula id from: https://graphics.stanford.edu/%7Eseander/bithacks.html#ParityParallel @@ -459,11 +433,6 @@ template parityImpl[T](value: T): int = when useGCC_builtins: - # Returns the number of set 1-bits in value. - proc builtin_popcount(x: cuint): cint {.importc: "__builtin_popcount", cdecl.} - proc builtin_popcountll(x: culonglong): cint {. - importc: "__builtin_popcountll", cdecl.} - # Returns the bit parity in value proc builtin_parity(x: cuint): cint {.importc: "__builtin_parity", cdecl.} proc builtin_parityll(x: culonglong): cint {.importc: "__builtin_parityll", cdecl.} @@ -481,14 +450,6 @@ when useGCC_builtins: proc builtin_ctzll(x: culonglong): cint {.importc: "__builtin_ctzll", cdecl.} elif useVCC_builtins: - # Counts the number of one bits (population count) in a 16-, 32-, or 64-byte unsigned integer. - func builtin_popcnt16(a2: uint16): uint16 {. - importc: "__popcnt16", header: "".} - func builtin_popcnt32(a2: uint32): uint32 {. - importc: "__popcnt", header: "".} - func builtin_popcnt64(a2: uint64): uint64 {. - importc: "__popcnt64", header: "".} - # Search the mask data from most significant bit (MSB) to least significant bit (LSB) for a set bit (1). func bitScanReverse(index: ptr culong, mask: culong): cuchar {. importc: "_BitScanReverse", header: "".} @@ -507,15 +468,6 @@ elif useVCC_builtins: index.int elif useICC_builtins: - - # Intel compiler intrinsics: http://fulla.fnal.gov/intel/compiler_c/main_cls/intref_cls/common/intref_allia_misc.htm - # see also: https://software.intel.com/en-us/node/523362 - # Count the number of bits set to 1 in an integer a, and return that count in dst. - func builtin_popcnt32(a: cint): cint {. - importc: "_popcnt", header: "".} - func builtin_popcnt64(a: uint64): cint {. - importc: "_popcnt64", header: "".} - # Returns the number of trailing 0-bits in x, starting at the least significant bit position. If x is 0, the result is undefined. func bitScanForward(p: ptr uint32, b: uint32): cuchar {. importc: "_BitScanForward", header: "".} @@ -533,37 +485,13 @@ elif useICC_builtins: discard fnc(index.addr, v) index.int - func countSetBits*(x: SomeInteger): int {.inline.} = ## Counts the set bits in an integer (also called `Hamming weight`:idx:). runnableExamples: doAssert countSetBits(0b0000_0011'u8) == 2 doAssert countSetBits(0b1010_1010'u8) == 4 - # TODO: figure out if ICC support _popcnt32/_popcnt64 on platform without POPCNT. - # like GCC and MSVC - when x is SomeSignedInt: - let x = x.toUnsigned - when nimvm: - result = forwardImpl(countSetBitsNim, x) - else: - when useGCC_builtins: - when sizeof(x) <= 4: result = builtin_popcount(x.cuint).int - else: result = builtin_popcountll(x.culonglong).int - elif useVCC_builtins: - when sizeof(x) <= 2: result = builtin_popcnt16(x.uint16).int - elif sizeof(x) <= 4: result = builtin_popcnt32(x.uint32).int - elif arch64: result = builtin_popcnt64(x.uint64).int - else: result = builtin_popcnt32((x.uint64 and 0xFFFFFFFF'u64).uint32).int + - builtin_popcnt32((x.uint64 shr 32'u64).uint32).int - elif useICC_builtins: - when sizeof(x) <= 4: result = builtin_popcnt32(x.cint).int - elif arch64: result = builtin_popcnt64(x.uint64).int - else: result = builtin_popcnt32((x.uint64 and 0xFFFFFFFF'u64).cint).int + - builtin_popcnt32((x.uint64 shr 32'u64).cint).int - else: - when sizeof(x) <= 4: result = countSetBitsNim(x.uint32) - else: result = countSetBitsNim(x.uint64) + result = countSetBitsImpl(x) func popcount*(x: SomeInteger): int {.inline.} = ## Alias for `countSetBits <#countSetBits,SomeInteger>`_ (Hamming weight). diff --git a/lib/std/private/vmutils.nim b/lib/std/private/vmutils.nim new file mode 100644 index 000000000..d54977b4e --- /dev/null +++ b/lib/std/private/vmutils.nim @@ -0,0 +1,17 @@ +template forwardImpl*(impl, arg) {.dirty.} = + when sizeof(x) <= 4: + when x is SomeSignedInt: + impl(cast[uint32](x.int32)) + else: + impl(x.uint32) + else: + when x is SomeSignedInt: + impl(cast[uint64](x.int64)) + else: + impl(x.uint64) + +template toUnsigned*(x: int8): uint8 = cast[uint8](x) +template toUnsigned*(x: int16): uint16 = cast[uint16](x) +template toUnsigned*(x: int32): uint32 = cast[uint32](x) +template toUnsigned*(x: int64): uint64 = cast[uint64](x) +template toUnsigned*(x: int): uint = cast[uint](x) diff --git a/lib/system/countbits_impl.nim b/lib/system/countbits_impl.nim index 6c85612e2..d3c003ff7 100644 --- a/lib/system/countbits_impl.nim +++ b/lib/system/countbits_impl.nim @@ -9,17 +9,86 @@ ## Contains the used algorithms for counting bits. -proc countBits32*(n: uint32): int {.compilerproc.} = +from std/private/vmutils import forwardImpl, toUnsigned + + +const useBuiltins* = not defined(noIntrinsicsBitOpts) +const noUndefined* = defined(noUndefinedBitOpts) +const useGCC_builtins* = (defined(gcc) or defined(llvm_gcc) or + defined(clang)) and useBuiltins +const useICC_builtins* = defined(icc) and useBuiltins +const useVCC_builtins* = defined(vcc) and useBuiltins + +template countBitsImpl(n: uint32): int = # generic formula is from: https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel var v = uint32(n) v = v - ((v shr 1'u32) and 0x55555555'u32) v = (v and 0x33333333'u32) + ((v shr 2'u32) and 0x33333333'u32) - result = (((v + (v shr 4'u32) and 0xF0F0F0F'u32) * 0x1010101'u32) shr 24'u32).int + (((v + (v shr 4'u32) and 0xF0F0F0F'u32) * 0x1010101'u32) shr 24'u32).int -proc countBits64*(n: uint64): int {.compilerproc, inline.} = +template countBitsImpl(n: uint64): int = # generic formula is from: https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel var v = uint64(n) v = v - ((v shr 1'u64) and 0x5555555555555555'u64) v = (v and 0x3333333333333333'u64) + ((v shr 2'u64) and 0x3333333333333333'u64) v = (v + (v shr 4'u64) and 0x0F0F0F0F0F0F0F0F'u64) - result = ((v * 0x0101010101010101'u64) shr 56'u64).int + ((v * 0x0101010101010101'u64) shr 56'u64).int + + +when useGCC_builtins: + # Returns the number of set 1-bits in value. + proc builtin_popcount(x: cuint): cint {.importc: "__builtin_popcount", cdecl.} + proc builtin_popcountll(x: culonglong): cint {. + importc: "__builtin_popcountll", cdecl.} + +elif useVCC_builtins: + # Counts the number of one bits (population count) in a 16-, 32-, or 64-byte unsigned integer. + func builtin_popcnt16(a2: uint16): uint16 {. + importc: "__popcnt16", header: "".} + func builtin_popcnt32(a2: uint32): uint32 {. + importc: "__popcnt", header: "".} + func builtin_popcnt64(a2: uint64): uint64 {. + importc: "__popcnt64", header: "".} + +elif useICC_builtins: + # Intel compiler intrinsics: http://fulla.fnal.gov/intel/compiler_c/main_cls/intref_cls/common/intref_allia_misc.htm + # see also: https://software.intel.com/en-us/node/523362 + # Count the number of bits set to 1 in an integer a, and return that count in dst. + func builtin_popcnt32(a: cint): cint {. + importc: "_popcnt", header: "".} + func builtin_popcnt64(a: uint64): cint {. + importc: "_popcnt64", header: "".} + + +func countSetBitsImpl*(x: SomeInteger): int {.inline.} = + ## Counts the set bits in an integer (also called `Hamming weight`:idx:). + # TODO: figure out if ICC support _popcnt32/_popcnt64 on platform without POPCNT. + # like GCC and MSVC + when x is SomeSignedInt: + let x = x.toUnsigned + when nimvm: + result = forwardImpl(countBitsImpl, x) + else: + when useGCC_builtins: + when sizeof(x) <= 4: result = builtin_popcount(x.cuint).int + else: result = builtin_popcountll(x.culonglong).int + elif useVCC_builtins: + when sizeof(x) <= 2: result = builtin_popcnt16(x.uint16).int + elif sizeof(x) <= 4: result = builtin_popcnt32(x.uint32).int + elif arch64: result = builtin_popcnt64(x.uint64).int + else: result = builtin_popcnt32((x.uint64 and 0xFFFFFFFF'u64).uint32).int + + builtin_popcnt32((x.uint64 shr 32'u64).uint32).int + elif useICC_builtins: + when sizeof(x) <= 4: result = builtin_popcnt32(x.cint).int + elif arch64: result = builtin_popcnt64(x.uint64).int + else: result = builtin_popcnt32((x.uint64 and 0xFFFFFFFF'u64).cint).int + + builtin_popcnt32((x.uint64 shr 32'u64).cint).int + else: + when sizeof(x) <= 4: result = countBitsImpl(x.uint32) + else: result = countBitsImpl(x.uint64) + +proc countBits32*(n: uint32): int {.compilerproc, inline.} = + result = countSetBitsImpl(n) + +proc countBits64*(n: uint64): int {.compilerproc, inline.} = + result = countSetBitsImpl(n) diff --git a/lib/system/sets.nim b/lib/system/sets.nim index 04e10ba04..103c8d343 100644 --- a/lib/system/sets.nim +++ b/lib/system/sets.nim @@ -12,7 +12,6 @@ type NimSet = array[0..4*2048-1, uint8] -# bitops can't be imported here, therefore the code duplication. proc cardSet(s: NimSet, len: int): int {.compilerproc, inline.} = var i = 0 From e5873b3a9300897443f0f2e2db128e3463089003 Mon Sep 17 00:00:00 2001 From: Saem Ghani Date: Mon, 22 Mar 2021 04:46:34 -0700 Subject: [PATCH 0462/2472] semTemplateDef and t17433 clean-ups (#17448) - use `doAssert` in t17433 - use setGenericParamsMisc in semTemplateDef akin to semProcAux - pragma handling in semTemplateDef inline with semProcAux --- compiler/sem.nim | 24 ++++++++++++++++++++++-- compiler/semstmts.nim | 18 ------------------ compiler/semtempl.nim | 36 ++++++++++++++++++------------------ tests/template/t17433.nim | 6 +++--- 4 files changed, 43 insertions(+), 41 deletions(-) diff --git a/compiler/sem.nim b/compiler/sem.nim index 5593f8b02..bb42b5c1b 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -485,11 +485,31 @@ proc semConstBoolExpr(c: PContext, n: PNode): PNode = if result.kind != nkIntLit: localError(c.config, n.info, errConstExprExpected) - proc semGenericStmt(c: PContext, n: PNode): PNode proc semConceptBody(c: PContext, n: PNode): PNode -include semtypes, semtempl, semgnrc, semstmts, semexprs +include semtypes + +proc setGenericParamsMisc(c: PContext; n: PNode) = + ## used by call defs (procs, templates, macros, ...) to analyse their generic + ## params, and store the originals in miscPos for better error reporting. + let orig = n[genericParamsPos] + + doAssert orig.kind in {nkEmpty, nkGenericParams} + + if n[genericParamsPos].kind == nkEmpty: + n[genericParamsPos] = newNodeI(nkGenericParams, n.info) + else: + # we keep the original params around for better error messages, see + # issue https://github.com/nim-lang/Nim/issues/1713 + n[genericParamsPos] = semGenericParamList(c, orig) + + if n[miscPos].kind == nkEmpty: + n[miscPos] = newTree(nkBracket, c.graph.emptyNode, orig) + else: + n[miscPos][1] = orig + +include semtempl, semgnrc, semstmts, semexprs proc addCodeForGenerics(c: PContext, n: PNode) = for i in c.lastGenericIdx..=", ">", "incl", "excl", "in", "notin", "isnot"] @@ -608,26 +610,23 @@ proc semTemplateDef(c: PContext, n: PNode): PNode = s.owner.name.s == "vm" and s.name.s == "stackTrace": incl(s.flags, sfCallsite) + s.ast = n + styleCheckDef(c.config, s) - onDef(n[0].info, s) + onDef(n[namePos].info, s) # check parameter list: #s.scope = c.currentScope pushOwner(c, s) openScope(c) - n[namePos] = newSymNode(s, n[namePos].info) - if n[pragmasPos].kind != nkEmpty: - pragma(c, s, n[pragmasPos], templatePragmas) + n[namePos] = newSymNode(s) + pragmaCallable(c, s, n, templatePragmas) + implicitPragmas(c, s, n.info, templatePragmas) - var gp: PNode - if n[genericParamsPos].kind != nkEmpty: - n[genericParamsPos] = semGenericParamList(c, n[genericParamsPos]) - gp = n[genericParamsPos] - else: - gp = newNodeI(nkGenericParams, n.info) + setGenericParamsMisc(c, n) # process parameters: var allUntyped = true if n[paramsPos].kind != nkEmpty: - semParamList(c, n[paramsPos], gp, s) + semParamList(c, n[paramsPos], n[genericParamsPos], s) # a template's parameters are not gensym'ed even if that was originally the # case as we determine whether it's a template parameter in the template # body by the absence of the sfGenSym flag: @@ -636,18 +635,21 @@ proc semTemplateDef(c: PContext, n: PNode): PNode = param.flags.incl sfTemplateParam param.flags.excl sfGenSym if param.typ.kind != tyUntyped: allUntyped = false - if gp.len > 0 and n[genericParamsPos].kind == nkEmpty: - # we have a list of implicit type parameters: - n[genericParamsPos] = gp else: s.typ = newTypeS(tyProc, c) # XXX why do we need tyTyped as a return type again? s.typ.n = newNodeI(nkFormalParams, n.info) rawAddSon(s.typ, newTypeS(tyTyped, c)) s.typ.n.add newNodeIT(nkType, n.info, s.typ[0]) + if n[genericParamsPos].safeLen == 0: + # restore original generic type params as no explicit or implicit were found + n[genericParamsPos] = n[miscPos][1] + n[miscPos] = c.graph.emptyNode if allUntyped: incl(s.flags, sfAllUntyped) + if n[patternPos].kind != nkEmpty: n[patternPos] = semPattern(c, n[patternPos]) + var ctx: TemplCtx ctx.toBind = initIntSet() ctx.toMixin = initIntSet() @@ -662,8 +664,6 @@ proc semTemplateDef(c: PContext, n: PNode): PNode = semIdeForTemplateOrGeneric(c, n[bodyPos], ctx.cursorInBody) closeScope(c) popOwner(c) - s.ast = n - result = n if sfCustomPragma in s.flags: if n[bodyPos].kind != nkEmpty: localError(c.config, n[bodyPos].info, errImplOfXNotAllowed % s.name.s) diff --git a/tests/template/t17433.nim b/tests/template/t17433.nim index 121f3c471..053d33ff0 100644 --- a/tests/template/t17433.nim +++ b/tests/template/t17433.nim @@ -7,10 +7,10 @@ from std/macros import expandMacros proc bar(a: typedesc): a = default(a) -assert bar(float) == 0.0 -assert bar(string) == "" +doAssert bar(float) == 0.0 +doAssert bar(string) == "" template main = proc baz(a: typedesc): a = default(a) - assert baz(float) == 0.0 + doAssert baz(float) == 0.0 main() From a75c4b70e86705c98a4006aa8475a10a0ebf94ec Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 22 Mar 2021 18:24:28 -0700 Subject: [PATCH 0463/2472] hint:cc goes to stderr (like all other hints) instead of stdout (#17465) Co-authored-by: ee7 <45465154+ee7@users.noreply.github.com> --- changelog.md | 1 + compiler/extccomp.nim | 15 +++++++-------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/changelog.md b/changelog.md index cc4f54fe0..00fc970bf 100644 --- a/changelog.md +++ b/changelog.md @@ -301,6 +301,7 @@ - Added `unsafeIsolate` and `extract` to `std/isolation`. +- `--hint:CC` now goes to stderr (like all other hints) instead of stdout. ## Tool changes diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index e95ed893b..e0fd5e206 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -281,6 +281,11 @@ const hExt* = ".h" +template writePrettyCmdsStderr(cmd) = + if cmd.len > 0: + flushDot(conf) + stderr.writeLine(cmd) + proc nameToCC*(name: string): TSystemCC = ## Returns the kind of compiler referred to by `name`, or ccNone ## if the name doesn't refer to any known compiler. @@ -848,11 +853,7 @@ proc callCCompiler*(conf: ConfigRef) = var script: Rope = nil var cmds: TStringSeq var prettyCmds: TStringSeq - let prettyCb = proc (idx: int) = - if prettyCmds[idx].len > 0: - flushDot(conf) - # xxx should probably use stderr like other compiler messages, not stdout - echo prettyCmds[idx] + let prettyCb = proc (idx: int) = writePrettyCmdsStderr(prettyCmds[idx]) for idx, it in conf.toCompile: # call the C compiler for the .c file: @@ -1108,9 +1109,7 @@ proc runJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile) = doAssert toCompile.kind == JArray var cmds: TStringSeq var prettyCmds: TStringSeq - let prettyCb = proc (idx: int) = - if prettyCmds[idx].len > 0: echo prettyCmds[idx] - + let prettyCb = proc (idx: int) = writePrettyCmdsStderr(prettyCmds[idx]) for c in toCompile: doAssert c.kind == JArray doAssert c.len >= 2 From 4f9aaee1d9a30976e4ca9bff6fb428960a6b92b0 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 23 Mar 2021 00:28:53 -0700 Subject: [PATCH 0464/2472] remove redundant void return in stdlib (#17464) --- lib/posix/linux.nim | 2 +- lib/posix/posix.nim | 2 +- lib/pure/concurrency/threadpool.nim | 4 ++-- lib/pure/coro.nim | 3 +-- lib/pure/json.nim | 2 +- lib/pure/net.nim | 2 +- lib/windows/winlean.nim | 8 ++++---- 7 files changed, 11 insertions(+), 12 deletions(-) diff --git a/lib/posix/linux.nim b/lib/posix/linux.nim index 5ce9bf2fb..2d6702e87 100644 --- a/lib/posix/linux.nim +++ b/lib/posix/linux.nim @@ -31,7 +31,7 @@ const CLONE_IO* = 0x80000000'i32 CLONE_STOPPED* {.deprecated.} = 0x02000000'i32 -# fn should be of type proc (a2: pointer): void {.cdecl.} +# fn should be of type proc (a2: pointer) {.cdecl.} proc clone*(fn: pointer; child_stack: pointer; flags: cint; arg: pointer; ptid: ptr Pid; tls: pointer; ctid: ptr Pid): cint {.importc, header: "".} diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 520d0caa0..035b4e8fb 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -447,7 +447,7 @@ proc pthread_spin_unlock*(a1: ptr Pthread_spinlock): cint {. proc pthread_testcancel*() {.importc, header: "".} -proc exitnow*(code: int): void {.importc: "_exit", header: "".} +proc exitnow*(code: int) {.importc: "_exit", header: "".} proc access*(a1: cstring, a2: cint): cint {.importc, header: "".} proc alarm*(a1: cint): cint {.importc, header: "".} proc chdir*(a1: cstring): cint {.importc, header: "".} diff --git a/lib/pure/concurrency/threadpool.nim b/lib/pure/concurrency/threadpool.nim index 905e668ce..9beb39522 100644 --- a/lib/pure/concurrency/threadpool.nim +++ b/lib/pure/concurrency/threadpool.nim @@ -451,14 +451,14 @@ proc preferSpawn*(): bool = ## <#spawnX.t>`_ instead. result = gSomeReady.counter > 0 -proc spawn*(call: sink typed): void {.magic: "Spawn".} +proc spawn*(call: sink typed) {.magic: "Spawn".} ## Always spawns a new task, so that the `call` is never executed on ## the calling thread. ## ## `call` has to be a proc call `p(...)` where `p` is gcsafe and has a ## return type that is either `void` or compatible with `FlowVar[T]`. -proc pinnedSpawn*(id: ThreadId; call: sink typed): void {.magic: "Spawn".} +proc pinnedSpawn*(id: ThreadId; call: sink typed) {.magic: "Spawn".} ## Always spawns a new task on the worker thread with `id`, so that ## the `call` is **always** executed on the thread. ## diff --git a/lib/pure/coro.nim b/lib/pure/coro.nim index 21675928d..af8acf268 100644 --- a/lib/pure/coro.nim +++ b/lib/pure/coro.nim @@ -273,8 +273,7 @@ proc start*(c: proc(), stacksize: int = defaultStackSize): CoroutineRef {.discar coro = cast[CoroutinePtr](alloc0(sizeof(Coroutine))) coro.execContext = CreateFiberEx(stacksize, stacksize, FIBER_FLAG_FLOAT_SWITCH, - (proc(p: pointer): void {.stdcall.} = runCurrentTask()), - nil) + (proc(p: pointer) {.stdcall.} = runCurrentTask()), nil) coro.stack.size = stacksize else: coro = cast[CoroutinePtr](alloc0(sizeof(Coroutine) + stacksize)) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 13cd3fb85..c09af32f3 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -1165,7 +1165,7 @@ when defined(nimFixedForwardGeneric): proc initFromJson[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) = assignDistinctImpl(dst, jsonNode, jsonPath) - proc detectIncompatibleType(typeExpr, lineinfoNode: NimNode): void = + proc detectIncompatibleType(typeExpr, lineinfoNode: NimNode) = if typeExpr.kind == nnkTupleConstr: error("Use a named tuple instead of: " & typeExpr.repr, lineinfoNode) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 37e6e8f5a..b37782271 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -200,7 +200,7 @@ when defined(nimHasStyleChecks): proc socketError*(socket: Socket, err: int = -1, async = false, lastError = (-1).OSErrorCode, - flags: set[SocketFlag] = {}): void {.gcsafe.} + flags: set[SocketFlag] = {}) {.gcsafe.} proc isDisconnectionError*(flags: set[SocketFlag], lastError: OSErrorCode): bool = diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index 1cdb20fe7..6e4c438e6 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -1030,7 +1030,7 @@ const PROCESS_QUERY_LIMITED_INFORMATION* = 0x00001000'i32 PROCESS_SET_LIMITED_INFORMATION* = 0x00002000'i32 type - WAITORTIMERCALLBACK* = proc(para1: pointer, para2: int32): void {.stdcall.} + WAITORTIMERCALLBACK* = proc(para1: pointer, para2: int32) {.stdcall.} proc postQueuedCompletionStatus*(CompletionPort: Handle, dwNumberOfBytesTransferred: DWORD, @@ -1112,7 +1112,7 @@ else: {.stdcall, dynlib: "kernel32", importc: "ReadConsoleInputW".} type - LPFIBER_START_ROUTINE* = proc (param: pointer): void {.stdcall.} + LPFIBER_START_ROUTINE* = proc (param: pointer) {.stdcall.} const FIBER_FLAG_FLOAT_SWITCH* = 0x01 @@ -1121,8 +1121,8 @@ proc CreateFiber*(stackSize: int, fn: LPFIBER_START_ROUTINE, param: pointer): po proc CreateFiberEx*(stkCommit: int, stkReserve: int, flags: int32, fn: LPFIBER_START_ROUTINE, param: pointer): pointer {.stdcall, discardable, dynlib: "kernel32", importc.} proc ConvertThreadToFiber*(param: pointer): pointer {.stdcall, discardable, dynlib: "kernel32", importc.} proc ConvertThreadToFiberEx*(param: pointer, flags: int32): pointer {.stdcall, discardable, dynlib: "kernel32", importc.} -proc DeleteFiber*(fiber: pointer): void {.stdcall, discardable, dynlib: "kernel32", importc.} -proc SwitchToFiber*(fiber: pointer): void {.stdcall, discardable, dynlib: "kernel32", importc.} +proc DeleteFiber*(fiber: pointer) {.stdcall, discardable, dynlib: "kernel32", importc.} +proc SwitchToFiber*(fiber: pointer) {.stdcall, discardable, dynlib: "kernel32", importc.} proc GetCurrentFiber*(): pointer {.stdcall, importc, header: "windows.h".} proc toFILETIME*(t: int64): FILETIME = From b50776dd2facae166eb9d1bdbcff205c01adc25b Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 23 Mar 2021 00:30:41 -0700 Subject: [PATCH 0465/2472] refs #16338 make thttpclient_ssl_remotenetwork.nim less flaky (#17457) --- .../thttpclient_ssl_remotenetwork.nim | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/tests/untestable/thttpclient_ssl_remotenetwork.nim b/tests/untestable/thttpclient_ssl_remotenetwork.nim index eea65bf0b..f56fe274b 100644 --- a/tests/untestable/thttpclient_ssl_remotenetwork.nim +++ b/tests/untestable/thttpclient_ssl_remotenetwork.nim @@ -115,12 +115,23 @@ when enableRemoteNetworking and (defined(nimTestsEnableFlaky) or not defined(win else: # this is unexpected + var fatal = true + var msg = "" if raised: - echo " $# ($#) raised: $#" % [desc, $category, exception_msg] + msg = " $# ($#) raised: $#" % [desc, $category, exception_msg] + if "500 Internal Server Error" in exception_msg: + # refs https://github.com/nim-lang/Nim/issues/16338#issuecomment-804300278 + # we got: `good (good) raised: 500 Internal Server Error` + fatal = false + msg.add " (http 500 => assuming this is not our problem)" else: - echo " $# ($#) did not raise" % [desc, $category] - if category in {good, dubious, bad}: + msg = " $# ($#) did not raise" % [desc, $category] + + if category in {good, dubious, bad} and fatal: + echo "D20210322T121353: error: " & msg fail() + else: + echo "D20210322T121353: warning: " & msg suite "SSL certificate check - httpclient": From d78ebe4a0eadb889255c77f6e375ecec9c706f19 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 23 Mar 2021 00:33:09 -0700 Subject: [PATCH 0466/2472] fix #17454 (#17461) --- lib/core/macros.nim | 20 +++++++++----------- tests/stdlib/tmacros.nim | 5 ++++- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 491235d8b..8b45ed86b 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -1287,7 +1287,15 @@ proc `body=`*(someProc: NimNode, val: NimNode) {.compileTime.} = else: badNodeKind someProc, "body=" -proc basename*(a: NimNode): NimNode {.compileTime, benign.} +proc basename*(a: NimNode): NimNode {.raises: [].} = + ## Pull an identifier from prefix/postfix expressions. + case a.kind + of nnkIdent: result = a + of nnkPostfix, nnkPrefix: result = a[1] + of nnkPragmaExpr: result = basename(a[0]) + else: + error("Do not know how to get basename of (" & treeRepr(a) & ")\n" & + repr(a), a) proc `$`*(node: NimNode): string {.compileTime.} = ## Get the string of an identifier node. @@ -1347,16 +1355,6 @@ proc insert*(a: NimNode; pos: int; b: NimNode) {.compileTime.} = a[i + 1] = a[i] a[pos] = b -proc basename*(a: NimNode): NimNode = - ## Pull an identifier from prefix/postfix expressions. - case a.kind - of nnkIdent: result = a - of nnkPostfix, nnkPrefix: result = a[1] - of nnkPragmaExpr: result = basename(a[0]) - else: - error("Do not know how to get basename of (" & treeRepr(a) & ")\n" & - repr(a), a) - proc `basename=`*(a: NimNode; val: string) {.compileTime.}= case a.kind of nnkIdent: diff --git a/tests/stdlib/tmacros.nim b/tests/stdlib/tmacros.nim index 0cc1e340c..effe1032f 100644 --- a/tests/stdlib/tmacros.nim +++ b/tests/stdlib/tmacros.nim @@ -1,4 +1,4 @@ -import macros +import std/macros block: # hasArgOfName macro m(u: untyped): untyped = @@ -7,3 +7,6 @@ block: # hasArgOfName doAssert not hasArgOfName(params u,"nonexistent") proc p(s: string; i,j,k: int; b: bool; xs,ys: seq[int] = @[]) {.m.} = discard + +block: # bug #17454 + proc f(v: NimNode): string {.raises: [].} = $v From f3a642710989c3f47d5c2131efd69e3dbfedc336 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 23 Mar 2021 00:34:04 -0700 Subject: [PATCH 0467/2472] refs #17292 fix `repr` with `do:` (#17449) * refs #17292 fix `repr` with `do:` * address comment --- compiler/renderer.nim | 13 +++++-- tests/stdlib/trepr.nim | 80 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 3 deletions(-) diff --git a/compiler/renderer.nim b/compiler/renderer.nim index 13ff6941f..c36eaaf11 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -1007,12 +1007,19 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) = of nkCall, nkConv, nkDotCall, nkPattern, nkObjConstr: if n.len > 1 and n.lastSon.kind in {nkStmtList, nkStmtListExpr}: accentedName(g, n[0]) - if n.len > 2: + var i = 1 + while i < n.len and n[i].kind notin {nkStmtList, nkStmtListExpr}: i.inc + if i > 1: put(g, tkParLe, "(") - gcomma(g, n, 1, -2) + gcomma(g, n, 1, i - 1 - n.len) put(g, tkParRi, ")") put(g, tkColon, ":") - gsub(g, n, n.len-1) + gsub(g, n, i) + for j in i+1 ..< n.len: + optNL(g) + put(g, tkDo, "do") + put(g, tkColon, ":") + gsub(g, n, j) elif n.len >= 1: case bracketKind(g, n[0]) of bkBracket: diff --git a/tests/stdlib/trepr.nim b/tests/stdlib/trepr.nim index 1d89d37b2..357854d67 100644 --- a/tests/stdlib/trepr.nim +++ b/tests/stdlib/trepr.nim @@ -6,6 +6,8 @@ discard """ # if excessive, could remove 'cpp' from targets from strutils import endsWith, contains +from std/macros import newLit +macro deb(a): string = newLit a.repr template main() = doAssert repr({3,5}) == "{3, 5}" @@ -65,5 +67,83 @@ template main() = else: doAssert reprOpenarray(arr) == "[1, 2, 3]" + block: # bug #17292 + template foo(a, b, c, d) = discard + block: + let a = deb: + foo(1, 2, 3, 4) + doAssert a == "\nfoo(1, 2, 3, 4)" + block: + let a = deb: + foo(1, 2, 3): 4 + doAssert a == """ + +foo(1, 2, 3): + 4""" + + block: + let a = deb: + foo(1, 2): 3 + do: 4 + doAssert a == """ + +foo(1, 2): + 3 +do: + 4""" + + block: + let a = deb: + foo(1): 3 + do: 3 + do: 4 + doAssert a == """ + +foo(1): + 3 +do: + 3 +do: + 4""" + + block: + let a = deb: + foo(1): + 3 + do: + discard + 3 + do: + discard + 4 + + doAssert a == """ + +foo(1): + 3 +do: + discard + 3 +do: + discard + 4""" + + block: + let a = deb: + foo: 1 + do: 2 + do: 3 + do: 4 + doAssert a == """ + +foo: + 1 +do: + 2 +do: + 3 +do: + 4""" + static: main() main() From 1d19cd660f39b9e76005b849ae1f7ca966278438 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 23 Mar 2021 00:37:15 -0700 Subject: [PATCH 0468/2472] fix #17458 tnetconnect.nim flaky (#17459) * fix tests * fix #17458 --- tests/stdlib/tnetconnect.nim | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/tests/stdlib/tnetconnect.nim b/tests/stdlib/tnetconnect.nim index e27499651..b74545710 100644 --- a/tests/stdlib/tnetconnect.nim +++ b/tests/stdlib/tnetconnect.nim @@ -1,22 +1,26 @@ discard """ - cmd: "nim c -r -d:ssl $file" - exitcode: 0 + matrix: "-d:ssl" """ import std/net +from std/strutils import `%` -# Issue 15215 - https://github.com/nim-lang/Nim/issues/15215 +# bug #15215 proc test() = - var - ctx = newContext() - socket = newSocket() + let ctx = newContext() - wrapSocket(ctx, socket) + proc fn(url: string) = + let socket = newSocket() + defer: close(socket) + connect(socket, url, Port(443), 5000) # typically 20 could be enough + send(socket, "GET / HTTP/1.0\nHost: $#\nConnection: close\n\n" % [url]) + wrapSocket(ctx, socket) - connect(socket, "www.nim-lang.org", Port(443), 5000) - - send(socket, "GET / HTTP/1.0\nHost: www.nim-lang.org\nConnection: close\n\n") - - close(socket) + # trying 2 sites makes it more resilent: refs #17458 this could give: + # Error: unhandled exception: Call to 'connect' timed out. [TimeoutError] + try: + fn("www.nim-lang.org") + except TimeoutError: + fn("www.google.com") test() From d6a16020fe98b467459544ee4cfafe61d681f918 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 23 Mar 2021 08:52:00 +0100 Subject: [PATCH 0469/2472] IC: backend: remember produced type information (#17440) --- compiler/ast.nim | 15 ++++---------- compiler/ccgtypes.nim | 14 ++++++++++--- compiler/ic/cbackend.nim | 43 ++++++++++++++++++++++++++++++++------- compiler/ic/ic.nim | 11 ++++++++-- compiler/ic/rodfiles.nim | 1 + compiler/modulegraphs.nim | 29 ++++++++++++++++++++++---- compiler/passes.nim | 5 +++++ compiler/sem.nim | 2 +- compiler/semdata.nim | 15 +------------- 9 files changed, 93 insertions(+), 42 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 50a2fb58c..1f3d5f129 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -1229,17 +1229,10 @@ proc newSym*(symKind: TSymKind, name: PIdent, id: ItemId, owner: PSym, result = PSym(name: name, kind: symKind, flags: {}, info: info, itemId: id, options: options, owner: owner, offset: defaultOffset) when false: - if id.item > 2141: - let s = getStackTrace() - const words = ["createTypeBoundOps", - "initOperators", - "generateInstance", - "semIdentDef", "addLocalDecl"] - for w in words: - if w in s: - x.inc w - return - x.inc "" + if id.module == 48 and id.item == 39: + writeStackTrace() + echo "kind ", symKind, " ", name.s + if owner != nil: echo owner.name.s proc astdef*(s: PSym): PNode = # get only the definition (initializer) portion of the ast diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 73ee9cb8a..e17a55542 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -1386,7 +1386,6 @@ proc genTypeInfoV2(m: BModule, t: PType; info: TLineInfo): Rope = let owner = t.skipTypes(typedescPtrs).itemId.module if owner != m.module.position and moduleOpenForCodegen(m, owner): # make sure the type info is created in the owner module - assert m.g.modules[owner] != nil discard genTypeInfoV2(m.g.modules[owner], origType, info) # reference the type info as extern here discard cgsym(m, "TNimTypeV2") @@ -1456,18 +1455,27 @@ proc genTypeInfoV1(m: BModule, t: PType; info: TLineInfo): Rope = result = "NTI$1$2_" % [rope(typeToC(t)), rope($sig)] m.typeInfoMarker[sig] = result - let owner = t.skipTypes(typedescPtrs).itemId.module + let old = m.g.graph.emittedTypeInfo.getOrDefault($result) + if old != FileIndex(0): + discard cgsym(m, "TNimType") + discard cgsym(m, "TNimNode") + declareNimType(m, "TNimType", result, old.int) + return prefixTI.rope & result & ")".rope + + var owner = t.skipTypes(typedescPtrs).itemId.module if owner != m.module.position and moduleOpenForCodegen(m, owner): # make sure the type info is created in the owner module - assert m.g.modules[owner] != nil discard genTypeInfoV1(m.g.modules[owner], origType, info) # reference the type info as extern here discard cgsym(m, "TNimType") discard cgsym(m, "TNimNode") declareNimType(m, "TNimType", result, owner) return prefixTI.rope & result & ")".rope + else: + owner = m.module.position.int32 m.g.typeInfoMarker[sig] = (str: result, owner: owner) + rememberEmittedTypeInfo(m.g.graph, FileIndex(owner), $result) case t.kind of tyEmpty, tyVoid: result = rope"0" diff --git a/compiler/ic/cbackend.nim b/compiler/ic/cbackend.nim index 88b2a9477..34ee59d52 100644 --- a/compiler/ic/cbackend.nim +++ b/compiler/ic/cbackend.nim @@ -18,7 +18,7 @@ ## also doing cross-module dependency tracking and DCE that we don't need ## anymore. DCE is now done as prepass over the entire packed module graph. -import std/[packedsets, algorithm] +import std/[packedsets, algorithm, tables] # std/intsets would give `UnusedImport`, pending https://github.com/nim-lang/Nim/issues/14246 import ".."/[ast, options, lineinfos, modulegraphs, cgendata, cgen, pathutils, extccomp, msgs] @@ -45,6 +45,10 @@ proc generateCodeForModule(g: ModuleGraph; m: var LoadedModule; alive: var Alive finalCodegenActions(g, bmod, newNodeI(nkStmtList, m.module.info)) +proc replayTypeInfo(g: ModuleGraph; m: var LoadedModule; origin: FileIndex) = + for x in mitems(m.fromDisk.emittedTypeInfo): + g.emittedTypeInfo[x] = origin + proc addFileToLink(config: ConfigRef; m: PSym) = let filename = AbsoluteFile toFullPath(config, m.position.FileIndex) let ext = @@ -59,11 +63,28 @@ proc addFileToLink(config: ConfigRef; m: PSym) = flags: {CfileFlag.Cached}) addFileToCompile(config, cf) -proc aliveSymsChanged(config: ConfigRef; position: int; alive: AliveSyms): bool = +when defined(debugDce): + import std / [os, packedsets] + +proc storeAliveSymsImpl(asymFile: AbsoluteFile; s: seq[int32]) = + var f = rodfiles.create(asymFile.string) + f.storeHeader() + f.storeSection aliveSymsSection + f.storeSeq(s) + close f + +template prepare {.dirty.} = let asymFile = toRodFile(config, AbsoluteFile toFullPath(config, position.FileIndex), ".alivesyms") var s = newSeqOfCap[int32](alive[position].len) for a in items(alive[position]): s.add int32(a) sort(s) + +proc storeAliveSyms(config: ConfigRef; position: int; alive: AliveSyms) = + prepare() + storeAliveSymsImpl(asymFile, s) + +proc aliveSymsChanged(config: ConfigRef; position: int; alive: AliveSyms): bool = + prepare() var f2 = rodfiles.open(asymFile.string) f2.loadHeader() f2.loadSection aliveSymsSection @@ -73,12 +94,17 @@ proc aliveSymsChanged(config: ConfigRef; position: int; alive: AliveSyms): bool if f2.err == ok and oldData == s: result = false else: + when defined(debugDce): + let oldAsSet = toPackedSet[int32](oldData) + let newAsSet = toPackedSet[int32](s) + echo "set of live symbols changed ", asymFile.changeFileExt("rod"), " ", position, " ", f2.err + echo "in old but not in new ", oldAsSet.difference(newAsSet) + echo "in new but not in old ", newAsSet.difference(oldAsSet) + + if execShellCmd(getAppFilename() & " rod " & quoteShell(asymFile.changeFileExt("rod"))) != 0: + echo "command failed" result = true - var f = rodfiles.create(asymFile.string) - f.storeHeader() - f.storeSection aliveSymsSection - f.storeSeq(s) - close f + storeAliveSymsImpl(asymFile, s) proc generateCode*(g: ModuleGraph) = ## The single entry point, generate C(++) code for the entire @@ -95,6 +121,8 @@ proc generateCode*(g: ModuleGraph) = assert false of storing, outdated: generateCodeForModule(g, g.packed[i], alive) + closeRodFile(g, g.packed[i].module) + storeAliveSyms(g.config, g.packed[i].module.position, alive) of loaded: # Even though this module didn't change, DCE might trigger a change. # Consider this case: Module A uses symbol S from B and B does not use @@ -104,3 +132,4 @@ proc generateCode*(g: ModuleGraph) = generateCodeForModule(g, g.packed[i], alive) else: addFileToLink(g.config, g.packed[i].module) + replayTypeInfo(g, g.packed[i], FileIndex(i)) diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim index 99a68e0f0..230b4d087 100644 --- a/compiler/ic/ic.nim +++ b/compiler/ic/ic.nim @@ -42,6 +42,8 @@ type methodsPerType*: seq[(PackedItemId, int, PackedItemId)] enumToStringProcs*: seq[(PackedItemId, PackedItemId)] + emittedTypeInfo*: seq[string] + sh*: Shared cfg: PackedConfig @@ -58,7 +60,7 @@ type config*: ConfigRef proc isActive*(e: PackedEncoder): bool = e.config != nil -proc disable*(e: var PackedEncoder) = e.config = nil +proc disable(e: var PackedEncoder) = e.config = nil template primConfigFields(fn: untyped) {.dirty.} = fn backend @@ -552,6 +554,7 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef loadSeqSection attachedOpsSection, m.attachedOps loadSeqSection methodsPerTypeSection, m.methodsPerType loadSeqSection enumToStringProcsSection, m.enumToStringProcs + loadSeqSection typeInfoSection, m.emittedTypeInfo close(f) result = f.err @@ -614,6 +617,7 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac storeSeqSection attachedOpsSection, m.attachedOps storeSeqSection methodsPerTypeSection, m.methodsPerType storeSeqSection enumToStringProcsSection, m.enumToStringProcs + storeSeqSection typeInfoSection, m.emittedTypeInfo close(f) encoder.disable() @@ -1139,7 +1143,10 @@ proc rodViewer*(rodfile: AbsoluteFile; config: ConfigRef, cache: IdentCache) = echo "all symbols" for i in 0..high(m.sh.syms): - echo " ", m.sh.strings[m.sh.syms[i].name], " local ID: ", i + if m.sh.syms[i].name != LitId(0): + echo " ", m.sh.strings[m.sh.syms[i].name], " local ID: ", i, " kind ", m.sh.syms[i].kind + else: + echo " local ID: ", i, " kind ", m.sh.syms[i].kind echo "symbols: ", m.sh.syms.len, " types: ", m.sh.types.len, " top level nodes: ", m.topLevel.nodes.len, " other nodes: ", m.bodies.nodes.len, diff --git a/compiler/ic/rodfiles.nim b/compiler/ic/rodfiles.nim index fa0a7c734..a518870f8 100644 --- a/compiler/ic/rodfiles.nim +++ b/compiler/ic/rodfiles.nim @@ -36,6 +36,7 @@ type attachedOpsSection methodsPerTypeSection enumToStringProcsSection + typeInfoSection # required by the backend aliveSymsSection # beware, this is stored in a `.alivesyms` file. RodFileError* = enum diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index de3773ca5..9ac76457c 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -11,9 +11,8 @@ ## represents a complete Nim project. Single modules can either be kept in RAM ## or stored in a rod-file. -import ast, astalgo, intsets, tables, options, lineinfos, hashes, idents, - btrees, md5, ropes, msgs - +import std / [intsets, tables, hashes, md5] +import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils import ic / [packed_ast, ic] type @@ -60,6 +59,7 @@ type attachedOps*: array[TTypeAttachedOp, Table[ItemId, PSym]] # Type ID, destructors, etc. methodsPerType*: Table[ItemId, seq[(int, LazySym)]] # Type ID, attached methods enumToStringProcs*: Table[ItemId, LazySym] + emittedTypeInfo*: Table[string, FileIndex] startupPackedConfig*: PackedConfig packageSyms*: TStrTable @@ -68,7 +68,6 @@ type importDeps*: Table[FileIndex, seq[FileIndex]] # explicit import module dependencies suggestMode*: bool # whether we are in nimsuggest mode or not. invalidTransitiveClosure: bool - systemModuleComplete*: bool inclToMod*: Table[FileIndex, FileIndex] # mapping of include file to the # first module that included it importStack*: seq[FileIndex] # The current import stack. Used for detecting recursive @@ -435,6 +434,7 @@ proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph = result.canonTypes = initTable[SigHash, PType]() result.symBodyHashes = initTable[int, SigHash]() result.operators = initOperators(result) + result.emittedTypeInfo = initTable[string, FileIndex]() proc resetAllModules*(g: ModuleGraph) = initStrTable(g.packageSyms) @@ -455,6 +455,27 @@ proc getModule*(g: ModuleGraph; fileIdx: FileIndex): PSym = elif fileIdx.int32 < g.ifaces.len: result = g.ifaces[fileIdx.int32].module +proc rememberEmittedTypeInfo*(g: ModuleGraph; m: FileIndex; ti: string) = + #assert(not isCachedModule(g, m.int32)) + if g.config.symbolFiles != disabledSf: + #assert g.encoders[m.int32].isActive + g.packed[m.int32].fromDisk.emittedTypeInfo.add ti + +proc closeRodFile*(g: ModuleGraph; m: PSym) = + if g.config.symbolFiles in {readOnlySf, v2Sf}: + # For stress testing we seek to reload the symbols from memory. This + # way much of the logic is tested but the test is reproducible as it does + # not depend on the hard disk contents! + let mint = m.position + saveRodFile(toRodFile(g.config, AbsoluteFile toFullPath(g.config, FileIndex(mint))), + g.encoders[mint], g.packed[mint].fromDisk) + elif g.config.symbolFiles == stressTest: + # debug code, but maybe a good idea for production? Could reduce the compiler's + # memory consumption considerably at the cost of more loads from disk. + let mint = m.position + simulateCachedModule(g, m, g.packed[mint].fromDisk) + g.packed[mint].status = loaded + proc dependsOn(a, b: int): int {.inline.} = (a shl 15) + b proc addDep*(g: ModuleGraph; m: PSym, dep: FileIndex) = diff --git a/compiler/passes.nim b/compiler/passes.nim index 11799b122..3debce1f6 100644 --- a/compiler/passes.nim +++ b/compiler/passes.nim @@ -187,4 +187,9 @@ proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator; closeParser(p) if s.kind != llsStdIn: break closePasses(graph, a) + if graph.config.backend notin {backendC, backendCpp, backendObjc}: + # We only write rod files here if no C-like backend is active. + # The C-like backends have been patched to support the IC mechanism. + # They are responsible for closing the rod files. See `cbackend.nim`. + closeRodFile(graph, module) result = true diff --git a/compiler/sem.nim b/compiler/sem.nim index bb42b5c1b..6b3f0c80d 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -661,7 +661,7 @@ proc myClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode = result.add(c.module.ast) popOwner(c) popProcCon(c) - saveRodFile(c) + sealRodFile(c) const semPass* = makePass(myOpen, myProcess, myClose, isFrontend = true) diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 5cd440cc1..c655047e2 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -560,23 +560,10 @@ proc addToGenericCache*(c: PContext; s: PSym; inst: PType) = if c.config.symbolFiles != disabledSf: storeTypeInst(c.encoder, c.packedRepr, s, inst) -proc saveRodFile*(c: PContext) = +proc sealRodFile*(c: PContext) = if c.config.symbolFiles != disabledSf: if c.graph.vm != nil: for (m, n) in PCtx(c.graph.vm).vmstateDiff: if m == c.module: addPragmaComputation(c, n) - if sfSystemModule in c.module.flags: - c.graph.systemModuleComplete = true c.idgen.sealed = true # no further additions are allowed - if c.config.symbolFiles != stressTest: - # For stress testing we seek to reload the symbols from memory. This - # way much of the logic is tested but the test is reproducible as it does - # not depend on the hard disk contents! - saveRodFile(toRodFile(c.config, AbsoluteFile toFullPath(c.config, FileIndex c.module.position)), - c.encoder, c.packedRepr) - else: - # debug code, but maybe a good idea for production? Could reduce the compiler's - # memory consumption considerably at the cost of more loads from disk. - simulateCachedModule(c.graph, c.module, c.packedRepr) - c.graph.packed[c.module.position].status = loaded From 64e6670de45f8f5a470e5cc3ad9ed2cb85f0b57c Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 23 Mar 2021 01:31:23 -0700 Subject: [PATCH 0470/2472] fix #16973 ; nim doc now shows correct, canonical import name in title (#16999) * nim doc now shows correct import name in title --- compiler/docgen.nim | 25 +++++++++++----- compiler/options.nim | 29 ++++++++++++------- .../expected/index.html | 4 +-- 3 files changed, 38 insertions(+), 20 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 57d5e7b01..dded231d7 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -63,7 +63,22 @@ proc prettyString(a: object): string = for k, v in fieldPairs(a): result.add k & ": " & $v & "\n" -proc presentationPath*(conf: ConfigRef, file: AbsoluteFile, isTitle = false): RelativeFile = +proc canonicalImport*(conf: ConfigRef, file: AbsoluteFile): string = + ##[ + Shows the canonical module import, e.g.: + system, std/tables, fusion/pointers, system/assertions, std/private/asciitables + ]## + var ret = getRelativePathFromConfigPath(conf, file, isTitle = true) + let dir = getNimbleFile(conf, $file).parentDir.AbsoluteDir + if not dir.isEmpty: + let relPath = relativeTo(file, dir) + if not relPath.isEmpty and (ret.isEmpty or relPath.string.len < ret.string.len): + ret = relPath + if ret.isEmpty: + ret = relativeTo(file, conf.projectPath) + result = ret.string.nativeToUnixPath.changeFileExt("") + +proc presentationPath*(conf: ConfigRef, file: AbsoluteFile): RelativeFile = ## returns a relative file that will be appended to outDir let file2 = $file template bail() = @@ -97,10 +112,7 @@ proc presentationPath*(conf: ConfigRef, file: AbsoluteFile, isTitle = false): Re bail() if isAbsolute(result.string): result = file.string.splitPath()[1].RelativeFile - if isTitle: - result = result.string.nativeToUnixPath.RelativeFile - else: - result = result.string.replace("..", dotdotMangle).RelativeFile + result = result.string.replace("..", dotdotMangle).RelativeFile doAssert not result.isEmpty doAssert not isAbsolute(result.string) @@ -1259,8 +1271,7 @@ proc genOutFile(d: PDoc, groupedToc = false): Rope = setIndexTerm(d[], external, "", title) else: # Modules get an automatic title for the HTML, but no entry in the index. - # better than `extractFilename(changeFileExt(d.filename, ""))` as it disambiguates dups - title = $presentationPath(d.conf, AbsoluteFile d.filename, isTitle = true).changeFileExt("") + title = canonicalImport(d.conf, AbsoluteFile d.filename) var subtitle = "".rope if d.meta[metaSubtitle] != "": dispA(d.conf, subtitle, "

    $1

    ", diff --git a/compiler/options.nim b/compiler/options.nim index a2d6a51b3..2d63043df 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -761,8 +761,25 @@ when (NimMajor, NimMinor) < (1, 1) or not declared(isRelativeTo): let ret = relativePath(path, base) result = path.len > 0 and not ret.startsWith ".." -proc getRelativePathFromConfigPath*(conf: ConfigRef; f: AbsoluteFile): RelativeFile = +const stdlibDirs = [ + "pure", "core", "arch", + "pure/collections", + "pure/concurrency", + "pure/unidecode", "impure", + "wrappers", "wrappers/linenoise", + "windows", "posix", "js"] + +const + pkgPrefix = "pkg/" + stdPrefix = "std/" + +proc getRelativePathFromConfigPath*(conf: ConfigRef; f: AbsoluteFile, isTitle = false): RelativeFile = let f = $f + if isTitle: + for dir in stdlibDirs: + let path = conf.libpath.string / dir / f.lastPathPart + if path.cmpPaths(f) == 0: + return RelativeFile(stdPrefix & f.splitFile.name) template search(paths) = for it in paths: let it = $it @@ -784,18 +801,8 @@ proc findFile*(conf: ConfigRef; f: string; suppressStdlib = false): AbsoluteFile result = rawFindFile2(conf, RelativeFile f.toLowerAscii) patchModule(conf) -const stdlibDirs = [ - "pure", "core", "arch", - "pure/collections", - "pure/concurrency", - "pure/unidecode", "impure", - "wrappers", "wrappers/linenoise", - "windows", "posix", "js"] - proc findModule*(conf: ConfigRef; modulename, currentModule: string): AbsoluteFile = # returns path to module - const pkgPrefix = "pkg/" - const stdPrefix = "std/" var m = addFileExt(modulename, NimExt) if m.startsWith(pkgPrefix): result = findFile(conf, m.substr(pkgPrefix.len), suppressStdlib = true) diff --git a/nimdoc/test_out_index_dot_html/expected/index.html b/nimdoc/test_out_index_dot_html/expected/index.html index 4646732f1..db7470050 100644 --- a/nimdoc/test_out_index_dot_html/expected/index.html +++ b/nimdoc/test_out_index_dot_html/expected/index.html @@ -17,7 +17,7 @@ -foo +nimdoc/test_out_index_dot_html/foo @@ -64,7 +64,7 @@ window.addEventListener('DOMContentLoaded', main);
    -

    foo

    +

    nimdoc/test_out_index_dot_html/foo

    From c719d79d232c1dc8d878b0799e1011519652016f Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 23 Mar 2021 17:49:02 +0800 Subject: [PATCH 0471/2472] add algorithm.merge (#16182) * add merge to algorithm * Apply suggestions from code review * Update lib/pure/algorithm.nim * Apply suggestions from code review * Update changelog.md Co-authored-by: ee7 <45465154+ee7@users.noreply.github.com> --- changelog.md | 3 + lib/pure/algorithm.nim | 94 +++++++++++++++++++++ tests/stdlib/talgorithm.nim | 159 +++++++++++++++++++++++++++++++++++- 3 files changed, 255 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 00fc970bf..78817cf85 100644 --- a/changelog.md +++ b/changelog.md @@ -212,6 +212,9 @@ - `std/options` changed `$some(3)` to `"some(3)"` instead of `"Some(3)"` and `$none(int)` to `"none(int)"` instead of `"None[int]"`. + +- Added `algorithm.merge`. + - Added `std/jsfetch` module [Fetch](https://developer.mozilla.org/docs/Web/API/Fetch_API) wrapper for JavaScript target. diff --git a/lib/pure/algorithm.nim b/lib/pure/algorithm.nim index a32df65bb..0410d65ee 100644 --- a/lib/pure/algorithm.nim +++ b/lib/pure/algorithm.nim @@ -42,6 +42,8 @@ runnableExamples: ## * `sequtils module`_ for working with the built-in seq type ## * `tables module`_ for sorting tables +import std/private/since + type SortOrder* = enum Descending, Ascending @@ -559,6 +561,98 @@ proc isSorted*[T](a: openArray[T], order = SortOrder.Ascending): bool = assert isSorted(e) == false isSorted(a, system.cmp[T], order) +proc merge*[T]( + result: var seq[T], + x, y: openArray[T], cmp: proc(x, y: T): int {.closure.} +) {.since: (1, 5, 1).} = + ## Merges two sorted `openArray`. `x` and `y` are assumed to be sorted. + ## If you do not wish to provide your own `cmp`, + ## you may use `system.cmp` or instead call the overloaded + ## version of `merge`, which uses `system.cmp`. + ## + ## .. note:: The original data of `result` is not cleared, + ## new data is appended to `result`. + ## + ## **See also:** + ## * `merge proc<#merge,seq[T],openArray[T],openArray[T]>`_ + runnableExamples: + let x = @[1, 3, 6] + let y = @[2, 3, 4] + + block: + var merged = @[7] # new data is appended to merged sequence + merged.merge(x, y, system.cmp[int]) + assert merged == @[7, 1, 2, 3, 3, 4, 6] + + block: + var merged = @[7] # if you only want new data, clear merged sequence first + merged.setLen(0) + merged.merge(x, y, system.cmp[int]) + assert merged.isSorted + assert merged == @[1, 2, 3, 3, 4, 6] + + import std/sugar + + var res: seq[(int, int)] + res.merge([(1, 1)], [(1, 2)], (a, b) => a[0] - b[0]) + assert res == @[(1, 1), (1, 2)] + + assert seq[int].default.dup(merge([1, 3], [2, 4])) == @[1, 2, 3, 4] + + let + sizeX = x.len + sizeY = y.len + oldLen = result.len + + result.setLen(oldLen + sizeX + sizeY) + + var + ix = 0 + iy = 0 + i = oldLen + + while true: + if ix == sizeX: + while iy < sizeY: + result[i] = y[iy] + inc i + inc iy + return + + if iy == sizeY: + while ix < sizeX: + result[i] = x[ix] + inc i + inc ix + return + + let itemX = x[ix] + let itemY = y[iy] + + if cmp(itemX, itemY) > 0: # to have a stable sort + result[i] = itemY + inc iy + else: + result[i] = itemX + inc ix + + inc i + +proc merge*[T](result: var seq[T], x, y: openArray[T]) {.inline, since: (1, 5, 1).} = + ## Shortcut version of `merge` that uses `system.cmp[T]` as the comparison function. + ## + ## **See also:** + ## * `merge proc<#merge,seq[T],openArray[T],openArray[T],proc(T,T)>`_ + runnableExamples: + let x = [5, 10, 15, 20, 25] + let y = [50, 40, 30, 20, 10].sorted + + var merged: seq[int] + merged.merge(x, y) + assert merged.isSorted + assert merged == @[5, 10, 10, 15, 20, 20, 25, 30, 40, 50] + merge(result, x, y, system.cmp) + proc product*[T](x: openArray[seq[T]]): seq[seq[T]] = ## Produces the Cartesian product of the array. ## Every element of the result is a combination of one element from each seq in `x`, diff --git a/tests/stdlib/talgorithm.nim b/tests/stdlib/talgorithm.nim index fecd85820..5bb5a6bdf 100644 --- a/tests/stdlib/talgorithm.nim +++ b/tests/stdlib/talgorithm.nim @@ -4,7 +4,8 @@ discard """ ''' """ #12928,10456 -import std/[sequtils, algorithm, json] + +import std/[sequtils, algorithm, json, sugar] proc test() = try: @@ -114,3 +115,159 @@ block: doAssert binarySearch(moreData, 6) == -1 doAssert binarySearch(moreData, 4711) == 4 doAssert binarySearch(moreData, 4712) == -1 + +# merge +proc main() = + block: + var x = @[1, 7, 8, 11, 21, 33, 45, 99] + var y = @[6, 7, 9, 12, 57, 66] + + var merged: seq[int] + merged.merge(x, y) + doAssert merged.isSorted + doAssert merged == sorted(x & y) + + block: + var x = @[111, 88, 76, 56, 45, 31, 22, 19, 11, 3] + var y = @[99, 85, 83, 82, 69, 64, 48, 42, 33, 31, 26, 13] + + var merged: seq[int] + merged.merge(x, y, (x, y) => -system.cmp(x, y)) + doAssert merged.isSorted((x, y) => -system.cmp(x, y)) + doAssert merged == sorted(x & y, SortOrder.Descending) + + block: + var x: seq[int] = @[] + var y = @[1] + + var merged: seq[int] + merged.merge(x, y) + doAssert merged.isSorted + doAssert merged.isSorted(SortOrder.Descending) + doAssert merged == @[1] + + block: + var x = [1, 3, 5, 5, 7] + var y: seq[int] = @[] + + var merged: seq[int] + merged.merge(x, y) + doAssert merged.isSorted + doAssert merged == @x + + block: + var x = [1, 3, 5, 5, 7] + var y: seq[int] = @[] + + var merged: seq[int] = @[1, 2, 3, 5, 6, 56, 99, 2, 34] + merged.merge(x, y) + doAssert merged == @[1, 2, 3, 5, 6, 56, 99, 2, 34, 1, 3, 5, 5, 7] + + + block: + var x: array[0, int] + var y = [1, 4, 6, 7, 9] + + var merged: seq[int] + merged.merge(x, y) + doAssert merged.isSorted + doAssert merged == @y + + block: + var x: array[0, int] + var y: array[0, int] + + var merged: seq[int] + merged.merge(x, y) + doAssert merged.isSorted + doAssert merged.len == 0 + + block: + var x: array[0, int] + var y: array[0, int] + + var merged: seq[int] = @[99, 99, 99] + merged.setLen(0) + merged.merge(x, y) + doAssert merged.isSorted + doAssert merged.len == 0 + + block: + var x: seq[int] + var y: seq[int] + + var merged: seq[int] + merged.merge(x, y) + doAssert merged.isSorted + doAssert merged.len == 0 + + block: + type + Record = object + id: int + + proc r(id: int): Record = + Record(id: id) + + proc cmp(x, y: Record): int = + if x.id == y.id: return 0 + if x.id < y.id: return -1 + result = 1 + + var x = @[r(-12), r(1), r(3), r(8), r(13), r(88)] + var y = @[r(4), r(7), r(12), r(13), r(77), r(99)] + + var merged: seq[Record] = @[] + merged.merge(x, y, cmp) + doAssert merged.isSorted(cmp) + doAssert merged.len == 12 + + block: + type + Record = object + id: int + + proc r(id: int): Record = + Record(id: id) + + proc ascendingCmp(x, y: Record): int = + if x.id == y.id: return 0 + if x.id < y.id: return -1 + result = 1 + + proc descendingCmp(x, y: Record): int = + if x.id == y.id: return 0 + if x.id < y.id: return 1 + result = -1 + + var x = @[r(-12), r(1), r(3), r(8), r(13), r(88)] + var y = @[r(4), r(7), r(12), r(13), r(77), r(99)] + + var merged: seq[Record] + merged.setLen(0) + merged.merge(x, y, ascendingCmp) + doAssert merged.isSorted(ascendingCmp) + doAssert merged == sorted(x & y, ascendingCmp) + + reverse(x) + reverse(y) + + merged.setLen(0) + merged.merge(x, y, descendingCmp) + doAssert merged.isSorted(descendingCmp) + doAssert merged == sorted(x & y, ascendingCmp, SortOrder.Descending) + + reverse(x) + reverse(y) + merged.setLen(0) + merged.merge(x, y, proc (x, y: Record): int = -descendingCmp(x, y)) + doAssert merged.isSorted(proc (x, y: Record): int = -descendingCmp(x, y)) + doAssert merged == sorted(x & y, ascendingCmp) + + + var x: seq[(int, int)] + x.merge([(1,1)], [(1,2)], (a,b) => a[0] - b[0]) + doAssert x == @[(1, 1), (1, 2)] + +static: main() +main() From c27cd83265558195316e0d3b6d13ccadcaecc728 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 23 Mar 2021 02:51:21 -0700 Subject: [PATCH 0472/2472] remove {.compileTime.} for procs with NimNode: redundant because of tfTriggersCompileTime (#17463) --- lib/core/macros.nim | 193 +++++++++++++++++++--------------------- lib/pure/asyncmacro.nim | 13 ++- lib/pure/htmlgen.nim | 4 +- lib/pure/json.nim | 4 +- lib/pure/sugar.nim | 2 +- lib/pure/xmltree.nim | 2 +- 6 files changed, 106 insertions(+), 112 deletions(-) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 8b45ed86b..5a556fc8d 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -404,29 +404,29 @@ proc warning*(msg: string, n: NimNode = nil) {.magic: "NWarning", benign.} proc hint*(msg: string, n: NimNode = nil) {.magic: "NHint", benign.} ## Writes a hint message at compile time. -proc newStrLitNode*(s: string): NimNode {.compileTime, noSideEffect.} = +proc newStrLitNode*(s: string): NimNode {.noSideEffect.} = ## Creates a string literal node from `s`. result = newNimNode(nnkStrLit) result.strVal = s -proc newCommentStmtNode*(s: string): NimNode {.compileTime, noSideEffect.} = +proc newCommentStmtNode*(s: string): NimNode {.noSideEffect.} = ## Creates a comment statement node. result = newNimNode(nnkCommentStmt) result.strVal = s -proc newIntLitNode*(i: BiggestInt): NimNode {.compileTime.} = +proc newIntLitNode*(i: BiggestInt): NimNode = ## Creates an int literal node from `i`. result = newNimNode(nnkIntLit) result.intVal = i -proc newFloatLitNode*(f: BiggestFloat): NimNode {.compileTime.} = +proc newFloatLitNode*(f: BiggestFloat): NimNode = ## Creates a float literal node from `f`. result = newNimNode(nnkFloatLit) result.floatVal = f {.push warnings: off.} -proc newIdentNode*(i: NimIdent): NimNode {.compileTime, deprecated.} = +proc newIdentNode*(i: NimIdent): NimNode {.deprecated: "use ident(string)".} = ## Creates an identifier node from `i`. result = newNimNode(nnkIdent) result.ident = i @@ -478,7 +478,7 @@ proc callsite*(): NimNode {.magic: "NCallSite", benign, deprecated: "Deprecated since v0.18.1; use varargs[untyped] in the macro prototype instead".} ## Returns the AST of the invocation expression that invoked this macro. -proc toStrLit*(n: NimNode): NimNode {.compileTime.} = +proc toStrLit*(n: NimNode): NimNode = ## Converts the AST `n` to the concrete Nim code and wraps that ## in a string literal node. return newStrLitNode(repr(n)) @@ -504,11 +504,11 @@ proc getFile(arg: NimNode): string {.magic: "NLineInfo", noSideEffect.} proc copyLineInfo*(arg: NimNode, info: NimNode) {.magic: "NLineInfo", noSideEffect.} ## Copy lineinfo from `info`. -proc lineInfoObj*(n: NimNode): LineInfo {.compileTime.} = +proc lineInfoObj*(n: NimNode): LineInfo = ## Returns `LineInfo` of `n`, using absolute path for `filename`. result = LineInfo(filename: n.getFile, line: n.getLine, column: n.getColumn) -proc lineInfo*(arg: NimNode): string {.compileTime.} = +proc lineInfo*(arg: NimNode): string = ## Return line info in the form `filepath(line, column)`. $arg.lineInfoObj @@ -522,14 +522,14 @@ proc internalErrorFlag*(): string {.magic: "NError", noSideEffect.} ## Some builtins set an error flag. This is then turned into a proper ## exception. **Note**: Ordinary application code should not call this. -proc parseExpr*(s: string): NimNode {.noSideEffect, compileTime.} = +proc parseExpr*(s: string): NimNode {.noSideEffect.} = ## Compiles the passed string to its AST representation. ## Expects a single expression. Raises `ValueError` for parsing errors. result = internalParseExpr(s) let x = internalErrorFlag() if x.len > 0: raise newException(ValueError, x) -proc parseStmt*(s: string): NimNode {.noSideEffect, compileTime.} = +proc parseStmt*(s: string): NimNode {.noSideEffect.} = ## Compiles the passed string to its AST representation. ## Expects one or more statements. Raises `ValueError` for parsing errors. result = internalParseStmt(s) @@ -626,25 +626,25 @@ proc quote*(bl: typed, op = "``"): NimNode {.magic: "QuoteAst", noSideEffect.} = doAssert y == 3 bar2() -proc expectKind*(n: NimNode, k: NimNodeKind) {.compileTime.} = +proc expectKind*(n: NimNode, k: NimNodeKind) = ## Checks that `n` is of kind `k`. If this is not the case, ## compilation aborts with an error message. This is useful for writing ## macros that check the AST that is passed to them. if n.kind != k: error("Expected a node of kind " & $k & ", got " & $n.kind, n) -proc expectMinLen*(n: NimNode, min: int) {.compileTime.} = +proc expectMinLen*(n: NimNode, min: int) = ## Checks that `n` has at least `min` children. If this is not the case, ## compilation aborts with an error message. This is useful for writing ## macros that check its number of arguments. if n.len < min: error("Expected a node with at least " & $min & " children, got " & $n.len, n) -proc expectLen*(n: NimNode, len: int) {.compileTime.} = +proc expectLen*(n: NimNode, len: int) = ## Checks that `n` has exactly `len` children. If this is not the case, ## compilation aborts with an error message. This is useful for writing ## macros that check its number of arguments. if n.len != len: error("Expected a node with " & $len & " children, got " & $n.len, n) -proc expectLen*(n: NimNode, min, max: int) {.compileTime.} = +proc expectLen*(n: NimNode, min, max: int) = ## Checks that `n` has a number of children in the range `min..max`. ## If this is not the case, compilation aborts with an error message. ## This is useful for writing macros that check its number of arguments. @@ -652,13 +652,12 @@ proc expectLen*(n: NimNode, min, max: int) {.compileTime.} = error("Expected a node with " & $min & ".." & $max & " children, got " & $n.len, n) proc newTree*(kind: NimNodeKind, - children: varargs[NimNode]): NimNode {.compileTime.} = + children: varargs[NimNode]): NimNode = ## Produces a new node with children. result = newNimNode(kind) result.add(children) -proc newCall*(theProc: NimNode, - args: varargs[NimNode]): NimNode {.compileTime.} = +proc newCall*(theProc: NimNode, args: varargs[NimNode]): NimNode = ## Produces a new call node. `theProc` is the proc that is called with ## the arguments `args[0..]`. result = newNimNode(nnkCall) @@ -667,7 +666,7 @@ proc newCall*(theProc: NimNode, {.push warnings: off.} -proc newCall*(theProc: NimIdent, args: varargs[NimNode]): NimNode {.compileTime, deprecated: +proc newCall*(theProc: NimIdent, args: varargs[NimNode]): NimNode {.deprecated: "Deprecated since v0.18.1; use 'newCall(string, ...)' or 'newCall(NimNode, ...)' instead".} = ## Produces a new call node. `theProc` is the proc that is called with ## the arguments `args[0..]`. @@ -678,128 +677,128 @@ proc newCall*(theProc: NimIdent, args: varargs[NimNode]): NimNode {.compileTime, {.pop.} proc newCall*(theProc: string, - args: varargs[NimNode]): NimNode {.compileTime.} = + args: varargs[NimNode]): NimNode = ## Produces a new call node. `theProc` is the proc that is called with ## the arguments `args[0..]`. result = newNimNode(nnkCall) result.add(newIdentNode(theProc)) result.add(args) -proc newLit*(c: char): NimNode {.compileTime.} = +proc newLit*(c: char): NimNode = ## Produces a new character literal node. result = newNimNode(nnkCharLit) result.intVal = ord(c) -proc newLit*(i: int): NimNode {.compileTime.} = +proc newLit*(i: int): NimNode = ## Produces a new integer literal node. result = newNimNode(nnkIntLit) result.intVal = i -proc newLit*(i: int8): NimNode {.compileTime.} = +proc newLit*(i: int8): NimNode = ## Produces a new integer literal node. result = newNimNode(nnkInt8Lit) result.intVal = i -proc newLit*(i: int16): NimNode {.compileTime.} = +proc newLit*(i: int16): NimNode = ## Produces a new integer literal node. result = newNimNode(nnkInt16Lit) result.intVal = i -proc newLit*(i: int32): NimNode {.compileTime.} = +proc newLit*(i: int32): NimNode = ## Produces a new integer literal node. result = newNimNode(nnkInt32Lit) result.intVal = i -proc newLit*(i: int64): NimNode {.compileTime.} = +proc newLit*(i: int64): NimNode = ## Produces a new integer literal node. result = newNimNode(nnkInt64Lit) result.intVal = i -proc newLit*(i: uint): NimNode {.compileTime.} = +proc newLit*(i: uint): NimNode = ## Produces a new unsigned integer literal node. result = newNimNode(nnkUIntLit) result.intVal = BiggestInt(i) -proc newLit*(i: uint8): NimNode {.compileTime.} = +proc newLit*(i: uint8): NimNode = ## Produces a new unsigned integer literal node. result = newNimNode(nnkUInt8Lit) result.intVal = BiggestInt(i) -proc newLit*(i: uint16): NimNode {.compileTime.} = +proc newLit*(i: uint16): NimNode = ## Produces a new unsigned integer literal node. result = newNimNode(nnkUInt16Lit) result.intVal = BiggestInt(i) -proc newLit*(i: uint32): NimNode {.compileTime.} = +proc newLit*(i: uint32): NimNode = ## Produces a new unsigned integer literal node. result = newNimNode(nnkUInt32Lit) result.intVal = BiggestInt(i) -proc newLit*(i: uint64): NimNode {.compileTime.} = +proc newLit*(i: uint64): NimNode = ## Produces a new unsigned integer literal node. result = newNimNode(nnkUInt64Lit) result.intVal = BiggestInt(i) -proc newLit*(b: bool): NimNode {.compileTime.} = +proc newLit*(b: bool): NimNode = ## Produces a new boolean literal node. result = if b: bindSym"true" else: bindSym"false" -proc newLit*(s: string): NimNode {.compileTime.} = +proc newLit*(s: string): NimNode = ## Produces a new string literal node. result = newNimNode(nnkStrLit) result.strVal = s when false: # the float type is not really a distinct type as described in https://github.com/nim-lang/Nim/issues/5875 - proc newLit*(f: float): NimNode {.compileTime.} = + proc newLit*(f: float): NimNode = ## Produces a new float literal node. result = newNimNode(nnkFloatLit) result.floatVal = f -proc newLit*(f: float32): NimNode {.compileTime.} = +proc newLit*(f: float32): NimNode = ## Produces a new float literal node. result = newNimNode(nnkFloat32Lit) result.floatVal = f -proc newLit*(f: float64): NimNode {.compileTime.} = +proc newLit*(f: float64): NimNode = ## Produces a new float literal node. result = newNimNode(nnkFloat64Lit) result.floatVal = f when declared(float128): - proc newLit*(f: float128): NimNode {.compileTime.} = + proc newLit*(f: float128): NimNode = ## Produces a new float literal node. result = newNimNode(nnkFloat128Lit) result.floatVal = f -proc newLit*(arg: enum): NimNode {.compileTime.} = +proc newLit*(arg: enum): NimNode = result = newCall( arg.typeof.getTypeInst[1], newLit(int(arg)) ) -proc newLit*[N,T](arg: array[N,T]): NimNode {.compileTime.} -proc newLit*[T](arg: seq[T]): NimNode {.compileTime.} -proc newLit*[T](s: set[T]): NimNode {.compileTime.} -proc newLit*[T: tuple](arg: T): NimNode {.compileTime.} +proc newLit*[N,T](arg: array[N,T]): NimNode +proc newLit*[T](arg: seq[T]): NimNode +proc newLit*[T](s: set[T]): NimNode +proc newLit*[T: tuple](arg: T): NimNode -proc newLit*(arg: object): NimNode {.compileTime.} = +proc newLit*(arg: object): NimNode = result = nnkObjConstr.newTree(arg.typeof.getTypeInst[1]) for a, b in arg.fieldPairs: result.add nnkExprColonExpr.newTree( newIdentNode(a), newLit(b) ) -proc newLit*(arg: ref object): NimNode {.compileTime.} = +proc newLit*(arg: ref object): NimNode = ## produces a new ref type literal node. result = nnkObjConstr.newTree(arg.typeof.getTypeInst[1]) for a, b in fieldPairs(arg[]): result.add nnkExprColonExpr.newTree(newIdentNode(a), newLit(b)) -proc newLit*[N,T](arg: array[N,T]): NimNode {.compileTime.} = +proc newLit*[N,T](arg: array[N,T]): NimNode = result = nnkBracket.newTree for x in arg: result.add newLit(x) -proc newLit*[T](arg: seq[T]): NimNode {.compileTime.} = +proc newLit*[T](arg: seq[T]): NimNode = let bracket = nnkBracket.newTree for x in arg: bracket.add newLit(x) @@ -812,7 +811,7 @@ proc newLit*[T](arg: seq[T]): NimNode {.compileTime.} = var typ = getTypeInst(typeof(arg))[1] result = newCall(typ,result) -proc newLit*[T](s: set[T]): NimNode {.compileTime.} = +proc newLit*[T](s: set[T]): NimNode = result = nnkCurly.newTree for x in s: result.add newLit(x) @@ -824,7 +823,7 @@ proc newLit*[T](s: set[T]): NimNode {.compileTime.} = proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} ## See `typetraits.isNamedTuple` -proc newLit*[T: tuple](arg: T): NimNode {.compileTime.} = +proc newLit*[T: tuple](arg: T): NimNode = ## use -d:nimHasWorkaround14720 to restore behavior prior to PR, forcing ## a named tuple even when `arg` is unnamed. result = nnkTupleConstr.newTree @@ -835,7 +834,7 @@ proc newLit*[T: tuple](arg: T): NimNode {.compileTime.} = for b in arg.fields: result.add newLit(b) -proc nestList*(op: NimNode; pack: NimNode): NimNode {.compileTime.} = +proc nestList*(op: NimNode; pack: NimNode): NimNode = ## Nests the list `pack` into a tree of call expressions: ## `[a, b, c]` is transformed into `op(a, op(c, d))`. ## This is also known as fold expression. @@ -845,7 +844,7 @@ proc nestList*(op: NimNode; pack: NimNode): NimNode {.compileTime.} = for i in countdown(pack.len - 2, 0): result = newCall(op, pack[i], result) -proc nestList*(op: NimNode; pack: NimNode; init: NimNode): NimNode {.compileTime.} = +proc nestList*(op: NimNode; pack: NimNode; init: NimNode): NimNode = ## Nests the list `pack` into a tree of call expressions: ## `[a, b, c]` is transformed into `op(a, op(c, d))`. ## This is also known as fold expression. @@ -887,21 +886,21 @@ proc treeTraverse(n: NimNode; res: var string; level = 0; isLisp = false, indent if isLisp: res.add(")") -proc treeRepr*(n: NimNode): string {.compileTime, benign.} = +proc treeRepr*(n: NimNode): string {.benign.} = ## Convert the AST `n` to a human-readable tree-like string. ## ## See also `repr`, `lispRepr`, and `astGenRepr`. result = "" n.treeTraverse(result, isLisp = false, indented = true) -proc lispRepr*(n: NimNode; indented = false): string {.compileTime, benign.} = +proc lispRepr*(n: NimNode; indented = false): string {.benign.} = ## Convert the AST `n` to a human-readable lisp-like string. ## ## See also `repr`, `treeRepr`, and `astGenRepr`. result = "" n.treeTraverse(result, isLisp = true, indented = indented) -proc astGenRepr*(n: NimNode): string {.compileTime, benign.} = +proc astGenRepr*(n: NimNode): string {.benign.} = ## Convert the AST `n` to the code required to generate that AST. ## ## See also `repr`, `treeRepr`, and `lispRepr`. @@ -1020,56 +1019,56 @@ macro dumpAstGen*(s: untyped): untyped = echo s.astGenRepr ## ## Also see `dumpTree` and `dumpLisp`. -proc newEmptyNode*(): NimNode {.compileTime, noSideEffect.} = +proc newEmptyNode*(): NimNode {.noSideEffect.} = ## Create a new empty node. result = newNimNode(nnkEmpty) -proc newStmtList*(stmts: varargs[NimNode]): NimNode {.compileTime.}= +proc newStmtList*(stmts: varargs[NimNode]): NimNode= ## Create a new statement list. result = newNimNode(nnkStmtList).add(stmts) -proc newPar*(exprs: varargs[NimNode]): NimNode {.compileTime.}= +proc newPar*(exprs: varargs[NimNode]): NimNode= ## Create a new parentheses-enclosed expression. newNimNode(nnkPar).add(exprs) -proc newBlockStmt*(label, body: NimNode): NimNode {.compileTime.} = +proc newBlockStmt*(label, body: NimNode): NimNode = ## Create a new block statement with label. return newNimNode(nnkBlockStmt).add(label, body) -proc newBlockStmt*(body: NimNode): NimNode {.compileTime.} = +proc newBlockStmt*(body: NimNode): NimNode = ## Create a new block: stmt. return newNimNode(nnkBlockStmt).add(newEmptyNode(), body) -proc newVarStmt*(name, value: NimNode): NimNode {.compileTime.} = +proc newVarStmt*(name, value: NimNode): NimNode = ## Create a new var stmt. return newNimNode(nnkVarSection).add( newNimNode(nnkIdentDefs).add(name, newNimNode(nnkEmpty), value)) -proc newLetStmt*(name, value: NimNode): NimNode {.compileTime.} = +proc newLetStmt*(name, value: NimNode): NimNode = ## Create a new let stmt. return newNimNode(nnkLetSection).add( newNimNode(nnkIdentDefs).add(name, newNimNode(nnkEmpty), value)) -proc newConstStmt*(name, value: NimNode): NimNode {.compileTime.} = +proc newConstStmt*(name, value: NimNode): NimNode = ## Create a new const stmt. newNimNode(nnkConstSection).add( newNimNode(nnkConstDef).add(name, newNimNode(nnkEmpty), value)) -proc newAssignment*(lhs, rhs: NimNode): NimNode {.compileTime.} = +proc newAssignment*(lhs, rhs: NimNode): NimNode = return newNimNode(nnkAsgn).add(lhs, rhs) -proc newDotExpr*(a, b: NimNode): NimNode {.compileTime.} = +proc newDotExpr*(a, b: NimNode): NimNode = ## Create new dot expression. ## a.dot(b) -> `a.b` return newNimNode(nnkDotExpr).add(a, b) -proc newColonExpr*(a, b: NimNode): NimNode {.compileTime.} = +proc newColonExpr*(a, b: NimNode): NimNode = ## Create new colon expression. ## newColonExpr(a, b) -> `a: b` newNimNode(nnkExprColonExpr).add(a, b) proc newIdentDefs*(name, kind: NimNode; - default = newEmptyNode()): NimNode {.compileTime.} = + default = newEmptyNode()): NimNode = ## Creates a new `nnkIdentDefs` node of a specific kind and value. ## ## `nnkIdentDefs` need to have at least three children, but they can have @@ -1100,11 +1099,11 @@ proc newIdentDefs*(name, kind: NimNode; ## newStrLitNode("Hello")) newNimNode(nnkIdentDefs).add(name, kind, default) -proc newNilLit*(): NimNode {.compileTime.} = +proc newNilLit*(): NimNode = ## New nil literal shortcut. result = newNimNode(nnkNilLit) -proc last*(node: NimNode): NimNode {.compileTime.} = node[node.len-1] +proc last*(node: NimNode): NimNode = node[node.len-1] ## Return the last item in nodes children. Same as `node[^1]`. @@ -1115,7 +1114,7 @@ const CallNodes* = {nnkCall, nnkInfix, nnkPrefix, nnkPostfix, nnkCommand, nnkCallStrLit, nnkHiddenCallConv} -proc expectKind*(n: NimNode; k: set[NimNodeKind]) {.compileTime.} = +proc expectKind*(n: NimNode; k: set[NimNodeKind]) = ## Checks that `n` is of kind `k`. If this is not the case, ## compilation aborts with an error message. This is useful for writing ## macros that check the AST that is passed to them. @@ -1125,7 +1124,7 @@ proc newProc*(name = newEmptyNode(); params: openArray[NimNode] = [newEmptyNode()]; body: NimNode = newStmtList(); procType = nnkProcDef; - pragmas: NimNode = newEmptyNode()): NimNode {.compileTime.} = + pragmas: NimNode = newEmptyNode()): NimNode = ## Shortcut for creating a new proc. ## ## The `params` array must start with the return type of the proc, @@ -1142,8 +1141,7 @@ proc newProc*(name = newEmptyNode(); newEmptyNode(), body) -proc newIfStmt*(branches: varargs[tuple[cond, body: NimNode]]): - NimNode {.compileTime.} = +proc newIfStmt*(branches: varargs[tuple[cond, body: NimNode]]): NimNode = ## Constructor for `if` statements. ## ## .. code-block:: nim @@ -1160,7 +1158,7 @@ proc newIfStmt*(branches: varargs[tuple[cond, body: NimNode]]): result.add(newTree(nnkElifBranch, i.cond, i.body)) proc newEnum*(name: NimNode, fields: openArray[NimNode], - public, pure: bool): NimNode {.compileTime.} = + public, pure: bool): NimNode = ## Creates a new enum. `name` must be an ident. Fields are allowed to be ## either idents or EnumFieldDef @@ -1203,7 +1201,7 @@ proc newEnum*(name: NimNode, fields: openArray[NimNode], return typeSect -proc copyChildrenTo*(src, dest: NimNode) {.compileTime.}= +proc copyChildrenTo*(src, dest: NimNode) = ## Copy all children from `src` to `dest`. for i in 0 ..< src.len: dest.add src[i].copyNimTree @@ -1211,7 +1209,7 @@ proc copyChildrenTo*(src, dest: NimNode) {.compileTime.}= template expectRoutine(node: NimNode) = expectKind(node, RoutineNodes) -proc name*(someProc: NimNode): NimNode {.compileTime.} = +proc name*(someProc: NimNode): NimNode = someProc.expectRoutine result = someProc[0] if result.kind == nnkPostfix: @@ -1222,21 +1220,21 @@ proc name*(someProc: NimNode): NimNode {.compileTime.} = elif result.kind == nnkAccQuoted: result = result[0] -proc `name=`*(someProc: NimNode; val: NimNode) {.compileTime.} = +proc `name=`*(someProc: NimNode; val: NimNode) = someProc.expectRoutine if someProc[0].kind == nnkPostfix: someProc[0][1] = val else: someProc[0] = val -proc params*(someProc: NimNode): NimNode {.compileTime.} = +proc params*(someProc: NimNode): NimNode = someProc.expectRoutine result = someProc[3] -proc `params=`* (someProc: NimNode; params: NimNode) {.compileTime.}= +proc `params=`* (someProc: NimNode; params: NimNode) = someProc.expectRoutine expectKind(params, nnkFormalParams) someProc[3] = params -proc pragma*(someProc: NimNode): NimNode {.compileTime.} = +proc pragma*(someProc: NimNode): NimNode = ## Get the pragma of a proc type. ## These will be expanded. if someProc.kind == nnkProcTy: @@ -1244,7 +1242,7 @@ proc pragma*(someProc: NimNode): NimNode {.compileTime.} = else: someProc.expectRoutine result = someProc[4] -proc `pragma=`*(someProc: NimNode; val: NimNode) {.compileTime.}= +proc `pragma=`*(someProc: NimNode; val: NimNode) = ## Set the pragma of a proc type. expectKind(val, {nnkEmpty, nnkPragma}) if someProc.kind == nnkProcTy: @@ -1253,7 +1251,7 @@ proc `pragma=`*(someProc: NimNode; val: NimNode) {.compileTime.}= someProc.expectRoutine someProc[4] = val -proc addPragma*(someProc, pragma: NimNode) {.compileTime.} = +proc addPragma*(someProc, pragma: NimNode) = ## Adds pragma to routine definition. someProc.expectKind(RoutineNodes + {nnkProcTy}) var pragmaNode = someProc.pragma @@ -1265,7 +1263,7 @@ proc addPragma*(someProc, pragma: NimNode) {.compileTime.} = template badNodeKind(n, f) = error("Invalid node kind " & $n.kind & " for macros.`" & $f & "`", n) -proc body*(someProc: NimNode): NimNode {.compileTime.} = +proc body*(someProc: NimNode): NimNode = case someProc.kind: of RoutineNodes: return someProc[6] @@ -1276,7 +1274,7 @@ proc body*(someProc: NimNode): NimNode {.compileTime.} = else: badNodeKind someProc, "body" -proc `body=`*(someProc: NimNode, val: NimNode) {.compileTime.} = +proc `body=`*(someProc: NimNode, val: NimNode) = case someProc.kind of RoutineNodes: someProc[6] = val @@ -1287,7 +1285,7 @@ proc `body=`*(someProc: NimNode, val: NimNode) {.compileTime.} = else: badNodeKind someProc, "body=" -proc basename*(a: NimNode): NimNode {.raises: [].} = +proc basename*(a: NimNode): NimNode = ## Pull an identifier from prefix/postfix expressions. case a.kind of nnkIdent: result = a @@ -1297,7 +1295,7 @@ proc basename*(a: NimNode): NimNode {.raises: [].} = error("Do not know how to get basename of (" & treeRepr(a) & ")\n" & repr(a), a) -proc `$`*(node: NimNode): string {.compileTime.} = +proc `$`*(node: NimNode): string = ## Get the string of an identifier node. case node.kind of nnkPostfix: @@ -1340,7 +1338,7 @@ template findChild*(n: NimNode; cond: untyped): NimNode {.dirty.} = break res -proc insert*(a: NimNode; pos: int; b: NimNode) {.compileTime.} = +proc insert*(a: NimNode; pos: int; b: NimNode) = ## Insert node `b` into node `a` at `pos`. if len(a)-1 < pos: # add some empty nodes first @@ -1355,7 +1353,7 @@ proc insert*(a: NimNode; pos: int; b: NimNode) {.compileTime.} = a[i + 1] = a[i] a[pos] = b -proc `basename=`*(a: NimNode; val: string) {.compileTime.}= +proc `basename=`*(a: NimNode; val: string) = case a.kind of nnkIdent: a.strVal = val @@ -1366,32 +1364,29 @@ proc `basename=`*(a: NimNode; val: string) {.compileTime.}= error("Do not know how to get basename of (" & treeRepr(a) & ")\n" & repr(a), a) -proc postfix*(node: NimNode; op: string): NimNode {.compileTime.} = +proc postfix*(node: NimNode; op: string): NimNode = newNimNode(nnkPostfix).add(ident(op), node) -proc prefix*(node: NimNode; op: string): NimNode {.compileTime.} = +proc prefix*(node: NimNode; op: string): NimNode = newNimNode(nnkPrefix).add(ident(op), node) proc infix*(a: NimNode; op: string; - b: NimNode): NimNode {.compileTime.} = + b: NimNode): NimNode = newNimNode(nnkInfix).add(ident(op), a, b) -proc unpackPostfix*(node: NimNode): tuple[node: NimNode; op: string] {. - compileTime.} = +proc unpackPostfix*(node: NimNode): tuple[node: NimNode; op: string] = node.expectKind nnkPostfix result = (node[1], $node[0]) -proc unpackPrefix*(node: NimNode): tuple[node: NimNode; op: string] {. - compileTime.} = +proc unpackPrefix*(node: NimNode): tuple[node: NimNode; op: string] = node.expectKind nnkPrefix result = (node[1], $node[0]) -proc unpackInfix*(node: NimNode): tuple[left: NimNode; op: string; - right: NimNode] {.compileTime.} = +proc unpackInfix*(node: NimNode): tuple[left: NimNode; op: string; right: NimNode] = expectKind(node, nnkInfix) result = (node[1], $node[0], node[2]) -proc copy*(node: NimNode): NimNode {.compileTime.} = +proc copy*(node: NimNode): NimNode = ## An alias for `copyNimTree<#copyNimTree,NimNode>`_. return node.copyNimTree() @@ -1416,14 +1411,14 @@ proc eqIdent*(a: NimNode; b: NimNode): bool {.magic: "EqIdent", noSideEffect.} ## (`nnkPostfix`) or quoted with backticks (`nnkAccQuoted`), ## these nodes will be unwrapped. -proc expectIdent*(n: NimNode, name: string) {.compileTime, since: (1,1).} = +proc expectIdent*(n: NimNode, name: string) {.since: (1,1).} = ## Check that `eqIdent(n,name)` holds true. If this is not the ## case, compilation aborts with an error message. This is useful ## for writing macros that check the AST that is passed to them. if not eqIdent(n, name): error("Expected identifier to be `" & name & "` here", n) -proc hasArgOfName*(params: NimNode; name: string): bool {.compileTime.}= +proc hasArgOfName*(params: NimNode; name: string): bool= ## Search `nnkFormalParams` for an argument. expectKind(params, nnkFormalParams) for i in 1.. Date: Tue, 23 Mar 2021 15:40:30 +0100 Subject: [PATCH 0473/2472] Fixes #17450 (#17474) * Fixes #17450 * Add missing test output --- compiler/injectdestructors.nim | 1 + tests/arc/tmovebug.nim | 14 ++++++++++++++ tests/destructor/tnewruntime_strutils.nim | 13 ++++++++++++- 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/compiler/injectdestructors.nim b/compiler/injectdestructors.nim index b65391252..152efd8a1 100644 --- a/compiler/injectdestructors.nim +++ b/compiler/injectdestructors.nim @@ -573,6 +573,7 @@ template processScopeExpr(c: var Con; s: var Scope; ret: PNode, processCall: unt let tmp = c.getTemp(s.parent[], ret.typ, ret.info) tmp.sym.flags.incl sfSingleUsedTemp let cpy = if hasDestructor(c, ret.typ): + s.parent[].final.add c.genDestroy(tmp) moveOrCopy(tmp, ret, c, s, isDecl = true) else: newTree(nkFastAsgn, tmp, p(ret, c, s, normal)) diff --git a/tests/arc/tmovebug.nim b/tests/arc/tmovebug.nim index 888027186..3ff1c4a0c 100644 --- a/tests/arc/tmovebug.nim +++ b/tests/arc/tmovebug.nim @@ -107,6 +107,8 @@ sink destroy copy destroy +(f: 1) +destroy ''' """ @@ -770,3 +772,15 @@ proc pair(): tuple[a: C, b: C] = discard pair() + +# bug #17450 +proc noConsume(x: OO) {.nosinks.} = echo x + +proc main3 = + var i = 1 + noConsume: + block: + OO(f: i) + +main3() + diff --git a/tests/destructor/tnewruntime_strutils.nim b/tests/destructor/tnewruntime_strutils.nim index 8e5378f77..9c8d41973 100644 --- a/tests/destructor/tnewruntime_strutils.nim +++ b/tests/destructor/tnewruntime_strutils.nim @@ -5,7 +5,8 @@ discard """ @[(input: @["KXSC", "BGMC"]), (input: @["PXFX"]), (input: @["WXRQ", "ZSCZD"])] 14 First tasks completed. -Second tasks completed.''' +Second tasks completed. +test1''' """ import strutils, os, std / wordwrap @@ -241,3 +242,13 @@ when true: test_string_b.setLen new_len_b echo "Second tasks completed." + +# bug #17450 +proc main = + var i = 1 + echo: + block: + "test" & $i + +main() + From 97f51ed7c2fc67bfba9aef9c4b22a1fb450a5cae Mon Sep 17 00:00:00 2001 From: Clyybber Date: Tue, 23 Mar 2021 15:44:20 +0100 Subject: [PATCH 0474/2472] Revert "Fixes #17450 (#17474)" (#17476) This reverts commit 5f0c52048970cb7449937bd19191638c9e9f0c8f. --- compiler/injectdestructors.nim | 1 - tests/arc/tmovebug.nim | 14 -------------- tests/destructor/tnewruntime_strutils.nim | 13 +------------ 3 files changed, 1 insertion(+), 27 deletions(-) diff --git a/compiler/injectdestructors.nim b/compiler/injectdestructors.nim index 152efd8a1..b65391252 100644 --- a/compiler/injectdestructors.nim +++ b/compiler/injectdestructors.nim @@ -573,7 +573,6 @@ template processScopeExpr(c: var Con; s: var Scope; ret: PNode, processCall: unt let tmp = c.getTemp(s.parent[], ret.typ, ret.info) tmp.sym.flags.incl sfSingleUsedTemp let cpy = if hasDestructor(c, ret.typ): - s.parent[].final.add c.genDestroy(tmp) moveOrCopy(tmp, ret, c, s, isDecl = true) else: newTree(nkFastAsgn, tmp, p(ret, c, s, normal)) diff --git a/tests/arc/tmovebug.nim b/tests/arc/tmovebug.nim index 3ff1c4a0c..888027186 100644 --- a/tests/arc/tmovebug.nim +++ b/tests/arc/tmovebug.nim @@ -107,8 +107,6 @@ sink destroy copy destroy -(f: 1) -destroy ''' """ @@ -772,15 +770,3 @@ proc pair(): tuple[a: C, b: C] = discard pair() - -# bug #17450 -proc noConsume(x: OO) {.nosinks.} = echo x - -proc main3 = - var i = 1 - noConsume: - block: - OO(f: i) - -main3() - diff --git a/tests/destructor/tnewruntime_strutils.nim b/tests/destructor/tnewruntime_strutils.nim index 9c8d41973..8e5378f77 100644 --- a/tests/destructor/tnewruntime_strutils.nim +++ b/tests/destructor/tnewruntime_strutils.nim @@ -5,8 +5,7 @@ discard """ @[(input: @["KXSC", "BGMC"]), (input: @["PXFX"]), (input: @["WXRQ", "ZSCZD"])] 14 First tasks completed. -Second tasks completed. -test1''' +Second tasks completed.''' """ import strutils, os, std / wordwrap @@ -242,13 +241,3 @@ when true: test_string_b.setLen new_len_b echo "Second tasks completed." - -# bug #17450 -proc main = - var i = 1 - echo: - block: - "test" & $i - -main() - From 833084b671d1df21bd1a958810fc2a1bd13ae1d6 Mon Sep 17 00:00:00 2001 From: Clyybber Date: Tue, 23 Mar 2021 16:30:49 +0100 Subject: [PATCH 0475/2472] Fixes #17450 (#17477) * Fixes #17450 * Add missing test output --- compiler/injectdestructors.nim | 1 + tests/arc/tmovebug.nim | 14 ++++++++++++++ tests/destructor/tnewruntime_strutils.nim | 13 ++++++++++++- 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/compiler/injectdestructors.nim b/compiler/injectdestructors.nim index b65391252..152efd8a1 100644 --- a/compiler/injectdestructors.nim +++ b/compiler/injectdestructors.nim @@ -573,6 +573,7 @@ template processScopeExpr(c: var Con; s: var Scope; ret: PNode, processCall: unt let tmp = c.getTemp(s.parent[], ret.typ, ret.info) tmp.sym.flags.incl sfSingleUsedTemp let cpy = if hasDestructor(c, ret.typ): + s.parent[].final.add c.genDestroy(tmp) moveOrCopy(tmp, ret, c, s, isDecl = true) else: newTree(nkFastAsgn, tmp, p(ret, c, s, normal)) diff --git a/tests/arc/tmovebug.nim b/tests/arc/tmovebug.nim index 888027186..3ff1c4a0c 100644 --- a/tests/arc/tmovebug.nim +++ b/tests/arc/tmovebug.nim @@ -107,6 +107,8 @@ sink destroy copy destroy +(f: 1) +destroy ''' """ @@ -770,3 +772,15 @@ proc pair(): tuple[a: C, b: C] = discard pair() + +# bug #17450 +proc noConsume(x: OO) {.nosinks.} = echo x + +proc main3 = + var i = 1 + noConsume: + block: + OO(f: i) + +main3() + diff --git a/tests/destructor/tnewruntime_strutils.nim b/tests/destructor/tnewruntime_strutils.nim index 8e5378f77..9c8d41973 100644 --- a/tests/destructor/tnewruntime_strutils.nim +++ b/tests/destructor/tnewruntime_strutils.nim @@ -5,7 +5,8 @@ discard """ @[(input: @["KXSC", "BGMC"]), (input: @["PXFX"]), (input: @["WXRQ", "ZSCZD"])] 14 First tasks completed. -Second tasks completed.''' +Second tasks completed. +test1''' """ import strutils, os, std / wordwrap @@ -241,3 +242,13 @@ when true: test_string_b.setLen new_len_b echo "Second tasks completed." + +# bug #17450 +proc main = + var i = 1 + echo: + block: + "test" & $i + +main() + From 368422c219dd11018a8a3265921ddc283666e9e2 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Tue, 23 Mar 2021 20:12:27 +0300 Subject: [PATCH 0476/2472] update nimgrep documentation (#17415) * update nimgrep documentation * Update doc/nimgrep_cmdline.txt Co-authored-by: Andreas Rumpf --- doc/nimgrep.rst | 66 ++++++++++++++--------- doc/nimgrep_cmdline.txt | 114 ++++++++++++++++++++++++++++++++++++++++ tools/nimgrep.nim | 101 +---------------------------------- 3 files changed, 157 insertions(+), 124 deletions(-) create mode 100644 doc/nimgrep_cmdline.txt diff --git a/doc/nimgrep.rst b/doc/nimgrep.rst index 5b0fe0dbb..52fc4d62a 100644 --- a/doc/nimgrep.rst +++ b/doc/nimgrep.rst @@ -4,17 +4,20 @@ nimgrep User's manual ========================= +.. default-role:: literal + :Author: Andreas Rumpf -:Version: 0.9 +:Version: 1.6.0 +.. contents:: -Nimgrep is a command line tool for search&replace tasks. It can search for +Nimgrep is a command line tool for search and replace tasks. It can search for regex or peg patterns and can search whole directories at once. User confirmation for every single replace operation can be requested. Nimgrep has particularly good support for Nim's -eccentric *style insensitivity*. Apart from that it is a generic text -manipulation tool. +eccentric *style insensitivity* (see option `-y` below). +Apart from that it is a generic text manipulation tool. Installation @@ -30,23 +33,38 @@ And copy the executable somewhere in your `$PATH`. Command line switches ===================== -Usage: - nimgrep [options] [pattern] [replacement] (file/directory)* -Options: - --find, -f find the pattern (default) - --replace, -r replace the pattern - --peg pattern is a peg - --re pattern is a regular expression (default); extended - syntax for the regular expression is always turned on - --recursive process directories recursively - --confirm confirm each occurrence/replacement; there is a chance - to abort any time without touching the file - --stdin read pattern from stdin (to avoid the shell's confusing - quoting rules) - --word, -w the match should have word boundaries (buggy for pegs!) - --ignoreCase, -i be case insensitive - --ignoreStyle, -y be style insensitive - --ext:EX1|EX2|... only search the files with the given extension(s) - --verbose be verbose: list every processed file - --help, -h shows this help - --version, -v shows the version +.. include:: nimgrep_cmdline.txt + +Examples +======== + +All examples below use default PCRE Regex patterns: + ++ To search recursively in Nim files using style-insensitive identifiers:: + + --recursive --ext:'nim|nims' --ignoreStyle + # short: -r --ext:'nim|nims' -y + + .. Note:: we used `'` quotes to avoid special treatment of `|` symbol + for shells like Bash + ++ To exclude version control directories (Git, Mercurial=hg, Subversion=svn) + from the search:: + + --excludeDir:'^\.git$' --excludeDir:'^\.hg$' --excludeDir:'^\.svn$' + # short: --ed:'^\.git$' --ed:'^\.hg$' --ed:'^\.svn$' + ++ To search only in paths containing the `tests` sub-directory recursively:: + + --recursive --includeDir:'(^|/)tests($|/)' + # short: -r --id:'(^|/)tests($|/)' + + .. Attention:: note the subtle difference between `--excludeDir` and + `--includeDir`: the former is applied to relative directory entries + and the latter is applied to the whole paths + ++ Nimgrep can search multi-line, e.g. to find files containing `import` + and then `strutils` use:: + + 'import(.|\n)*?strutils' + diff --git a/doc/nimgrep_cmdline.txt b/doc/nimgrep_cmdline.txt new file mode 100644 index 000000000..bbcbcf530 --- /dev/null +++ b/doc/nimgrep_cmdline.txt @@ -0,0 +1,114 @@ + +Usage: + +* To search:: + nimgrep [options] PATTERN [(FILE/DIRECTORY)*/-] +* To replace:: + nimgrep [options] PATTERN --replace REPLACEMENT (FILE/DIRECTORY)*/- +* To list file names:: + nimgrep [options] --filenames [PATTERN] [(FILE/DIRECTORY)*] + +Positional arguments, from left to right: +1) PATTERN is either Regex (default) or Peg if `--peg` is specified. + PATTERN and REPLACEMENT should be skipped when `--stdin` is specified. +2) REPLACEMENT supports `$1`, `$#` notations for captured groups in PATTERN. + + .. DANGER:: `--replace` mode **DOES NOT** ask confirmation + unless `--confirm` is specified! + +3) Final arguments are a list of paths (FILE/DIRECTORY) or a standalone + minus `-` or not specified (empty): + + * empty, current directory `.` is assumed (not with `--replace`) + + .. Note:: so when no FILE/DIRECTORY/`-` is specified nimgrep + does **not** read the pipe, but searches files in the current + dir instead! + * `-`, read buffer once from stdin: pipe or terminal input; + in `--replace` mode the result is directed to stdout; + it's not compatible with `--stdin`, `--filenames`, or `--confirm` + + + For any given DIRECTORY nimgrep searches only its immediate files without + traversing sub-directories unless `--recursive` is specified. + +In replacement mode we require all 3 positional arguments to avoid damaging. + +Options: +* Mode of operation: + --find, -f find the PATTERN (default) + --replace, -! replace the PATTERN to REPLACEMENT, rewriting the files + --confirm confirm each occurrence/replacement; there is a chance + to abort any time without touching the file + --filenames just list filenames. Provide a PATTERN to find it in + the filenames (not in the contents of a file) or run + with empty pattern to just list all files:: + nimgrep --filenames # In current dir + nimgrep --filenames "" DIRECTORY + # Note empty pattern "", lists all files in DIRECTORY + +* Interprete patterns: + --peg PATTERN and PAT are Peg + --re PATTERN and PAT are regular expressions (default) + --rex, -x use the "extended" syntax for the regular expression + so that whitespace is not significant + --word, -w matches should have word boundaries (buggy for pegs!) + --ignoreCase, -i be case insensitive in PATTERN and PAT + --ignoreStyle, -y be style insensitive in PATTERN and PAT + .. Note:: PATERN and patterns PAT (see below in other options) are all either + Regex or Peg simultaneously and options `--rex`, `--word`, `--ignoreCase`, + and `--ignoreStyle` are applied to all of them. + +* File system walk: + --recursive, -r process directories recursively + --follow follow all symlinks when processing recursively + --ext:EX1|EX2|... only search the files with the given extension(s), + empty one ("--ext") means files with missing extension + --noExt:EX1|... exclude files having given extension(s), use empty one to + skip files with no extension (like some binary files are) + --includeFile:PAT search only files whose names contain pattern PAT + --excludeFile:PAT skip files whose names contain pattern PAT + --includeDir:PAT search only files with their whole directory path + containing PAT + --excludeDir:PAT skip directories whose name (not path) + contain pattern PAT + --if,--ef,--id,--ed abbreviations of the 4 options above + --sortTime, -s[:asc|desc] + order files by the last modification time (default: off): + ascending (recent files go last) or descending + +* Filter file content: + --match:PAT select files containing a (not displayed) match of PAT + --noMatch:PAT select files not containing any match of PAT + --bin:on|off|only process binary files? (detected by \0 in first 1K bytes) + (default: on - binary and text files treated the same way) + --text, -t process only text files, the same as `--bin:off` + +* Represent results: + --nocolor output will be given without any colors + --color[:on] force color even if output is redirected (default: auto) + --colorTheme:THEME select color THEME from `simple` (default), + `bnw` (black and white), `ack`, or `gnu` (GNU grep) + --count only print counts of matches for files that matched + --context:N, -c:N print N lines of leading context before every match and + N lines of trailing context after it (default N: 0) + --afterContext:N, -a:N + print N lines of trailing context after every match + --beforeContext:N, -b:N + print N lines of leading context before every match + --group, -g group matches by file + --newLine, -l display every matching line starting from a new line + --cols[:N] limit max displayed columns/width of output lines from + files by N characters, cropping overflows (default: off) + --cols:auto, -% calculate columns from terminal width for every line + --onlyAscii, -@ display only printable ASCII Latin characters 0x20-0x7E + substitutions: 0 -> ^@, 1 -> ^A, ... 0x1F -> ^_, + 0x7F -> '7F, ..., 0xFF -> 'FF + +* Miscellaneous: + --threads:N, -j:N speed up search by N additional workers (default: 0, off) + --stdin read PATTERN from stdin (to avoid the shell's confusing + quoting rules) and, if `--replace` given, REPLACEMENT + --verbose be verbose: list every processed file + --help, -h shows this help + --version, -v shows the version diff --git a/tools/nimgrep.nim b/tools/nimgrep.nim index cb46f30b8..a368c40ef 100644 --- a/tools/nimgrep.nim +++ b/tools/nimgrep.nim @@ -16,106 +16,7 @@ const Version & """ (c) 2012-2020 Andreas Rumpf - -Usage: -* To search: - nimgrep [options] PATTERN [(FILE/DIRECTORY)*/-] -* To replace: - nimgrep [options] PATTERN --replace REPLACEMENT (FILE/DIRECTORY)*/- -* To list file names: - nimgrep [options] --filenames [PATTERN] [(FILE/DIRECTORY)*] - -Positional arguments, from left to right: -* PATERN is either Regex (default) or Peg if --peg is specified. - PATTERN and REPLACEMENT should be skipped when --stdin is specified. -* REPLACEMENT supports $1, $# notations for captured groups in PATTERN. - Note: --replace mode DOES NOT ask confirmation unless --confirm is specified! -* Final arguments are a list of paths (FILE/DIRECTORY) or a standalone - minus '-' (pipe) or not specified (empty). Note for the empty case: when - no FILE/DIRECTORY/- is specified nimgrep DOES NOT read the pipe, but - searches files in the current dir instead! - - read buffer once from stdin: pipe or terminal input; - in --replace mode the result is directed to stdout; - it's not compatible with --stdin, --filenames, --confirm - (empty) current directory '.' is assumed (not with --replace) - For any given DIRECTORY nimgrep searches only its immediate files without - traversing sub-directories unless --recursive is specified. - In replacement mode all 3 positional arguments are required to avoid damaging. - -Options: -* Mode of operation: - --find, -f find the PATTERN (default) - --replace, -! replace the PATTERN to REPLACEMENT, rewriting the files - --confirm confirm each occurrence/replacement; there is a chance - to abort any time without touching the file - --filenames just list filenames. Provide a PATTERN to find it in - the filenames (not in the contents of a file) or run - with empty pattern to just list all files: - nimgrep --filenames # In current directory - nimgrep --filenames "" DIRECTORY # Note empty pattern "" - -* Interprete patterns: - --peg PATTERN and PAT are Peg - --re PATTERN and PAT are regular expressions (default) - --rex, -x use the "extended" syntax for the regular expression - so that whitespace is not significant - --word, -w matches should have word boundaries (buggy for pegs!) - --ignoreCase, -i be case insensitive in PATTERN and PAT - --ignoreStyle, -y be style insensitive in PATTERN and PAT - NOTE: PATERN and patterns PAT (see below in other options) are all either - Regex or Peg simultaneously and options --rex, --word, --ignoreCase, - --ignoreStyle are applied to all of them. - -* File system walk: - --recursive, -r process directories recursively - --follow follow all symlinks when processing recursively - --ext:EX1|EX2|... only search the files with the given extension(s), - empty one ("--ext") means files with missing extension - --noExt:EX1|... exclude files having given extension(s), use empty one to - skip files with no extension (like some binary files are) - --includeFile:PAT search only files whose names contain pattern PAT - --excludeFile:PAT skip files whose names contain pattern PAT - --includeDir:PAT search only files with whole directory path containing PAT - --excludeDir:PAT skip directories whose name (not path) contain pattern PAT - --if,--ef,--id,--ed abbreviations of 4 options above - --sortTime order files by the last modification time (default: off): - -s[:asc|desc] ascending (recent files go last) or descending - -* Filter file content: - --match:PAT select files containing a (not displayed) match of PAT - --noMatch:PAT select files not containing any match of PAT - --bin:on|off|only process binary files? (detected by \0 in first 1K bytes) - (default: on - binary and text files treated the same way) - --text, -t process only text files, the same as --bin:off - -* Represent results: - --nocolor output will be given without any colors - --color[:on] force color even if output is redirected (default: auto) - --colorTheme:THEME select color THEME from 'simple' (default), - 'bnw' (black and white) ,'ack', or 'gnu' (GNU grep) - --count only print counts of matches for files that matched - --context:N, -c:N print N lines of leading context before every match and - N lines of trailing context after it (default N: 0) - --afterContext:N, - -a:N print N lines of trailing context after every match - --beforeContext:N, - -b:N print N lines of leading context before every match - --group, -g group matches by file - --newLine, -l display every matching line starting from a new line - --cols[:N] limit max displayed columns/width of output lines from - files by N characters, cropping overflows (default: off) - --cols:auto, -% calculate columns from terminal width for every line - --onlyAscii, -@ display only printable ASCII Latin characters 0x20-0x7E - substitutions: 0 -> ^@, 1 -> ^A, ... 0x1F -> ^_, - 0x7F -> '7F, ..., 0xFF -> 'FF -* Miscellaneous: - --threads:N, -j:N speed up search by N additional workers (default: 0, off) - --stdin read PATTERN from stdin (to avoid the shell's confusing - quoting rules) and, if --replace given, REPLACEMENT - --verbose be verbose: list every processed file - --help, -h shows this help - --version, -v shows the version -""" +""" & slurp "../doc/nimgrep_cmdline.txt" # Limitations / ideas / TODO: # * No unicode support with --cols From 28221e8a33e111d8653c4ff21fad47a7ec6ef54d Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 24 Mar 2021 03:18:36 +0800 Subject: [PATCH 0477/2472] followup #16182 (#17472) * fix nim js cmp fails at CT * followup #16182 --- lib/pure/algorithm.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pure/algorithm.nim b/lib/pure/algorithm.nim index 0410d65ee..f4b3f46be 100644 --- a/lib/pure/algorithm.nim +++ b/lib/pure/algorithm.nim @@ -336,12 +336,12 @@ template `<-`(a, b) = else: copyMem(addr(a), addr(b), sizeof(T)) -proc merge[T](a, b: var openArray[T], lo, m, hi: int, +proc mergeAlt[T](a, b: var openArray[T], lo, m, hi: int, cmp: proc (x, y: T): int {.closure.}, order: SortOrder) = # Optimization: If max(left) <= min(right) there is nothing to do! # 1 2 3 4 ## 5 6 7 8 # -> O(n) for sorted arrays. - # On random data this saves up to 40% of merge calls. + # On random data this saves up to 40% of mergeAlt calls. if cmp(a[m], a[m+1]) * order <= 0: return var j = lo # copy a[j..m] into b: @@ -424,7 +424,7 @@ func sort*[T](a: var openArray[T], while s < n: var m = n-1-s while m >= 0: - merge(a, b, max(m-s+1, 0), m, m+s, cmp, order) + mergeAlt(a, b, max(m-s+1, 0), m, m+s, cmp, order) dec(m, s*2) s = s*2 From dbde97f649b0a4d947e15eddfacbde761cd49885 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 23 Mar 2021 23:46:19 +0100 Subject: [PATCH 0478/2472] fixes #16076 (#17486) --- compiler/semfold.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/semfold.nim b/compiler/semfold.nim index a3b9a8dde..0d4f6a0bd 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -685,7 +685,8 @@ proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode of nkDerefExpr, nkHiddenDeref: let a = getConstExpr(m, n[0], idgen, g) if a != nil and a.kind == nkNilLit: - localError(g.config, n.info, "nil dereference is not allowed") + result = nil + #localError(g.config, n.info, "nil dereference is not allowed") of nkCast: var a = getConstExpr(m, n[1], idgen, g) if a == nil: return From 8ccde68f132be4dba330eb6ec50f4679e564efac Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 24 Mar 2021 01:01:31 +0100 Subject: [PATCH 0479/2472] closes #17085; boehm has problems with large allocations, that is expected and we don't care (#17487) --- testament/categories.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/categories.nim b/testament/categories.nim index acb3ee0a8..86aa1f8af 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -173,7 +173,7 @@ proc gcTests(r: var TResults, cat: Category, options: string) = test "stackrefleak" test "cyclecollector" - test "trace_globals" + testWithoutBoehm "trace_globals" proc longGCTests(r: var TResults, cat: Category, options: string) = when defined(windows): From c015ecdc37221b374647b3b1503d80519e253d94 Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 24 Mar 2021 15:16:41 +0800 Subject: [PATCH 0480/2472] fix #17490 (#17491) --- lib/pure/json.nim | 14 ++++++++++---- tests/stdlib/tjson.nim | 11 +++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 048ed2797..833403d68 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -927,7 +927,7 @@ when defined(js): proc parseNativeJson(x: cstring): JSObject {.importjs: "JSON.parse(#)".} - proc getVarType(x: JSObject): JsonNodeKind = + proc getVarType(x: JSObject, isRawNumber: var bool): JsonNodeKind = result = JNull case $getProtoName(x) # TODO: Implicit returns fail here. of "[object Array]": return JArray @@ -937,6 +937,7 @@ when defined(js): if isSafeInteger(x): return JInt else: + isRawNumber = true return JString else: return JFloat @@ -946,13 +947,13 @@ when defined(js): else: assert false proc len(x: JSObject): int = - assert x.getVarType == JArray asm """ `result` = `x`.length; """ proc convertObject(x: JSObject): JsonNode = - case getVarType(x) + var isRawNumber = false + case getVarType(x, isRawNumber) of JArray: result = newJArray() for i in 0 ..< x.len: @@ -973,7 +974,12 @@ when defined(js): result = newJFloat(x.to(float)) of JString: # Dunno what to do with isUnquoted here - result = newJString($x.to(cstring)) + if isRawNumber: + var value: cstring + {.emit: "`value` = `x`.toString();".} + result = newJRawNumber($value) + else: + result = newJString($x.to(cstring)) of JBool: result = newJBool(x.to(bool)) of JNull: diff --git a/tests/stdlib/tjson.nim b/tests/stdlib/tjson.nim index ceb9efb0e..e538baf4f 100644 --- a/tests/stdlib/tjson.nim +++ b/tests/stdlib/tjson.nim @@ -300,3 +300,14 @@ block: # bug #17383 when not defined(js): testRoundtrip(int64.high): "9223372036854775807" testRoundtrip(uint64.high): "18446744073709551615" + + +block: + let a = "18446744073709551615" + let b = a.parseJson + doAssert b.kind == JString + let c = $b + when defined(js): + doAssert c == "18446744073709552000" + else: + doAssert c == "18446744073709551615" From 13a203001434db8f6cc9e4deab10dbfc7f57bd94 Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 24 Mar 2021 15:49:05 +0800 Subject: [PATCH 0481/2472] follow up #17486 (#17492) * fix nim js cmp fails at CT * follow up #17486 * test more branches * better --- tests/iter/t16076.nim | 45 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 tests/iter/t16076.nim diff --git a/tests/iter/t16076.nim b/tests/iter/t16076.nim new file mode 100644 index 000000000..2eb409068 --- /dev/null +++ b/tests/iter/t16076.nim @@ -0,0 +1,45 @@ +discard """ + targets: "c js" +""" + +proc main() = + block: # bug #17485 + type + O = ref object + i: int + + iterator t(o: O): int = + if o != nil: + yield o.i + yield 0 + + proc m = + var data = "" + for i in t(nil): + data.addInt i + + doAssert data == "0" + + m() + + + block: # bug #16076 + type + R = ref object + z: int + + var data = "" + + iterator foo(x: int; y: R = nil): int {.inline.} = + if y == nil: + yield x + else: + yield y.z + + for b in foo(10): + data.addInt b + + doAssert data == "10" + +static: main() +main() From 465a41c3083668c37f4bbee1f00fd709d7f35298 Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 24 Mar 2021 16:44:24 +0800 Subject: [PATCH 0482/2472] [isolation]fix empty ref object bug (#17471) * fix nim js cmp fails at CT * [Minor]fix empty ref object for isolation * Update compiler/isolation_check.nim * Update compiler/isolation_check.nim Co-authored-by: Clyybber Co-authored-by: Clyybber --- compiler/isolation_check.nim | 8 ++++++-- tests/stdlib/tisolation.nim | 8 ++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/compiler/isolation_check.nim b/compiler/isolation_check.nim index 01f0a002a..9c8ca58b4 100644 --- a/compiler/isolation_check.nim +++ b/compiler/isolation_check.nim @@ -100,7 +100,12 @@ proc checkIsolate*(n: PNode): bool = for it in n: result = checkIsolate(it.lastSon) if not result: break - of nkCaseStmt, nkObjConstr: + of nkCaseStmt: + for i in 1.. Date: Wed, 24 Mar 2021 09:47:11 +0100 Subject: [PATCH 0483/2472] make unary minus part of number literals, refs #17020 (#17488) * make unary minus part of number literals, refs #17020 * fixes regression --- changelog.md | 4 +- compiler/lexer.nim | 87 +++++++++++++++++++++++------------- compiler/semtypes.nim | 2 +- doc/manual.rst | 43 +++++++++++++++--- tests/lexer/tunary_minus.nim | 76 +++++++++++++++++++++++++++++++ 5 files changed, 173 insertions(+), 39 deletions(-) create mode 100644 tests/lexer/tunary_minus.nim diff --git a/changelog.md b/changelog.md index 78817cf85..c0cd97024 100644 --- a/changelog.md +++ b/changelog.md @@ -212,7 +212,7 @@ - `std/options` changed `$some(3)` to `"some(3)"` instead of `"Some(3)"` and `$none(int)` to `"none(int)"` instead of `"None[int]"`. - + - Added `algorithm.merge`. @@ -263,6 +263,8 @@ - `typedesc[Foo]` now renders as such instead of `type Foo` in compiler messages. +- The unary minus in `-1` is now part of the integer literal, it is now parsed as a single token. + This implies that edge cases like `-128'i8` finally work correctly. ## Compiler changes diff --git a/compiler/lexer.nim b/compiler/lexer.nim index 729ba3435..bcd3f0076 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -26,6 +26,7 @@ const SymStartChars*: set[char] = {'a'..'z', 'A'..'Z', '\x80'..'\xFF'} OpChars*: set[char] = {'+', '-', '*', '/', '\\', '<', '>', '!', '?', '^', '.', '|', '=', '%', '&', '$', '@', '~', ':'} + UnaryMinusWhitelist = {' ', '\t', '\n', '\r', ',', ';', '(', '[', '{'} # don't forget to update the 'highlite' module if these charsets should change @@ -51,22 +52,22 @@ type tkVar = "var", tkWhen = "when", tkWhile = "while", tkXor = "xor", tkYield = "yield", # end of keywords - tkIntLit = "tkIntLit", tkInt8Lit = "tkInt8Lit", tkInt16Lit = "tkInt16Lit", + tkIntLit = "tkIntLit", tkInt8Lit = "tkInt8Lit", tkInt16Lit = "tkInt16Lit", tkInt32Lit = "tkInt32Lit", tkInt64Lit = "tkInt64Lit", - tkUIntLit = "tkUIntLit", tkUInt8Lit = "tkUInt8Lit", tkUInt16Lit = "tkUInt16Lit", + tkUIntLit = "tkUIntLit", tkUInt8Lit = "tkUInt8Lit", tkUInt16Lit = "tkUInt16Lit", tkUInt32Lit = "tkUInt32Lit", tkUInt64Lit = "tkUInt64Lit", tkFloatLit = "tkFloatLit", tkFloat32Lit = "tkFloat32Lit", tkFloat64Lit = "tkFloat64Lit", tkFloat128Lit = "tkFloat128Lit", tkStrLit = "tkStrLit", tkRStrLit = "tkRStrLit", tkTripleStrLit = "tkTripleStrLit", - tkGStrLit = "tkGStrLit", tkGTripleStrLit = "tkGTripleStrLit", tkCharLit = "tkCharLit", - + tkGStrLit = "tkGStrLit", tkGTripleStrLit = "tkGTripleStrLit", tkCharLit = "tkCharLit", + tkParLe = "(", tkParRi = ")", tkBracketLe = "[", tkBracketRi = "]", tkCurlyLe = "{", tkCurlyRi = "}", tkBracketDotLe = "[.", tkBracketDotRi = ".]", tkCurlyDotLe = "{.", tkCurlyDotRi = ".}", tkParDotLe = "(.", tkParDotRi = ".)", tkComma = ",", tkSemiColon = ";", - tkColon = ":", tkColonColon = "::", tkEquals = "=", + tkColon = ":", tkColonColon = "::", tkEquals = "=", tkDot = ".", tkDotDot = "..", tkBracketLeColon = "[:", tkOpr, tkComment, tkAccent = "`", # these are fake tokens used by renderer.nim @@ -348,6 +349,14 @@ proc getNumber(L: var Lexer, result: var Token) = startpos = L.bufpos tokenBegin(result, startpos) + var isPositive = true + if L.buf[L.bufpos] == '-': + eatChar(L, result) + isPositive = true + + template setNumber(field, value) = + field = (if isPositive: value else: -value) + # First stage: find out base, make verifications, build token literal string # {'c', 'C'} is added for deprecation reasons to provide a clear error message if L.buf[L.bufpos] == '0' and L.buf[L.bufpos + 1] in baseCodeChars + {'c', 'C', 'O'}: @@ -459,7 +468,7 @@ proc getNumber(L: var Lexer, result: var Token) = # Third stage, extract actual number L.bufpos = startpos # restore position - var pos: int = startpos + var pos = startpos try: if (L.buf[pos] == '0') and (L.buf[pos + 1] in baseCodeChars): inc(pos, 2) @@ -500,20 +509,20 @@ proc getNumber(L: var Lexer, result: var Token) = internalError(L.config, getLineInfo(L), "getNumber") case result.tokType - of tkIntLit, tkInt64Lit: result.iNumber = xi - of tkInt8Lit: result.iNumber = ashr(xi shl 56, 56) - of tkInt16Lit: result.iNumber = ashr(xi shl 48, 48) - of tkInt32Lit: result.iNumber = ashr(xi shl 32, 32) - of tkUIntLit, tkUInt64Lit: result.iNumber = xi - of tkUInt8Lit: result.iNumber = xi and 0xff - of tkUInt16Lit: result.iNumber = xi and 0xffff - of tkUInt32Lit: result.iNumber = xi and 0xffffffff + of tkIntLit, tkInt64Lit: setNumber result.iNumber, xi + of tkInt8Lit: setNumber result.iNumber, ashr(xi shl 56, 56) + of tkInt16Lit: setNumber result.iNumber, ashr(xi shl 48, 48) + of tkInt32Lit: setNumber result.iNumber, ashr(xi shl 32, 32) + of tkUIntLit, tkUInt64Lit: setNumber result.iNumber, xi + of tkUInt8Lit: setNumber result.iNumber, xi and 0xff + of tkUInt16Lit: setNumber result.iNumber, xi and 0xffff + of tkUInt32Lit: setNumber result.iNumber, xi and 0xffffffff of tkFloat32Lit: - result.fNumber = (cast[PFloat32](addr(xi)))[] + setNumber result.fNumber, (cast[PFloat32](addr(xi)))[] # note: this code is endian neutral! # XXX: Test this on big endian machine! of tkFloat64Lit, tkFloatLit: - result.fNumber = (cast[PFloat64](addr(xi)))[] + setNumber result.fNumber, (cast[PFloat64](addr(xi)))[] else: internalError(L.config, getLineInfo(L), "getNumber") # Bounds checks. Non decimal literals are allowed to overflow the range of @@ -521,12 +530,13 @@ proc getNumber(L: var Lexer, result: var Token) = # below checks of signed sizes against uint*.high is deliberate: # (0x80'u8 = 128, 0x80'i8 = -128, etc == OK) if result.tokType notin floatTypes: - let outOfRange = case result.tokType: - of tkUInt8Lit, tkUInt16Lit, tkUInt32Lit: result.iNumber != xi - of tkInt8Lit: (xi > BiggestInt(uint8.high)) - of tkInt16Lit: (xi > BiggestInt(uint16.high)) - of tkInt32Lit: (xi > BiggestInt(uint32.high)) - else: false + let outOfRange = + case result.tokType + of tkUInt8Lit, tkUInt16Lit, tkUInt32Lit: result.iNumber != xi + of tkInt8Lit: (xi > BiggestInt(uint8.high)) + of tkInt16Lit: (xi > BiggestInt(uint16.high)) + of tkInt32Lit: (xi > BiggestInt(uint32.high)) + else: false if outOfRange: #echo "out of range num: ", result.iNumber, " vs ", xi @@ -557,23 +567,23 @@ proc getNumber(L: var Lexer, result: var Token) = raise newException(ValueError, "invalid integer: " & $result.literal) result.iNumber = iNumber - # Explicit bounds checks. Only T.high needs to be considered - # since result.iNumber can't be negative. + # Explicit bounds checks. let outOfRange = case result.tokType - of tkInt8Lit: result.iNumber > int8.high - of tkUInt8Lit: result.iNumber > BiggestInt(uint8.high) - of tkInt16Lit: result.iNumber > int16.high - of tkUInt16Lit: result.iNumber > BiggestInt(uint16.high) - of tkInt32Lit: result.iNumber > int32.high - of tkUInt32Lit: result.iNumber > BiggestInt(uint32.high) + of tkInt8Lit: result.iNumber > int8.high or result.iNumber < int8.low + of tkUInt8Lit: result.iNumber > BiggestInt(uint8.high) or result.iNumber < 0 + of tkInt16Lit: result.iNumber > int16.high or result.iNumber < int16.low + of tkUInt16Lit: result.iNumber > BiggestInt(uint16.high) or result.iNumber < 0 + of tkInt32Lit: result.iNumber > int32.high or result.iNumber < int32.low + of tkUInt32Lit: result.iNumber > BiggestInt(uint32.high) or result.iNumber < 0 else: false - if outOfRange: lexMessageLitNum(L, "number out of range: '$1'", startpos) + if outOfRange: + lexMessageLitNum(L, "number out of range: '$1'", startpos) # Promote int literal to int64? Not always necessary, but more consistent if result.tokType == tkIntLit: - if result.iNumber > high(int32): + if result.iNumber > high(int32) or result.iNumber < low(int32): result.tokType = tkInt64Lit except ValueError: @@ -1278,6 +1288,19 @@ proc rawGetTok*(L: var Lexer, tok: var Token) = let c = L.buf[L.bufpos] if c in SymChars+{'_'}: lexMessage(L, errGenerated, "invalid token: no whitespace between number and identifier") + of '-': + if L.buf[L.bufpos+1] in {'0'..'9'} and + (L.bufpos-1 == 0 or L.buf[L.bufpos-1] in UnaryMinusWhitelist): + # x)-23 # binary minus + # ,-23 # unary minus + # \n-78 # unary minus? Yes. + # =-3 # parsed as `=-` anyway + getNumber(L, tok) + let c = L.buf[L.bufpos] + if c in SymChars+{'_'}: + lexMessage(L, errGenerated, "invalid token: no whitespace between number and identifier") + else: + getOperator(L, tok) else: if c in OpChars: getOperator(L, tok) diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index dcab9a884..0fce7b417 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -299,7 +299,7 @@ proc semArrayIndex(c: PContext, n: PNode): PType = result = makeRangeWithStaticExpr(c, e.typ.n) elif e.kind in {nkIntLit..nkUInt64Lit}: if e.intVal < 0: - localError(c.config, n[1].info, + localError(c.config, n.info, "Array length can't be negative, but was " & $e.intVal) result = makeRangeType(c, 0, e.intVal-1, n.info, e.typ) elif e.kind == nkSym and e.typ.kind == tyStatic: diff --git a/doc/manual.rst b/doc/manual.rst index 1bb47f28b..db1251630 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -499,10 +499,11 @@ Numerical constants are of a single type and have the form:: hexdigit = digit | 'A'..'F' | 'a'..'f' octdigit = '0'..'7' bindigit = '0'..'1' - HEX_LIT = '0' ('x' | 'X' ) hexdigit ( ['_'] hexdigit )* - DEC_LIT = digit ( ['_'] digit )* - OCT_LIT = '0' 'o' octdigit ( ['_'] octdigit )* - BIN_LIT = '0' ('b' | 'B' ) bindigit ( ['_'] bindigit )* + unary_minus = '-' # See the section about unary minus + HEX_LIT = unary_minus? '0' ('x' | 'X' ) hexdigit ( ['_'] hexdigit )* + DEC_LIT = unary_minus? digit ( ['_'] digit )* + OCT_LIT = unary_minus? '0' 'o' octdigit ( ['_'] octdigit )* + BIN_LIT = unary_minus? '0' ('b' | 'B' ) bindigit ( ['_'] bindigit )* INT_LIT = HEX_LIT | DEC_LIT @@ -521,7 +522,7 @@ Numerical constants are of a single type and have the form:: UINT64_LIT = INT_LIT ['\''] ('u' | 'U') '64' exponent = ('e' | 'E' ) ['+' | '-'] digit ( ['_'] digit )* - FLOAT_LIT = digit (['_'] digit)* (('.' digit (['_'] digit)* [exponent]) |exponent) + FLOAT_LIT = unary_minus? digit (['_'] digit)* (('.' digit (['_'] digit)* [exponent]) |exponent) FLOAT32_SUFFIX = ('f' | 'F') ['32'] FLOAT32_LIT = HEX_LIT '\'' FLOAT32_SUFFIX | (FLOAT_LIT | DEC_LIT | OCT_LIT | BIN_LIT) ['\''] FLOAT32_SUFFIX @@ -535,6 +536,38 @@ for readability. Integer and floating-point literals may be given in decimal (no prefix), binary (prefix `0b`), octal (prefix `0o`), and hexadecimal (prefix `0x`) notation. +The fact that the unary minus `-` in a number literal like `-1` is considered +to be part of the literal is a late addition to the language. The rationale is that +an expression `-128'i8` should be valid and without this special case, this would +be impossible -- `128` is not a valid `int8` value, only `-128` is. + +For the `unary_minus` rule there are further restrictions that are not covered +in the formal grammar. For `-` to be part of the number literal its immediately +preceeding character has to be in the +set `{' ', '\t', '\n', '\r', ',', ';', '(', '[', '{'}`. This set was designed to +cover most cases in a natural manner. + +In the following examples, `-1` is a single token: + +.. code-block:: nim + + echo -1 + echo(-1) + echo [-1] + echo 3,-1 + + "abc";-1 + +In the following examples, `-1` is parsed as two separate tokens (as `- 1`): + +.. code-block:: nim + + echo x-1 + echo (int)-1 + echo [a]-1 + "abc"-1 + + There exists a literal for each numerical type that is defined. The suffix starting with an apostrophe ('\'') is called a `type suffix`:idx:. Literals without a type suffix are of an integer type diff --git a/tests/lexer/tunary_minus.nim b/tests/lexer/tunary_minus.nim new file mode 100644 index 000000000..89f1b79ef --- /dev/null +++ b/tests/lexer/tunary_minus.nim @@ -0,0 +1,76 @@ +discard """ + targets: "c cpp js" +""" + +# Test numeric literals and handling of minus symbol + +import std/[macros, strutils] + +macro lispReprStr*(a: untyped): untyped = newLit(a.lispRepr) + +macro assertAST*(expected: string, struct: untyped): untyped = + var ast = newLit(struct.treeRepr) + result = quote do: + if `ast` != `expected`: + doAssert false, "\nGot:\n" & `ast`.indent(2) & "\nExpected:\n" & `expected`.indent(2) + +const one = 1 +const minusOne = `-`(one) + +# border cases that *should* generate compiler errors: +assertAST dedent """ + StmtList + Asgn + Ident "x" + Command + IntLit 4 + IntLit -1""": + x = 4 -1 +assertAST dedent """ + StmtList + VarSection + IdentDefs + Ident "x" + Ident "uint" + IntLit -1""": + var x: uint = -1 +template bad() = + x = 4 -1 +doAssert not compiles(bad()) + +template main = + block: # check when a minus (-) is a negative sign for a literal + doAssert -1 == minusOne: + "unable to parse a spaced-prefixed negative int" + doAssert lispReprStr(-1) == """(IntLit -1)""" + doAssert -1.0'f64 == minusOne.float64 + doAssert lispReprStr(-1.000'f64) == """(Float64Lit -1.0)""" + doAssert lispReprStr( -1.000'f64) == """(Float64Lit -1.0)""" + doAssert [-1].contains(minusOne): + "unable to handle negatives after square bracket" + doAssert lispReprStr([-1]) == """(Bracket (IntLit -1))""" + doAssert (-1, 2)[0] == minusOne: + "unable to handle negatives after parenthesis" + doAssert lispReprStr((-1, 2)) == """(Par (IntLit -1) (IntLit 2))""" + proc x(): int = + var a = 1;-1 # the -1 should act as the return value + doAssert x() == minusOne: + "unable to handle negatives after semi-colon" + + block: # check when a minus (-) is an unary op + doAssert -one == minusOne: + "unable to a negative prior to identifier" + + block: # check when a minus (-) is a a subtraction op + doAssert 4-1 == 3: + "unable to handle subtraction sans surrounding spaces with a numeric literal" + doAssert 4-one == 3: + "unable to handle subtraction sans surrounding spaces with an identifier" + doAssert 4 - 1 == 3: + "unable to handle subtraction with surrounding spaces with a numeric literal" + doAssert 4 - one == 3: + "unable to handle subtraction with surrounding spaces with an identifier" + + +static: main() +main() From e94aec20da354d4d3bb6db6e4b840549aa0e7bf1 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 24 Mar 2021 02:36:15 -0700 Subject: [PATCH 0484/2472] fix #17325 linux 32bit CI; fix #17085 flaky test (#17469) --- azure-pipelines.yml | 10 +++++----- lib/pure/json.nim | 7 ++++--- tests/gc/trace_globals.nim | 13 ++++++++++++- tests/stdlib/tjsonutils.nim | 2 +- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f65f8ca11..9006cf6e5 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -17,11 +17,11 @@ jobs: Linux_amd64: vmImage: 'ubuntu-16.04' CPU: amd64 - # Linux_i386: - # # bug #17325: fails on 'ubuntu-16.04' because it now errors with: - # # g++-multilib : Depends: gcc-multilib (>= 4:5.3.1-1ubuntu1) but it is not going to be installed - # vmImage: 'ubuntu-18.04' - # CPU: i386 + Linux_i386: + # bug #17325: fails on 'ubuntu-16.04' because it now errors with: + # g++-multilib : Depends: gcc-multilib (>= 4:5.3.1-1ubuntu1) but it is not going to be installed + vmImage: 'ubuntu-18.04' + CPU: i386 OSX_amd64: vmImage: 'macOS-10.15' CPU: amd64 diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 833403d68..7ee5be9c5 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -1075,12 +1075,13 @@ when defined(nimFixedForwardGeneric): dst = jsonNode.copy proc initFromJson[T: SomeInteger](dst: var T; jsonNode: JsonNode, jsonPath: var string) = - when T is uint|uint64: + when T is uint|uint64 or (not defined(js) and int.sizeof == 4): + verifyJsonKind(jsonNode, {JInt, JString}, jsonPath) case jsonNode.kind of JString: - dst = T(parseBiggestUInt(jsonNode.str)) + let x = parseBiggestUInt(jsonNode.str) + dst = cast[T](x) else: - verifyJsonKind(jsonNode, {JInt}, jsonPath) dst = T(jsonNode.num) else: verifyJsonKind(jsonNode, {JInt}, jsonPath) diff --git a/tests/gc/trace_globals.nim b/tests/gc/trace_globals.nim index d91bc5f35..f62a15692 100644 --- a/tests/gc/trace_globals.nim +++ b/tests/gc/trace_globals.nim @@ -1,11 +1,20 @@ discard """ - output: '''10000000 + output: ''' +10000000 10000000 10000000''' """ # bug #17085 +#[ +refs https://github.com/nim-lang/Nim/issues/17085#issuecomment-786466595 +with --gc:boehm, this warning sometimes gets generated: +Warning: Repeated allocation of very large block (appr. size 14880768): +May lead to memory leak and poor performance. +nim CI now runs this test with `testWithoutBoehm` to avoid running it with --gc:boehm. +]# + proc init(): string = for a in 0..<10000000: result.add 'c' @@ -16,6 +25,8 @@ proc f() = var c {.global.} = init() echo a.len + # `echo` intentional according to + # https://github.com/nim-lang/Nim/pull/17469/files/0c9e94cb6b9ebca9da7cb19a063fba7aa409748e#r600016573 echo b.len echo c.len diff --git a/tests/stdlib/tjsonutils.nim b/tests/stdlib/tjsonutils.nim index 5cd975cab..63bf97704 100644 --- a/tests/stdlib/tjsonutils.nim +++ b/tests/stdlib/tjsonutils.nim @@ -71,7 +71,7 @@ template fn() = block: let a = (int32.high, uint32.high) testRoundtrip(a): "[2147483647,4294967295]" - when not defined(js): + when int.sizeof > 4: block: let a = (int64.high, uint64.high) testRoundtrip(a): "[9223372036854775807,18446744073709551615]" From 1590d145757416788e99023ab50c5afae6f8ea36 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 24 Mar 2021 02:58:29 -0700 Subject: [PATCH 0485/2472] fix #17260 render `\` properly in nim doc, rst2html (#17315) --- lib/packages/docutils/rst.nim | 21 ++++++++++++++++----- lib/system.nim | 4 ++-- nimdoc/rst2html/expected/rst_examples.html | 2 +- tests/stdlib/trstgen.nim | 22 ++++++++++++++++++++-- 4 files changed, 39 insertions(+), 10 deletions(-) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index b5eef7610..11768c7da 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -852,6 +852,9 @@ proc isInlineMarkupEnd(p: RstParser, markup: string): bool = if not result: return # Rule 4: if p.idx > 0: + # see bug #17260; for now `\` must be written ``\``, likewise with sequences + # ending in an un-escaped `\`; `\\` is legal but not `\\\` for example; + # for this reason we can't use `["``", "`"]` here. if markup != "``" and prevTok(p).symbol == "\\": result = false @@ -1089,11 +1092,19 @@ proc parseUntil(p: var RstParser, father: PRstNode, postfix: string, if isInlineMarkupEnd(p, postfix): inc p.idx break - elif interpretBackslash: - parseBackslash(p, father) else: - father.add(newLeaf(p)) - inc p.idx + if postfix == "`": + if prevTok(p).symbol == "\\" and currentTok(p).symbol == "`": + father.sons[^1] = newLeaf(p) # instead, we should use lookahead + else: + father.add(newLeaf(p)) + inc p.idx + else: + if interpretBackslash: + parseBackslash(p, father) + else: + father.add(newLeaf(p)) + inc p.idx of tkAdornment, tkWord, tkOther: father.add(newLeaf(p)) inc p.idx @@ -1243,7 +1254,7 @@ proc parseInline(p: var RstParser, father: PRstNode) = father.add(n) elif isInlineMarkupStart(p, "`"): var n = newRstNode(rnInterpretedText) - parseUntil(p, n, "`", true) + parseUntil(p, n, "`", false) # bug #17260 n = parsePostfix(p, n) father.add(n) elif isInlineMarkupStart(p, "|"): diff --git a/lib/system.nim b/lib/system.nim index 574043125..4c31f20a2 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2827,7 +2827,7 @@ when declared(initDebugger): proc addEscapedChar*(s: var string, c: char) {.noSideEffect, inline.} = ## Adds a char to string `s` and applies the following escaping: ## - ## * replaces any `\` by `\\` + ## * replaces any ``\`` by `\\` ## * replaces any `'` by `\'` ## * replaces any `"` by `\"` ## * replaces any `\a` by `\\a` @@ -2838,7 +2838,7 @@ proc addEscapedChar*(s: var string, c: char) {.noSideEffect, inline.} = ## * replaces any `\f` by `\\f` ## * replaces any `\r` by `\\r` ## * replaces any `\e` by `\\e` - ## * replaces any other character not in the set `{'\21..'\126'} + ## * replaces any other character not in the set `{\21..\126}` ## by `\xHH` where `HH` is its hexadecimal value. ## ## The procedure has been designed so that its output is usable for many diff --git a/nimdoc/rst2html/expected/rst_examples.html b/nimdoc/rst2html/expected/rst_examples.html index 23d192009..8253289a4 100644 --- a/nimdoc/rst2html/expected/rst_examples.html +++ b/nimdoc/rst2html/expected/rst_examples.html @@ -300,7 +300,7 @@ stmt = IND{>} stmt ^+ IND{=} DED # list of statements

    However, this does not work. The problem is that the procedure should not only return, but return and continue after an iteration has finished. This return and continue is called a yield statement. Now the only thing left to do is to replace the proc keyword by iterator and here it is - our first iterator:

    - +
    A1 headerA2 | not fooled
    C1C2 bold
    D1 code |D2
    D1 code \|D2
    E1 | text
    F2 without pipe

    not in table

    diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index cf82cdf91..d6055cc11 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -196,9 +196,15 @@ suite "RST/Markdown general": | | F2 without pipe not in table""" let output1 = input1.toHtml - doAssert output1 == """ + #[ + TODO: `\|` inside a table cell should render as `|` + `|` outside a table cell should render as `\|` + consistently with markdown, see https://stackoverflow.com/a/66557930/1426932 + ]# + doAssert output1 == """ +
    A1 headerA2 | not fooled
    - +
    A1 headerA2 | not fooled
    C1C2 bold
    D1 code |D2
    D1 code \|D2
    E1 | text
    F2 without pipe

    not in table

    @@ -549,6 +555,18 @@ let x = 1 let output2 = input2.toHtml doAssert "foo.bar""" + check """`foo\`\`bar`""".toHtml == """foo``bar""" + check """`foo\`bar`""".toHtml == """foo`bar""" + check """`\`bar`""".toHtml == """`bar""" + check """`a\b\x\\ar`""".toHtml == """a\b\x\\ar""" + + test "inline literal": + check """``foo.bar``""".toHtml == """foo.bar""" + check """``foo\bar``""".toHtml == """foo\bar""" + check """``f\`o\\o\b`ar``""".toHtml == """f\`o\\o\b`ar""" + test "RST comments": let input1 = """ Check that comment disappears: From 7366a3da37605b230823dd4b6db07abb70dbd40b Mon Sep 17 00:00:00 2001 From: Saem Ghani Date: Wed, 24 Mar 2021 03:55:58 -0700 Subject: [PATCH 0486/2472] potential fix for semgeneric formal params (#17494) marked locations where analysis of return formal param is done prior to args. This might fix some subtle bugs. --- compiler/isolation_check.nim | 3 +++ compiler/reorder.nim | 1 + compiler/semgnrc.nim | 6 ++++-- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/compiler/isolation_check.nim b/compiler/isolation_check.nim index 9c8ca58b4..43db5d59b 100644 --- a/compiler/isolation_check.nim +++ b/compiler/isolation_check.nim @@ -85,6 +85,9 @@ proc checkIsolate*(n: PNode): bool = of nkCharLit..nkNilLit: result = true of nkCallKinds: + # XXX: as long as we don't update the analysis while examining arguments + # we can do an early check of the return type, otherwise this is a + # bug and needs to be moved below if n[0].typ.flags * {tfGcSafe, tfNoSideEffect} == {}: return false for i in 1.. Date: Wed, 24 Mar 2021 14:46:19 +0100 Subject: [PATCH 0487/2472] custom integer literals (#17489) * user defined integer literals; refs #17020 * updated renderer.nim * use mlexerutils helper * imported all test cases from https://github.com/nim-lang/Nim/pull/17020 * final grammar updated --- changelog.md | 2 + compiler/docgen.nim | 2 +- compiler/lexer.nim | 342 +++++++++++------------ compiler/parser.nim | 18 +- compiler/renderer.nim | 18 +- compiler/semstmts.nim | 2 +- doc/grammar.txt | 3 +- doc/manual.rst | 54 +++- tests/lexer/mlexerutils.nim | 9 + tests/lexer/tcustom_numeric_literals.nim | 150 ++++++++++ tests/lexer/tstrlits.nim | 19 -- tests/lexer/tunary_minus.nim | 8 +- 12 files changed, 409 insertions(+), 218 deletions(-) create mode 100644 tests/lexer/mlexerutils.nim create mode 100644 tests/lexer/tcustom_numeric_literals.nim delete mode 100644 tests/lexer/tstrlits.nim diff --git a/changelog.md b/changelog.md index c0cd97024..922b8470a 100644 --- a/changelog.md +++ b/changelog.md @@ -266,6 +266,8 @@ - The unary minus in `-1` is now part of the integer literal, it is now parsed as a single token. This implies that edge cases like `-128'i8` finally work correctly. +- Custom numeric literals are now supported. + ## Compiler changes diff --git a/compiler/docgen.nim b/compiler/docgen.nim index dded231d7..b24a24a2b 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -416,7 +416,7 @@ proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var Rope; renderFlags: TRe of tkOpr: dispA(d.conf, result, "$1", "\\spanOperator{$1}", [escLit]) - of tkStrLit..tkTripleStrLit: + of tkStrLit..tkTripleStrLit, tkCustomLit: dispA(d.conf, result, "$1", "\\spanStringLit{$1}", [escLit]) of tkCharLit: diff --git a/compiler/lexer.nim b/compiler/lexer.nim index bcd3f0076..b34b010c2 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -60,6 +60,7 @@ type tkFloat64Lit = "tkFloat64Lit", tkFloat128Lit = "tkFloat128Lit", tkStrLit = "tkStrLit", tkRStrLit = "tkRStrLit", tkTripleStrLit = "tkTripleStrLit", tkGStrLit = "tkGStrLit", tkGTripleStrLit = "tkGTripleStrLit", tkCharLit = "tkCharLit", + tkCustomLit = "tkCustomLit", tkParLe = "(", tkParRi = ")", tkBracketLe = "[", tkBracketRi = "]", tkCurlyLe = "{", tkCurlyRi = "}", @@ -313,8 +314,7 @@ proc getNumber(L: var Lexer, result: var Token) = proc lexMessageLitNum(L: var Lexer, msg: string, startpos: int, msgKind = errGenerated) = # Used to get slightly human friendlier err messages. - const literalishChars = {'A'..'F', 'a'..'f', '0'..'9', 'X', 'x', 'o', 'O', - 'c', 'C', 'b', 'B', '_', '.', '\'', 'd', 'i', 'u'} + const literalishChars = {'A'..'Z', 'a'..'z', '0'..'9', '_', '.', '\''} var msgPos = L.bufpos var t: Token t.literal = "" @@ -326,15 +326,14 @@ proc getNumber(L: var Lexer, result: var Token) = t.literal.add(L.buf[L.bufpos]) inc(L.bufpos) matchChars(L, t, literalishChars) - if L.buf[L.bufpos] in {'\'', 'f', 'F', 'd', 'D', 'i', 'I', 'u', 'U'}: - inc(L.bufpos) + if L.buf[L.bufpos] in literalishChars: t.literal.add(L.buf[L.bufpos]) + inc(L.bufpos) matchChars(L, t, {'0'..'9'}) L.bufpos = msgPos lexMessage(L, msgKind, msg % t.literal) var - startpos, endpos: int xi: BiggestInt isBase10 = true numDigits = 0 @@ -346,7 +345,7 @@ proc getNumber(L: var Lexer, result: var Token) = result.tokType = tkIntLit # int literal until we know better result.literal = "" result.base = base10 - startpos = L.bufpos + let startpos = L.bufpos tokenBegin(result, startpos) var isPositive = true @@ -395,201 +394,187 @@ proc getNumber(L: var Lexer, result: var Token) = discard matchUnderscoreChars(L, result, {'0'..'9'}) if L.buf[L.bufpos] in {'e', 'E'}: result.tokType = tkFloatLit - eatChar(L, result, 'e') + eatChar(L, result) if L.buf[L.bufpos] in {'+', '-'}: eatChar(L, result) discard matchUnderscoreChars(L, result, {'0'..'9'}) - endpos = L.bufpos + let endpos = L.bufpos # Second stage, find out if there's a datatype suffix and handle it var postPos = endpos + if L.buf[postPos] in {'\'', 'f', 'F', 'd', 'D', 'i', 'I', 'u', 'U'}: + let errPos = postPos + var customLitPossible = false if L.buf[postPos] == '\'': inc(postPos) + customLitPossible = true - case L.buf[postPos] - of 'f', 'F': - inc(postPos) - if (L.buf[postPos] == '3') and (L.buf[postPos + 1] == '2'): - result.tokType = tkFloat32Lit - inc(postPos, 2) - elif (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'): - result.tokType = tkFloat64Lit - inc(postPos, 2) - elif (L.buf[postPos] == '1') and - (L.buf[postPos + 1] == '2') and - (L.buf[postPos + 2] == '8'): - result.tokType = tkFloat128Lit - inc(postPos, 3) - else: # "f" alone defaults to float32 - result.tokType = tkFloat32Lit - of 'd', 'D': # ad hoc convenience shortcut for f64 - inc(postPos) - result.tokType = tkFloat64Lit - of 'i', 'I': - inc(postPos) - if (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'): - result.tokType = tkInt64Lit - inc(postPos, 2) - elif (L.buf[postPos] == '3') and (L.buf[postPos + 1] == '2'): - result.tokType = tkInt32Lit - inc(postPos, 2) - elif (L.buf[postPos] == '1') and (L.buf[postPos + 1] == '6'): - result.tokType = tkInt16Lit - inc(postPos, 2) - elif (L.buf[postPos] == '8'): - result.tokType = tkInt8Lit - inc(postPos) + if L.buf[postPos] in SymChars: + var suffixAsLower = newStringOfCap(10) + var suffix = newStringOfCap(10) + while true: + let c = L.buf[postPos] + suffix.add c + suffixAsLower.add toLowerAscii(c) + inc postPos + if L.buf[postPos] notin SymChars+{'_'}: break + case suffix + of "f", "f32": result.tokType = tkFloat32Lit + of "d", "f64": result.tokType = tkFloat64Lit + of "f128": result.tokType = tkFloat128Lit + of "i8": result.tokType = tkInt8Lit + of "i16": result.tokType = tkInt16Lit + of "i32": result.tokType = tkInt32Lit + of "i64": result.tokType = tkInt64Lit + of "u": result.tokType = tkUIntLit + of "u8": result.tokType = tkUInt8Lit + of "u16": result.tokType = tkUInt16Lit + of "u32": result.tokType = tkUInt32Lit + of "u64": result.tokType = tkUInt64Lit else: - lexMessageLitNum(L, "invalid number: '$1'", startpos) - of 'u', 'U': - inc(postPos) - if (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'): - result.tokType = tkUInt64Lit - inc(postPos, 2) - elif (L.buf[postPos] == '3') and (L.buf[postPos + 1] == '2'): - result.tokType = tkUInt32Lit - inc(postPos, 2) - elif (L.buf[postPos] == '1') and (L.buf[postPos + 1] == '6'): - result.tokType = tkUInt16Lit - inc(postPos, 2) - elif (L.buf[postPos] == '8'): - result.tokType = tkUInt8Lit - inc(postPos) - else: - result.tokType = tkUIntLit + if customLitPossible: + # remember the position of the ``'`` so that the parser doesn't + # have to reparse the custom literal: + result.iNumber = len(result.literal) + result.literal.add '\'' + result.literal.add suffix + result.tokType = tkCustomLit + else: + lexMessageLitNum(L, "invalid number suffix: '$1'", errPos) else: - lexMessageLitNum(L, "invalid number: '$1'", startpos) + lexMessageLitNum(L, "invalid number suffix: '$1'", errPos) # Is there still a literalish char awaiting? Then it's an error! if L.buf[postPos] in literalishChars or (L.buf[postPos] == '.' and L.buf[postPos + 1] in {'0'..'9'}): lexMessageLitNum(L, "invalid number: '$1'", startpos) - # Third stage, extract actual number - L.bufpos = startpos # restore position - var pos = startpos - try: - if (L.buf[pos] == '0') and (L.buf[pos + 1] in baseCodeChars): - inc(pos, 2) - xi = 0 # it is a base prefix + if result.tokType != tkCustomLit: + # Third stage, extract actual number + L.bufpos = startpos # restore position + var pos = startpos + try: + if (L.buf[pos] == '0') and (L.buf[pos + 1] in baseCodeChars): + inc(pos, 2) + xi = 0 # it is a base prefix - case L.buf[pos - 1] - of 'b', 'B': - result.base = base2 - while pos < endpos: - if L.buf[pos] != '_': - xi = `shl`(xi, 1) or (ord(L.buf[pos]) - ord('0')) - inc(pos) - # 'c', 'C' is deprecated - of 'o', 'c', 'C': - result.base = base8 - while pos < endpos: - if L.buf[pos] != '_': - xi = `shl`(xi, 3) or (ord(L.buf[pos]) - ord('0')) - inc(pos) - of 'x', 'X': - result.base = base16 - while pos < endpos: - case L.buf[pos] - of '_': + case L.buf[pos - 1] + of 'b', 'B': + result.base = base2 + while pos < endpos: + if L.buf[pos] != '_': + xi = `shl`(xi, 1) or (ord(L.buf[pos]) - ord('0')) inc(pos) - of '0'..'9': - xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('0')) + # 'c', 'C' is deprecated + of 'o', 'c', 'C': + result.base = base8 + while pos < endpos: + if L.buf[pos] != '_': + xi = `shl`(xi, 3) or (ord(L.buf[pos]) - ord('0')) inc(pos) - of 'a'..'f': - xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('a') + 10) - inc(pos) - of 'A'..'F': - xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('A') + 10) - inc(pos) - else: - break + of 'x', 'X': + result.base = base16 + while pos < endpos: + case L.buf[pos] + of '_': + inc(pos) + of '0'..'9': + xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('0')) + inc(pos) + of 'a'..'f': + xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('a') + 10) + inc(pos) + of 'A'..'F': + xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('A') + 10) + inc(pos) + else: + break + else: + internalError(L.config, getLineInfo(L), "getNumber") + + case result.tokType + of tkIntLit, tkInt64Lit: setNumber result.iNumber, xi + of tkInt8Lit: setNumber result.iNumber, ashr(xi shl 56, 56) + of tkInt16Lit: setNumber result.iNumber, ashr(xi shl 48, 48) + of tkInt32Lit: setNumber result.iNumber, ashr(xi shl 32, 32) + of tkUIntLit, tkUInt64Lit: setNumber result.iNumber, xi + of tkUInt8Lit: setNumber result.iNumber, xi and 0xff + of tkUInt16Lit: setNumber result.iNumber, xi and 0xffff + of tkUInt32Lit: setNumber result.iNumber, xi and 0xffffffff + of tkFloat32Lit: + setNumber result.fNumber, (cast[PFloat32](addr(xi)))[] + # note: this code is endian neutral! + # XXX: Test this on big endian machine! + of tkFloat64Lit, tkFloatLit: + setNumber result.fNumber, (cast[PFloat64](addr(xi)))[] + else: internalError(L.config, getLineInfo(L), "getNumber") + + # Bounds checks. Non decimal literals are allowed to overflow the range of + # the datatype as long as their pattern don't overflow _bitwise_, hence + # below checks of signed sizes against uint*.high is deliberate: + # (0x80'u8 = 128, 0x80'i8 = -128, etc == OK) + if result.tokType notin floatTypes: + let outOfRange = + case result.tokType + of tkUInt8Lit, tkUInt16Lit, tkUInt32Lit: result.iNumber != xi + of tkInt8Lit: (xi > BiggestInt(uint8.high)) + of tkInt16Lit: (xi > BiggestInt(uint16.high)) + of tkInt32Lit: (xi > BiggestInt(uint32.high)) + else: false + + if outOfRange: + #echo "out of range num: ", result.iNumber, " vs ", xi + lexMessageLitNum(L, "number out of range: '$1'", startpos) + else: - internalError(L.config, getLineInfo(L), "getNumber") + case result.tokType + of floatTypes: + result.fNumber = parseFloat(result.literal) + of tkUInt64Lit, tkUIntLit: + var iNumber: uint64 + var len: int + try: + len = parseBiggestUInt(result.literal, iNumber) + except ValueError: + raise newException(OverflowDefect, "number out of range: " & $result.literal) + if len != result.literal.len: + raise newException(ValueError, "invalid integer: " & $result.literal) + result.iNumber = cast[int64](iNumber) + else: + var iNumber: int64 + var len: int + try: + len = parseBiggestInt(result.literal, iNumber) + except ValueError: + raise newException(OverflowDefect, "number out of range: " & $result.literal) + if len != result.literal.len: + raise newException(ValueError, "invalid integer: " & $result.literal) + result.iNumber = iNumber - case result.tokType - of tkIntLit, tkInt64Lit: setNumber result.iNumber, xi - of tkInt8Lit: setNumber result.iNumber, ashr(xi shl 56, 56) - of tkInt16Lit: setNumber result.iNumber, ashr(xi shl 48, 48) - of tkInt32Lit: setNumber result.iNumber, ashr(xi shl 32, 32) - of tkUIntLit, tkUInt64Lit: setNumber result.iNumber, xi - of tkUInt8Lit: setNumber result.iNumber, xi and 0xff - of tkUInt16Lit: setNumber result.iNumber, xi and 0xffff - of tkUInt32Lit: setNumber result.iNumber, xi and 0xffffffff - of tkFloat32Lit: - setNumber result.fNumber, (cast[PFloat32](addr(xi)))[] - # note: this code is endian neutral! - # XXX: Test this on big endian machine! - of tkFloat64Lit, tkFloatLit: - setNumber result.fNumber, (cast[PFloat64](addr(xi)))[] - else: internalError(L.config, getLineInfo(L), "getNumber") - - # Bounds checks. Non decimal literals are allowed to overflow the range of - # the datatype as long as their pattern don't overflow _bitwise_, hence - # below checks of signed sizes against uint*.high is deliberate: - # (0x80'u8 = 128, 0x80'i8 = -128, etc == OK) - if result.tokType notin floatTypes: + # Explicit bounds checks. let outOfRange = case result.tokType - of tkUInt8Lit, tkUInt16Lit, tkUInt32Lit: result.iNumber != xi - of tkInt8Lit: (xi > BiggestInt(uint8.high)) - of tkInt16Lit: (xi > BiggestInt(uint16.high)) - of tkInt32Lit: (xi > BiggestInt(uint32.high)) + of tkInt8Lit: result.iNumber > int8.high or result.iNumber < int8.low + of tkUInt8Lit: result.iNumber > BiggestInt(uint8.high) or result.iNumber < 0 + of tkInt16Lit: result.iNumber > int16.high or result.iNumber < int16.low + of tkUInt16Lit: result.iNumber > BiggestInt(uint16.high) or result.iNumber < 0 + of tkInt32Lit: result.iNumber > int32.high or result.iNumber < int32.low + of tkUInt32Lit: result.iNumber > BiggestInt(uint32.high) or result.iNumber < 0 else: false if outOfRange: - #echo "out of range num: ", result.iNumber, " vs ", xi lexMessageLitNum(L, "number out of range: '$1'", startpos) - else: - case result.tokType - of floatTypes: - result.fNumber = parseFloat(result.literal) - of tkUInt64Lit, tkUIntLit: - var iNumber: uint64 - var len: int - try: - len = parseBiggestUInt(result.literal, iNumber) - except ValueError: - raise newException(OverflowDefect, "number out of range: " & $result.literal) - if len != result.literal.len: - raise newException(ValueError, "invalid integer: " & $result.literal) - result.iNumber = cast[int64](iNumber) - else: - var iNumber: int64 - var len: int - try: - len = parseBiggestInt(result.literal, iNumber) - except ValueError: - raise newException(OverflowDefect, "number out of range: " & $result.literal) - if len != result.literal.len: - raise newException(ValueError, "invalid integer: " & $result.literal) - result.iNumber = iNumber + # Promote int literal to int64? Not always necessary, but more consistent + if result.tokType == tkIntLit: + if result.iNumber > high(int32) or result.iNumber < low(int32): + result.tokType = tkInt64Lit - # Explicit bounds checks. - let outOfRange = - case result.tokType - of tkInt8Lit: result.iNumber > int8.high or result.iNumber < int8.low - of tkUInt8Lit: result.iNumber > BiggestInt(uint8.high) or result.iNumber < 0 - of tkInt16Lit: result.iNumber > int16.high or result.iNumber < int16.low - of tkUInt16Lit: result.iNumber > BiggestInt(uint16.high) or result.iNumber < 0 - of tkInt32Lit: result.iNumber > int32.high or result.iNumber < int32.low - of tkUInt32Lit: result.iNumber > BiggestInt(uint32.high) or result.iNumber < 0 - else: false - - if outOfRange: - lexMessageLitNum(L, "number out of range: '$1'", startpos) - - # Promote int literal to int64? Not always necessary, but more consistent - if result.tokType == tkIntLit: - if result.iNumber > high(int32) or result.iNumber < low(int32): - result.tokType = tkInt64Lit - - except ValueError: - lexMessageLitNum(L, "invalid number: '$1'", startpos) - except OverflowDefect, RangeDefect: - lexMessageLitNum(L, "number out of range: '$1'", startpos) + except ValueError: + lexMessageLitNum(L, "invalid number: '$1'", startpos) + except OverflowDefect, RangeDefect: + lexMessageLitNum(L, "number out of range: '$1'", startpos) tokenEnd(result, postPos-1) L.bufpos = postPos @@ -830,8 +815,9 @@ proc getString(L: var Lexer, tok: var Token, mode: StringMode) = inc(pos) L.bufpos = pos -proc getCharacter(L: var Lexer, tok: var Token) = +proc getCharacter(L: var Lexer; tok: var Token) = tokenBegin(tok, L.bufpos) + let startPos = L.bufpos inc(L.bufpos) # skip ' var c = L.buf[L.bufpos] case c @@ -842,10 +828,16 @@ proc getCharacter(L: var Lexer, tok: var Token) = else: tok.literal = $c inc(L.bufpos) - if L.buf[L.bufpos] != '\'': - lexMessage(L, errGenerated, "missing closing ' for character literal") - tokenEndIgnore(tok, L.bufpos) - inc(L.bufpos) # skip ' + if L.buf[L.bufpos] == '\'': + tokenEndIgnore(tok, L.bufpos) + inc(L.bufpos) # skip ' + else: + if startPos > 0 and L.buf[startPos-1] == '`': + tok.literal = "'" + L.bufpos = startPos+1 + else: + lexMessage(L, errGenerated, "missing closing ' for character literal") + tokenEndIgnore(tok, L.bufpos) proc getSymbol(L: var Lexer, tok: var Token) = var h: Hash = 0 diff --git a/compiler/parser.nim b/compiler/parser.nim index fe857c81b..b9a6ffb8c 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -355,7 +355,7 @@ proc parseSymbol(p: var Parser, mode = smNormal): PNode = let node = newNodeI(nkIdent, lineinfo) node.ident = p.lex.cache.getIdent(accm) result.add(node) - of tokKeywordLow..tokKeywordHigh, tkSymbol, tkIntLit..tkCharLit: + of tokKeywordLow..tokKeywordHigh, tkSymbol, tkIntLit..tkCustomLit: result.add(newIdentNodeP(p.lex.cache.getIdent($p.tok), p)) getTok(p) else: @@ -627,7 +627,7 @@ proc identOrLiteral(p: var Parser, mode: PrimaryMode): PNode = #| | UINT_LIT | UINT8_LIT | UINT16_LIT | UINT32_LIT | UINT64_LIT #| | FLOAT_LIT | FLOAT32_LIT | FLOAT64_LIT #| | STR_LIT | RSTR_LIT | TRIPLESTR_LIT - #| | CHAR_LIT + #| | CHAR_LIT | CUSTOM_NUMERIC_LIT #| | NIL #| generalizedLit = GENERALIZED_STR_LIT | GENERALIZED_TRIPLESTR_LIT #| identOrLiteral = generalizedLit | symbol | literal @@ -710,6 +710,14 @@ proc identOrLiteral(p: var Parser, mode: PrimaryMode): PNode = of tkCharLit: result = newIntNodeP(nkCharLit, ord(p.tok.literal[0]), p) getTok(p) + of tkCustomLit: + let splitPos = p.tok.iNumber.int + let str = newStrNodeP(nkRStrLit, p.tok.literal.substr(0, splitPos-1), p) + let callee = newIdentNodeP(getIdent(p.lex.cache, p.tok.literal.substr(splitPos)), p) + result = newNodeP(nkDotExpr, p) + result.add str + result.add callee + getTok(p) of tkNil: result = newNodeP(nkNilLit, p) getTok(p) @@ -807,7 +815,7 @@ proc primarySuffix(p: var Parser, r: PNode, result = commandExpr(p, result, mode) break result = namedParams(p, result, nkCurlyExpr, tkCurlyRi) - of tkSymbol, tkAccent, tkIntLit..tkCharLit, tkNil, tkCast, + of tkSymbol, tkAccent, tkIntLit..tkCustomLit, tkNil, tkCast, tkOpr, tkDotDot, tkVar, tkOut, tkStatic, tkType, tkEnum, tkTuple, tkObject, tkProc: # XXX: In type sections we allow the free application of the @@ -1097,7 +1105,7 @@ proc isExprStart(p: Parser): bool = case p.tok.tokType of tkSymbol, tkAccent, tkOpr, tkNot, tkNil, tkCast, tkIf, tkFor, tkProc, tkFunc, tkIterator, tkBind, tkBuiltInMagics, - tkParLe, tkBracketLe, tkCurlyLe, tkIntLit..tkCharLit, tkVar, tkRef, tkPtr, + tkParLe, tkBracketLe, tkCurlyLe, tkIntLit..tkCustomLit, tkVar, tkRef, tkPtr, tkTuple, tkObject, tkWhen, tkCase, tkOut: result = true else: result = false @@ -1498,7 +1506,7 @@ proc parseReturnOrRaise(p: var Parser, kind: TNodeKind): PNode = #| yieldStmt = 'yield' optInd expr? #| discardStmt = 'discard' optInd expr? #| breakStmt = 'break' optInd expr? - #| continueStmt = 'break' optInd expr? + #| continueStmt = 'continue' optInd expr? result = newNodeP(kind, p) getTok(p) if p.tok.tokType == tkComment: diff --git a/compiler/renderer.nim b/compiler/renderer.nim index c36eaaf11..9ca485f6e 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -942,7 +942,7 @@ proc skipHiddenNodes(n: PNode): PNode = else: break proc accentedName(g: var TSrcGen, n: PNode) = - const backticksNeeded = OpChars + {'[', '{'} + const backticksNeeded = OpChars + {'[', '{', '\''} if n == nil: return let isOperator = if n.kind == nkIdent and n.ident.s.len > 0 and n.ident.s[0] in backticksNeeded: true @@ -976,6 +976,11 @@ proc infixArgument(g: var TSrcGen, n: PNode, i: int) = if needsParenthesis: put(g, tkParRi, ")") +proc isCustomLit(n: PNode): bool = + n.len == 2 and n[0].kind == nkRStrLit and + (n[1].kind == nkIdent and n[1].ident.s.startsWith('\'')) or + (n[1].kind == nkSym and n[1].sym.name.s.startsWith('\'')) + proc gsub(g: var TSrcGen, n: PNode, c: TContext) = if isNil(n): return var @@ -1195,9 +1200,14 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) = gcomma(g, n, c) put(g, tkBracketRi, "]") of nkDotExpr: - gsub(g, n, 0) - put(g, tkDot, ".") - gsub(g, n, 1) + if isCustomLit(n): + put(g, tkCustomLit, n[0].strVal) + gsub(g, n, 1) + else: + gsub(g, n, 0) + put(g, tkDot, ".") + if n.len > 1: + accentedName(g, n[1]) of nkBind: putWithSpace(g, tkBind, "bind") gsub(g, n, 0) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index ff8f68ed0..fee43162e 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -1524,7 +1524,7 @@ proc semProcAnnotation(c: PContext, prc: PNode; return proc semInferredLambda(c: PContext, pt: TIdTable, n: PNode): PNode {.nosinks.} = - ## used for resolving 'auto' in lambdas based on their callsite + ## used for resolving 'auto' in lambdas based on their callsite var n = n let original = n[namePos].sym let s = original #copySym(original, false) diff --git a/doc/grammar.txt b/doc/grammar.txt index 0d5eef179..d4f4a0515 100644 --- a/doc/grammar.txt +++ b/doc/grammar.txt @@ -46,7 +46,7 @@ literal = | INT_LIT | INT8_LIT | INT16_LIT | INT32_LIT | INT64_LIT | UINT_LIT | UINT8_LIT | UINT16_LIT | UINT32_LIT | UINT64_LIT | FLOAT_LIT | FLOAT32_LIT | FLOAT64_LIT | STR_LIT | RSTR_LIT | TRIPLESTR_LIT - | CHAR_LIT + | CHAR_LIT | CUSTOM_NUMERIC_LIT | NIL generalizedLit = GENERALIZED_STR_LIT | GENERALIZED_TRIPLESTR_LIT identOrLiteral = generalizedLit | symbol | literal @@ -100,6 +100,7 @@ postExprBlocks = ':' stmt? ( IND{=} doBlock | IND{=} 'of' exprList ':' stmt | IND{=} 'elif' expr ':' stmt | IND{=} 'except' exprList ':' stmt + | IND{=} 'finally' ':' stmt | IND{=} 'else' ':' stmt )* exprStmt = simpleExpr (( '=' optInd expr colonBody? ) diff --git a/doc/manual.rst b/doc/manual.rst index db1251630..a882eb945 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -490,11 +490,17 @@ this. Another reason is that Nim can thus support `array[char, int]` or type is used for Unicode characters, it can represent any Unicode character. `Rune` is declared in the `unicode module `_. +A character literal that does not end in ``'`` interpreted as ``'`` if there +is a preceeding backtick token. There must be no whitespace between the preceeding +backtick token and the character literal. This special rule ensures that a declaration +like ``proc `'customLiteral`(s: string)`` is valid. See also +`Custom Numeric Literals <#custom-numeric-literals>`_. -Numerical constants -------------------- -Numerical constants are of a single type and have the form:: +Numeric Literals +---------------- + +Numeric literals have the form:: hexdigit = digit | 'A'..'F' | 'a'..'f' octdigit = '0'..'7' @@ -530,8 +536,10 @@ Numerical constants are of a single type and have the form:: FLOAT64_LIT = HEX_LIT '\'' FLOAT64_SUFFIX | (FLOAT_LIT | DEC_LIT | OCT_LIT | BIN_LIT) ['\''] FLOAT64_SUFFIX + CUSTOM_NUMERIC_LIT = (FLOAT_LIT | DEC_LIT | OCT_LIT | BIN_LIT) '\'' CUSTOM_NUMERIC_SUFFIX -As can be seen in the productions, numerical constants can contain underscores + +As can be seen in the productions, numeric literals can contain underscores for readability. Integer and floating-point literals may be given in decimal (no prefix), binary (prefix `0b`), octal (prefix `0o`), and hexadecimal (prefix `0x`) notation. @@ -579,7 +587,7 @@ is optional if it is not ambiguous (only hexadecimal floating-point literals with a type suffix can be ambiguous). -The type suffixes are: +The pre-defined type suffixes are: ================= ========================= Type Suffix Resulting type of literal @@ -611,6 +619,42 @@ the bit width of the datatype, it is accepted. Hence: 0b10000000'u8 == 0x80'u8 == 128, but, 0b10000000'i8 == 0x80'i8 == -1 instead of causing an overflow error. + +Custom Numeric Literals +~~~~~~~~~~~~~~~~~~~~~~~ + +If the suffix is not predefined, then the suffix is assumed to be a call +to a proc, template, macro or other callable identifier that is passed the +string containing the literal. The callable identifier needs to be declared +with a special ``'`` prefix: + +.. code-block:: nim + + import strutils + type u4 = distinct uint8 # a 4-bit unsigned integer aka "nibble" + proc `'u4`(n: string): u4 = + # The leading ' is required. + result = (parseInt(n) and 0x0F).u4 + + var x = 5'u4 + +More formally, a custom numeric literal `123'custom` is transformed +to r"123".`'custom` in the parsing step. There is no AST node kind that +corresponds to this transformation. The transformation naturally handles +the case that additional parameters are passed to the callee: + +.. code-block:: nim + + import strutils + type u4 = distinct uint8 # a 4-bit unsigned integer aka "nibble" + proc `'u4`(n: string; moreData: int): u4 = + result = (parseInt(n) and 0x0F).u4 + + var x = 5'u4(123) + +Custom numeric literals are covered by the grammar rule named `CUSTOM_NUMERIC_LIT`. + + Operators --------- diff --git a/tests/lexer/mlexerutils.nim b/tests/lexer/mlexerutils.nim new file mode 100644 index 000000000..eae7a0006 --- /dev/null +++ b/tests/lexer/mlexerutils.nim @@ -0,0 +1,9 @@ +import macros + +macro lispReprStr*(a: untyped): untyped = newLit(a.lispRepr) + +macro assertAST*(expected: string, struct: untyped): untyped = + var ast = newLit(struct.treeRepr) + result = quote do: + if `ast` != `expected`: + doAssert false, "\nGot:\n" & `ast`.indent(2) & "\nExpected:\n" & `expected`.indent(2) \ No newline at end of file diff --git a/tests/lexer/tcustom_numeric_literals.nim b/tests/lexer/tcustom_numeric_literals.nim new file mode 100644 index 000000000..a2f355b4d --- /dev/null +++ b/tests/lexer/tcustom_numeric_literals.nim @@ -0,0 +1,150 @@ +discard """ + targets: "c cpp js" +""" + +# Test tkStrNumLit + +import std/[macros, strutils] +import mlexerutils + +# AST checks + +assertAST dedent """ + StmtList + ProcDef + AccQuoted + Ident "\'" + Ident "wrap" + Empty + Empty + FormalParams + Ident "string" + IdentDefs + Ident "number" + Ident "string" + Empty + Empty + Empty + StmtList + Asgn + Ident "result" + Infix + Ident "&" + Infix + Ident "&" + StrLit "[[" + Ident "number" + StrLit "]]"""": + proc `'wrap`(number: string): string = + result = "[[" & number & "]]" + +assertAST dedent """ + StmtList + DotExpr + RStrLit "-38383839292839283928392839283928392839283.928493849385935898243e-50000" + Ident "\'wrap"""": + -38383839292839283928392839283928392839283.928493849385935898243e-50000'wrap + +proc `'wrap`(number: string): string = "[[" & number & "]]" +doAssert lispReprStr(-1'wrap) == """(DotExpr (RStrLit "-1") (Ident "\'wrap"))""" + +template main = + block: # basic suffix usage + template `'twrap`(number: string): untyped = + number.`'wrap` + proc extraContext(): string = + 22.40'wrap + proc `*`(left, right: string): string = + result = left & "times" & right + proc `+`(left, right: string): string = + result = left & "plus" & right + + doAssert 1'wrap == "[[1]]" + doAssert -1'wrap == "[[-1]]": + "unable to resolve a negative integer-suffix pattern" + doAssert 12345.67890'wrap == "[[12345.67890]]" + doAssert 1'wrap*1'wrap == "[[1]]times[[1]]": + "unable to resolve an operator between two suffixed numeric literals" + doAssert 1'wrap+ -1'wrap == "[[1]]plus[[-1]]": # will generate a compiler warning about inconsistent spacing + "unable to resolve a negative suffixed numeric literal following an operator" + doAssert 1'wrap + -1'wrap == "[[1]]plus[[-1]]" + doAssert 1'twrap == "[[1]]" + doAssert extraContext() == "[[22.40]]": + "unable to return a suffixed numeric literal by an implicit return" + doAssert 0x5a3a'wrap == "[[0x5a3a]]" + doAssert 0o5732'wrap == "[[0o5732]]" + doAssert 0b0101111010101'wrap == "[[0b0101111010101]]" + doAssert -38383839292839283928392839283928392839283.928493849385935898243e-50000'wrap == "[[-38383839292839283928392839283928392839283.928493849385935898243e-50000]]" + doAssert 1234.56'wrap == "[[1234.56]]": + "unable to properly account for context with suffixed numeric literals" + + block: # verify that the i64, f32, etc builtin suffixes still parse correctly + const expectedF32: float32 = 123.125 + proc `'f9`(number: string): string = # proc starts with 'f' just like 'f32' + "[[" & number & "]]" + proc `'f32a`(number: string): string = # looks even more like 'f32' + "[[" & number & "]]" + proc `'d9`(number: string): string = # proc starts with 'd' just like the d suffix + "[[" & number & "]]" + proc `'i9`(number: string): string = # proc starts with 'i' just like 'i64' + "[[" & number & "]]" + proc `'u9`(number: string): string = # proc starts with 'u' just like 'u8' + "[[" & number & "]]" + + doAssert 123.125f32 == expectedF32: + "failing to support non-quoted legacy f32 floating point suffix" + doAssert 123.125'f32 == expectedF32 + doAssert 123.125e0'f32 == expectedF32 + doAssert 1234.56'wrap == 1234.56'f9 + doAssert 1234.56'wrap == 1234.56'f32a + doAssert 1234.56'wrap == 1234.56'd9 + doAssert 1234.56'wrap == 1234.56'i9 + doAssert 1234.56'wrap == 1234.56'u9 + doAssert lispReprStr(1234.56'u9) == """(DotExpr (RStrLit "1234.56") (Ident "\'u9"))""": + "failed to properly build AST for suffix that starts with u" + doAssert -128'i8 == (-128).int8 + + block: # case checks + doAssert 1E2 == 100: + "lexer not handling upper-case exponent" + doAssert 1.0E2 == 100.0 + doAssert 1e2 == 100 + doAssert 0xdeadBEEF'wrap == "[[0xdeadBEEF]]": + "lexer not maintaining original case" + doAssert 0.1E12'wrap == "[[0.1E12]]" + doAssert 0.0e12'wrap == "[[0.0e12]]" + doAssert 0.0e+12'wrap == "[[0.0e+12]]" + doAssert 0.0e-12'wrap == "[[0.0e-12]]" + doAssert 0e-12'wrap == "[[0e-12]]" + + block: # macro and template usage + template `'foo`(a: string): untyped = (a, 2) + doAssert -12'foo == ("-12", 2) + template `'fooplus`(a: string, b: int): untyped = (a, b) + doAssert -12'fooplus(2) == ("-12", 2) + template `'fooplusopt`(a: string, b: int = 99): untyped = (a, b) + doAssert -12'fooplusopt(2) == ("-12", 2) + doAssert -12'fooplusopt() == ("-12", 99) + doAssert -12'fooplusopt == ("-12", 99) + macro `'bar`(a: static string): untyped = + var infix = newNimNode(nnkInfix) + infix.add newIdentNode("&") + infix.add newLit("got ") + infix.add newLit(a.repr) + result = newNimNode(nnkStmtList) + result.add infix + doAssert -12'bar == "got \"-12\"" + macro deb(a): untyped = newLit(a.repr) + doAssert deb(-12'bar) == "-12'bar" + # macro metawrap(): untyped = + # func wrap1(a: string): string = "{" & a & "}" + # func `'wrap2`(a: string): string = "{" & a & "}" + # result = quote do: + # let a1 = wrap1"-128" + # let a2 = -128'wrap2 + # metawrap() + # doAssert a1 == "{-128}" + # doAssert a2 == "{-128}" + +static: main() +main() diff --git a/tests/lexer/tstrlits.nim b/tests/lexer/tstrlits.nim deleted file mode 100644 index 8e8250a5b..000000000 --- a/tests/lexer/tstrlits.nim +++ /dev/null @@ -1,19 +0,0 @@ -discard """ - output: "a\"\"long string\"\"\"\"\"abc\"def_'2'●𝌆𝌆A" -""" -# Test the new different string literals - -const - tripleEmpty = """"long string"""""""" # "long string """"" - - rawQuote = r"a""" - - raw = r"abc""def" - - escaped = "\x5f'\50'\u25cf\u{1D306}\u{1d306}\u{41}" - - -stdout.write(rawQuote) -stdout.write(tripleEmpty) -stdout.write(raw) -stdout.writeLine(escaped) diff --git a/tests/lexer/tunary_minus.nim b/tests/lexer/tunary_minus.nim index 89f1b79ef..0aa861d53 100644 --- a/tests/lexer/tunary_minus.nim +++ b/tests/lexer/tunary_minus.nim @@ -6,13 +6,7 @@ discard """ import std/[macros, strutils] -macro lispReprStr*(a: untyped): untyped = newLit(a.lispRepr) - -macro assertAST*(expected: string, struct: untyped): untyped = - var ast = newLit(struct.treeRepr) - result = quote do: - if `ast` != `expected`: - doAssert false, "\nGot:\n" & `ast`.indent(2) & "\nExpected:\n" & `expected`.indent(2) +import mlexerutils const one = 1 const minusOne = `-`(one) From 355985ac89dd13d5b317744b30842155d9321565 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 24 Mar 2021 22:29:42 +0100 Subject: [PATCH 0488/2472] custom integer literals bugfixes (#17499) * custom integer literals bugfixes * make nimsuggest compile again --- changelog.md | 2 +- compiler/lexer.nim | 9 +++++---- doc/manual.rst | 6 +++++- tests/lexer/tunary_minus.nim | 3 +++ 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/changelog.md b/changelog.md index 922b8470a..f180e90f9 100644 --- a/changelog.md +++ b/changelog.md @@ -266,7 +266,7 @@ - The unary minus in `-1` is now part of the integer literal, it is now parsed as a single token. This implies that edge cases like `-128'i8` finally work correctly. -- Custom numeric literals are now supported. +- Custom numeric literals (e.g. `-128'bignum`) are now supported. ## Compiler changes diff --git a/compiler/lexer.nim b/compiler/lexer.nim index b34b010c2..ecf6cb0d9 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -345,13 +345,14 @@ proc getNumber(L: var Lexer, result: var Token) = result.tokType = tkIntLit # int literal until we know better result.literal = "" result.base = base10 - let startpos = L.bufpos - tokenBegin(result, startpos) + tokenBegin(result, L.bufpos) var isPositive = true if L.buf[L.bufpos] == '-': eatChar(L, result) - isPositive = true + isPositive = false + + let startpos = L.bufpos template setNumber(field, value) = field = (if isPositive: value else: -value) @@ -419,7 +420,7 @@ proc getNumber(L: var Lexer, result: var Token) = suffixAsLower.add toLowerAscii(c) inc postPos if L.buf[postPos] notin SymChars+{'_'}: break - case suffix + case suffixAsLower of "f", "f32": result.tokType = tkFloat32Lit of "d", "f64": result.tokType = tkFloat64Lit of "f128": result.tokType = tkFloat128Lit diff --git a/doc/manual.rst b/doc/manual.rst index a882eb945..51c434c7b 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -490,7 +490,7 @@ this. Another reason is that Nim can thus support `array[char, int]` or type is used for Unicode characters, it can represent any Unicode character. `Rune` is declared in the `unicode module `_. -A character literal that does not end in ``'`` interpreted as ``'`` if there +A character literal that does not end in ``'`` is interpreted as ``'`` if there is a preceeding backtick token. There must be no whitespace between the preceeding backtick token and the character literal. This special rule ensures that a declaration like ``proc `'customLiteral`(s: string)`` is valid. See also @@ -538,6 +538,9 @@ Numeric literals have the form:: CUSTOM_NUMERIC_LIT = (FLOAT_LIT | DEC_LIT | OCT_LIT | BIN_LIT) '\'' CUSTOM_NUMERIC_SUFFIX + # CUSTOM_NUMERIC_SUFFIX is any Nim identifier that is not + # a pre-defined type suffix. + As can be seen in the productions, numeric literals can contain underscores for readability. Integer and floating-point literals may be given in decimal (no @@ -653,6 +656,7 @@ the case that additional parameters are passed to the callee: var x = 5'u4(123) Custom numeric literals are covered by the grammar rule named `CUSTOM_NUMERIC_LIT`. +A custom numeric literal is a single token. Operators diff --git a/tests/lexer/tunary_minus.nim b/tests/lexer/tunary_minus.nim index 0aa861d53..639911fcd 100644 --- a/tests/lexer/tunary_minus.nim +++ b/tests/lexer/tunary_minus.nim @@ -51,6 +51,9 @@ template main = doAssert x() == minusOne: "unable to handle negatives after semi-colon" + doAssert -0b111 == -7 + doAssert -0xff == -255 + block: # check when a minus (-) is an unary op doAssert -one == minusOne: "unable to a negative prior to identifier" From 76d391a512b53b731941d6b355fe3431fc6b74c1 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 25 Mar 2021 14:06:30 +0800 Subject: [PATCH 0489/2472] remove unnecessary check (#17502) --- compiler/spawn.nim | 2 -- 1 file changed, 2 deletions(-) diff --git a/compiler/spawn.nim b/compiler/spawn.nim index 54ed51dbc..1635f9f83 100644 --- a/compiler/spawn.nim +++ b/compiler/spawn.nim @@ -371,8 +371,6 @@ proc wrapProcForSpawn*(g: ModuleGraph; idgen: IdGenerator; owner: PSym; spawnExp fn = indirectAccess(castExpr, field, n.info) elif fn.kind == nkSym and fn.sym.kind == skIterator: localError(g.config, n.info, "iterator in spawn environment is not allowed") - elif fn.typ.callConv == ccClosure: - localError(g.config, n.info, "closure in spawn environment is not allowed") call.add(fn) var varSection = newNodeI(nkVarSection, n.info) From 045400ad92f98363ef8931aea29165b48d07ac16 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 25 Mar 2021 14:07:29 +0800 Subject: [PATCH 0490/2472] ref https://github.com/nim-lang/Nim/pull/17333 (#17495) --- doc/nep1.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/nep1.rst b/doc/nep1.rst index 3bae6a00b..bdf8e0eab 100644 --- a/doc/nep1.rst +++ b/doc/nep1.rst @@ -157,7 +157,8 @@ English word To use Notes ------------------- ------------ -------------------------------------- initialize initFoo initializes a value type `Foo` new newFoo initializes a reference type `Foo` - via `new` + via `new` or a value type `Foo` + with reference semantics. this or self self for method like procs, e.g.: `proc fun(self: Foo, a: int)` rationale: `self` is more unique in English From 46364e63cd0c292b3a803d1975d48db73f46e8e6 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Thu, 25 Mar 2021 10:15:05 +0300 Subject: [PATCH 0491/2472] fix RST parsing after option lists (#17442) --- lib/packages/docutils/rst.nim | 73 ++++++++++++++++++++++++++++++++++- tests/stdlib/trstgen.nim | 49 +++++++++++++++++++++++ 2 files changed, 121 insertions(+), 1 deletion(-) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 11768c7da..adc83c9af 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -82,6 +82,14 @@ ## * ***triple emphasis*** (bold and italic) using \*\*\* ## * ``:idx:`` role for \`interpreted text\` to include the link to this ## text into an index (example: `Nim index`_). +## * double slash `//` in option lists serves as a prefix for any option that +## starts from a word (without any leading symbols like `-`, `--`, `/`):: +## +## //compile compile the project +## //doc generate documentation +## +## Here the dummy `//` will disappear, while options ``compile`` +## and ``doc`` will be left in the final document. ## ## .. [cmp:Sphinx] similar but different from the directives of ## Python `Sphinx directives`_ extensions @@ -548,6 +556,67 @@ proc pushInd(p: var RstParser, ind: int) = proc popInd(p: var RstParser) = if p.indentStack.len > 1: setLen(p.indentStack, p.indentStack.len - 1) +# Working with indentation in rst.nim +# ----------------------------------- +# +# Every line break has an associated tkIndent. +# The tokenizer writes back the first column of next non-blank line +# in all preceeding tkIndent tokens to the `ival` field of tkIndent. +# +# RST document is separated into body elements (B.E.), every of which +# has a dedicated handler proc (or block of logic when B.E. is a block quote) +# that should follow the next rule: +# Every B.E. handler proc should finish at tkIndent (newline) +# after its B.E. finishes. +# Then its callers (which is `parseSection` or another B.E. handler) +# check for tkIndent ival (without necessity to advance `p.idx`) +# and decide themselves whether they continue processing or also stop. +# +# An example:: +# +# L RST text fragment indentation +# +--------------------+ +# 1 | | <- (empty line at the start of file) no tokens +# 2 |First paragraph. | <- tkIndent has ival=0, and next tkWord has col=0 +# 3 | | <- tkIndent has ival=0 +# 4 |* bullet item and | <- tkIndent has ival=0, and next tkPunct has col=0 +# 5 | its continuation | <- tkIndent has ival=2, and next tkWord has col=2 +# 6 | | <- tkIndent has ival=4 +# 7 | Block quote | <- tkIndent has ival=4, and next tkWord has col=4 +# 8 | | <- tkIndent has ival=0 +# 9 | | <- tkIndent has ival=0 +# 10|Final paragraph | <- tkIndent has ival=0, and tkWord has col=0 +# +--------------------+ +# C:01234 +# +# Here parser starts with initial `indentStack=[0]` and then calls the +# 1st `parseSection`: +# +# - `parseSection` calls `parseParagraph` and "First paragraph" is parsed +# - bullet list handler is started at reaching ``*`` (L4 C0), it +# starts bullet item logic (L4 C2), which calls `pushInd(p, ind=2)`, +# then calls `parseSection` (2nd call, nested) which parses +# paragraph "bullet list and its continuation" and then starts +# a block quote logic (L7 C4). +# The block quote logic calls calls `pushInd(p, ind=4)` and +# calls `parseSection` again, so a (simplified) sequence of calls now is:: +# +# parseSection -> parseBulletList -> +# parseSection (+block quote logic) -> parseSection +# +# 3rd `parseSection` finishes, block quote logic calls `popInd(p)`, +# it returns to bullet item logic, which sees that next tkIndent has +# ival=0 and stops there since the required indentation for a bullet item +# is 2 and 0<2; the bullet item logic calls `popInd(p)`. +# Then bullet list handler checks that next tkWord (L10 C0) has the +# right indentation but does not have ``*`` so stops at tkIndent (L10). +# - 1st `parseSection` invocation calls `parseParagraph` and the +# "Final paragraph" is parsed. +# +# If a B.E. handler has advanced `p.idx` past tkIndent to check +# whether it should continue its processing or not, and decided not to, +# then this B.E. handler should step back (e.g. do `dec p.idx`). + proc initParser(p: var RstParser, sharedState: PSharedState) = p.indentStack = @[0] p.tok = @[] @@ -1912,8 +1981,9 @@ proc parseBulletList(p: var RstParser): PRstNode = proc parseOptionList(p: var RstParser): PRstNode = result = newRstNodeA(p, rnOptionList) + let col = currentTok(p).col while true: - if isOptionList(p): + if currentTok(p).col == col and isOptionList(p): var a = newRstNode(rnOptionGroup) var b = newRstNode(rnDescription) var c = newRstNode(rnOptionListItem) @@ -1936,6 +2006,7 @@ proc parseOptionList(p: var RstParser): PRstNode = c.add(b) result.add(c) else: + dec p.idx # back to tkIndent break proc parseDefinitionList(p: var RstParser): PRstNode = diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index d6055cc11..0f3890faa 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -1277,6 +1277,55 @@ Test1 let refline = "Ref. " & ref1 & "! and " & ref2 & ";and " & ref3 & "." doAssert refline in output1 + test "Option lists 1": + # check that "* b" is not consumed by previous bullet item because of + # incorrect indentation handling in option lists + let input = dedent """ + * a + -m desc + -n very long + desc + * b""" + let output = input.toHtml + check(output.count("") == 2) + check(output.count("-mdesc""" in output) + check("""-nvery long desc""" in + output) + + test "Option lists 2": + # check that 2nd option list is not united with the 1st + let input = dedent """ + * a + -m desc + -n very long + desc + -d option""" + let output = input.toHtml + check(output.count("-mdesc""" in output) + check("""-nvery long desc""" in + output) + check("""-doption""" in + output) + + test "Option list 3 (double /)": + let input = dedent """ + * a + //compile compile1 + //doc doc1 + cont + -d option""" + let output = input.toHtml + check(output.count("compilecompile1""" in output) + check("""docdoc1 cont""" in + output) + check("""-doption""" in + output) suite "RST/Code highlight": test "Basic Python code highlight": let pythonCode = """ From 6e0a293dab195c973a45b38d22fdc97575c025fc Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 25 Mar 2021 15:28:06 +0800 Subject: [PATCH 0492/2472] re-enable fidget (#17229) --- testament/important_packages.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 1dd7c69ca..66a2fa09e 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -49,6 +49,7 @@ pkg "chronicles", "nim c -o:chr -r chronicles.nim" # when not defined(osx): # testdatagram.nim(560, 54): Check failed # pkg "chronos", "nim c -r -d:release tests/testall" # pending https://github.com/nim-lang/Nim/issues/17130 + pkg "cligen", "nim c --path:. -r cligen.nim" pkg "combparser", "nimble test --gc:orc" pkg "compactdict" @@ -59,7 +60,7 @@ pkg "delaunay" pkg "docopt" pkg "easygl", "nim c -o:egl -r src/easygl.nim", "https://github.com/jackmott/easygl" pkg "elvis" -# pkg "fidget" # pending https://github.com/treeform/fidget/issues/133 +pkg "fidget" pkg "fragments", "nim c -r fragments/dsl.nim" pkg "fusion" pkg "gara" From 8573160a44439ffe3fbbdd354188db9c976a2081 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 25 Mar 2021 21:05:50 +0800 Subject: [PATCH 0493/2472] lent support for tables (#17505) --- lib/pure/collections/tables.nim | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index f77642349..9387807c0 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -702,7 +702,7 @@ iterator mpairs*[A, B](t: var Table[A, B]): (A, var B) = yield (t.data[h].key, t.data[h].val) assert(len(t) == L, "the length of the table changed while iterating over it") -iterator keys*[A, B](t: Table[A, B]): A = +iterator keys*[A, B](t: Table[A, B]): lent A = ## Iterates over any key in the table `t`. ## ## See also: @@ -723,7 +723,7 @@ iterator keys*[A, B](t: Table[A, B]): A = yield t.data[h].key assert(len(t) == L, "the length of the table changed while iterating over it") -iterator values*[A, B](t: Table[A, B]): B = +iterator values*[A, B](t: Table[A, B]): lent B = ## Iterates over any value in the table `t`. ## ## See also: @@ -1146,7 +1146,7 @@ iterator mpairs*[A, B](t: TableRef[A, B]): (A, var B) = yield (t.data[h].key, t.data[h].val) assert(len(t) == L, "the length of the table changed while iterating over it") -iterator keys*[A, B](t: TableRef[A, B]): A = +iterator keys*[A, B](t: TableRef[A, B]): lent A = ## Iterates over any key in the table `t`. ## ## See also: @@ -1167,7 +1167,7 @@ iterator keys*[A, B](t: TableRef[A, B]): A = yield t.data[h].key assert(len(t) == L, "the length of the table changed while iterating over it") -iterator values*[A, B](t: TableRef[A, B]): B = +iterator values*[A, B](t: TableRef[A, B]): lent B = ## Iterates over any value in the table `t`. ## ## See also: @@ -1722,7 +1722,7 @@ iterator mpairs*[A, B](t: var OrderedTable[A, B]): (A, var B) = yield (t.data[h].key, t.data[h].val) assert(len(t) == L, "the length of the table changed while iterating over it") -iterator keys*[A, B](t: OrderedTable[A, B]): A = +iterator keys*[A, B](t: OrderedTable[A, B]): lent A = ## Iterates over any key in the table `t` in insertion order. ## ## See also: @@ -1743,7 +1743,7 @@ iterator keys*[A, B](t: OrderedTable[A, B]): A = yield t.data[h].key assert(len(t) == L, "the length of the table changed while iterating over it") -iterator values*[A, B](t: OrderedTable[A, B]): B = +iterator values*[A, B](t: OrderedTable[A, B]): lent B = ## Iterates over any value in the table `t` in insertion order. ## ## See also: @@ -2130,7 +2130,7 @@ iterator mpairs*[A, B](t: OrderedTableRef[A, B]): (A, var B) = yield (t.data[h].key, t.data[h].val) assert(len(t) == L, "the length of the table changed while iterating over it") -iterator keys*[A, B](t: OrderedTableRef[A, B]): A = +iterator keys*[A, B](t: OrderedTableRef[A, B]): lent A = ## Iterates over any key in the table `t` in insertion order. ## ## See also: @@ -2151,7 +2151,7 @@ iterator keys*[A, B](t: OrderedTableRef[A, B]): A = yield t.data[h].key assert(len(t) == L, "the length of the table changed while iterating over it") -iterator values*[A, B](t: OrderedTableRef[A, B]): B = +iterator values*[A, B](t: OrderedTableRef[A, B]): lent B = ## Iterates over any value in the table `t` in insertion order. ## ## See also: @@ -2543,7 +2543,7 @@ iterator mpairs*[A](t: var CountTable[A]): (A, var int) = yield (t.data[h].key, t.data[h].val) assert(len(t) == L, "the length of the table changed while iterating over it") -iterator keys*[A](t: CountTable[A]): A = +iterator keys*[A](t: CountTable[A]): lent A = ## Iterates over any key in the table `t`. ## ## See also: From 42e895feb11515689aaca96f58fcffb366d34bb7 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 26 Mar 2021 12:20:58 +0800 Subject: [PATCH 0494/2472] close #15696 (#17518) --- tests/vm/tvmmisc.nim | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/vm/tvmmisc.nim b/tests/vm/tvmmisc.nim index a485bac2e..f542fa560 100644 --- a/tests/vm/tvmmisc.nim +++ b/tests/vm/tvmmisc.nim @@ -255,6 +255,31 @@ block: doAssert e == @[] doAssert f == @[] + +block: # bug #10815 + type + Opcode = enum + iChar, iSet + + Inst = object + case code: Opcode + of iChar: + c: char + of iSet: + cs: set[char] + + Patt = seq[Inst] + + + proc `$`(p: Patt): string = + discard + + proc P(): Patt = + result.add Inst(code: iSet) + + const a = P() + doAssert $a == "" + import tables block: # bug #8007 From 8049d0befc8ac7689e98f266df69b5f268fb7683 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 25 Mar 2021 23:07:33 -0700 Subject: [PATCH 0495/2472] testament: remove unused tfile,tline,tcolumn (#17515) --- testament/specs.nim | 8 -------- testament/testament.nim | 30 ++---------------------------- 2 files changed, 2 insertions(+), 36 deletions(-) diff --git a/testament/specs.nim b/testament/specs.nim index 6b80fe41d..b5e6de7c2 100644 --- a/testament/specs.nim +++ b/testament/specs.nim @@ -79,8 +79,6 @@ type sortoutput*: bool output*: string line*, column*: int - tfile*: string - tline*, tcolumn*: int exitCode*: int msg*: string ccodeCheck*: seq[string] @@ -277,12 +275,6 @@ proc parseSpec*(filename: string): TSpec = if result.msg.len == 0 and result.nimout.len == 0: result.parseErrors.addLine "errormsg or msg needs to be specified before column" discard parseInt(e.value, result.column) - of "tfile": - result.tfile = e.value - of "tline": - discard parseInt(e.value, result.tline) - of "tcolumn": - discard parseInt(e.value, result.tcolumn) of "output": if result.outputCheck != ocSubstr: result.outputCheck = ocEqual diff --git a/testament/testament.nim b/testament/testament.nim index f45d4043a..6fc7e32ce 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -82,12 +82,6 @@ type let pegLineError = peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' ('Error') ':' \s* {.*}" - - pegLineTemplate = - peg""" - {[^(]*} '(' {\d+} ', ' {\d+} ') ' - 'template/generic instantiation' ( ' of `' [^`]+ '`' )? ' from here' .* - """ pegOtherError = peg"'Error:' \s* {.*}" pegOfInterest = pegLineError / pegOtherError @@ -165,7 +159,6 @@ proc callCompiler(cmdTemplate, filename, options, nimcache: string, let outp = p.outputStream var suc = "" var err = "" - var tmpl = "" var x = newStringOfCap(120) result.nimout = "" while true: @@ -174,9 +167,6 @@ proc callCompiler(cmdTemplate, filename, options, nimcache: string, if x =~ pegOfInterest: # `err` should contain the last error/warning message err = x - elif x =~ pegLineTemplate and err == "": - # `tmpl` contains the last template expansion before the error - tmpl = x elif x.isSuccess: suc = x elif not running(p): @@ -187,14 +177,7 @@ proc callCompiler(cmdTemplate, filename, options, nimcache: string, result.output = "" result.line = 0 result.column = 0 - result.tfile = "" - result.tline = 0 - result.tcolumn = 0 result.err = reNimcCrash - if tmpl =~ pegLineTemplate: - result.tfile = extractFilename(matches[0]) - result.tline = parseInt(matches[1]) - result.tcolumn = parseInt(matches[2]) if err =~ pegLineError: result.file = extractFilename(matches[0]) result.line = parseInt(matches[1]) @@ -374,22 +357,13 @@ proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest, target: TTarg r.addResult(test, target, expected.msg, given.msg, reMsgsDiffer) elif expected.nimout.len > 0 and not greedyOrderedSubsetLines(expected.nimout, given.nimout): r.addResult(test, target, expected.nimout, given.nimout, reMsgsDiffer) - elif expected.tfile == "" and extractFilename(expected.file) != extractFilename(given.file) and + elif extractFilename(expected.file) != extractFilename(given.file) and "internal error:" notin expected.msg: r.addResult(test, target, expected.file, given.file, reFilesDiffer) elif expected.line != given.line and expected.line != 0 or expected.column != given.column and expected.column != 0: r.addResult(test, target, $expected.line & ':' & $expected.column, - $given.line & ':' & $given.column, - reLinesDiffer) - elif expected.tfile != "" and extractFilename(expected.tfile) != extractFilename(given.tfile) and - "internal error:" notin expected.msg: - r.addResult(test, target, expected.tfile, given.tfile, reFilesDiffer) - elif expected.tline != given.tline and expected.tline != 0 or - expected.tcolumn != given.tcolumn and expected.tcolumn != 0: - r.addResult(test, target, $expected.tline & ':' & $expected.tcolumn, - $given.tline & ':' & $given.tcolumn, - reLinesDiffer) + $given.line & ':' & $given.column, reLinesDiffer) else: r.addResult(test, target, expected.msg, given.msg, reSuccess) inc(r.passed) From e3e97421438da27267b159b922d7338fa92f1cd8 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 26 Mar 2021 08:52:26 +0100 Subject: [PATCH 0496/2472] added a 'koch ic' command for easier adhoc testing of IC (#17508) * added a 'koch ic' command for easier adhoc testing of IC * IC: progress * IC: enable generics test * make tests green --- compiler/ccgtypes.nim | 7 ++----- compiler/cgen.nim | 30 ++++++++++++++++++++++++++++++ compiler/ic/cbackend.nim | 12 +++++++++++- compiler/ic/ic.nim | 22 +++++++++++++++++----- compiler/ic/rodfiles.nim | 1 + compiler/modulegraphs.nim | 16 ++++++++++++++++ koch.nim | 21 +++++++++++++++++++-- tests/ic/tgenerics.nim | 1 - 8 files changed, 96 insertions(+), 14 deletions(-) diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index e17a55542..b0c3aa3b1 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -1357,9 +1357,6 @@ proc genTypeInfoV2Impl(m: BModule, t, origType: PType, name: Rope; info: TLineIn if t.kind == tyObject and t.len > 0 and t[0] != nil and optEnableDeepCopy in m.config.globalOptions: discard genTypeInfoV1(m, t, info) -proc moduleOpenForCodegen(m: BModule; module: int32): bool {.inline.} = - result = module < m.g.modules.len and m.g.modules[module] != nil - proc genTypeInfoV2(m: BModule, t: PType; info: TLineInfo): Rope = let origType = t # distinct types can have their own destructors @@ -1384,7 +1381,7 @@ proc genTypeInfoV2(m: BModule, t: PType; info: TLineInfo): Rope = m.typeInfoMarkerV2[sig] = result let owner = t.skipTypes(typedescPtrs).itemId.module - if owner != m.module.position and moduleOpenForCodegen(m, owner): + if owner != m.module.position and moduleOpenForCodegen(m.g.graph, FileIndex owner): # make sure the type info is created in the owner module discard genTypeInfoV2(m.g.modules[owner], origType, info) # reference the type info as extern here @@ -1463,7 +1460,7 @@ proc genTypeInfoV1(m: BModule, t: PType; info: TLineInfo): Rope = return prefixTI.rope & result & ")".rope var owner = t.skipTypes(typedescPtrs).itemId.module - if owner != m.module.position and moduleOpenForCodegen(m, owner): + if owner != m.module.position and moduleOpenForCodegen(m.g.graph, FileIndex owner): # make sure the type info is created in the owner module discard genTypeInfoV1(m.g.modules[owner], origType, info) # reference the type info as extern here diff --git a/compiler/cgen.nim b/compiler/cgen.nim index b88999088..7dfab6a42 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -22,6 +22,7 @@ when not defined(leanCompiler): import strutils except `%` # collides with ropes.`%` +from ic / ic import ModuleBackendFlag from modulegraphs import ModuleGraph, PPassContext from lineinfos import warnGcMem, errXMustBeCompileTime, hintDependency, errGenerated, errCannotOpenFile @@ -1501,6 +1502,33 @@ proc genMainProc(m: BModule) = if m.config.cppCustomNamespace.len > 0: m.s[cfsProcs].add openNamespaceNim(m.config.cppCustomNamespace) +proc registerInitProcs*(g: BModuleList; m: PSym; flags: set[ModuleBackendFlag]) = + ## Called from the IC backend. + if HasDatInitProc in flags: + let datInit = getSomeNameForModule(m) & "DatInit000" + g.mainModProcs.addf("N_LIB_PRIVATE N_NIMCALL(void, $1)(void);$N", [datInit]) + g.mainDatInit.addf("\t$1();$N", [datInit]) + if HasModuleInitProc in flags: + let init = getSomeNameForModule(m) & "Init000" + g.mainModProcs.addf("N_LIB_PRIVATE N_NIMCALL(void, $1)(void);$N", [init]) + let initCall = "\t$1();$N" % [init] + if sfMainModule in m.flags: + g.mainModInit.add(initCall) + elif sfSystemModule in m.flags: + g.mainDatInit.add(initCall) # systemInit must called right after systemDatInit if any + else: + g.otherModsInit.add(initCall) + +proc whichInitProcs*(m: BModule): set[ModuleBackendFlag] = + # called from IC. + result = {} + if m.hcrOn or m.preInitProc.s(cpsInit).len > 0 or m.preInitProc.s(cpsStmts).len > 0: + result.incl HasModuleInitProc + for i in cfsTypeInit1..cfsDynLibInit: + if m.s[i].len != 0: + result.incl HasDatInitProc + break + proc registerModuleToMain(g: BModuleList; m: BModule) = let init = m.getInitName @@ -1595,6 +1623,7 @@ proc genDatInitCode(m: BModule) = if moduleDatInitRequired: m.s[cfsDatInitProc].add(prc) + #rememberFlag(m.g.graph, m.module, HasDatInitProc) # Very similar to the contents of symInDynamicLib - basically only the # things needed for the hot code reloading runtime procs to be loaded @@ -1725,6 +1754,7 @@ proc genInitCode(m: BModule) = if moduleInitRequired or sfMainModule in m.module.flags: m.s[cfsInitProc].add(prc) + #rememberFlag(m.g.graph, m.module, HasModuleInitProc) genDatInitCode(m) diff --git a/compiler/ic/cbackend.nim b/compiler/ic/cbackend.nim index 34ee59d52..9282ac466 100644 --- a/compiler/ic/cbackend.nim +++ b/compiler/ic/cbackend.nim @@ -44,9 +44,11 @@ proc generateCodeForModule(g: ModuleGraph; m: var LoadedModule; alive: var Alive cgen.genTopLevelStmt(bmod, n) finalCodegenActions(g, bmod, newNodeI(nkStmtList, m.module.info)) + m.fromDisk.backendFlags = cgen.whichInitProcs(bmod) proc replayTypeInfo(g: ModuleGraph; m: var LoadedModule; origin: FileIndex) = for x in mitems(m.fromDisk.emittedTypeInfo): + #echo "found type ", x, " for file ", int(origin) g.emittedTypeInfo[x] = origin proc addFileToLink(config: ConfigRef; m: PSym) = @@ -112,12 +114,16 @@ proc generateCode*(g: ModuleGraph) = resetForBackend(g) var alive = computeAliveSyms(g.packed, g.config) + when false: + for i in 0..high(g.packed): + echo i, " is of status ", g.packed[i].status, " ", toFullPath(g.config, FileIndex(i)) + for i in 0..high(g.packed): # case statement here to enforce exhaustive checks. case g.packed[i].status of undefined: discard "nothing to do" - of loading: + of loading, stored: assert false of storing, outdated: generateCodeForModule(g, g.packed[i], alive) @@ -133,3 +139,7 @@ proc generateCode*(g: ModuleGraph) = else: addFileToLink(g.config, g.packed[i].module) replayTypeInfo(g, g.packed[i], FileIndex(i)) + + if g.backend == nil: + g.backend = cgendata.newModuleList(g) + registerInitProcs(BModuleList(g.backend), g.packed[i].module, g.packed[i].fromDisk.backendFlags) diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim index 230b4d087..8017d8b40 100644 --- a/compiler/ic/ic.nim +++ b/compiler/ic/ic.nim @@ -22,6 +22,10 @@ type options: TOptions globalOptions: TGlobalOptions + ModuleBackendFlag* = enum + HasDatInitProc + HasModuleInitProc + PackedModule* = object ## the parts of a PackedEncoder that are part of the .rod file definedSymbols: string includes: seq[(LitId, string)] # first entry is the module filename itself @@ -43,6 +47,7 @@ type enumToStringProcs*: seq[(PackedItemId, PackedItemId)] emittedTypeInfo*: seq[string] + backendFlags*: set[ModuleBackendFlag] sh*: Shared cfg: PackedConfig @@ -556,6 +561,9 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef loadSeqSection enumToStringProcsSection, m.enumToStringProcs loadSeqSection typeInfoSection, m.emittedTypeInfo + f.loadSection backendFlagsSection + f.loadPrim m.backendFlags + close(f) result = f.err @@ -619,6 +627,9 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac storeSeqSection enumToStringProcsSection, m.enumToStringProcs storeSeqSection typeInfoSection, m.emittedTypeInfo + f.storeSection backendFlagsSection + f.storePrim m.backendFlags + close(f) encoder.disable() if f.err != ok: @@ -646,7 +657,8 @@ type storing, # state is strictly for stress-testing purposes loading, loaded, - outdated + outdated, + stored # store is complete, no further additions possible LoadedModule* = object status*: ModuleStatus @@ -673,7 +685,7 @@ proc toFileIndexCached*(c: var PackedDecoder; g: PackedModuleGraph; thisModule: proc translateLineInfo(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; x: PackedLineInfo): TLineInfo = - assert g[thisModule].status in {loaded, storing} + assert g[thisModule].status in {loaded, storing, stored} result = TLineInfo(line: x.line, col: x.col, fileIndex: toFileIndexCached(c, g, thisModule, x.file)) @@ -806,7 +818,7 @@ proc loadSym(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; s: result = nil else: let si = moduleIndex(c, g, thisModule, s) - assert g[si].status in {loaded, storing} + assert g[si].status in {loaded, storing, stored} if not g[si].symsInit: g[si].symsInit = true setLen g[si].syms, g[si].fromDisk.sh.syms.len @@ -852,7 +864,7 @@ proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; t result = nil else: let si = moduleIndex(c, g, thisModule, t) - assert g[si].status in {loaded, storing} + assert g[si].status in {loaded, storing, stored} assert t.item > 0 if not g[si].typesInit: @@ -950,7 +962,7 @@ proc needsRecompile(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache of loading, loaded: # For loading: Assume no recompile is required. result = false - of outdated, storing: + of outdated, storing, stored: result = true proc moduleFromRodFile*(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; diff --git a/compiler/ic/rodfiles.nim b/compiler/ic/rodfiles.nim index a518870f8..39a5b6b73 100644 --- a/compiler/ic/rodfiles.nim +++ b/compiler/ic/rodfiles.nim @@ -37,6 +37,7 @@ type methodsPerTypeSection enumToStringProcsSection typeInfoSection # required by the backend + backendFlagsSection aliveSymsSection # beware, this is stored in a `.alivesyms` file. RodFileError* = enum diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index 9ac76457c..e6f5267cd 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -455,11 +455,25 @@ proc getModule*(g: ModuleGraph; fileIdx: FileIndex): PSym = elif fileIdx.int32 < g.ifaces.len: result = g.ifaces[fileIdx.int32].module +proc moduleOpenForCodegen*(g: ModuleGraph; m: FileIndex): bool {.inline.} = + if g.config.symbolFiles == disabledSf: + result = true + else: + result = g.packed[m.int32].status notin {undefined, stored, loaded} + proc rememberEmittedTypeInfo*(g: ModuleGraph; m: FileIndex; ti: string) = #assert(not isCachedModule(g, m.int32)) if g.config.symbolFiles != disabledSf: #assert g.encoders[m.int32].isActive + assert g.packed[m.int32].status != stored g.packed[m.int32].fromDisk.emittedTypeInfo.add ti + #echo "added typeinfo ", m.int32, " ", ti, " suspicious ", not g.encoders[m.int32].isActive + +proc rememberFlag*(g: ModuleGraph; m: PSym; flag: ModuleBackendFlag) = + if g.config.symbolFiles != disabledSf: + #assert g.encoders[m.int32].isActive + assert g.packed[m.position].status != stored + g.packed[m.position].fromDisk.backendFlags.incl flag proc closeRodFile*(g: ModuleGraph; m: PSym) = if g.config.symbolFiles in {readOnlySf, v2Sf}: @@ -469,6 +483,8 @@ proc closeRodFile*(g: ModuleGraph; m: PSym) = let mint = m.position saveRodFile(toRodFile(g.config, AbsoluteFile toFullPath(g.config, FileIndex(mint))), g.encoders[mint], g.packed[mint].fromDisk) + g.packed[mint].status = stored + elif g.config.symbolFiles == stressTest: # debug code, but maybe a good idea for production? Could reduce the compiler's # memory consumption considerably at the cost of more loads from disk. diff --git a/koch.nim b/koch.nim index bf7fb1e62..6f1da8165 100644 --- a/koch.nim +++ b/koch.nim @@ -458,8 +458,8 @@ proc temp(args: string) = inc i let d = getAppDir() - var output = d / "compiler" / "nim".exe - var finalDest = d / "bin" / "nim_temp".exe + let output = d / "compiler" / "nim".exe + let finalDest = d / "bin" / "nim_temp".exe # 125 is the magic number to tell git bisect to skip the current commit. var (bootArgs, programArgs) = splitArgs(args) if "doc" notin programArgs and @@ -483,6 +483,22 @@ proc xtemp(cmd: string) = finally: copyExe(d / "bin" / "nim_backup".exe, d / "bin" / "nim".exe) +proc icTest(args: string) = + temp("") + let inp = os.parseCmdLine(args)[0] + let content = readFile(inp) + let nimExe = getAppDir() / "bin" / "nim_temp".exe + var i = 0 + for fragment in content.split("#!EDIT!#"): + let file = inp.replace(".nim", "_temp.nim") + writeFile(file, fragment) + var cmd = nimExe & " cpp --ic:on --listcmd " + if i == 0: + cmd.add "-f " + cmd.add quoteShell(file) + exec(cmd) + inc i + proc buildDrNim(args: string) = if not dirExists("dist/nimz3"): exec("git clone https://github.com/zevv/nimz3.git dist/nimz3") @@ -705,6 +721,7 @@ when isMainModule: of "fusion": let suffix = if latest: HeadHash else: FusionStableHash exec("nimble install -y fusion@$#" % suffix) + of "ic": icTest(op.cmdLineRest) else: showHelp() break of cmdEnd: break diff --git a/tests/ic/tgenerics.nim b/tests/ic/tgenerics.nim index bc5c05f4f..138799e85 100644 --- a/tests/ic/tgenerics.nim +++ b/tests/ic/tgenerics.nim @@ -1,6 +1,5 @@ discard """ output: "bar" - disabled: "true" """ import tables From bb3769975b7cbbe2f84c288d7f3363b6bad9ca0e Mon Sep 17 00:00:00 2001 From: Clyybber Date: Fri, 26 Mar 2021 13:05:51 +0100 Subject: [PATCH 0497/2472] Fix #17199 (#17348) * don't zero out in a move in the VM * Add testcases for #17199 * Update tests/arc/tarcmisc.nim Co-authored-by: Timothee Cour * Update tests/vm/tissues.nim Co-authored-by: Timothee Cour * Fix test failures * Fix test * Fix tests Co-authored-by: Andreas Rumpf Co-authored-by: Timothee Cour --- compiler/vmgen.nim | 7 ++++--- tests/arc/tarcmisc.nim | 32 ++++++++++++++++++++++++++++ tests/stdlib/tisolation.nim | 42 ++++++++++++++++++++++++------------- 3 files changed, 63 insertions(+), 18 deletions(-) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 7d7382d18..bd80df219 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1354,9 +1354,10 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = if dest < 0: dest = c.getTemp(arg.typ) gABC(c, arg, whichAsgnOpc(arg, requiresCopy=false), dest, a) # XXX use ldNullOpcode() here? - c.gABx(n, opcLdNull, a, c.genType(arg.typ)) - c.gABx(n, opcNodeToReg, a, a) - c.genAsgnPatch(arg, a) + # Don't zero out the arg for now #17199 + # c.gABx(n, opcLdNull, a, c.genType(arg.typ)) + # c.gABx(n, opcNodeToReg, a, a) + # c.genAsgnPatch(arg, a) c.freeTemp(a) of mNodeId: c.genUnaryABC(n, dest, opcNodeId) diff --git a/tests/arc/tarcmisc.nim b/tests/arc/tarcmisc.nim index 55803085f..82751f44c 100644 --- a/tests/arc/tarcmisc.nim +++ b/tests/arc/tarcmisc.nim @@ -390,3 +390,35 @@ proc newPixelBuffer(): PixelBuffer = discard newPixelBuffer() + +# bug #17199 + +proc passSeq(data: seq[string]) = + # used the system.& proc initially + let wat = data & "hello" + +proc test2 = + let name = @["hello", "world"] + passSeq(name) + doAssert name == @["hello", "world"] + +static: test2() # was buggy +test2() + +proc merge(x: sink seq[string], y: sink string): seq[string] = + newSeq(result, x.len + 1) + for i in 0..x.len-1: + result[i] = move(x[i]) + result[x.len] = move(y) + +proc passSeq2(data: seq[string]) = + # used the system.& proc initially + let wat = merge(data, "hello") + +proc test3 = + let name = @["hello", "world"] + passSeq2(name) + doAssert name == @["hello", "world"] + +static: test3() # was buggy +test3() diff --git a/tests/stdlib/tisolation.nim b/tests/stdlib/tisolation.nim index e43fd558a..c3857f483 100644 --- a/tests/stdlib/tisolation.nim +++ b/tests/stdlib/tisolation.nim @@ -7,7 +7,7 @@ import std/[isolation, json] -proc main() = +proc main(moveZeroesOut: static bool) = block: type Empty = ref object @@ -19,64 +19,76 @@ proc main() = block: # string literals var data = isolate("string") doAssert data.extract == "string" - doAssert data.extract == "" + if moveZeroesOut: + doAssert data.extract == "" block: # string literals var data = isolate("") doAssert data.extract == "" - doAssert data.extract == "" + if moveZeroesOut: + doAssert data.extract == "" block: var src = "string" var data = isolate(move src) doAssert data.extract == "string" - doAssert src.len == 0 + if moveZeroesOut: + doAssert src.len == 0 block: # int literals var data = isolate(1) doAssert data.extract == 1 - doAssert data.extract == 0 + if moveZeroesOut: + doAssert data.extract == 0 block: # float literals var data = isolate(1.6) doAssert data.extract == 1.6 - doAssert data.extract == 0.0 + if moveZeroesOut: + doAssert data.extract == 0.0 block: var data = isolate(@["1", "2"]) doAssert data.extract == @["1", "2"] - doAssert data.extract == @[] + if moveZeroesOut: + doAssert data.extract == @[] block: var data = isolate(@["1", "2", "3", "4", "5"]) doAssert data.extract == @["1", "2", "3", "4", "5"] - doAssert data.extract == @[] + if moveZeroesOut: + doAssert data.extract == @[] block: var data = isolate(@["", ""]) doAssert data.extract == @["", ""] - doAssert data.extract == @[] + if moveZeroesOut: + doAssert data.extract == @[] block: var src = @["1", "2"] var data = isolate(move src) doAssert data.extract == @["1", "2"] - doAssert src.len == 0 + if moveZeroesOut: + doAssert src.len == 0 block: var data = isolate(@[1, 2]) doAssert data.extract == @[1, 2] - doAssert data.extract == @[] + if moveZeroesOut: + doAssert data.extract == @[] block: var data = isolate(["1", "2"]) doAssert data.extract == ["1", "2"] - doAssert data.extract == ["", ""] + if moveZeroesOut: + doAssert data.extract == ["", ""] block: var data = isolate([1, 2]) doAssert data.extract == [1, 2] - doAssert data.extract == [0, 0] + if moveZeroesOut: + doAssert data.extract == [0, 0] block: type @@ -119,5 +131,5 @@ proc main() = doAssert $x == """@[(value: "1234")]""" -static: main() -main() +static: main(moveZeroesOut = false) +main(moveZeroesOut = true) From add771b051128b5eef09b4d77015fbeacf994700 Mon Sep 17 00:00:00 2001 From: narimiran Date: Fri, 26 Mar 2021 13:41:43 +0100 Subject: [PATCH 0498/2472] disable package 'chronicles' until 'chronos' is fixed --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 66a2fa09e..eb0196f70 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -45,7 +45,7 @@ pkg "c2nim", "nim c testsuite/tester.nim" pkg "cascade" pkg "cello" pkg "chroma" -pkg "chronicles", "nim c -o:chr -r chronicles.nim" +# pkg "chronicles", "nim c -o:chr -r chronicles.nim" # when not defined(osx): # testdatagram.nim(560, 54): Check failed # pkg "chronos", "nim c -r -d:release tests/testall" # pending https://github.com/nim-lang/Nim/issues/17130 From 3e03f6733548fecb328214e0c8632fa4916992ed Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 26 Mar 2021 16:27:55 +0100 Subject: [PATCH 0499/2472] cleaned up the internal documentation (#17524) --- doc/intern.rst | 605 +++++++++------------------------------- lib/system/cellsets.nim | 35 ++- lib/system/gc.nim | 47 ++++ 3 files changed, 220 insertions(+), 467 deletions(-) diff --git a/doc/intern.rst b/doc/intern.rst index 2456b25fd..1e18d975c 100644 --- a/doc/intern.rst +++ b/doc/intern.rst @@ -32,28 +32,29 @@ Path Purpose `doc` the documentation; it is a bunch of reStructuredText files `lib` the Nim library -`web` website of Nim; generated by `nimweb` - from the `*.txt` and `*.nimf` files ============ =================================================== Bootstrapping the compiler ========================== +**Note**: Add ``.`` to your PATH so that `koch` can be used without the `./`. + Compiling the compiler is a simple matter of running:: nim c koch.nim - ./koch boot + koch boot -d:release -For a release version use:: +For a debug version use:: nim c koch.nim - ./koch boot -d:release + koch boot + And for a debug version compatible with GDB:: nim c koch.nim - ./koch boot --debuginfo --linedir:on + koch boot --debuginfo --linedir:on The `koch` program is Nim's maintenance script. It is a replacement for make and shell scripting with the advantage that it is much more portable. @@ -61,12 +62,105 @@ More information about its options can be found in the `koch `_ documentation. +Developing the compiler +======================= + +To create a new compiler for each run, use `koch temp`:: + + koch temp c test.nim + +`koch temp` creates a debug build of the compiler, which is useful +to create stacktraces for compiler debugging. + +You can of course use GDB or Visual Studio to debug the +compiler (via `--debuginfo --lineDir:on`). However, there +are also lots of procs that aid in debugging: + + +.. code-block:: nim + + # dealing with PNode: + echo renderTree(someNode) + debug(someNode) # some JSON representation + + # dealing with PType: + echo typeToString(someType) + debug(someType) + + # dealing with PSym: + echo symbol.name.s + debug(symbol) + + # pretty prints the Nim ast, but annotates symbol IDs: + echo renderTree(someNode, {renderIds}) + if `??`(conf, n.info, "temp.nim"): + # only output when it comes from "temp.nim" + echo renderTree(n) + if `??`(conf, n.info, "temp.nim"): + # why does it process temp.nim here? + writeStackTrace() + +These procs may not already be imported by the module you're editing. +You can import them directly for debugging: + +.. code-block:: nim + from astalgo import debug + from types import typeToString + from renderer import renderTree + from msgs import `??` + + +The compiler's architecture +=========================== + +Nim uses the classic compiler architecture: A lexer/scanner feds tokens to a +parser. The parser builds a syntax tree that is used by the code generators. +This syntax tree is the interface between the parser and the code generator. +It is essential to understand most of the compiler's code. + +Semantic analysis is separated from parsing. + +.. include:: filelist.txt + + +The syntax tree +--------------- +The syntax tree consists of nodes which may have an arbitrary number of +children. Types and symbols are represented by other nodes, because they +may contain cycles. The AST changes its shape after semantic checking. This +is needed to make life easier for the code generators. See the "ast" module +for the type definitions. The `macros `_ module contains many +examples how the AST represents each syntactic structure. + + +Bisecting for regressions +========================= + +`koch temp` returns 125 as the exit code in case the compiler +compilation fails. This exit code tells `git bisect` to skip the +current commit.:: + + git bisect start bad-commit good-commit + git bisect run ./koch temp -r c test-source.nim + +You can also bisect using custom options to build the compiler, for example if +you don't need a debug version of the compiler (which runs slower), you can replace +`./koch temp` by explicit compilation command, see `Rebuilding the compiler`_. + + +Runtimes +======== + +Nim has two different runtimes, the "old runtime" and the "new runtime". The old +runtime supports the old GCs (markAndSweep, refc, Boehm), the new runtime supports +ARC/ORC. The new runtime is active `when defined(nimV2)`. + + Coding Guidelines ================= -* Use CamelCase, not underscored_identifiers. -* Indent with two spaces. -* Max line length is 80 characters. +* We follow Nim's official style guide, see ``_. +* Max line length is 100 characters. * Provide spaces around binary operators if that enhances readability. * Use a space after a colon, but not before it. * [deprecated] Start types with a capital `T`, unless they are @@ -86,9 +180,9 @@ POSIX-compliant systems on conventional hardware are usually pretty easy to port: Add the platform to `platform` (if it is not already listed there), check that the OS, System modules work and recompile Nim. -The only case where things aren't as easy is when the garbage -collector needs some assembler tweaking to work. The standard -version of the GC uses C's `setjmp` function to store all registers +The only case where things aren't as easy is when old runtime's garbage +collectors need some assembler tweaking to work. The default +implementation uses C's `setjmp` function to store all registers on the hardware stack. It may be necessary that the new platform needs to replace this generic code by some assembler code. @@ -96,136 +190,33 @@ replace this generic code by some assembler code. Runtime type information ======================== +**Note**: This section describes the "old runtime". + *Runtime type information* (RTTI) is needed for several aspects of the Nim programming language: Garbage collection - The most important reason for RTTI. Generating - traversal procedures produces bigger code and is likely to be slower on - modern hardware as dynamic procedure binding is hard to predict. + The old GCs use the RTTI for traversing abitrary Nim types, but usually + only the `marker` field which contains a proc that does the traversal. Complex assignments Sequences and strings are implemented as pointers to resizeable buffers, but Nim requires copying for - assignments. Apart from RTTI the compiler could generate copy procedures - for any type that needs one. However, this would make the code bigger and - the RTTI is likely already there for the GC. + assignments. Apart from RTTI the compiler also generates copy procedures + as a specialization. We already know the type information as a graph in the compiler. Thus we need to serialize this graph as RTTI for C code generation. Look at the file `lib/system/hti.nim` for more information. -Rebuilding the compiler + +Magics and compilerProcs ======================== -After an initial build via `sh build_all.sh` on posix or `build_all.bat` on windows, -you can rebuild the compiler as follows: - -* `nim c koch` if you need to rebuild koch -* `./koch boot -d:release` this ensures the compiler can rebuild itself - (use `koch` instead of `./koch` on windows), which builds the compiler 3 times. - -A faster approach if you don't need to run the full bootstrapping implied by `koch boot`, -is the following: - -* `pathto/nim c --lib:lib -d:release -o:bin/nim_temp compiler/nim.nim` - -Where `pathto/nim` is any nim binary sufficiently recent (e.g. `bin/nim_cources` -built during bootstrap or `$HOME/.nimble/bin/nim` installed by `choosenim 1.2.0`) - -You can pass any additional options such as `-d:leanCompiler` if you don't need -certain features or `-d:debug --stacktrace:on --excessiveStackTrace --stackTraceMsgs` -for debugging the compiler. See also `Debugging the compiler`_. - -Debugging the compiler -====================== - -You can of course use GDB or Visual Studio to debug the -compiler (via `--debuginfo --lineDir:on`). However, there -are also lots of procs that aid in debugging: - - -.. code-block:: nim - # pretty prints the Nim AST - echo renderTree(someNode) - # outputs some JSON representation - debug(someNode) - # pretty prints some type - echo typeToString(someType) - debug(someType) - echo symbol.name.s - debug(symbol) - # pretty prints the Nim ast, but annotates symbol IDs: - echo renderTree(someNode, {renderIds}) - if `??`(conf, n.info, "temp.nim"): - # only output when it comes from "temp.nim" - echo renderTree(n) - if `??`(conf, n.info, "temp.nim"): - # why does it process temp.nim here? - writeStackTrace() - -These procs may not be imported by a module. You can import them directly for debugging: - -.. code-block:: nim - from astalgo import debug - from types import typeToString - from renderer import renderTree - from msgs import `??` - -To create a new compiler for each run, use `koch temp`:: - - ./koch temp c /tmp/test.nim - -`koch temp` creates a debug build of the compiler, which is useful -to create stacktraces for compiler debugging. See also -`Rebuilding the compiler`_ if you need more control. - -Bisecting for regressions -========================= - -`koch temp` returns 125 as the exit code in case the compiler -compilation fails. This exit code tells `git bisect` to skip the -current commit.:: - - git bisect start bad-commit good-commit - git bisect run ./koch temp -r c test-source.nim - -You can also bisect using custom options to build the compiler, for example if -you don't need a debug version of the compiler (which runs slower), you can replace -`./koch temp` by explicit compilation command, see `Rebuilding the compiler`_. - -The compiler's architecture -=========================== - -Nim uses the classic compiler architecture: A lexer/scanner feds tokens to a -parser. The parser builds a syntax tree that is used by the code generator. -This syntax tree is the interface between the parser and the code generator. -It is essential to understand most of the compiler's code. - -In order to compile Nim correctly, type-checking has to be separated from -parsing. Otherwise generics cannot work. - -.. include:: filelist.txt - - -The syntax tree ---------------- -The syntax tree consists of nodes which may have an arbitrary number of -children. Types and symbols are represented by other nodes, because they -may contain cycles. The AST changes its shape after semantic checking. This -is needed to make life easier for the code generators. See the "ast" module -for the type definitions. The `macros `_ module contains many -examples how the AST represents each syntactic structure. - - -How the RTL is compiled -======================= - The `system` module contains the part of the RTL which needs support by -compiler magic (and the stuff that needs to be in it because the spec -says so). The C code generator generates the C code for it, just like any other +compiler magic. The C code generator generates the C code for it, just like any other module. However, calls to some procedures like `addInt` are inserted by -the CCG. Therefore the module `magicsys` contains a table (`compilerprocs`) +the generator. Therefore there is a table (`compilerprocs`) with all symbols that are marked as `compilerproc`. `compilerprocs` are needed by the code generator. A `magic` proc is not the same as a `compilerproc`: A `magic` is a proc that needs compiler magic for its @@ -233,325 +224,6 @@ semantic checking, a `compilerproc` is a proc that is used by the code generator. -Compilation cache -================= - -The implementation of the compilation cache is tricky: There are lots -of issues to be solved for the front- and backend. - - -General approach: AST replay ----------------------------- - -We store a module's AST of a successful semantic check in a SQLite -database. There are plenty of features that require a sub sequence -to be re-applied, for example: - -.. code-block:: nim - {.compile: "foo.c".} # even if the module is loaded from the DB, - # "foo.c" needs to be compiled/linked. - -The solution is to **re-play** the module's top level statements. -This solves the problem without having to special case the logic -that fills the internal seqs which are affected by the pragmas. - -In fact, this describes how the AST should be stored in the database, -as a "shallow" tree. Let's assume we compile module `m` with the -following contents: - -.. code-block:: nim - import std/strutils - - var x*: int = 90 - {.compile: "foo.c".} - proc p = echo "p" - proc q = echo "q" - static: - echo "static" - -Conceptually this is the AST we store for the module: - -.. code-block:: nim - import std/strutils - - var x* - {.compile: "foo.c".} - proc p - proc q - static: - echo "static" - -The symbol's `ast` field is loaded lazily, on demand. This is where most -savings come from, only the shallow outer AST is reconstructed immediately. - -It is also important that the replay involves the `import` statement so -that dependencies are resolved properly. - - -Shared global compiletime state -------------------------------- - -Nim allows `.global, compiletime` variables that can be filled by macro -invocations across different modules. This feature breaks modularity in a -severe way. Plenty of different solutions have been proposed: - -- Restrict the types of global compiletime variables to `Set[T]` or - similar unordered, only-growable collections so that we can track - the module's write effects to these variables and reapply the changes - in a different order. -- In every module compilation, reset the variable to its default value. -- Provide a restrictive API that can load/save the compiletime state to - a file. - -(These solutions are not mutually exclusive.) - -Since we adopt the "replay the top level statements" idea, the natural -solution to this problem is to emit pseudo top level statements that -reflect the mutations done to the global variable. However, this is -MUCH harder than it sounds, for example `squeaknim` uses this -snippet: - -.. code-block:: nim - apicall.add(") module: '" & dllName & "'>\C" & - "\t^self externalCallFailed\C!\C\C") - stCode.add(st & "\C\t\"Generated by NimSqueak\"\C\t" & apicall) - -We can "replay" `stCode.add` only if the values of `st` -and `apicall` are known. And even then a hash table's `add` with its -hashing mechanism is too hard to replay. - -In practice, things are worse still, consider `someGlobal[i][j].add arg`. -We only know the root is `someGlobal` but the concrete path to the data -is unknown as is the value that is added. We could compute a "diff" between -the global states and use that to compute a symbol patchset, but this is -quite some work, expensive to do at runtime (it would need to run after -every module has been compiled) and would also break for hash tables. - -We need an API that hides the complex aliasing problems by not relying -on Nim's global variables. The obvious solution is to use string keys -instead of global variables: - -.. code-block:: nim - - proc cachePut*(key: string; value: string) - proc cacheGet*(key: string): string - -However, the values being strings/json is quite problematic: Many -lookup tables that are built at compiletime embed *proc vars* and -types which have no obvious string representation... Seems like -AST diffing is still the best idea as it will not require to use -an alien API and works with some existing Nimble packages, at least. - -On the other hand, in Nim's future I would like to replace the VM -by native code. A diff algorithm wouldn't work for that. -Instead the native code would work with an API like `put`, `get`: - -.. code-block:: nim - - proc cachePut*(key: string; value: NimNode) - proc cacheGet*(key: string): NimNode - -The API should embrace the AST diffing notion: See the -module `macrocache` for the final details. - - - -Methods and type converters ---------------------------- - -In the following -sections *global* means *shared between modules* or *property of the whole -program*. - -Nim contains language features that are *global*. The best example for that -are multi methods: Introducing a new method with the same name and some -compatible object parameter means that the method's dispatcher needs to take -the new method into account. So the dispatching logic is only completely known -after the whole program has been translated! - -Other features that are *implicitly* triggered cause problems for modularity -too. Type converters fall into this category: - -.. code-block:: nim - # module A - converter toBool(x: int): bool = - result = x != 0 - -.. code-block:: nim - # module B - import A - - if 1: - echo "ugly, but should work" - -If in the above example module `B` is re-compiled, but `A` is not then -`B` needs to be aware of `toBool` even though `toBool` is not referenced -in `B` *explicitly*. - -Both the multi method and the type converter problems are solved by the -AST replay implementation. - - -Generics -~~~~~~~~ - -We cache generic instantiations and need to ensure this caching works -well with the incremental compilation feature. Since the cache is -attached to the `PSym` datastructure, it should work without any -special logic. - - -Backend issues --------------- - -- Init procs must not be "forgotten" to be called. -- Files must not be "forgotten" to be linked. -- Method dispatchers are global. -- DLL loading via `dlsym` is global. -- Emulated thread vars are global. - -However the biggest problem is that dead code elimination breaks modularity! -To see why, consider this scenario: The module `G` (for example the huge -Gtk2 module...) is compiled with dead code elimination turned on. So none -of `G`'s procs is generated at all. - -Then module `B` is compiled that requires `G.P1`. Ok, no problem, -`G.P1` is loaded from the symbol file and `G.c` now contains `G.P1`. - -Then module `A` (that depends on `B` and `G`) is compiled and `B` -and `G` are left unchanged. `A` requires `G.P2`. - -So now `G.c` MUST contain both `P1` and `P2`, but we haven't even -loaded `P1` from the symbol file, nor do we want to because we then quickly -would restore large parts of the whole program. - - -Solution -~~~~~~~~ - -The backend must have some logic so that if the currently processed module -is from the compilation cache, the `ast` field is not accessed. Instead -the generated C(++) for the symbol's body needs to be cached too and -inserted back into the produced C file. This approach seems to deal with -all the outlined problems above. - - -Debugging Nim's memory management -================================= - -The following paragraphs are mostly a reminder for myself. Things to keep -in mind: - -* If an assertion in Nim's memory manager or GC fails, the stack trace - keeps allocating memory! Thus a stack overflow may happen, hiding the - real issue. -* What seem to be C code generation problems is often a bug resulting from - not producing prototypes, so that some types default to `cint`. Testing - without the `-w` option helps! - - -The Garbage Collector -===================== - -Introduction ------------- - -I use the term *cell* here to refer to everything that is traced -(sequences, refs, strings). -This section describes how the GC works. - -The basic algorithm is *Deferrent Reference Counting* with cycle detection. -References on the stack are not counted for better performance and easier C -code generation. - -Each cell has a header consisting of a RC and a pointer to its type -descriptor. However the program does not know about these, so they are placed at -negative offsets. In the GC code the type `PCell` denotes a pointer -decremented by the right offset, so that the header can be accessed easily. It -is extremely important that `pointer` is not confused with a `PCell` -as this would lead to a memory corruption. - - -The CellSet data structure --------------------------- - -The GC depends on an extremely efficient datastructure for storing a -set of pointers - this is called a `TCellSet` in the source code. -Inserting, deleting and searching are done in constant time. However, -modifying a `TCellSet` during traversal leads to undefined behaviour. - -.. code-block:: Nim - type - TCellSet # hidden - - proc cellSetInit(s: var TCellSet) # initialize a new set - proc cellSetDeinit(s: var TCellSet) # empty the set and free its memory - proc incl(s: var TCellSet, elem: PCell) # include an element - proc excl(s: var TCellSet, elem: PCell) # exclude an element - - proc `in`(elem: PCell, s: TCellSet): bool # tests membership - - iterator elements(s: TCellSet): (elem: PCell) - - -All the operations have to perform efficiently. Because a Cellset can -become huge a hash table alone is not suitable for this. - -We use a mixture of bitset and hash table for this. The hash table maps *pages* -to a page descriptor. The page descriptor contains a bit for any possible cell -address within this page. So including a cell is done as follows: - -- Find the page descriptor for the page the cell belongs to. -- Set the appropriate bit in the page descriptor indicating that the - cell points to the start of a memory block. - -Removing a cell is analogous - the bit has to be set to zero. -Single page descriptors are never deleted from the hash table. This is not -needed as the data structures needs to be rebuilt periodically anyway. - -Complete traversal is done in this way:: - - for each page descriptor d: - for each bit in d: - if bit == 1: - traverse the pointer belonging to this bit - - -Further complications ---------------------- - -In Nim the compiler cannot always know if a reference -is stored on the stack or not. This is caused by var parameters. -Consider this example: - -.. code-block:: Nim - proc setRef(r: var ref TNode) = - new(r) - - proc usage = - var - r: ref TNode - setRef(r) # here we should not update the reference counts, because - # r is on the stack - setRef(r.left) # here we should update the refcounts! - -We have to decide at runtime whether the reference is on the stack or not. -The generated code looks roughly like this: - -.. code-block:: C - void setref(TNode** ref) { - unsureAsgnRef(ref, newObj(TNode_TI, sizeof(TNode))) - } - void usage(void) { - setRef(&r) - setRef(&r->left) - } - -Note that for systems with a continuous stack (which most systems have) -the check whether the ref is on the stack is very cheap (only two -comparisons). - - Code generation for closures ============================ @@ -598,14 +270,14 @@ This should produce roughly this code: .. code-block:: nim type - PEnv = ref object + Env = ref object x: int # data - proc anon(y: int, c: PEnv): int = + proc anon(y: int, c: Env): int = return y + c.x proc add(x: int): tuple[prc, data] = - var env: PEnv + var env: Env new env env.x = x result = (anon, env) @@ -630,25 +302,25 @@ This should produce roughly this code: .. code-block:: nim type - PEnvX = ref object + EnvX = ref object x: int # data - PEnvY = ref object + EnvY = ref object y: int - ex: PEnvX + ex: EnvX - proc lambdaZ(z: int, ey: PEnvY): int = + proc lambdaZ(z: int, ey: EnvY): int = return ey.ex.x + ey.y + z - proc lambdaY(y: int, ex: PEnvX): tuple[prc, data: PEnvY] = - var ey: PEnvY + proc lambdaY(y: int, ex: EnvX): tuple[prc, data: EnvY] = + var ey: EnvY new ey ey.y = y ey.ex = ex result = (lambdaZ, ey) - proc add(x: int): tuple[prc, data: PEnvX] = - var ex: PEnvX + proc add(x: int): tuple[prc, data: EnvX] = + var ex: EnvX ex.x = x result = (labmdaY, ex) @@ -663,17 +335,11 @@ More useful is escape analysis and stack allocation of the environment, however. -Alternative ------------ - -Process the closure of all inner procs in one pass and accumulate the -environments. This is however not always possible. - - Accumulator ----------- .. code-block:: nim + proc getAccumulator(start: int): proc (): int {.closure} = var i = start return lambda: int = @@ -708,20 +374,26 @@ backend somehow. We deal with this by modifying `s.ast[paramPos]` to contain the formal hidden parameter, but not `s.typ`! -Integer literals: ------------------ +Notes on type and AST representation +==================================== + +To be expanded. + + +Integer literals +---------------- In Nim, there is a redundant way to specify the type of an integer literal. First of all, it should be unsurprising that every node has a node kind. The node of an integer literal can be any of the -following values: +following values:: nkIntLit, nkInt8Lit, nkInt16Lit, nkInt32Lit, nkInt64Lit, nkUIntLit, nkUInt8Lit, nkUInt16Lit, nkUInt32Lit, nkUInt64Lit On top of that, there is also the `typ` field for the type. It the kind of the `typ` field can be one of the following ones, and it -should be matching the literal kind: +should be matching the literal kind:: tyInt, tyInt8, tyInt16, tyInt32, tyInt64, tyUInt, tyUInt8, tyUInt16, tyUInt32, tyUInt64 @@ -777,6 +449,7 @@ pointing back to the integer literal node in the ast containing the integer value. These are the properties that hold true for integer literal types. +:: n.kind == nkIntLit n.typ.kind == tyInt n.typ.n == n diff --git a/lib/system/cellsets.nim b/lib/system/cellsets.nim index 779f1a91f..a0f1fabf9 100644 --- a/lib/system/cellsets.nim +++ b/lib/system/cellsets.nim @@ -7,7 +7,40 @@ # distribution, for details about the copyright. # -# Efficient set of pointers for the GC (and repr) + +#[ + +Efficient set of pointers for the GC (and repr) +----------------------------------------------- + +The GC depends on an extremely efficient datastructure for storing a +set of pointers - this is called a `CellSet` in the source code. +Inserting, deleting and searching are done in constant time. However, +modifying a `CellSet` during traversal leads to undefined behaviour. + +All operations on a CellSet have to perform efficiently. Because a Cellset can +become huge a hash table alone is not suitable for this. + +We use a mixture of bitset and hash table for this. The hash table maps *pages* +to a page descriptor. The page descriptor contains a bit for any possible cell +address within this page. So including a cell is done as follows: + +- Find the page descriptor for the page the cell belongs to. +- Set the appropriate bit in the page descriptor indicating that the + cell points to the start of a memory block. + +Removing a cell is analogous - the bit has to be set to zero. +Single page descriptors are never deleted from the hash table. This is not +needed as the data structures needs to be rebuilt periodically anyway. + +Complete traversal is done in this way:: + + for each page descriptor d: + for each bit in d: + if bit == 1: + traverse the pointer belonging to this bit + +]# when defined(gcOrc) or defined(gcArc): type diff --git a/lib/system/gc.nim b/lib/system/gc.nim index 1f7164266..ae29f3466 100644 --- a/lib/system/gc.nim +++ b/lib/system/gc.nim @@ -12,6 +12,53 @@ # Refcounting + Mark&Sweep. Complex algorithms avoided. # Been there, done that, didn't work. +#[ + +A *cell* is anything that is traced by the GC +(sequences, refs, strings, closures). + +The basic algorithm is *Deferrent Reference Counting* with cycle detection. +References on the stack are not counted for better performance and easier C +code generation. + +Each cell has a header consisting of a RC and a pointer to its type +descriptor. However the program does not know about these, so they are placed at +negative offsets. In the GC code the type `PCell` denotes a pointer +decremented by the right offset, so that the header can be accessed easily. It +is extremely important that `pointer` is not confused with a `PCell`. + +In Nim the compiler cannot always know if a reference +is stored on the stack or not. This is caused by var parameters. +Consider this example: + +.. code-block:: Nim + proc setRef(r: var ref TNode) = + new(r) + + proc usage = + var + r: ref TNode + setRef(r) # here we should not update the reference counts, because + # r is on the stack + setRef(r.left) # here we should update the refcounts! + +We have to decide at runtime whether the reference is on the stack or not. +The generated code looks roughly like this: + +.. code-block:: C + void setref(TNode** ref) { + unsureAsgnRef(ref, newObj(TNode_TI, sizeof(TNode))) + } + void usage(void) { + setRef(&r) + setRef(&r->left) + } + +Note that for systems with a continuous stack (which most systems have) +the check whether the ref is on the stack is very cheap (only two +comparisons). +]# + {.push profiler:off.} const From b1b767ed0db19b337ab41fab9241227d23aaae38 Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 27 Mar 2021 16:51:57 +0800 Subject: [PATCH 0500/2472] close #17157 add testcase (#17531) --- tests/proc/t17157.nim | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 tests/proc/t17157.nim diff --git a/tests/proc/t17157.nim b/tests/proc/t17157.nim new file mode 100644 index 000000000..020e93fce --- /dev/null +++ b/tests/proc/t17157.nim @@ -0,0 +1,6 @@ +discard """ + errormsg: "'untyped' is only allowed in templates and macros or magic procs" +""" + +template something(op: proc (v: untyped): void): void = + discard From 8ed6de4fe6517dc4db38ab5dea898c0016d1c08a Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 27 Mar 2021 02:22:42 -0700 Subject: [PATCH 0501/2472] fix flaky tnetconnect which sometimes failed with: No route to host [OSError] (#17526) --- tests/stdlib/tnetconnect.nim | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/stdlib/tnetconnect.nim b/tests/stdlib/tnetconnect.nim index b74545710..3a80df19f 100644 --- a/tests/stdlib/tnetconnect.nim +++ b/tests/stdlib/tnetconnect.nim @@ -17,10 +17,11 @@ proc test() = wrapSocket(ctx, socket) # trying 2 sites makes it more resilent: refs #17458 this could give: - # Error: unhandled exception: Call to 'connect' timed out. [TimeoutError] + # * Call to 'connect' timed out. [TimeoutError] + # * No route to host [OSError] try: fn("www.nim-lang.org") - except TimeoutError: + except TimeoutError, OSError: fn("www.google.com") test() From a65189a739c59fcc0d7b3f8fdcb2cf8bed432f68 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 27 Mar 2021 02:28:11 -0700 Subject: [PATCH 0502/2472] nnkArglist => nnkArgList + special case stylecheck:error (#17529) * nnkArglist => nnkArgList * special case stylecheck:error --- compiler/linter.nim | 8 +++++--- compiler/msgs.nim | 6 +++--- doc/astspec.txt | 2 +- doc/manual_experimental.rst | 2 +- lib/core/macros.nim | 2 +- lib/pure/htmlgen.nim | 2 +- lib/std/private/underscored_calls.nim | 2 +- tests/astspec/tastspec.nim | 2 +- tests/casestmt/tincompletecaseobject.nim | 4 ++-- tests/macros/tmacros_issues.nim | 2 +- 10 files changed, 17 insertions(+), 15 deletions(-) diff --git a/compiler/linter.nim b/compiler/linter.nim index 1ae1fb097..9af4f468e 100644 --- a/compiler/linter.nim +++ b/compiler/linter.nim @@ -125,9 +125,11 @@ proc styleCheckUse*(conf: ConfigRef; info: TLineInfo; s: PSym) = return let newName = s.name.s - let oldName = differs(conf, info, newName) - if oldName.len > 0: - lintReport(conf, info, newName, oldName) + let badName = differs(conf, info, newName) + if badName.len > 0: + # special rules for historical reasons + let forceHint = badName == "nnkArgList" and newName == "nnkArglist" or badName == "nnkArglist" and newName == "nnkArgList" + lintReport(conf, info, newName, badName, forceHint = forceHint) proc checkPragmaUse*(conf: ConfigRef; info: TLineInfo; w: TSpecialWord; pragmaName: string) = let wanted = $w diff --git a/compiler/msgs.nim b/compiler/msgs.nim index bbe40507f..24855ae18 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -612,9 +612,9 @@ template internalAssert*(conf: ConfigRef, e: bool) = let arg = info2.toFileLineCol internalErrorImpl(conf, unknownLineInfo, arg, info2) -template lintReport*(conf: ConfigRef; info: TLineInfo, beau, got: string) = - let m = "'$2' should be: '$1'" % [beau, got] - let msg = if optStyleError in conf.globalOptions: errGenerated else: hintName +template lintReport*(conf: ConfigRef; info: TLineInfo, beau, got: string, forceHint = false) = + let m = "'$1' should be: '$2'" % [got, beau] + let msg = if optStyleError in conf.globalOptions and not forceHint: errGenerated else: hintName liMessage(conf, info, msg, m, doNothing, instLoc()) proc quotedFilename*(conf: ConfigRef; i: TLineInfo): Rope = diff --git a/doc/astspec.txt b/doc/astspec.txt index c49f7bcc2..b4b8b34b5 100644 --- a/doc/astspec.txt +++ b/doc/astspec.txt @@ -1135,7 +1135,7 @@ AST: .. code-block:: nim # ... nnkTypeClassTy( # note this isn't nnkConceptTy! - nnkArglist( + nnkArgList( # ... idents for x, y, z ) # ... diff --git a/doc/manual_experimental.rst b/doc/manual_experimental.rst index cf2e0c247..fc46a2a14 100644 --- a/doc/manual_experimental.rst +++ b/doc/manual_experimental.rst @@ -1283,7 +1283,7 @@ all the arguments, but also the matched operators in reverse polish notation: echo x + y * z - x This passes the expression `x + y * z - x` to the `optM` macro as -an `nnkArglist` node containing:: +an `nnkArgList` node containing:: Arglist Sym "x" diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 5a556fc8d..8d6258e80 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -78,7 +78,7 @@ type nnkSharedTy, # 'shared T' nnkEnumTy, nnkEnumFieldDef, - nnkArglist, nnkPattern + nnkArgList, nnkPattern nnkHiddenTryStmt, nnkClosure, nnkGotoState, diff --git a/lib/pure/htmlgen.nim b/lib/pure/htmlgen.nim index 163f9303b..89eb24bb9 100644 --- a/lib/pure/htmlgen.nim +++ b/lib/pure/htmlgen.nim @@ -322,7 +322,7 @@ macro html*(e: varargs[untyped]): untyped = macro hr*(): untyped = ## Generates the HTML `hr` element. - result = xmlCheckedTag(newNimNode(nnkArglist), "hr", commonAttr, "", true) + result = xmlCheckedTag(newNimNode(nnkArgList), "hr", commonAttr, "", true) macro i*(e: varargs[untyped]): untyped = ## Generates the HTML `i` element. diff --git a/lib/std/private/underscored_calls.nim b/lib/std/private/underscored_calls.nim index 6d0a99ab5..f0bcbcc74 100644 --- a/lib/std/private/underscored_calls.nim +++ b/lib/std/private/underscored_calls.nim @@ -39,7 +39,7 @@ proc underscoredCall(n, arg0: NimNode): NimNode = result.add arg0 proc underscoredCalls*(result, calls, arg0: NimNode) = - expectKind calls, {nnkArglist, nnkStmtList, nnkStmtListExpr} + expectKind calls, {nnkArgList, nnkStmtList, nnkStmtListExpr} for call in calls: if call.kind in {nnkStmtList, nnkStmtListExpr}: diff --git a/tests/astspec/tastspec.nim b/tests/astspec/tastspec.nim index 3413d32f3..e2cfed277 100644 --- a/tests/astspec/tastspec.nim +++ b/tests/astspec/tastspec.nim @@ -929,7 +929,7 @@ static: (x & y & z) is string ast.peelOff({nnkStmtList, nnkTypeSection}).matchAst: - of nnkTypeDef(_, _, nnkTypeClassTy(nnkArglist, _, _, nnkStmtList)): + of nnkTypeDef(_, _, nnkTypeClassTy(nnkArgList, _, _, nnkStmtList)): # note this isn't nnkConceptTy! echo "ok" diff --git a/tests/casestmt/tincompletecaseobject.nim b/tests/casestmt/tincompletecaseobject.nim index 909ee4e1c..aa5deda7a 100644 --- a/tests/casestmt/tincompletecaseobject.nim +++ b/tests/casestmt/tincompletecaseobject.nim @@ -1,6 +1,6 @@ discard """ errormsg: ''' -not all cases are covered; missing: {nnkComesFrom, nnkDotCall, nnkHiddenCallConv, nnkVarTuple, nnkCurlyExpr, nnkRange, nnkCheckedFieldExpr, nnkDerefExpr, nnkElifExpr, nnkElseExpr, nnkLambda, nnkDo, nnkBind, nnkClosedSymChoice, nnkHiddenSubConv, nnkConv, nnkStaticExpr, nnkAddr, nnkHiddenAddr, nnkHiddenDeref, nnkObjDownConv, nnkObjUpConv, nnkChckRangeF, nnkChckRange64, nnkChckRange, nnkStringToCString, nnkCStringToString, nnkFastAsgn, nnkGenericParams, nnkFormalParams, nnkOfInherit, nnkImportAs, nnkConverterDef, nnkMacroDef, nnkTemplateDef, nnkIteratorDef, nnkOfBranch, nnkElifBranch, nnkExceptBranch, nnkElse, nnkAsmStmt, nnkTypeDef, nnkFinally, nnkContinueStmt, nnkImportStmt, nnkImportExceptStmt, nnkExportStmt, nnkExportExceptStmt, nnkFromStmt, nnkIncludeStmt, nnkUsingStmt, nnkBlockExpr, nnkStmtListType, nnkBlockType, nnkWith, nnkWithout, nnkTypeOfExpr, nnkObjectTy, nnkTupleTy, nnkTupleClassTy, nnkTypeClassTy, nnkStaticTy, nnkRecList, nnkRecCase, nnkRecWhen, nnkVarTy, nnkConstTy, nnkMutableTy, nnkDistinctTy, nnkProcTy, nnkIteratorTy, nnkSharedTy, nnkEnumTy, nnkEnumFieldDef, nnkArglist, nnkPattern, nnkReturnToken, nnkClosure, nnkGotoState, nnkState, nnkBreakState, nnkFuncDef, nnkTupleConstr} +not all cases are covered; missing: {nnkComesFrom, nnkDotCall, nnkHiddenCallConv, nnkVarTuple, nnkCurlyExpr, nnkRange, nnkCheckedFieldExpr, nnkDerefExpr, nnkElifExpr, nnkElseExpr, nnkLambda, nnkDo, nnkBind, nnkClosedSymChoice, nnkHiddenSubConv, nnkConv, nnkStaticExpr, nnkAddr, nnkHiddenAddr, nnkHiddenDeref, nnkObjDownConv, nnkObjUpConv, nnkChckRangeF, nnkChckRange64, nnkChckRange, nnkStringToCString, nnkCStringToString, nnkFastAsgn, nnkGenericParams, nnkFormalParams, nnkOfInherit, nnkImportAs, nnkConverterDef, nnkMacroDef, nnkTemplateDef, nnkIteratorDef, nnkOfBranch, nnkElifBranch, nnkExceptBranch, nnkElse, nnkAsmStmt, nnkTypeDef, nnkFinally, nnkContinueStmt, nnkImportStmt, nnkImportExceptStmt, nnkExportStmt, nnkExportExceptStmt, nnkFromStmt, nnkIncludeStmt, nnkUsingStmt, nnkBlockExpr, nnkStmtListType, nnkBlockType, nnkWith, nnkWithout, nnkTypeOfExpr, nnkObjectTy, nnkTupleTy, nnkTupleClassTy, nnkTypeClassTy, nnkStaticTy, nnkRecList, nnkRecCase, nnkRecWhen, nnkVarTy, nnkConstTy, nnkMutableTy, nnkDistinctTy, nnkProcTy, nnkIteratorTy, nnkSharedTy, nnkEnumTy, nnkEnumFieldDef, nnkArgList, nnkPattern, nnkReturnToken, nnkClosure, nnkGotoState, nnkState, nnkBreakState, nnkFuncDef, nnkTupleConstr} ''' """ @@ -62,7 +62,7 @@ type nnkSharedTy, # 'shared T' nnkEnumTy, nnkEnumFieldDef, - nnkArglist, nnkPattern + nnkArgList, nnkPattern nnkReturnToken, nnkClosure, nnkGotoState, diff --git a/tests/macros/tmacros_issues.nim b/tests/macros/tmacros_issues.nim index 19c706a82..a964f46ba 100644 --- a/tests/macros/tmacros_issues.nim +++ b/tests/macros/tmacros_issues.nim @@ -64,7 +64,7 @@ block t7723: block t8706: macro varargsLen(args:varargs[untyped]): untyped = - doAssert args.kind == nnkArglist + doAssert args.kind == nnkArgList doAssert args.len == 0 result = newLit(args.len) From 30959e24bd8840f16472d1f621e1a349d9ccae78 Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Sat, 27 Mar 2021 10:36:39 +0100 Subject: [PATCH 0503/2472] Update security policy (#17522) * Update security policy * Update security.md Co-authored-by: Dominik Picheta Co-authored-by: Andreas Rumpf Co-authored-by: Dominik Picheta --- security.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/security.md b/security.md index e8d31b1b9..72a1a3e3d 100644 --- a/security.md +++ b/security.md @@ -2,7 +2,10 @@ ## Supported Versions -Security fixes are provided in new releases and bugfix releases. +Security advisories are published at: +https://github.com/nim-lang/security/security/advisories?state=published + +Security fixes are provided in new releases and in bugfix releases. We do not backport security fixes to older releases. @@ -10,8 +13,8 @@ We do not backport security fixes to older releases. ## Reporting a Vulnerability -Please do not report vulnerabilities via GitHub issues. - -If you have discovered a vulnerability, it is best to notify us about it via +If you have discovered a vulnerability, please notify us about it via security@nim-lang.org in order to set up a meeting where we can discuss the next steps. + +Please do not report vulnerabilities via GitHub issues. From cf5ce7616b3b2dcbe734dee7f48133c5dcf332f9 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 27 Mar 2021 12:21:57 +0100 Subject: [PATCH 0504/2472] IC: bugfix (#17533) * cleanups * IC: bugfix --- compiler/ic/ic.nim | 7 +++++-- compiler/transf.nim | 6 ++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim index 8017d8b40..b5fed7d4e 100644 --- a/compiler/ic/ic.nim +++ b/compiler/ic/ic.nim @@ -28,6 +28,7 @@ type PackedModule* = object ## the parts of a PackedEncoder that are part of the .rod file definedSymbols: string + moduleFlags: TSymFlags includes: seq[(LitId, string)] # first entry is the module filename itself imports: seq[LitId] # the modules this module depends on toReplay: PackedTree # pragmas and VM specific state to replay. @@ -139,6 +140,7 @@ proc initEncoder*(c: var PackedEncoder; m: var PackedModule; moduleSym: PSym; co m.sh = Shared() c.thisModule = moduleSym.itemId.module c.config = config + m.moduleFlags = moduleSym.flags m.bodies = newTreeFrom(m.topLevel) m.toReplay = newTreeFrom(m.topLevel) @@ -511,6 +513,7 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef f.loadSection configSection f.loadPrim m.definedSymbols + f.loadPrim m.moduleFlags f.loadPrim m.cfg if f.err == ok and not configIdentical(m, config) and not ignoreConfig: @@ -581,6 +584,7 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac f.storeHeader() f.storeSection configSection f.storePrim m.definedSymbols + f.storePrim m.moduleFlags f.storePrim m.cfg template storeSeqSection(section, data) {.dirty.} = @@ -909,8 +913,7 @@ proc setupLookupTables(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCa info: newLineInfo(fileIdx, 1, 1), position: int(fileIdx)) m.module.owner = newPackage(conf, cache, fileIdx) - if fileIdx == conf.projectMainIdx2: - m.module.flags.incl sfMainModule + m.module.flags = m.fromDisk.moduleFlags proc loadToReplayNodes(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; fileIdx: FileIndex; m: var LoadedModule) = diff --git a/compiler/transf.nim b/compiler/transf.nim index d2d9156aa..48c737f46 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -29,8 +29,7 @@ proc transformBody*(g: ModuleGraph; idgen: IdGenerator, prc: PSym, cache: bool): import closureiters, lambdalifting type - PTransCon = ref TTransCon - TTransCon{.final.} = object # part of TContext; stackable + PTransCon = ref object # part of TContext; stackable mapping: TIdNodeTable # mapping from symbols to nodes owner: PSym # current owner forStmt: PNode # current for stmt @@ -40,7 +39,7 @@ type # if we encounter the 2nd yield statement next: PTransCon # for stacking - TTransfContext = object + PTransf = ref object module: PSym transCon: PTransCon # top of a TransCon stack inlining: int # > 0 if we are in inlining context (copy vars) @@ -49,7 +48,6 @@ type deferDetected, tooEarly: bool graph: ModuleGraph idgen: IdGenerator - PTransf = ref TTransfContext proc newTransNode(a: PNode): PNode {.inline.} = result = shallowCopy(a) From 9c52009de61a119dda2def870e5fb21581dafc9a Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 28 Mar 2021 01:12:11 +0800 Subject: [PATCH 0505/2472] close #7012 add testcase (#17537) --- tests/tuples/t7012.nim | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 tests/tuples/t7012.nim diff --git a/tests/tuples/t7012.nim b/tests/tuples/t7012.nim new file mode 100644 index 000000000..32d441ddd --- /dev/null +++ b/tests/tuples/t7012.nim @@ -0,0 +1,7 @@ +discard """ + errormsg: "illegal recursion in type 'Node'" +""" + +type Node[T] = tuple + next: ref Node[T] +var n: Node[int] \ No newline at end of file From d3705b253c155a6f129a7f26ea9435a0ab45143f Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 28 Mar 2021 01:18:35 +0800 Subject: [PATCH 0506/2472] close #5792 with testcase (#17536) --- tests/sets/t5792.nim | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 tests/sets/t5792.nim diff --git a/tests/sets/t5792.nim b/tests/sets/t5792.nim new file mode 100644 index 000000000..efc38b1bc --- /dev/null +++ b/tests/sets/t5792.nim @@ -0,0 +1,17 @@ +discard """ + matrix: "--gc:refc; --gc:arc" +""" + +type + T = enum + a + b + c + U = object + case k: T + of a: + x: int + of {b, c} - {a}: + y: int + +discard U(k: b, y: 1) From bbe4cf4703903081553fe066ece5a57a5f190493 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 27 Mar 2021 12:03:14 -0700 Subject: [PATCH 0507/2472] followup custom literals (#17500) --- compiler/condsyms.nim | 1 + compiler/ic/packed_ast.nim | 1 + compiler/lexer.nim | 25 ++++----- doc/manual.rst | 2 +- tests/lexer/tcustom_numeric_literals.nim | 69 ++++++++++++++++++------ tests/lexer/tstrlits.nim | 19 +++++++ tests/lexer/tunary_minus.nim | 10 +++- 7 files changed, 94 insertions(+), 33 deletions(-) create mode 100644 tests/lexer/tstrlits.nim diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index aa955e763..8d11dae7c 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -130,3 +130,4 @@ proc initDefines*(symbols: StringTableRef) = defineSymbol("nimHasWarningAsError") defineSymbol("nimHasHintAsError") defineSymbol("nimHasSpellSuggest") + defineSymbol("nimHasCustomLiterals") diff --git a/compiler/ic/packed_ast.nim b/compiler/ic/packed_ast.nim index 353cc3a42..1075664cb 100644 --- a/compiler/ic/packed_ast.nim +++ b/compiler/ic/packed_ast.nim @@ -333,6 +333,7 @@ template symId*(n: NodePos): SymId = SymId tree.nodes[n.int].operand proc firstSon*(n: NodePos): NodePos {.inline.} = NodePos(n.int+1) when false: + # xxx `nkStrLit` or `nkStrLit..nkTripleStrLit:` below? proc strLit*(tree: PackedTree; n: NodePos): lent string = assert n.kind == nkStrLit result = tree.sh.strings[LitId tree.nodes[n.int].operand] diff --git a/compiler/lexer.nim b/compiler/lexer.nim index ecf6cb0d9..cca7765e5 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -412,14 +412,12 @@ proc getNumber(L: var Lexer, result: var Token) = customLitPossible = true if L.buf[postPos] in SymChars: - var suffixAsLower = newStringOfCap(10) var suffix = newStringOfCap(10) while true: - let c = L.buf[postPos] - suffix.add c - suffixAsLower.add toLowerAscii(c) + suffix.add L.buf[postPos] inc postPos if L.buf[postPos] notin SymChars+{'_'}: break + let suffixAsLower = suffix.toLowerAscii case suffixAsLower of "f", "f32": result.tokType = tkFloat32Lit of "d", "f64": result.tokType = tkFloat64Lit @@ -433,16 +431,15 @@ proc getNumber(L: var Lexer, result: var Token) = of "u16": result.tokType = tkUInt16Lit of "u32": result.tokType = tkUInt32Lit of "u64": result.tokType = tkUInt64Lit + elif customLitPossible: + # remember the position of the `'` so that the parser doesn't + # have to reparse the custom literal: + result.iNumber = len(result.literal) + result.literal.add '\'' + result.literal.add suffix + result.tokType = tkCustomLit else: - if customLitPossible: - # remember the position of the ``'`` so that the parser doesn't - # have to reparse the custom literal: - result.iNumber = len(result.literal) - result.literal.add '\'' - result.literal.add suffix - result.tokType = tkCustomLit - else: - lexMessageLitNum(L, "invalid number suffix: '$1'", errPos) + lexMessageLitNum(L, "invalid number suffix: '$1'", errPos) else: lexMessageLitNum(L, "invalid number suffix: '$1'", errPos) @@ -467,7 +464,7 @@ proc getNumber(L: var Lexer, result: var Token) = if L.buf[pos] != '_': xi = `shl`(xi, 1) or (ord(L.buf[pos]) - ord('0')) inc(pos) - # 'c', 'C' is deprecated + # 'c', 'C' is deprecated (a warning is issued elsewhere) of 'o', 'c', 'C': result.base = base8 while pos < endpos: diff --git a/doc/manual.rst b/doc/manual.rst index 51c434c7b..fdc744bc7 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -536,7 +536,7 @@ Numeric literals have the form:: FLOAT64_LIT = HEX_LIT '\'' FLOAT64_SUFFIX | (FLOAT_LIT | DEC_LIT | OCT_LIT | BIN_LIT) ['\''] FLOAT64_SUFFIX - CUSTOM_NUMERIC_LIT = (FLOAT_LIT | DEC_LIT | OCT_LIT | BIN_LIT) '\'' CUSTOM_NUMERIC_SUFFIX + CUSTOM_NUMERIC_LIT = (FLOAT_LIT | INT_LIT) '\'' CUSTOM_NUMERIC_SUFFIX # CUSTOM_NUMERIC_SUFFIX is any Nim identifier that is not # a pre-defined type suffix. diff --git a/tests/lexer/tcustom_numeric_literals.nim b/tests/lexer/tcustom_numeric_literals.nim index a2f355b4d..17933950a 100644 --- a/tests/lexer/tcustom_numeric_literals.nim +++ b/tests/lexer/tcustom_numeric_literals.nim @@ -46,6 +46,7 @@ assertAST dedent """ -38383839292839283928392839283928392839283.928493849385935898243e-50000'wrap proc `'wrap`(number: string): string = "[[" & number & "]]" +proc wrap2(number: string): string = "[[" & number & "]]" doAssert lispReprStr(-1'wrap) == """(DotExpr (RStrLit "-1") (Ident "\'wrap"))""" template main = @@ -126,25 +127,59 @@ template main = doAssert -12'fooplusopt(2) == ("-12", 2) doAssert -12'fooplusopt() == ("-12", 99) doAssert -12'fooplusopt == ("-12", 99) - macro `'bar`(a: static string): untyped = - var infix = newNimNode(nnkInfix) - infix.add newIdentNode("&") - infix.add newLit("got ") - infix.add newLit(a.repr) - result = newNimNode(nnkStmtList) - result.add infix - doAssert -12'bar == "got \"-12\"" + macro `'bar`(a: static string): untyped = newLit(a.repr) + doAssert -12'bar == "\"-12\"" macro deb(a): untyped = newLit(a.repr) doAssert deb(-12'bar) == "-12'bar" - # macro metawrap(): untyped = - # func wrap1(a: string): string = "{" & a & "}" - # func `'wrap2`(a: string): string = "{" & a & "}" - # result = quote do: - # let a1 = wrap1"-128" - # let a2 = -128'wrap2 - # metawrap() - # doAssert a1 == "{-128}" - # doAssert a2 == "{-128}" + + block: # bug 1 from https://github.com/nim-lang/Nim/pull/17020#issuecomment-803193947 + macro deb1(a): untyped = newLit a.repr + macro deb2(a): untyped = newLit a.lispRepr + doAssert deb1(-12'wrap) == "-12'wrap" + doAssert deb1(-12'nonexistant) == "-12'nonexistant" + doAssert deb2(-12'nonexistant) == """(DotExpr (RStrLit "-12") (Ident "\'nonexistant"))""" + when false: # xxx bug: + # this holds: + doAssert deb2(-12.wrap2) == """(DotExpr (IntLit -12) (Sym "wrap2"))""" + doAssert deb2(-12'wrap) == """(DotExpr (RStrLit "-12") (Sym "\'wrap"))""" + # but instead this should hold: + doAssert deb2(-12.wrap2) == """(DotExpr (IntLit -12) (Ident "wrap2"))""" + doAssert deb2(-12'wrap) == """(DotExpr (RStrLit "-12") (Ident "\'wrap"))""" + + block: # bug 2 from https://github.com/nim-lang/Nim/pull/17020#issuecomment-803193947 + template toSuf(`'suf`): untyped = + let x = -12'suf + x + doAssert toSuf(`'wrap`) == "[[-12]]" + + block: # bug 10 from https://github.com/nim-lang/Nim/pull/17020#issuecomment-803193947 + proc `myecho`(a: auto): auto = a + template fn1(): untyped = + let a = "abc" + -12'wrap + template fn2(): untyped = + `myecho` -12'wrap + template fn3(): untyped = + -12'wrap + doAssert fn1() == "[[-12]]" + doAssert fn2() == "[[-12]]" + doAssert fn3() == "[[-12]]" + + when false: # xxx this fails; bug 9 from https://github.com/nim-lang/Nim/pull/17020#issuecomment-803193947 + #[ + possible workaround: use `genAst` (https://github.com/nim-lang/Nim/pull/17426) and this: + let a3 = `'wrap3`("-128") + ]# + block: + macro metawrap(): untyped = + func wrap1(a: string): string = "{" & a & "}" + func `'wrap3`(a: string): string = "{" & a & "}" + result = quote do: + let a1 = wrap1"-128" + let a2 = -128'wrap3 + metawrap() + doAssert a1 == "{-128}" + doAssert a2 == "{-128}" static: main() main() diff --git a/tests/lexer/tstrlits.nim b/tests/lexer/tstrlits.nim new file mode 100644 index 000000000..8e8250a5b --- /dev/null +++ b/tests/lexer/tstrlits.nim @@ -0,0 +1,19 @@ +discard """ + output: "a\"\"long string\"\"\"\"\"abc\"def_'2'●𝌆𝌆A" +""" +# Test the new different string literals + +const + tripleEmpty = """"long string"""""""" # "long string """"" + + rawQuote = r"a""" + + raw = r"abc""def" + + escaped = "\x5f'\50'\u25cf\u{1D306}\u{1d306}\u{41}" + + +stdout.write(rawQuote) +stdout.write(tripleEmpty) +stdout.write(raw) +stdout.writeLine(escaped) diff --git a/tests/lexer/tunary_minus.nim b/tests/lexer/tunary_minus.nim index 639911fcd..87b3cb52d 100644 --- a/tests/lexer/tunary_minus.nim +++ b/tests/lexer/tunary_minus.nim @@ -51,8 +51,17 @@ template main = doAssert x() == minusOne: "unable to handle negatives after semi-colon" + block: doAssert -0b111 == -7 doAssert -0xff == -255 + doAssert -128'i8 == (-128).int8 + doAssert $(-128'i8) == "-128" + doAssert -32768'i16 == int16.low + doAssert -2147483648'i32 == int32.low + when int.sizeof > 4: + doAssert -9223372036854775808 == int.low + when not defined(js): + doAssert -9223372036854775808 == int64.low block: # check when a minus (-) is an unary op doAssert -one == minusOne: @@ -68,6 +77,5 @@ template main = doAssert 4 - one == 3: "unable to handle subtraction with surrounding spaces with an identifier" - static: main() main() From d96469037ecc1ee7982b420cbe6ce34e91d18b01 Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Sat, 27 Mar 2021 18:43:24 -0600 Subject: [PATCH 0508/2472] Fixed embedded nimscript imports (#17425) --- compiler/options.nim | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/compiler/options.nim b/compiler/options.nim index 2d63043df..2c8d43733 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -812,10 +812,11 @@ proc findModule*(conf: ConfigRef; modulename, currentModule: string): AbsoluteFi for candidate in stdlibDirs: let path = (conf.libpath.string / candidate / stripped) if fileExists(path): - m = path + result = AbsoluteFile path break - let currentPath = currentModule.splitFile.dir - result = AbsoluteFile currentPath / m + else: # If prefixed with std/ why would we add the current module path! + let currentPath = currentModule.splitFile.dir + result = AbsoluteFile currentPath / m if not fileExists(result): result = findFile(conf, m) patchModule(conf) From b77a420d3eb118cc92443e925f48996fe419f662 Mon Sep 17 00:00:00 2001 From: rockcavera Date: Sun, 28 Mar 2021 02:11:51 -0300 Subject: [PATCH 0509/2472] set const arch64 to fix compiling with vcc/icc (#17539) --- lib/system/countbits_impl.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/system/countbits_impl.nim b/lib/system/countbits_impl.nim index d3c003ff7..e0338f8f9 100644 --- a/lib/system/countbits_impl.nim +++ b/lib/system/countbits_impl.nim @@ -18,6 +18,7 @@ const useGCC_builtins* = (defined(gcc) or defined(llvm_gcc) or defined(clang)) and useBuiltins const useICC_builtins* = defined(icc) and useBuiltins const useVCC_builtins* = defined(vcc) and useBuiltins +const arch64 = sizeof(int) == 8 template countBitsImpl(n: uint32): int = # generic formula is from: https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel From 6a355a4db05addb01fc02df0243c540f100bf107 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 28 Mar 2021 00:29:06 -0700 Subject: [PATCH 0510/2472] nim: unbreak CI; testament: add allowedFailure logic for tests that may fail but should still run (#17513) --- testament/categories.nim | 7 +++-- testament/important_packages.nim | 47 ++++++++++++++++---------------- testament/testament.nim | 16 +++++++---- 3 files changed, 40 insertions(+), 30 deletions(-) diff --git a/testament/categories.nim b/testament/categories.nim index 86aa1f8af..af6331dde 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -448,7 +448,10 @@ proc testNimblePackages(r: var TResults; cat: Category; packageFilter: string) = (outp, status) = execCmdEx(cmd, workingDir = workingDir2) status == QuitSuccess if not ok: - addResult(r, test, targetC, "", cmd & "\n" & outp, reFailed) + if pkg.allowFailure: + inc r.passed + inc r.failedButAllowed + addResult(r, test, targetC, "", cmd & "\n" & outp, reFailed, allowFailure = pkg.allowFailure) continue outp @@ -461,7 +464,7 @@ proc testNimblePackages(r: var TResults; cat: Category; packageFilter: string) = discard tryCommand("nimble install --depsOnly -y", maxRetries = 3) discard tryCommand(pkg.cmd, reFailed = reBuildFailed) inc r.passed - r.addResult(test, targetC, "", "", reSuccess) + r.addResult(test, targetC, "", "", reSuccess, allowFailure = pkg.allowFailure) errors = r.total - r.passed if errors == 0: diff --git a/testament/important_packages.nim b/testament/important_packages.nim index eb0196f70..8a8d6ffd0 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -22,22 +22,26 @@ When this is the case, a workaround is to test this package here by adding `--pa type NimblePackage* = object name*, cmd*, url*: string useHead*: bool + allowFailure*: bool + ## When true, we still run the test but the test is allowed to fail. + ## This is useful for packages that currently fail but that we still want to + ## run in CI, e.g. so that we can monitor when they start working again and + ## are reminded about those failures without making CI fail for unrelated PRs. var packages*: seq[NimblePackage] -proc pkg(name: string; cmd = "nimble test"; url = "", useHead = true) = - packages.add NimblePackage(name: name, cmd: cmd, url: url, useHead: useHead) +proc pkg(name: string; cmd = "nimble test"; url = "", useHead = true, allowFailure = false) = + packages.add NimblePackage(name: name, cmd: cmd, url: url, useHead: useHead, allowFailure: allowFailure) -# pkg "alea" +pkg "alea", allowFailure = true pkg "argparse" -when false: - pkg "arraymancer", "nim c tests/tests_cpu.nim" -# pkg "ast_pattern_matching", "nim c -r --oldgensym:on tests/test1.nim" +pkg "arraymancer", "nim c tests/tests_cpu.nim", allowFailure = true +pkg "ast_pattern_matching", "nim c -r --oldgensym:on tests/test1.nim", allowFailure = true pkg "awk" pkg "bigints", url = "https://github.com/Araq/nim-bigints" pkg "binaryheap", "nim c -r binaryheap.nim" pkg "BipBuffer" -# pkg "blscurve" # pending https://github.com/status-im/nim-blscurve/issues/39 +pkg "blscurve", allowFailure = true # pending https://github.com/status-im/nim-blscurve/issues/39 pkg "bncurve" pkg "brainfuck", "nim c -d:release -r tests/compile.nim" pkg "bump", "nim c --gc:arc --path:. -r tests/tbump.nim", "https://github.com/disruptek/bump" @@ -45,16 +49,13 @@ pkg "c2nim", "nim c testsuite/tester.nim" pkg "cascade" pkg "cello" pkg "chroma" -# pkg "chronicles", "nim c -o:chr -r chronicles.nim" -# when not defined(osx): # testdatagram.nim(560, 54): Check failed -# pkg "chronos", "nim c -r -d:release tests/testall" - # pending https://github.com/nim-lang/Nim/issues/17130 - +pkg "chronicles", "nim c -o:chr -r chronicles.nim", allowFailure = true # pending https://github.com/status-im/nim-chronos/issues/169 +pkg "chronos", "nim c -r -d:release tests/testall", allowFailure = true # pending https://github.com/nim-lang/Nim/issues/17130 pkg "cligen", "nim c --path:. -r cligen.nim" pkg "combparser", "nimble test --gc:orc" pkg "compactdict" pkg "comprehension", "nimble test", "https://github.com/alehander42/comprehension" -# pkg "criterion" # pending https://github.com/disruptek/criterion/issues/3 (wrongly closed) +pkg "criterion", allowFailure = true # pending https://github.com/disruptek/criterion/issues/3 (wrongly closed) pkg "dashing", "nim c tests/functional.nim" pkg "delaunay" pkg "docopt" @@ -66,12 +67,12 @@ pkg "fusion" pkg "gara" pkg "glob" pkg "ggplotnim", "nim c -d:noCairo -r tests/tests.nim" -# pkg "gittyup", "nimble test", "https://github.com/disruptek/gittyup" +pkg "gittyup", "nimble test", "https://github.com/disruptek/gittyup", allowFailure = true pkg "gnuplot", "nim c gnuplot.nim" # pkg "gram", "nim c -r --gc:arc --define:danger tests/test.nim", "https://github.com/disruptek/gram" # pending https://github.com/nim-lang/Nim/issues/16509 pkg "hts", "nim c -o:htss src/hts.nim" -# pkg "httpauth" +pkg "httpauth", allowFailure = true pkg "illwill", "nimble examples" pkg "inim" pkg "itertools", "nim doc src/itertools.nim" @@ -87,28 +88,28 @@ pkg "memo" pkg "msgpack4nim", "nim c -r tests/test_spec.nim" pkg "nake", "nim c nakefile.nim" pkg "neo", "nim c -d:blas=openblas tests/all.nim" -# pkg "nesm", "nimble tests" # notice plural 'tests' -# pkg "nico" +pkg "nesm", "nimble tests", allowFailure = true # notice plural 'tests' +pkg "nico", allowFailure = true pkg "nicy", "nim c -r src/nicy.nim" pkg "nigui", "nim c -o:niguii -r src/nigui.nim" pkg "nimcrypto", "nim r --path:. tests/testall.nim" # `--path:.` workaround needed, see D20210308T165435 pkg "NimData", "nim c -o:nimdataa src/nimdata.nim" pkg "nimes", "nim c src/nimes.nim" pkg "nimfp", "nim c -o:nfp -r src/fp.nim" -# pkg "nimgame2", "nim c nimgame2/nimgame.nim" # XXX Doesn't work with deprecated 'randomize', will create a PR. +pkg "nimgame2", "nim c nimgame2/nimgame.nim", allowFailure = true # XXX Doesn't work with deprecated 'randomize', will create a PR. pkg "nimgen", "nim c -o:nimgenn -r src/nimgen/runcfg.nim" pkg "nimlsp" pkg "nimly", "nim c -r tests/test_readme_example.nim" -# pkg "nimongo", "nimble test_ci" -# pkg "nimph", "nimble test", "https://github.com/disruptek/nimph" +pkg "nimongo", "nimble test_ci", allowFailure = true +pkg "nimph", "nimble test", "https://github.com/disruptek/nimph", allowFailure = true pkg "nimpy", "nim c -r tests/nimfrompy.nim" pkg "nimquery" pkg "nimsl" pkg "nimsvg" pkg "nimterop", "nimble minitest" pkg "nimwc", "nim c nimwc.nim" -# pkg "nimx", "nim c --threads:on test/main.nim" -# pkg "nitter", "nim c src/nitter.nim", "https://github.com/zedeus/nitter" +pkg "nimx", "nim c --threads:on test/main.nim", allowFailure = true +pkg "nitter", "nim c src/nitter.nim", "https://github.com/zedeus/nitter", allowFailure = true pkg "norm", "nim c -r tests/sqlite/trows.nim" pkg "npeg", "nimble testarc" pkg "numericalnim", "nim c -r tests/test_integrate.nim" @@ -150,7 +151,7 @@ pkg "unicodedb", "nim c -d:release -r tests/tests.nim" pkg "unicodeplus", "nim c -d:release -r tests/tests.nim" pkg "unpack" pkg "websocket", "nim c websocket.nim" -# pkg "winim" +pkg "winim", allowFailure = true pkg "with" pkg "ws" pkg "yaml", "nim build" diff --git a/testament/testament.nim b/testament/testament.nim index 6fc7e32ce..462c0a57c 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -66,7 +66,8 @@ proc isNimRepoTests(): bool = type Category = distinct string TResults = object - total, passed, skipped: int + total, passed, failedButAllowed, skipped: int + ## xxx rename passed to passedOrAllowedFailure data: string TTest = object name: string @@ -212,6 +213,7 @@ proc callCCompiler(cmdTemplate, filename, options: string, proc initResults: TResults = result.total = 0 result.passed = 0 + result.failedButAllowed = 0 result.skipped = 0 result.data = "" @@ -239,16 +241,20 @@ template maybeStyledEcho(args: varargs[untyped]): untyped = proc `$`(x: TResults): string = - result = ("Tests passed: $1 / $3
    \n" & - "Tests skipped: $2 / $3
    \n") % - [$x.passed, $x.skipped, $x.total] + result = """ +Tests passed or allowed to fail: $2 / $1
    +Tests failed and allowed to fail: $3 / $1
    +Tests skipped: $4 / $1
    +""" % [$x.total, $x.passed, $x.failedButAllowed, $x.skipped] proc addResult(r: var TResults, test: TTest, target: TTarget, - expected, given: string, successOrig: TResultEnum) = + expected, given: string, successOrig: TResultEnum, allowFailure = false) = # test.name is easier to find than test.name.extractFilename # A bit hacky but simple and works with tests/testament/tshould_not_work.nim var name = test.name.replace(DirSep, '/') name.add ' ' & $target + if allowFailure: + name.add " (allowed to fail) " if test.options.len > 0: name.add ' ' & test.options let duration = epochTime() - test.startTime From c86fdfa1ee2cbba201b0a6182212ced5e218dc65 Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 28 Mar 2021 15:57:22 +0800 Subject: [PATCH 0511/2472] add testcase for #9466 (#17538) --- tests/notnil/tnotnil5.nim | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 tests/notnil/tnotnil5.nim diff --git a/tests/notnil/tnotnil5.nim b/tests/notnil/tnotnil5.nim new file mode 100644 index 000000000..2dcb7f7c3 --- /dev/null +++ b/tests/notnil/tnotnil5.nim @@ -0,0 +1,28 @@ +discard """ + matrix: "--threads:on" +""" + +{.experimental: "parallel".} +{.experimental: "notnil".} +import threadpool + +type + AO = object + x: int + + A = ref AO not nil + +proc process(a: A): A = + return A(x: a.x+1) + +proc processMany(ayys: openArray[A]): seq[A] = + var newAs: seq[FlowVar[A]] + + parallel: + for a in ayys: + newAs.add(spawn process(a)) + for newAflow in newAs: + let newA = ^newAflow + if isNil(newA): + return @[] + result.add(newA) From eb3ed44009fdfea3e67608b68a049c265124d643 Mon Sep 17 00:00:00 2001 From: ee7 <45465154+ee7@users.noreply.github.com> Date: Sun, 28 Mar 2021 12:44:16 +0200 Subject: [PATCH 0512/2472] system: fix link in `delete` proc (#17550) The `delete` proc had a "See also" link that linked to itself. --- lib/system.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/system.nim b/lib/system.nim index 4c31f20a2..27f976119 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1321,7 +1321,7 @@ proc delete*[T](x: var seq[T], i: Natural) {.noSideEffect.} = ## This is an `O(n)` operation. ## ## See also: - ## * `del <#delete,seq[T],Natural>`_ for O(1) operation + ## * `del <#del,seq[T],Natural>`_ for O(1) operation ## ## .. code-block:: Nim ## var i = @[1, 2, 3, 4, 5] From 207bcabdf29dbaa954a3db778cbd9e4d59ca37ae Mon Sep 17 00:00:00 2001 From: Ardek Romak Date: Sun, 28 Mar 2021 19:57:40 +0300 Subject: [PATCH 0513/2472] Add a getter for all defined Sections in parsecfg (#15450) --- changelog.md | 2 ++ lib/pure/parsecfg.nim | 6 ++++++ tests/stdlib/tparsecfg.nim | 23 +++++++++-------------- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/changelog.md b/changelog.md index f180e90f9..d3335c2a3 100644 --- a/changelog.md +++ b/changelog.md @@ -4,6 +4,8 @@ ## Standard library additions and changes +- Added `sections` iterator in `parsecfg`. + - Make custom op in macros.quote work for all statements. - On Windows the SSL library now checks for valid certificates. diff --git a/lib/pure/parsecfg.nim b/lib/pure/parsecfg.nim index 23bb09ac4..7f22a9857 100644 --- a/lib/pure/parsecfg.nim +++ b/lib/pure/parsecfg.nim @@ -174,6 +174,7 @@ runnableExamples: import strutils, lexbase, streams, tables import std/private/decode_helpers +import std/private/since include "system/inclrtl" @@ -649,3 +650,8 @@ proc delSectionKey*(dict: var Config, section, key: string) = dict.del(section) else: dict[section].del(key) + +iterator sections*(dict: Config): lent string {.since: (1, 5).} = + ## Iterates through the sections in the `dict`. + for section in dict.keys: + yield section diff --git a/tests/stdlib/tparsecfg.nim b/tests/stdlib/tparsecfg.nim index 5c077bbda..b2e57ac3d 100644 --- a/tests/stdlib/tparsecfg.nim +++ b/tests/stdlib/tparsecfg.nim @@ -2,7 +2,7 @@ discard """ targets: "c js" """ -import parsecfg, streams +import parsecfg, streams, sequtils when not defined(js): from stdtest/specialpaths import buildDir @@ -39,19 +39,14 @@ var ss = newStringStream() dict1.writeConfig(ss) ## Reading a configuration file. -var dict2 = loadConfig(newStringStream(ss.data)) -var charset = dict2.getSectionValue("", "charset") -var threads = dict2.getSectionValue("Package", "--threads") -var pname = dict2.getSectionValue("Package", "name") -var name = dict2.getSectionValue("Author", "name") -var qq = dict2.getSectionValue("Author", "qq") -var email = dict2.getSectionValue("Author", "email") -doAssert charset == "utf-8" -doAssert threads == "on" -doAssert pname == "hello" -doAssert name == "lihf8515" -doAssert qq == "10214028" -doAssert email == "lihaifeng@wxm.com" +let dict2 = loadConfig(newStringStream(ss.data)) +doAssert dict2.getSectionValue("", "charset") == "utf-8" +doAssert dict2.getSectionValue("Package", "--threads") == "on" +doAssert dict2.getSectionValue("Package", "name") == "hello" +doAssert dict2.getSectionValue("Author", "name") == "lihf8515" +doAssert dict2.getSectionValue("Author", "qq") == "10214028" +doAssert dict2.getSectionValue("Author", "email") == "lihaifeng@wxm.com" +doAssert toSeq(dict2.sections) == @["", "Package", "Author"] ## Modifying a configuration file. var dict3 = loadConfig(newStringStream(ss.data)) From a5600e49df71463cda6bd0a062e02c8bdd48da0c Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 29 Mar 2021 07:40:37 +0800 Subject: [PATCH 0514/2472] close #9622 add testcase (#17557) * fix nim js cmp fails at CT * close #9622 add testcase --- tests/vm/t9622.nim | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 tests/vm/t9622.nim diff --git a/tests/vm/t9622.nim b/tests/vm/t9622.nim new file mode 100644 index 000000000..214ab0f19 --- /dev/null +++ b/tests/vm/t9622.nim @@ -0,0 +1,30 @@ +discard """ + targets: "c cpp" + matrix: "--gc:refc; --gc:arc" +""" + +type + GlobNodeKind = enum + LiteralIdent, + Group + + GlobNode = object + case kind: GlobNodeKind + of LiteralIdent: + value: string + of Group: + values: seq[string] + + PathSegment = object + children: seq[GlobNode] + + GlobPattern = seq[PathSegment] + +proc parseImpl(): GlobPattern = + if result.len == 0: + result.add PathSegment() + result[^1].children.add GlobNode(kind: LiteralIdent) + +block: + const pattern = parseImpl() + doAssert $pattern == """@[(children: @[(kind: LiteralIdent, value: "")])]""" From a36816fc047d4ab2c4f80056c207f117a635a5d4 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 29 Mar 2021 09:11:07 +0800 Subject: [PATCH 0515/2472] follow up #17539 (#17548) * fix nim js cmp fails at CT * follow up #17539 --- lib/pure/bitops.nim | 1 - lib/system/countbits_impl.nim | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/pure/bitops.nim b/lib/pure/bitops.nim index 377602e75..4ef190c1b 100644 --- a/lib/pure/bitops.nim +++ b/lib/pure/bitops.nim @@ -413,7 +413,6 @@ func fastlog2Nim(x: uint64): int {.inline.} = import system/countbits_impl -const arch64 = sizeof(int) == 8 const useBuiltinsRotate = (defined(amd64) or defined(i386)) and (defined(gcc) or defined(clang) or defined(vcc) or (defined(icl) and not defined(cpp))) and useBuiltins diff --git a/lib/system/countbits_impl.nim b/lib/system/countbits_impl.nim index e0338f8f9..184f97bd4 100644 --- a/lib/system/countbits_impl.nim +++ b/lib/system/countbits_impl.nim @@ -18,7 +18,7 @@ const useGCC_builtins* = (defined(gcc) or defined(llvm_gcc) or defined(clang)) and useBuiltins const useICC_builtins* = defined(icc) and useBuiltins const useVCC_builtins* = defined(vcc) and useBuiltins -const arch64 = sizeof(int) == 8 +const arch64* = sizeof(int) == 8 template countBitsImpl(n: uint32): int = # generic formula is from: https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel From e5be216ccbf1e3453e0076f2f260ddda85a5f27a Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 29 Mar 2021 11:21:27 +0800 Subject: [PATCH 0516/2472] [docs]close #12580 (#17549) --- lib/system/memalloc.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/system/memalloc.nim b/lib/system/memalloc.nim index fca1db91d..f8ebc8c5f 100644 --- a/lib/system/memalloc.nim +++ b/lib/system/memalloc.nim @@ -188,8 +188,8 @@ when hasAlloc and not defined(js): cast[ptr T](realloc(p, T.sizeof * newSize)) proc dealloc*(p: pointer) {.noconv, compilerproc, rtl, benign, raises: [], tags: [].} = - ## Frees the memory allocated with `alloc`, `alloc0` or - ## `realloc`. + ## Frees the memory allocated with `alloc`, `alloc0`, + ## `realloc`, `create` or `createU`. ## ## **This procedure is dangerous!** ## If one forgets to free the memory a leak occurs; if one tries to From b36182b0a4ca13d90c1a19de5b9945289c4f25fd Mon Sep 17 00:00:00 2001 From: Danil Yarantsev Date: Mon, 29 Mar 2021 11:49:19 +0300 Subject: [PATCH 0517/2472] Free the certificate after checking in `checkCertName` (#17558) [backport:1.2] * Fix small leak in checkCertName * Size is not needed either * Free the certificate after checking --- lib/pure/net.nim | 7 ++++--- lib/wrappers/openssl.nim | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index b37782271..bb1a14cfd 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -770,10 +770,11 @@ when defineSsl: raiseSSLError("No SSL certificate found.") const X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT = 0x1.cuint - const size = 1024 - var peername: string = newString(size) + # https://www.openssl.org/docs/man1.1.1/man3/X509_check_host.html let match = certificate.X509_check_host(hostname.cstring, hostname.len.cint, - X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT, peername) + X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT, nil) + # https://www.openssl.org/docs/man1.1.1/man3/SSL_get_peer_certificate.html + X509_free(certificate) if match != 1: raiseSSLError("SSL Certificate check failed.") diff --git a/lib/wrappers/openssl.nim b/lib/wrappers/openssl.nim index 313ce7d19..ec4740bab 100644 --- a/lib/wrappers/openssl.nim +++ b/lib/wrappers/openssl.nim @@ -809,6 +809,8 @@ when not defined(nimDisableCertificateValidation) and not defined(windows): proc X509_check_host*(cert: PX509, name: cstring, namelen: cint, flags:cuint, peername: cstring): cint {.cdecl, dynlib: DLLSSLName, importc.} + proc X509_free*(cert: PX509) {.cdecl, dynlib: DLLSSLName, importc.} + # Certificates store type PX509_STORE* = SslPtr From 085580a2c8f303acf1654731ebc7dba7e6e501a7 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 29 Mar 2021 16:50:58 +0800 Subject: [PATCH 0518/2472] fix #14010(fix std/registry + arc) (#17551) * fix nim js cmp fails at CT * fix #14010 * Update tests/stdlib/tregistry.nim Co-authored-by: Timothee Cour * Update tregistry.nim Co-authored-by: Timothee Cour --- lib/windows/registry.nim | 18 ++++++++++-------- tests/stdlib/tregistry.nim | 15 +++++++++++++++ 2 files changed, 25 insertions(+), 8 deletions(-) create mode 100644 tests/stdlib/tregistry.nim diff --git a/lib/windows/registry.nim b/lib/windows/registry.nim index c17f2f455..ab7d0b1fe 100644 --- a/lib/windows/registry.nim +++ b/lib/windows/registry.nim @@ -54,16 +54,18 @@ proc getUnicodeValue*(path, key: string; handle: HKEY): string = var newHandle: HKEY call regOpenKeyEx(handle, hh, 0, KEY_READ or KEY_WOW64_64KEY, newHandle) call regGetValue(newHandle, nil, kk, flags, nil, nil, addr bufsize) - var res = newWideCString("", bufsize) - call regGetValue(newHandle, nil, kk, flags, nil, cast[pointer](res), - addr bufsize) - result = res $ bufsize + if bufSize > 0: + var res = newWideCString(bufsize) + call regGetValue(newHandle, nil, kk, flags, nil, addr res[0], + addr bufsize) + result = res $ bufsize call regCloseKey(newHandle) else: - var res = newWideCString("", bufsize) - call regGetValue(handle, hh, kk, flags, nil, cast[pointer](res), - addr bufsize) - result = res $ bufsize + if bufSize > 0: + var res = newWideCString(bufsize) + call regGetValue(handle, hh, kk, flags, nil, addr res[0], + addr bufsize) + result = res $ bufsize proc regSetValue(key: HKEY, lpSubKey, lpValueName: WideCString, dwType: int32; lpData: WideCString; cbData: int32): int32 {. diff --git a/tests/stdlib/tregistry.nim b/tests/stdlib/tregistry.nim new file mode 100644 index 000000000..8bf084f6d --- /dev/null +++ b/tests/stdlib/tregistry.nim @@ -0,0 +1,15 @@ +discard """ + disabled: "unix" + matrix: "--gc:refc; --gc:arc" +""" + +when defined(windows): + import std/registry + + block: # bug #14010 + let path = "Environment" + let key = "D20210328T202842_key" + let val = "D20210328T202842_val" + let handle = HKEY_CURRENT_USER + setUnicodeValue("Environment", key, val, handle) + doAssert getUnicodeValue(path, key, handle) == val From e45b1e3f36f2c4778967410137bb87b8691e6c8c Mon Sep 17 00:00:00 2001 From: ben Date: Mon, 29 Mar 2021 10:24:15 +0100 Subject: [PATCH 0519/2472] Fix net.readLine infine loop #17534 (#17535) * Fix net.readLine infine loop #17534 * fix #17534 add return to template --- lib/pure/net.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index bb1a14cfd..ca109dc22 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -1535,6 +1535,7 @@ proc readLine*(socket: Socket, line: var string, timeout = -1, if flags.isDisconnectionError(lastError): setLen(line, 0) socket.socketError(n, lastError = lastError, flags = flags) + return var waited: Duration From ab57e7f244fb44fbd283de1fcb16c9c9a6fece42 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 29 Mar 2021 12:42:34 +0200 Subject: [PATCH 0520/2472] added a simple 'koch branchdone' command to speed up my workflow (#17563) --- koch.nim | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/koch.nim b/koch.nim index 6f1da8165..3b00d3564 100644 --- a/koch.nim +++ b/koch.nim @@ -664,6 +664,13 @@ proc showHelp() = quit(HelpText % [VersionAsString & spaces(44-len(VersionAsString)), CompileDate, CompileTime], QuitSuccess) +proc branchDone() = + let thisBranch = execProcess("git symbolic-ref --short HEAD").strip() + if thisBranch != "devel" and thisBranch != "": + exec("git checkout devel") + exec("git branch -D " & thisBranch) + exec("git pull --rebase") + when isMainModule: var op = initOptParser() var @@ -722,6 +729,7 @@ when isMainModule: let suffix = if latest: HeadHash else: FusionStableHash exec("nimble install -y fusion@$#" % suffix) of "ic": icTest(op.cmdLineRest) + of "branchdone": branchDone() else: showHelp() break of cmdEnd: break From 9e3960e83dea1b00fe670fc78e74cfd6fafd53b2 Mon Sep 17 00:00:00 2001 From: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Date: Mon, 29 Mar 2021 12:44:31 +0200 Subject: [PATCH 0521/2472] Small doc improvements for `std/with` (#17556) --- lib/std/with.nim | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/std/with.nim b/lib/std/with.nim index 79afd61a4..c7338b4e4 100644 --- a/lib/std/with.nim +++ b/lib/std/with.nim @@ -12,15 +12,16 @@ ## and https://github.com/nim-lang/RFCs/issues/192 for details leading to this ## particular design. ## -## **Since** version 1.2. +## **Since:** version 1.2. import macros, private / underscored_calls macro with*(arg: typed; calls: varargs[untyped]): untyped = - ## This macro provides the `chaining`:idx: of function calls. + ## This macro provides `chaining`:idx: of function calls. ## It does so by patching every call in `calls` to ## use `arg` as the first argument. - ## **This evaluates `arg` multiple times!** + ## + ## .. caution:: This evaluates `arg` multiple times! runnableExamples: var x = "yay" with x: From 8b26b3ad0d6c9458ac31170a95d956347a55627e Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 29 Mar 2021 03:46:53 -0700 Subject: [PATCH 0522/2472] remove docutils.nimble (not a real nimble package, and affected canonical imports) (#17544) * remove docutils.nimble (not a real nimble package, and affected canonical imports) * cleanup redundant references --- lib/packages/docutils/docutils.nimble | 4 ---- lib/packages/docutils/docutils.nimble.old | 7 +++++++ lib/packages/docutils/highlite.nim | 2 -- lib/packages/docutils/rst.nim | 3 --- lib/packages/docutils/rstast.nim | 2 -- lib/packages/docutils/rstgen.nim | 2 -- 6 files changed, 7 insertions(+), 13 deletions(-) delete mode 100644 lib/packages/docutils/docutils.nimble create mode 100644 lib/packages/docutils/docutils.nimble.old diff --git a/lib/packages/docutils/docutils.nimble b/lib/packages/docutils/docutils.nimble deleted file mode 100644 index f1683c515..000000000 --- a/lib/packages/docutils/docutils.nimble +++ /dev/null @@ -1,4 +0,0 @@ -version = "0.10.0" -author = "Andreas Rumpf" -description = "Nim's reStructuredText processor." -license = "MIT" diff --git a/lib/packages/docutils/docutils.nimble.old b/lib/packages/docutils/docutils.nimble.old new file mode 100644 index 000000000..f97c3bdde --- /dev/null +++ b/lib/packages/docutils/docutils.nimble.old @@ -0,0 +1,7 @@ +# xxx disabled this as this isn't really a nimble package and it affects logic +# used to compute canonical imports, refs https://github.com/nim-lang/Nim/pull/16999#issuecomment-805442914 + +version = "0.10.0" +author = "Andreas Rumpf" +description = "Nim's reStructuredText processor." +license = "MIT" diff --git a/lib/packages/docutils/highlite.nim b/lib/packages/docutils/highlite.nim index 8c91e0a8e..2655a7ea3 100644 --- a/lib/packages/docutils/highlite.nim +++ b/lib/packages/docutils/highlite.nim @@ -11,8 +11,6 @@ ## Currently only few languages are supported, other languages may be added. ## The interface supports one language nested in another. ## -## **Note:** Import `packages/docutils/highlite` to use this module -## ## You can use this to build your own syntax highlighting, check this example: ## ## .. code::nim diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index adc83c9af..6845f61b9 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -142,9 +142,6 @@ ## See `packages/docutils/rstgen module `_ to know how to ## generate HTML or Latex strings to embed them into your documents. ## -## .. Tip:: Import ``packages/docutils/rst`` to use this module -## programmatically. -## ## .. _quick introduction: https://docutils.sourceforge.io/docs/user/rst/quickstart.html ## .. _RST reference: https://docutils.sourceforge.io/docs/user/rst/quickref.html ## .. _RST specification: https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim index c68df7daa..ff34da2d1 100644 --- a/lib/packages/docutils/rstast.nim +++ b/lib/packages/docutils/rstast.nim @@ -8,8 +8,6 @@ # ## This module implements an AST for the `reStructuredText`:idx: parser. -## -## **Note:** Import ``packages/docutils/rstast`` to use this module import strutils, json diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index c52a0fdcc..30c7f3080 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -38,8 +38,6 @@ ## * The same goes for footnotes/citations links: they point to themselves. ## No backreferences are generated since finding all references of a footnote ## can be done by simply searching for [footnoteName]. -## -## .. Tip: Import ``packages/docutils/rstgen`` to use this module import strutils, os, hashes, strtabs, rstast, rst, highlite, tables, sequtils, algorithm, parseutils From c6dc9c02551e93fd4c53e48fd7fa8a0df7843267 Mon Sep 17 00:00:00 2001 From: Saem Ghani Date: Mon, 29 Mar 2021 03:48:00 -0700 Subject: [PATCH 0523/2472] fixes #17437 - crash where error reporting > 1 (#17547) * fixes #17437 * Fix bug reference comment Co-authored-by: Timothee Cour * [skip ci] describe why we have hasError Co-authored-by: Timothee Cour --- compiler/semobjconstr.nim | 16 ++++++++++++---- tests/objects/t17437.nim | 22 ++++++++++++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 tests/objects/t17437.nim diff --git a/compiler/semobjconstr.nim b/compiler/semobjconstr.nim index 792488c9f..dea187ea7 100644 --- a/compiler/semobjconstr.nim +++ b/compiler/semobjconstr.nim @@ -380,7 +380,7 @@ proc semObjConstr(c: PContext, n: PNode, flags: TExprFlags): PNode = if t == nil: localError(c.config, n.info, errGenerated, "object constructor needs an object type") - return + return errorNode(c, result) t = skipTypes(t, {tyGenericInst, tyAlias, tySink, tyOwned}) if t.kind == tyRef: @@ -392,17 +392,19 @@ proc semObjConstr(c: PContext, n: PNode, flags: TExprFlags): PNode = result.typ.flags.incl tfHasOwned if t.kind != tyObject: localError(c.config, n.info, errGenerated, "object constructor needs an object type") - return + return errorNode(c, result) # Check if the object is fully initialized by recursively testing each # field (if this is a case object, initialized fields in two different # branches will be reported as an error): var constrCtx = initConstrContext(t, result) let initResult = semConstructTypeAux(c, constrCtx, flags) + var hasError = false # needed to split error detect/report for better msgs # It's possible that the object was not fully initialized while # specifying a .requiresInit. pragma: if constrCtx.missingFields.len > 0: + hasError = true localError(c.config, result.info, "The $1 type requires the following fields to be initialized: $2.", [t.sym.name.s, listSymbolNames(constrCtx.missingFields)]) @@ -415,6 +417,7 @@ proc semObjConstr(c: PContext, n: PNode, flags: TExprFlags): PNode = if nfSem notin field.flags: if field.kind != nkExprColonExpr: invalidObjConstr(c, field) + hasError = true continue let id = considerQuotedIdent(c, field[0]) # This node was not processed. There are two possible reasons: @@ -423,10 +426,15 @@ proc semObjConstr(c: PContext, n: PNode, flags: TExprFlags): PNode = let prevId = considerQuotedIdent(c, result[j][0]) if prevId.id == id.id: localError(c.config, field.info, errFieldInitTwice % id.s) - return + hasError = true + break # 2) No such field exists in the constructed type localError(c.config, field.info, errUndeclaredFieldX % id.s) - return + hasError = true + break if initResult == initFull: incl result.flags, nfAllFieldsSet + + # wrap in an error see #17437 + if hasError: result = errorNode(c, result) diff --git a/tests/objects/t17437.nim b/tests/objects/t17437.nim new file mode 100644 index 000000000..b5c0e0525 --- /dev/null +++ b/tests/objects/t17437.nim @@ -0,0 +1,22 @@ +discard """ + cmd: "nim check $file" + errormsg: "" + nimout: ''' +t17437.nim(20, 16) Error: undeclared identifier: 'x' +t17437.nim(20, 16) Error: expression 'x' has no type (or is ambiguous) +t17437.nim(20, 19) Error: incorrect object construction syntax +t17437.nim(20, 19) Error: incorrect object construction syntax +t17437.nim(20, 12) Error: expression '' has no type (or is ambiguous) +''' +""" + +# bug #17437 invalid object construction should result in error + +type + V = ref object + x, y: int + +proc m = + var v = V(x: x, y) + +m() From 89e0e0f27fe937b0eb7386f6f49f4b66f1d27edb Mon Sep 17 00:00:00 2001 From: Danil Yarantsev Date: Mon, 29 Mar 2021 14:06:23 +0300 Subject: [PATCH 0524/2472] Small doc fixes in net (#17566) --- lib/pure/net.nim | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index ca109dc22..9be9c6acb 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -1427,7 +1427,7 @@ proc recv*(socket: Socket, data: var string, size: int, timeout = -1, flags = {SocketFlag.SafeDisconn}): int = ## Higher-level version of `recv`. ## - ## Reads **up to** `size` bytes from `socket` into `buf`. + ## Reads **up to** `size` bytes from `socket` into `data`. ## ## For buffered sockets this function will attempt to read all the requested ## data. It will read this data in `BufferSize` chunks. @@ -1444,8 +1444,6 @@ proc recv*(socket: Socket, data: var string, size: int, timeout = -1, ## A timeout may be specified in milliseconds, if enough data is not received ## within the time specified a TimeoutError exception will be raised. ## - ## **Note**: `data` must be initialised. - ## ## .. warning:: Only the `SafeDisconn` flag is currently supported. data.setLen(size) result = @@ -1464,7 +1462,7 @@ proc recv*(socket: Socket, size: int, timeout = -1, flags = {SocketFlag.SafeDisconn}): string {.inline.} = ## Higher-level version of `recv` which returns a string. ## - ## Reads **up to** `size` bytes from `socket` into `buf`. + ## Reads **up to** `size` bytes from `socket` into the result. ## ## For buffered sockets this function will attempt to read all the requested ## data. It will read this data in `BufferSize` chunks. From 04520c0ce4ed38a3fbae37731a4979480144c77b Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 29 Mar 2021 04:07:14 -0700 Subject: [PATCH 0525/2472] remove std/ prefix from doc/lib.rst now that canonical import is shown in module docs (#17543) * remove std/ prefix from doc/lib.rst now that canonical import is shown in module docs * cleanup top docs for std/sha1 --- doc/lib.rst | 26 +++++++++++++------------- lib/std/sha1.nim | 27 ++++++++++----------------- 2 files changed, 23 insertions(+), 30 deletions(-) diff --git a/doc/lib.rst b/doc/lib.rst index 62e02815c..15b4f70d6 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -90,14 +90,14 @@ Algorithms * `algorithm `_ This module implements some common generic algorithms like sort or binary search. -* `std/enumutils `_ +* `enumutils `_ This module adds functionality for the built-in `enum` type. * `sequtils `_ This module implements operations for the built-in `seq` type which were inspired by functional programming languages. -* `std/setutils `_ +* `setutils `_ This module adds functionality for the built-in `set` type. @@ -125,7 +125,7 @@ Collections * `options `_ The option type encapsulates an optional value. -* `std/packedsets `_ +* `packedsets `_ Efficient implementation of a set of ordinals as a sparse bit set. * `sets `_ @@ -147,7 +147,7 @@ String handling * `cstrutils `_ Utilities for `cstring` handling. -* `std/editdistance `_ +* `editdistance `_ This module contains an algorithm to compute the edit distance between two Unicode strings. @@ -197,14 +197,14 @@ String handling It provides a single proc that does Unicode to ASCII transliterations. Based on Python's Unidecode module. -* `std/wordwrap `_ +* `wordwrap `_ This module contains an algorithm to wordwrap a Unicode string. Time handling ------------- -* `std/monotimes `_ +* `monotimes `_ The `monotimes` module implements monotonic timestamps. * `times `_ @@ -272,7 +272,7 @@ Math libraries * `random `_ Fast and tiny random number generator. -* `std/sysrand `_ +* `sysrand `_ Cryptographically secure pseudorandom number generator. * `rationals `_ @@ -281,7 +281,7 @@ Math libraries * `stats `_ Statistical analysis -* `std/sums `_ +* `sums `_ Accurate summation functions. @@ -357,7 +357,7 @@ Parsers * `json `_ High-performance JSON parser. -* `std/jsonutils `_ +* `jsonutils `_ This module implements a hookable (de)serialization for arbitrary types. * `lexbase `_ @@ -448,7 +448,7 @@ Hashing produce a globally distributed unique ID. This implementation was extracted from the Mongodb interface and it thus binary compatible with a Mongo OID. -* `std/sha1 `_ +* `sha1 `_ This module implements a sha1 encoder and decoder. @@ -465,7 +465,7 @@ Miscellaneous * `coro `_ This module implements experimental coroutines in Nim. -* `std/enumerate `_ +* `enumerate `_ This module implements `enumerate` syntactic sugar based on Nim's macro system. * `logging `_ @@ -480,10 +480,10 @@ Miscellaneous * `unittest `_ Implements a Unit testing DSL. -* `std/varints `_ +* `varints `_ Decode variable-length integers that are compatible with SQLite. -* `std/with `_ +* `with `_ This module implements the `with` macro for easy function chaining. diff --git a/lib/std/sha1.nim b/lib/std/sha1.nim index b74b285f8..9c5efed1f 100644 --- a/lib/std/sha1.nim +++ b/lib/std/sha1.nim @@ -6,33 +6,26 @@ # See the file "copying.txt", included in this # distribution, for details about the copyright. # -## **Note:** Import `std/sha1` to use this module. -## ## [SHA-1 (Secure Hash Algorithm 1)](https://en.wikipedia.org/wiki/SHA-1) ## is a cryptographic hash function which takes an input and produces ## a 160-bit (20-byte) hash value known as a message digest. ## -## Basic usage -## =========== -## -runnableExamples: - let accessName = secureHash("John Doe") - assert $accessName == "AE6E4D1209F17B460503904FAD297B31E9CF6362" - -## .. code-block:: -## let -## a = secureHashFile("myFile.nim") -## b = parseSecureHash("10DFAEBF6BFDBC7939957068E2EFACEC4972933C") -## -## if a == b: -## echo "Files match" -## ## See also ## ======== ## * `base64 module`_ implements a Base64 encoder and decoder ## * `hashes module`_ for efficient computations of hash values for diverse Nim types ## * `md5 module`_ implements the MD5 checksum algorithm +runnableExamples: + let accessName = secureHash("John Doe") + assert $accessName == "AE6E4D1209F17B460503904FAD297B31E9CF6362" + +runnableExamples("-r:off"): + let + a = secureHashFile("myFile.nim") + b = parseSecureHash("10DFAEBF6BFDBC7939957068E2EFACEC4972933C") + assert a == b, "files don't match" + import std/strutils from std/endians import bigEndian32, bigEndian64 From 81e54c1d30b7651851b9de56d76b9af81e192504 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 29 Mar 2021 20:22:29 +0800 Subject: [PATCH 0526/2472] Add `hasClosure` to `std/typetraits` (#17501) * fix nim js cmp fails at CT * Add `hasClosure` to `std/effecttraits` * type * Update changelog.md Co-authored-by: Timothee Cour * Update lib/std/effecttraits.nim Co-authored-by: Timothee Cour Co-authored-by: Andreas Rumpf --- changelog.md | 2 + compiler/vmops.nim | 4 ++ lib/pure/typetraits.nim | 11 +++++ tests/stdlib/ttypetraits.nim | 92 ++++++++++++++++++++++++++++++++++++ 4 files changed, 109 insertions(+) create mode 100644 tests/stdlib/ttypetraits.nim diff --git a/changelog.md b/changelog.md index d3335c2a3..be130f780 100644 --- a/changelog.md +++ b/changelog.md @@ -246,6 +246,8 @@ - Added dollar `$` and `len` for `jsre.RegExp`. +- Added `hasClosure` to `std/typetraits`. + ## Language changes diff --git a/compiler/vmops.nim b/compiler/vmops.nim index 5748b41b3..04356fc76 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -316,3 +316,7 @@ proc registerAdditionalOps*(c: PCtx) = let fn = getNode(a, 0) setResult(a, (fn.typ != nil and tfNoSideEffect in fn.typ.flags) or (fn.kind == nkSym and fn.sym.kind == skFunc)) + + registerCallback c, "stdlib.typetraits.hasClosureImpl", proc (a: VmArgs) = + let fn = getNode(a, 0) + setResult(a, fn.kind == nkClosure or (fn.typ != nil and fn.typ.callConv == ccClosure)) diff --git a/lib/pure/typetraits.nim b/lib/pure/typetraits.nim index 6827e23b1..2527dc26e 100644 --- a/lib/pure/typetraits.nim +++ b/lib/pure/typetraits.nim @@ -263,3 +263,14 @@ since (1, 1): type T2 = T genericParamsImpl(T2) + + +proc hasClosureImpl(n: NimNode): bool = discard "see compiler/vmops.nim" + +proc hasClosure*(fn: NimNode): bool {.since: (1, 5, 1).} = + ## Return true if the func/proc/etc `fn` has `closure`. + ## `fn` has to be a resolved symbol of kind `nnkSym`. This + ## implies that the macro that calls this proc should accept `typed` + ## arguments and not `untyped` arguments. + expectKind fn, nnkSym + result = hasClosureImpl(fn) diff --git a/tests/stdlib/ttypetraits.nim b/tests/stdlib/ttypetraits.nim new file mode 100644 index 000000000..de8259ab0 --- /dev/null +++ b/tests/stdlib/ttypetraits.nim @@ -0,0 +1,92 @@ +discard """ + targets: "c cpp js" +""" + +import std/typetraits + + + +macro testClosure(fn: typed, flag: static bool) = + if flag: + doAssert hasClosure(fn) + else: + doAssert not hasClosure(fn) + +block: + proc h1() = + echo 1 + + testClosure(h1, false) + + proc h2() {.nimcall.} = + echo 2 + + testClosure(h2, false) + + +block: + proc fn(): auto = + proc hello() {.nimcall.} = + echo 3 + hello + + let name = fn() + testClosure(name, false) + +block: + proc fn(): auto = + proc hello() = + echo 3 + hello + + let name = fn() + testClosure(name, false) + +block: + proc fn(): auto = + var x = 0 + proc hello() = + echo 3 + inc x + hello + + let name = fn() + testClosure(name, true) + +block: + proc fn(): auto = + var x = 0 + proc hello() {.closure.} = + echo 3 + inc x + hello + + let name = fn() + testClosure(name, true) + +block: + proc fn(): auto = + var x = 0 + proc hello() {.closure.} = + echo 3 + inc x + hello + + let name = fn() + testClosure(name, true) + + let name2 = name + testClosure(name2, true) + +block: + iterator hello(): int = + yield 1 + + testClosure(hello, false) + +when not defined(js): + block: + iterator hello(): int {.closure.}= + yield 1 + + testClosure(hello, true) From e2269f9216e7d46bd446448c7bbeb5fd8ca38d1f Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 29 Mar 2021 20:39:49 +0800 Subject: [PATCH 0527/2472] [docs]fix #17473 (#17565) * fix nim js cmp fails at CT * Add `hasClosure` to `std/effecttraits` * type * Update changelog.md Co-authored-by: Timothee Cour * fix #14011 * Delete ttypetraits.nim * Apply suggestions from code review * fix #17473 * Revert "fix #14011" This reverts commit 0eed97a84b172b198bf4e6de69c04b84ef9d9f93. * Update lib/system.nim Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Co-authored-by: Timothee Cour Co-authored-by: Andreas Rumpf Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- lib/pure/strutils.nim | 12 ++++++++++-- lib/system.nim | 5 +++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 4098749ce..3680b11de 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -2198,13 +2198,21 @@ func insertSep*(s: string, sep = '_', digits = 3): string {.rtl, func escape*(s: string, prefix = "\"", suffix = "\""): string {.rtl, extern: "nsuEscape".} = - ## Escapes a string `s`. See `system.addEscapedChar - ## `_ for the escaping scheme. + ## Escapes a string `s`. + ## + ## .. note:: The escaping scheme is different from + ## `system.addEscapedChar`. + ## + ## * replaces `'\0'..'\31'` and `'\127'..'\255'` by `\xHH` where `HH` is its hexadecimal value + ## * replaces ``\`` by `\\` + ## * replaces `'` by `\'` + ## * replaces `"` by `\"` ## ## The resulting string is prefixed with `prefix` and suffixed with `suffix`. ## Both may be empty strings. ## ## See also: + ## * `addEscapedChar proc`_ ## * `unescape func<#unescape,string,string,string>`_ for the opposite ## operation result = newStringOfCap(s.len + s.len shr 2) diff --git a/lib/system.nim b/lib/system.nim index 27f976119..ba27da0c7 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2839,12 +2839,13 @@ proc addEscapedChar*(s: var string, c: char) {.noSideEffect, inline.} = ## * replaces any `\r` by `\\r` ## * replaces any `\e` by `\\e` ## * replaces any other character not in the set `{\21..\126}` - ## by `\xHH` where `HH` is its hexadecimal value. + ## by `\xHH` where `HH` is its hexadecimal value ## ## The procedure has been designed so that its output is usable for many ## different common syntaxes. ## - ## **Note**: This is **not correct** for producing Ansi C code! + ## .. warning:: This is **not correct** for producing ANSI C code! + ## case c of '\a': s.add "\\a" # \x07 of '\b': s.add "\\b" # \x08 From 1a407402a4614fc9100617f71c2d3c9b8a8367a5 Mon Sep 17 00:00:00 2001 From: Clyybber Date: Mon, 29 Mar 2021 16:03:54 +0200 Subject: [PATCH 0528/2472] Typo in vm.nim --- compiler/vm.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/vm.nim b/compiler/vm.nim index c2ff798a7..d97d2d6e6 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -2221,7 +2221,7 @@ proc setupCompileTimeVar*(module: PSym; idgen: IdGenerator; g: ModuleGraph; n: P discard evalConstExprAux(module, idgen, g, nil, n, emStaticStmt) proc prepareVMValue(arg: PNode): PNode = - ## strip nkExprColonExpr from tuple values recurively. That is how + ## strip nkExprColonExpr from tuple values recursively. That is how ## they are expected to be stored in the VM. # Early abort without copy. No transformation takes place. From cfff27529e4ec129daad602d945a2b222145e922 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 29 Mar 2021 16:23:19 +0200 Subject: [PATCH 0529/2472] added nkError to the AST (#17567) * added nkError to the AST * Update lib/core/macros.nim Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Update compiler/ast.nim Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- compiler/ast.nim | 1 + compiler/errorhandling.nim | 79 +++++++++++++++++++++++++++++++++++ compiler/renderer.nim | 4 ++ compiler/sempass2.nim | 4 +- lib/core/macros.nim | 3 +- tests/misc/tinvalidnewseq.nim | 2 +- 6 files changed, 90 insertions(+), 3 deletions(-) create mode 100644 compiler/errorhandling.nim diff --git a/compiler/ast.nim b/compiler/ast.nim index 1f3d5f129..7d13956e9 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -221,6 +221,7 @@ type nkBreakState, # special break statement for easier code generation nkFuncDef, # a func nkTupleConstr # a tuple constructor + nkError # erroneous AST node nkModuleRef # for .rod file support: A (moduleId, itemId) pair nkReplayAction # for .rod file support: A replay action nkNilRodNode # for .rod file support: a 'nil' PNode diff --git a/compiler/errorhandling.nim b/compiler/errorhandling.nim new file mode 100644 index 000000000..d7092d477 --- /dev/null +++ b/compiler/errorhandling.nim @@ -0,0 +1,79 @@ +# +# +# The Nim Compiler +# (c) Copyright 2021 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module contains support code for new-styled error +## handling via an `nkError` node kind. + +import ast, renderer, options, lineinfos, strutils, types + +type + ErrorKind* = enum ## expand as you need. + RawTypeMismatchError + ExpressionCannotBeCalled + CustomError + WrongNumberOfArguments + AmbiguousCall + +proc errorSubNode*(n: PNode): PNode = + case n.kind + of nkEmpty..nkNilLit: + result = nil + of nkError: + result = n + else: + result = nil + for i in 0.. 1 + let wrongNode = n[0] + case ErrorKind(n[1].intVal) + of RawTypeMismatchError: + result = "type mismatch" + of ExpressionCannotBeCalled: + result = "expression '$1' cannot be called" % wrongNode[0].renderTree + of CustomError: + result = n[2].strVal + of WrongNumberOfArguments: + result = "wrong number of arguments" + of AmbiguousCall: + let a = n[2].sym + let b = n[3].sym + var args = "(" + for i in 1.. 1: args.add(", ") + args.add(typeToString(wrongNode[i].typ)) + args.add(")") + result = "ambiguous call; both $1 and $2 match for: $3" % [ + getProcHeader(config, a), + getProcHeader(config, b), + args] diff --git a/compiler/renderer.nim b/compiler/renderer.nim index 9ca485f6e..9a599f6fc 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -1645,6 +1645,10 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) = gsons(g, n, c, 0) of nkTypeClassTy: gTypeClassTy(g, n) + of nkError: + putWithSpace(g, tkSymbol, "error") + #gcomma(g, n, c) + gsub(g, n[0], c) else: #nkNone, nkExplicitTypeListCall: internalError(g.config, n.info, "rnimsyn.gsub(" & $n.kind & ')') diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index f269afe4c..a1572c85e 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -10,7 +10,7 @@ import intsets, ast, astalgo, msgs, renderer, magicsys, types, idents, trees, wordrecg, strutils, options, guards, lineinfos, semfold, semdata, - modulegraphs, varpartitions, typeallowed, nilcheck + modulegraphs, varpartitions, typeallowed, nilcheck, errorhandling when defined(useDfa): import dfa @@ -1136,6 +1136,8 @@ proc track(tracked: PEffects, n: PNode) = dec tracked.leftPartOfAsgn for i in 1 ..< n.len: track(tracked, n[i]) inc tracked.leftPartOfAsgn + of nkError: + localError(tracked.config, n.info, errorToString(tracked.config, n)) else: for i in 0.. Date: Mon, 29 Mar 2021 07:25:27 -0700 Subject: [PATCH 0530/2472] fix https://github.com/nim-lang/RFCs/issues/352: show top-level import for top-level runnableExamples in generated docs (#17542) * fix https://github.com/nim-lang/RFCs/issues/352: show top-level import for top-level runnableExamples in generated docs * use canonical imoprt * fix test --- compiler/docgen.nim | 25 +++++++++++++------- nimdoc/testproject/expected/testproject.html | 12 ++++++---- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index b24a24a2b..f1a9750b7 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -505,7 +505,7 @@ proc runAllExamples(d: PDoc) = proc quoted(a: string): string = result.addQuoted(a) -proc prepareExample(d: PDoc; n: PNode): tuple[rdoccmd: string, code: string] = +proc prepareExample(d: PDoc; n: PNode, topLevel: bool): tuple[rdoccmd: string, code: string] = ## returns `rdoccmd` and source code for this runnableExamples var rdoccmd = "" if n.len < 2 or n.len > 3: globalError(d.conf, n.info, "runnableExamples invalid") @@ -556,7 +556,14 @@ $3 if rdoccmd notin d.exampleGroups: d.exampleGroups[rdoccmd] = ExampleGroup(rdoccmd: rdoccmd, docCmd: d.conf.docCmd, index: d.exampleGroups.len) d.exampleGroups[rdoccmd].code.add "import $1\n" % outp.string.quoted - result = (rdoccmd, code) + + var codeShown: string + if topLevel: # refs https://github.com/nim-lang/RFCs/issues/352 + let title = canonicalImport(d.conf, AbsoluteFile d.filename) + codeShown = "import $#\n$#" % [title, code] + else: + codeShown = code + result = (rdoccmd, codeShown) when false: proc extractImports(n: PNode; result: PNode) = if n.kind in {nkImportStmt, nkImportExceptStmt, nkFromStmt}: @@ -576,7 +583,7 @@ type RunnableState = enum rsRunnable rsDone -proc getAllRunnableExamplesImpl(d: PDoc; n: PNode, dest: var Rope, state: RunnableState): RunnableState = +proc getAllRunnableExamplesImpl(d: PDoc; n: PNode, dest: var Rope, state: RunnableState, topLevel: bool): RunnableState = ##[ Simple state machine to tell whether we render runnableExamples and doc comments. This is to ensure that we can interleave runnableExamples and doc comments freely; @@ -601,7 +608,7 @@ proc getAllRunnableExamplesImpl(d: PDoc; n: PNode, dest: var Rope, state: Runnab of nkCallKinds: if isRunnableExamples(n[0]) and n.len >= 2 and n.lastSon.kind == nkStmtList and state in {rsStart, rsComment, rsRunnable}: - let (rdoccmd, code) = prepareExample(d, n) + let (rdoccmd, code) = prepareExample(d, n, topLevel) var msg = "Example:" if rdoccmd.len > 0: msg.add " cmd: " & rdoccmd dispA(d.conf, dest, "\n

    $1

    \n", @@ -652,19 +659,19 @@ proc getRoutineBody(n: PNode): PNode = proc getAllRunnableExamples(d: PDoc, n: PNode, dest: var Rope) = var n = n var state = rsStart - template fn(n2) = - state = getAllRunnableExamplesImpl(d, n2, dest, state) + template fn(n2, topLevel) = + state = getAllRunnableExamplesImpl(d, n2, dest, state, topLevel) dest.add genComment(d, n).rope case n.kind of routineDefs: n = n.getRoutineBody case n.kind - of nkCommentStmt, nkCallKinds: fn(n) + of nkCommentStmt, nkCallKinds: fn(n, topLevel = false) else: for i in 0..This is the top level module.

    Example:

    -
    import subdir / subdir_b / utils
+
    import testproject
+import subdir / subdir_b / utils
 doAssert bar(3, 4) == 7
 foo(enumValueA, enumValueB)
 # bug #11078
 for x in "xx": discard
    top2
 
    Example:
    
-
    discard "in top2"
    top2 after
+
    import testproject
+discard "in top2"
    top2 after
 
    Example:
    
-
    discard "in top3"
    top3 after
+
    import testproject
+discard "in top3"
    top3 after
 
    Example:
    
-
    assert 3*2 == 6
    
+
    import testproject
+assert 3*2 == 6
    
   
    
 
    Imports
    
 
    

From 7ad49950bd5ec05f145dc43ae35f51127ea76366 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Mon, 29 Mar 2021 23:54:28 +0800
Subject: [PATCH 0531/2472] [os:standalone]fix #14011 (#17564)

---
 lib/system/chcks.nim       | 10 ++++++++--
 tests/gc/panicoverride.nim | 14 ++++++++++++++
 tests/gc/tstandalone.nim   | 14 ++++++++++++++
 3 files changed, 36 insertions(+), 2 deletions(-)
 create mode 100644 tests/gc/panicoverride.nim
 create mode 100644 tests/gc/tstandalone.nim

diff --git a/lib/system/chcks.nim b/lib/system/chcks.nim
index e0a8bd78e..255d97cb2 100644
--- a/lib/system/chcks.nim
+++ b/lib/system/chcks.nim
@@ -29,10 +29,16 @@ proc raiseFieldError(f: string) {.compilerproc, noinline.} =
   sysFatal(FieldDefect, f)
 
 proc raiseRangeErrorI(i, a, b: BiggestInt) {.compilerproc, noinline.} =
-  sysFatal(RangeDefect, "value out of range: " & $i & " notin " & $a & " .. " & $b)
+  when defined(standalone):
+    sysFatal(RangeDefect, "value out of range")
+  else:
+    sysFatal(RangeDefect, "value out of range: " & $i & " notin " & $a & " .. " & $b)
 
 proc raiseRangeErrorF(i, a, b: float) {.compilerproc, noinline.} =
-  sysFatal(RangeDefect, "value out of range: " & $i & " notin " & $a & " .. " & $b)
+  when defined(standalone):
+    sysFatal(RangeDefect, "value out of range")
+  else:
+    sysFatal(RangeDefect, "value out of range: " & $i & " notin " & $a & " .. " & $b)
 
 proc raiseRangeErrorU(i, a, b: uint64) {.compilerproc, noinline.} =
   # todo: better error reporting
diff --git a/tests/gc/panicoverride.nim b/tests/gc/panicoverride.nim
new file mode 100644
index 000000000..0f28b0b72
--- /dev/null
+++ b/tests/gc/panicoverride.nim
@@ -0,0 +1,14 @@
+
+proc printf(frmt: cstring) {.varargs, importc, header: "", cdecl.}
+proc exit(code: int) {.importc, header: "", cdecl.}
+
+{.push stack_trace: off, profiler:off.}
+
+proc rawoutput(s: string) =
+  printf("%s\n", s)
+
+proc panic(s: string) {.noreturn.} =
+  rawoutput(s)
+  exit(1)
+
+{.pop.}
\ No newline at end of file
diff --git a/tests/gc/tstandalone.nim b/tests/gc/tstandalone.nim
new file mode 100644
index 000000000..41dad9ba4
--- /dev/null
+++ b/tests/gc/tstandalone.nim
@@ -0,0 +1,14 @@
+discard """
+  matrix: "--os:standalone --gc:none"
+  exitcode: 1
+  output: "value out of range"
+"""
+
+type
+  rangeType = range[0..1]
+
+var
+  r: rangeType = 0
+  i = 2
+
+r = rangeType(i)

From 3f9c51a332dc798e1f4d61480ca8b6048ab281cd Mon Sep 17 00:00:00 2001
From: flywind 
Date: Tue, 30 Mar 2021 00:28:55 +0800
Subject: [PATCH 0532/2472] [nim check]fix #17460 (#17569)

---
 compiler/semstmts.nim    |  2 ++
 tests/errmsgs/t17460.nim | 19 +++++++++++++++++++
 2 files changed, 21 insertions(+)
 create mode 100644 tests/errmsgs/t17460.nim

diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim
index fee43162e..bbfd49e2a 100644
--- a/compiler/semstmts.nim
+++ b/compiler/semstmts.nim
@@ -726,6 +726,8 @@ proc semForVars(c: PContext, n: PNode; flags: TExprFlags): PNode =
       if n[0].kind == nkVarTuple:
         if n[0].len-1 != iterAfterVarLent.len:
           localError(c.config, n[0].info, errWrongNumberOfVariables)
+          return errorNode(c, n)
+
         for i in 0..
Date: Mon, 29 Mar 2021 19:32:49 +0300
Subject: [PATCH 0533/2472] RST: enable parsing of prefix roles (ref #17340)
 (#17514)

---
 doc/contributing.rst          |  9 ++++++++
 lib/packages/docutils/rst.nim | 38 +++++++++++++++++---------------
 tests/stdlib/trstgen.nim      | 41 +++++++++++++++++++++++++++++++++++
 3 files changed, 70 insertions(+), 18 deletions(-)

diff --git a/doc/contributing.rst b/doc/contributing.rst
index b7188a436..279a4ee94 100644
--- a/doc/contributing.rst
+++ b/doc/contributing.rst
@@ -285,6 +285,15 @@ or
 
 the first is preferred.
 
+When you specify an *RST role* (highlighting/interpretation marker) do it
+in the postfix form for uniformity, that is after \`text in backticks\`.
+For example an ``:idx:`` role for referencing a topic ("SQLite" in the
+example below) from `Nim Index`_ can be used in doc comment this way:
+
+.. code-block:: nim
+  ## A higher level `SQLite`:idx: database wrapper.
+
+.. _`Nim Index`: https://nim-lang.org/docs/theindex.html
 
 Best practices
 ==============
diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim
index 6845f61b9..d331c2c12 100644
--- a/lib/packages/docutils/rst.nim
+++ b/lib/packages/docutils/rst.nim
@@ -68,7 +68,7 @@
 ##     substitution references, standalone hyperlinks,
 ##     internal links (inline and outline)
 ##   + \`interpreted text\` with roles ``:literal:``, ``:strong:``,
-##     ``emphasis``, ``:sub:``/``:subscript:``, ``:sup:``/``:supscript:``
+##     ``emphasis``, ``:sub:``/``:subscript:``, ``:sup:``/``:superscript:``
 ##     (see `RST roles list`_ for description).
 ##   + inline internal targets
 ##
@@ -1018,6 +1018,16 @@ proc fixupEmbeddedRef(n, a, b: PRstNode) =
   for i in countup(0, sep - incr): a.add(n.sons[i])
   for i in countup(sep + 1, n.len - 2): b.add(n.sons[i])
 
+proc whichRole(sym: string): RstNodeKind =
+  case sym
+  of "idx": result = rnIdx
+  of "literal": result = rnInlineLiteral
+  of "strong": result = rnStrongEmphasis
+  of "emphasis": result = rnEmphasis
+  of "sub", "subscript": result = rnSub
+  of "sup", "superscript": result = rnSup
+  else: result = rnGeneralRole
+
 proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode =
   var newKind = n.kind
   var newSons = n.sons
@@ -1042,22 +1052,8 @@ proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode =
     result = newRstNode(newKind, newSons)
   elif match(p, p.idx, ":w:"):
     # a role:
-    if nextTok(p).symbol == "idx":
-      newKind = rnIdx
-    elif nextTok(p).symbol == "literal":
-      newKind = rnInlineLiteral
-    elif nextTok(p).symbol == "strong":
-      newKind = rnStrongEmphasis
-    elif nextTok(p).symbol == "emphasis":
-      newKind = rnEmphasis
-    elif nextTok(p).symbol == "sub" or
-        nextTok(p).symbol == "subscript":
-      newKind = rnSub
-    elif nextTok(p).symbol == "sup" or
-        nextTok(p).symbol == "supscript":
-      newKind = rnSup
-    else:
-      newKind = rnGeneralRole
+    newKind = whichRole(nextTok(p).symbol)
+    if newKind == rnGeneralRole:
       let newN = newRstNode(rnInner, n.sons)
       newSons = @[newN, newLeaf(nextTok(p).symbol)]
     inc p.idx, 3
@@ -1318,6 +1314,12 @@ proc parseInline(p: var RstParser, father: PRstNode) =
       var n = newRstNode(rnInlineLiteral)
       parseUntil(p, n, "``", false)
       father.add(n)
+    elif match(p, p.idx, ":w:") and p.tok[p.idx+3].symbol == "`":
+      let k = whichRole(nextTok(p).symbol)
+      let n = newRstNode(k)
+      inc p.idx, 3
+      parseUntil(p, n, "`", false) # bug #17260
+      father.add(n)
     elif isInlineMarkupStart(p, "`"):
       var n = newRstNode(rnInterpretedText)
       parseUntil(p, n, "`", false) # bug #17260
@@ -1677,7 +1679,7 @@ proc whichSection(p: RstParser): RstNodeKind =
     elif predNL(p) and
         currentTok(p).symbol in ["+", "*", "-"] and nextTok(p).kind == tkWhite:
       result = rnBulletList
-    elif match(p, p.idx, ":w:") and predNL(p):
+    elif match(p, p.idx, ":w:E") and predNL(p):
       # (currentTok(p).symbol == ":")
       result = rnFieldList
     elif match(p, p.idx, "(e) ") or match(p, p.idx, "e) ") or
diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim
index 0f3890faa..da01c30d2 100644
--- a/tests/stdlib/trstgen.nim
+++ b/tests/stdlib/trstgen.nim
@@ -1326,6 +1326,47 @@ Test1
           output)
     check("""-doption""" in
           output)
+
+  test "Roles: subscript prefix/postfix":
+    let expected = "See some text."
+    check "See :subscript:`some text`.".toHtml == expected
+    check "See `some text`:subscript:.".toHtml == expected
+
+  test "Roles: correct parsing from beginning of line":
+    let expected = "3He is an isotope of helium."
+    check """:superscript:`3`\ He is an isotope of helium.""".toHtml == expected
+    check """:sup:`3`\ He is an isotope of helium.""".toHtml == expected
+    check """`3`:sup:\ He is an isotope of helium.""".toHtml == expected
+    check """`3`:superscript:\ He is an isotope of helium.""".toHtml == expected
+
+  test "(not) Roles: check escaping 1":
+    let expected = """See :subscript:""" &
+                   """some text."""
+    check """See \:subscript:`some text`.""".toHtml == expected
+    check """See :subscript\:`some text`.""".toHtml == expected
+
+  test "(not) Roles: check escaping 2":
+    check("""See :subscript:\`some text\`.""".toHtml ==
+          "See :subscript:`some text`.")
+
+  test "Field list":
+    check(":field: text".toHtml ==
+            """""" &
+            """""" &
+            """""" &
+            """""" & "\n
    field: text
    ")
+
+  test "Field list: body after newline":
+    let output = dedent """
+      :field:
+        text1""".toHtml
+    check "field:" in output
+    check "" in output
+
+  test "Field list (incorrect)":
+    check ":field:text".toHtml == ":field:text"
+
 suite "RST/Code highlight":
   test "Basic Python code highlight":
     let pythonCode = """

From 1f1ef85eb0980f2e1077638a502c0754c3e2b80d Mon Sep 17 00:00:00 2001
From: flywind 
Date: Tue, 30 Mar 2021 02:48:06 +0800
Subject: [PATCH 0534/2472] [std/uri]fix #17481 (#17568)

---
 lib/pure/uri.nim      |  2 +-
 tests/stdlib/turi.nim | 11 +++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/lib/pure/uri.nim b/lib/pure/uri.nim
index 920529ecf..4ac04d02e 100644
--- a/lib/pure/uri.nim
+++ b/lib/pure/uri.nim
@@ -294,7 +294,7 @@ func parseUri*(uri: string, result: var Uri) =
   var i = 0
 
   # Check if this is a reference URI (relative URI)
-  let doubleSlash = uri.len > 1 and uri[1] == '/'
+  let doubleSlash = uri.len > 1 and uri[0] == '/' and uri[1] == '/'
   if i < uri.len and uri[i] == '/':
     # Make sure `uri` doesn't begin with '//'.
     if not doubleSlash:
diff --git a/tests/stdlib/turi.nim b/tests/stdlib/turi.nim
index 1a6f37520..d1a30bbfc 100644
--- a/tests/stdlib/turi.nim
+++ b/tests/stdlib/turi.nim
@@ -283,5 +283,16 @@ template main() =
     doAssert toSeq(decodeQuery("a=1&b=0")) == @[("a", "1"), ("b", "0")]
     doAssert toSeq(decodeQuery("a=1&b=2c=6")) == @[("a", "1"), ("b", "2c=6")]
 
+  block: # bug #17481
+    let u1 = parseUri("./")
+    let u2 = parseUri("./path")
+    let u3 = parseUri("a/path")
+    doAssert u1.scheme.len == 0
+    doAssert u1.path == "./"
+    doAssert u2.scheme.len == 0
+    doAssert u2.path == "./path"
+    doAssert u3.scheme.len == 0
+    doAssert u3.path == "a/path"
+
 static: main()
 main()

From ed126794b6af8b39960a7acc603cee1abc73da4a Mon Sep 17 00:00:00 2001
From: Clyybber 
Date: Mon, 29 Mar 2021 21:16:11 +0200
Subject: [PATCH 0535/2472] Fix #17412 (#17560)

* Fix #17412

* Address review
---
 compiler/ccgtypes.nim        | 28 ++-----------------------
 compiler/ccgutils.nim        | 40 ++++++++++++++++++++++++++++++++++--
 compiler/liftdestructors.nim | 11 +++++++++-
 3 files changed, 50 insertions(+), 29 deletions(-)

diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim
index b0c3aa3b1..761fd5bf8 100644
--- a/compiler/ccgtypes.nim
+++ b/compiler/ccgtypes.nim
@@ -256,32 +256,8 @@ proc addAbiCheck(m: BModule, t: PType, name: Rope) =
     m.s[cfsTypeInfo].addf("NIM_STATIC_ASSERT(sizeof($1) == $2, $3);$n", [name, rope(size), msg2.rope])
     # see `testCodegenABICheck` for example error message it generates
 
-proc ccgIntroducedPtr(conf: ConfigRef; s: PSym, retType: PType): bool =
-  var pt = skipTypes(s.typ, typedescInst)
-  assert skResult != s.kind
-
-  if tfByRef in pt.flags: return true
-  elif tfByCopy in pt.flags: return false
-  case pt.kind
-  of tyObject:
-    if s.typ.sym != nil and sfForward in s.typ.sym.flags:
-      # forwarded objects are *always* passed by pointers for consistency!
-      result = true
-    elif (optByRef in s.options) or (getSize(conf, pt) > conf.target.floatSize * 3):
-      result = true           # requested anyway
-    elif (tfFinal in pt.flags) and (pt[0] == nil):
-      result = false          # no need, because no subtyping possible
-    else:
-      result = true           # ordinary objects are always passed by reference,
-                              # otherwise casting doesn't work
-  of tyTuple:
-    result = (getSize(conf, pt) > conf.target.floatSize*3) or (optByRef in s.options)
-  else:
-    result = false
-  # first parameter and return type is 'lent T'? --> use pass by pointer
-  if s.position == 0 and retType != nil and retType.kind == tyLent:
-    result = not (pt.kind in {tyVar, tyArray, tyOpenArray, tyVarargs, tyRef, tyPtr, tyPointer} or
-      pt.kind == tySet and mapSetType(conf, pt) == ctArray)
+from ccgutils import ccgIntroducedPtr
+export ccgIntroducedPtr
 
 proc fillResult(conf: ConfigRef; param: PNode) =
   fillLoc(param.sym.loc, locParam, param, ~"Result",
diff --git a/compiler/ccgutils.nim b/compiler/ccgutils.nim
index f2a8c1e36..06b75a20f 100644
--- a/compiler/ccgutils.nim
+++ b/compiler/ccgutils.nim
@@ -10,8 +10,8 @@
 # This module declares some helpers for the C code generator.
 
 import
-  ast, hashes, strutils, msgs, wordrecg,
-  platform, trees, options
+  ast, types, hashes, strutils, msgs, wordrecg,
+  platform, trees, options, cgendata
 
 proc getPragmaStmt*(n: PNode, w: TSpecialWord): PNode =
   case n.kind
@@ -106,3 +106,39 @@ proc mangle*(name: string): string =
       requiresUnderscore = true
   if requiresUnderscore:
     result.add "_"
+
+proc mapSetType(conf: ConfigRef; typ: PType): TCTypeKind =
+  case int(getSize(conf, typ))
+  of 1: result = ctInt8
+  of 2: result = ctInt16
+  of 4: result = ctInt32
+  of 8: result = ctInt64
+  else: result = ctArray
+
+proc ccgIntroducedPtr*(conf: ConfigRef; s: PSym, retType: PType): bool =
+  var pt = skipTypes(s.typ, typedescInst)
+  assert skResult != s.kind
+
+  if tfByRef in pt.flags: return true
+  elif tfByCopy in pt.flags: return false
+  case pt.kind
+  of tyObject:
+    if s.typ.sym != nil and sfForward in s.typ.sym.flags:
+      # forwarded objects are *always* passed by pointers for consistency!
+      result = true
+    elif (optByRef in s.options) or (getSize(conf, pt) > conf.target.floatSize * 3):
+      result = true           # requested anyway
+    elif (tfFinal in pt.flags) and (pt[0] == nil):
+      result = false          # no need, because no subtyping possible
+    else:
+      result = true           # ordinary objects are always passed by reference,
+                              # otherwise casting doesn't work
+  of tyTuple:
+    result = (getSize(conf, pt) > conf.target.floatSize*3) or (optByRef in s.options)
+  else:
+    result = false
+  # first parameter and return type is 'lent T'? --> use pass by pointer
+  if s.position == 0 and retType != nil and retType.kind == tyLent:
+    result = not (pt.kind in {tyVar, tyArray, tyOpenArray, tyVarargs, tyRef, tyPtr, tyPointer} or
+      pt.kind == tySet and mapSetType(conf, pt) == ctArray)
+
diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim
index 980e77e4b..bb24fed34 100644
--- a/compiler/liftdestructors.nim
+++ b/compiler/liftdestructors.nim
@@ -11,7 +11,7 @@
 ## (``=sink``, ``=``, ``=destroy``, ``=deepCopy``).
 
 import modulegraphs, lineinfos, idents, ast, renderer, semdata,
-  sighashes, lowerings, options, types, msgs, magicsys, tables
+  sighashes, lowerings, options, types, msgs, magicsys, tables, ccgutils
 
 from trees import isCaseObj
 
@@ -230,6 +230,15 @@ proc fillBodyObjT(c: var TLiftCtx; t: PType, body, x, y: PNode) =
     # for every field (dependent on dest.kind):
     #   `=` dest.field, src.field
     # =destroy(blob)
+    var dummy = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), nextSymId c.idgen, c.fn, c.info)
+    dummy.typ = y.typ
+    if ccgIntroducedPtr(c.g.config, dummy, y.typ):
+      # Because of potential aliasing when the src param is passed by ref, we need to check for equality here,
+      # because the wasMoved(dest) call would zero out src, if dest aliases src.
+      var cond = newTree(nkCall, newSymNode(c.g.getSysMagic(c.info, "==", mEqRef)),
+        newTreeIT(nkAddr, c.info, makePtrType(c.fn, x.typ, c.idgen), x), newTreeIT(nkAddr, c.info, makePtrType(c.fn, y.typ, c.idgen), y))
+      cond.typ = getSysType(c.g, x.info, tyBool)
+      body.add genIf(c, cond, newTreeI(nkReturnStmt, c.info, newNodeI(nkEmpty, c.info)))
     var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), nextSymId c.idgen, c.fn, c.info)
     temp.typ = x.typ
     incl(temp.flags, sfFromGeneric)

From 35655cd1899ad2d85ca93ad7482572f0f345d8fb Mon Sep 17 00:00:00 2001
From: Dylan Modesitt 
Date: Mon, 29 Mar 2021 19:12:58 -0400
Subject: [PATCH 0536/2472] Add hasDataBuffered to asyncnet (#16000)

Co-authored-by: flywind 
Co-authored-by: Timothee Cour 
---
 changelog.md          | 2 ++
 lib/pure/asyncnet.nim | 5 +++++
 2 files changed, 7 insertions(+)

diff --git a/changelog.md b/changelog.md
index be130f780..2fbff8b41 100644
--- a/changelog.md
+++ b/changelog.md
@@ -246,6 +246,8 @@
 
 - Added dollar `$` and `len` for `jsre.RegExp`.
 
+- Added `hasDataBuffered` to `asyncnet`.
+
 - Added `hasClosure` to `std/typetraits`.
 
 
diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim
index f36c427de..7aeb99777 100644
--- a/lib/pure/asyncnet.nim
+++ b/lib/pure/asyncnet.nim
@@ -650,6 +650,11 @@ proc bindAddr*(socket: AsyncSocket, port = Port(0), address = "") {.
     raiseOSError(osLastError())
   freeaddrinfo(aiList)
 
+proc hasDataBuffered*(s: AsyncSocket): bool {.since: (1, 5).} =
+  ## Determines whether an AsyncSocket has data buffered.
+  # xxx dedup with std/net
+  s.isBuffered and s.bufLen > 0 and s.currPos != s.bufLen
+
 when defined(posix):
 
   proc connectUnix*(socket: AsyncSocket, path: string): owned(Future[void]) =

From 159c06e0451f85842b2168886565fa57496a2c68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arne=20D=C3=B6ring?= 
Date: Tue, 30 Mar 2021 02:06:51 +0200
Subject: [PATCH 0537/2472] unify tuple expressions (#13793)

* unify tuple expressions

* fix test

* fix test

* apply feedback

* Handle empty tuples

* Fix rendering named unary tuple

* Protect static NimNode against stripping

* Slightly less hacky

* Revert "Slightly less hacky"

This reverts commit 170c5aec0addc029f637afbc948700ca006b7942.

* Slightly less hacky

* Cleanup

* Fix test

* Fix another test

* Add condsym

* Rebase fallout

* changelog: Move from compiler changes to language changes

* Add stricter tests

* Add empty tuple example to doc/astspec

* Fix test

Co-authored-by: Clyybber 
---
 changelog.md                      |  3 +++
 compiler/condsyms.nim             |  1 +
 compiler/parser.nim               | 15 ++++++-----
 compiler/renderer.nim             |  2 +-
 compiler/vm.nim                   |  6 +++++
 doc/astspec.txt                   | 45 ++++++++++++++++++++++++++++---
 tests/ast_pattern_matching.nim    |  4 +--
 tests/astspec/tastspec.nim        | 36 ++++++++++++++++++++-----
 tests/compiler/tprefixmatches.nim |  2 +-
 tests/lexer/tunary_minus.nim      |  2 +-
 tests/macros/tdumpast.nim         | 31 +++++++++++++++++++++
 tests/parser/tpostexprblocks.nim  |  6 ++---
 tests/parser/ttypemodifiers.nim   | 15 +++++------
 13 files changed, 136 insertions(+), 32 deletions(-)

diff --git a/changelog.md b/changelog.md
index 2fbff8b41..7452e3182 100644
--- a/changelog.md
+++ b/changelog.md
@@ -274,6 +274,9 @@
 
 - Custom numeric literals (e.g. `-128'bignum`) are now supported.
 
+- Tuple expressions are now parsed consistently as
+  `nnkTupleConstr` node. Will affect macros expecting nodes to be of `nnkPar`.
+
 
 ## Compiler changes
 
diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim
index 8d11dae7c..9ba97a8ed 100644
--- a/compiler/condsyms.nim
+++ b/compiler/condsyms.nim
@@ -131,3 +131,4 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimHasHintAsError")
   defineSymbol("nimHasSpellSuggest")
   defineSymbol("nimHasCustomLiterals")
+  defineSymbol("nimHasUnifiedTuple")
diff --git a/compiler/parser.nim b/compiler/parser.nim
index b9a6ffb8c..83ea4cbdc 100644
--- a/compiler/parser.nim
+++ b/compiler/parser.nim
@@ -421,10 +421,9 @@ proc exprColonEqExprListAux(p: var Parser, endTok: TokType, result: PNode) =
     var a = exprColonEqExpr(p)
     result.add(a)
     if p.tok.tokType != tkComma: break
-    getTok(p)
-    # (1,) produces a tuple expression
-    if endTok == tkParRi and p.tok.tokType == tkParRi and result.kind == nkPar:
+    elif result.kind == nkPar:
       result.transitionSonsKind(nkTupleConstr)
+    getTok(p)
     skipComment(p, a)
   optPar(p)
   eat(p, endTok)
@@ -584,7 +583,10 @@ proc parsePar(p: var Parser): PNode =
     semiStmtList(p, result)
   elif p.tok.tokType == tkCurlyDotLe:
     result.add(parseStmtPragma(p))
-  elif p.tok.tokType != tkParRi:
+  elif p.tok.tokType == tkParRi:
+    # Empty tuple '()'
+    result.transitionSonsKind(nkTupleConstr)
+  else:
     var a = simpleExpr(p)
     if p.tok.tokType == tkDo:
       result = postExprBlocks(p, a)
@@ -605,13 +607,14 @@ proc parsePar(p: var Parser): PNode =
       semiStmtList(p, result)
     else:
       a = colonOrEquals(p, a)
+      if a.kind == nkExprColonExpr:
+        result.transitionSonsKind(nkTupleConstr)
       result.add(a)
       if p.tok.tokType == tkComma:
         getTok(p)
         skipComment(p, a)
         # (1,) produces a tuple expression:
-        if p.tok.tokType == tkParRi:
-          result.transitionSonsKind(nkTupleConstr)
+        result.transitionSonsKind(nkTupleConstr)
         # progress guaranteed
         while p.tok.tokType != tkParRi and p.tok.tokType != tkEof:
           var a = exprColonEqExpr(p)
diff --git a/compiler/renderer.nim b/compiler/renderer.nim
index 9a599f6fc..b9ec48185 100644
--- a/compiler/renderer.nim
+++ b/compiler/renderer.nim
@@ -1182,7 +1182,7 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
   of nkTupleConstr:
     put(g, tkParLe, "(")
     gcomma(g, n, c)
-    if n.len == 1: put(g, tkComma, ",")
+    if n.len == 1 and n[0].kind != nkExprColonExpr: put(g, tkComma, ",")
     put(g, tkParRi, ")")
   of nkCurly:
     put(g, tkCurlyLe, "{")
diff --git a/compiler/vm.nim b/compiler/vm.nim
index d97d2d6e6..baaf0f14e 100644
--- a/compiler/vm.nim
+++ b/compiler/vm.nim
@@ -2228,6 +2228,12 @@ proc prepareVMValue(arg: PNode): PNode =
   if arg.kind in nkLiterals:
     return arg
 
+  if arg.kind == nkExprColonExpr and arg[0].typ != nil and
+     arg[0].typ.sym != nil and arg[0].typ.sym.magic == mPNimrodNode:
+    # Poor mans way of protecting static NimNodes
+    # XXX: Maybe we need a nkNimNode?
+    return arg
+
   result = copyNode(arg)
   if arg.kind == nkTupleConstr:
     for child in arg:
diff --git a/doc/astspec.txt b/doc/astspec.txt
index b4b8b34b5..92212bf8c 100644
--- a/doc/astspec.txt
+++ b/doc/astspec.txt
@@ -329,19 +329,56 @@ AST:
 Parentheses
 -----------
 
-Parentheses for affecting operator precedence or tuple construction
-are built with the ``nnkPar`` node.
+Parentheses for affecting operator precedence use the ``nnkPar`` node.
 
 Concrete syntax:
 
 .. code-block:: nim
-  (1, 2, (3))
+  (a + b) * c
 
 AST:
 
 .. code-block:: nim
-  nnkPar(nnkIntLit(1), nnkIntLit(2), nnkPar(nnkIntLit(3)))
+  nnkInfix(nnkIdent("*"),
+    nnkPar(
+      nnkInfix(nnkIdent("+"), nnkIdent("a"), nnkIdent("b"))),
+    nnkIdent("c"))
 
+Tuple Constructors
+------------------
+
+Nodes for tuple construction are built with the ``nnkTupleConstr`` node.
+
+Concrete syntax:
+
+.. code-block:: nim
+  (1, 2, 3)
+  (a: 1, b: 2, c: 3)
+  ()
+
+AST:
+
+.. code-block:: nim
+  nnkTupleConstr(nnkIntLit(1), nnkIntLit(2), nnkIntLit(3))
+  nnkTupleConstr(
+    nnkExprColonExpr(nnkIdent("a"), nnkIntLit(1)),
+    nnkExprColonExpr(nnkIdent("b"), nnkIntLit(2)),
+    nnkExprColonExpr(nnkIdent("c"), nnkIntLit(3)))
+
+Since the one tuple would be syntactically identical to parentheses
+with an expression in them, the parser expects a trailing comma for
+them. For tuple constructors with field names, this is not necessary.
+
+.. code-block:: nim
+  (1,)
+  (a: 1)
+
+AST:
+
+.. code-block:: nim
+  nnkTupleConstr(nnkIntLit(1))
+  nnkTupleConstr(
+    nnkExprColonExpr(nnkIdent("a"), nnkIntLit(1)))
 
 Curly braces
 ------------
diff --git a/tests/ast_pattern_matching.nim b/tests/ast_pattern_matching.nim
index c08234b9e..d209493b6 100644
--- a/tests/ast_pattern_matching.nim
+++ b/tests/ast_pattern_matching.nim
@@ -133,8 +133,8 @@ proc matchLengthKind*(arg: NimNode; kind: NimNodeKind; length: int): MatchingErr
   matchLengthKind(arg, {kind}, length)
 
 proc matchValue(arg: NimNode; kind: set[NimNodeKind]; value: SomeInteger): MatchingError {.compileTime.} =
-  let kindFail   = not(kind.card == 0 or arg.kind in kind)
-  let valueFail  = arg.intVal != int(value)
+  template kindFail: bool  = not(kind.card == 0 or arg.kind in kind)
+  template valueFail: bool = arg.intVal != int(value)
   if kindFail or valueFail:
     result.node = arg
     result.kind = WrongKindValue
diff --git a/tests/astspec/tastspec.nim b/tests/astspec/tastspec.nim
index e2cfed277..33a245b1b 100644
--- a/tests/astspec/tastspec.nim
+++ b/tests/astspec/tastspec.nim
@@ -327,19 +327,43 @@ static:
 
   testArrayAccessOperator(x[y])
 
-
-
   ## Parentheses
 
   scope:
-
     let ast = myquote:
-      (1, 2, (3))
+      (a + b) * c
 
     ast.matchAst:
-    of nnkPar(nnkIntLit(intVal = 1), nnkIntLit(intVal = 2), nnkPar(nnkIntLit(intVal = 3))):
-      echo "ok"
+    of nnkInfix(ident"*", nnkPar(nnkInfix(ident"+", ident"a", ident"b")), ident"c"):
+      echo "parentheses ok"
 
+  ## Tuple Constructors
+
+  scope:
+    let ast = myquote:
+      (1, 2, 3)
+      (a: 1, b: 2, c: 3)
+      (1,)
+      (a: 1)
+      ()
+
+    for it in ast:
+      echo it.lispRepr
+      it.matchAst:
+      of nnkTupleConstr(nnkIntLit(intVal = 1), nnkIntLit(intVal = 2), nnkIntLit(intVal = 3)):
+        echo "simple tuple ok"
+      of nnkTupleConstr(
+        nnkExprColonExpr(ident"a", nnkIntLit(intVal = 1)),
+        nnkExprColonExpr(ident"b", nnkIntLit(intVal = 2)),
+        nnkExprColonExpr(ident"c", nnkIntLit(intVal = 3))
+      ):
+        echo "named tuple ok"
+      of nnkTupleConstr(nnkIntLit(intVal = 1)):
+        echo "one tuple ok"
+      of nnkTupleConstr(nnkExprColonExpr(ident"a", nnkIntLit(intVal = 1))):
+        echo "named one tuple ok"
+      of nnkTupleConstr():
+       echo "empty tuple ok"
 
   ## Curly braces
 
diff --git a/tests/compiler/tprefixmatches.nim b/tests/compiler/tprefixmatches.nim
index a89a6f613..6a3186729 100644
--- a/tests/compiler/tprefixmatches.nim
+++ b/tests/compiler/tprefixmatches.nim
@@ -5,7 +5,7 @@ macro check(val, body: untyped): untyped =
   result = newStmtList()
   expectKind body, nnkStmtList
   for b in body:
-    expectKind b, nnkPar
+    expectKind b, nnkTupleConstr
     expectLen b, 2
     let p = b[0]
     let s = b[1]
diff --git a/tests/lexer/tunary_minus.nim b/tests/lexer/tunary_minus.nim
index 87b3cb52d..1641e918c 100644
--- a/tests/lexer/tunary_minus.nim
+++ b/tests/lexer/tunary_minus.nim
@@ -45,7 +45,7 @@ template main =
     doAssert lispReprStr([-1]) == """(Bracket (IntLit -1))"""
     doAssert (-1, 2)[0] == minusOne:
       "unable to handle negatives after parenthesis"
-    doAssert lispReprStr((-1, 2)) == """(Par (IntLit -1) (IntLit 2))"""
+    doAssert lispReprStr((-1, 2)) == """(TupleConstr (IntLit -1) (IntLit 2))"""
     proc x(): int =
       var a = 1;-1  # the -1 should act as the return value
     doAssert x() == minusOne:
diff --git a/tests/macros/tdumpast.nim b/tests/macros/tdumpast.nim
index b9d224ab8..b6bcbe8f0 100644
--- a/tests/macros/tdumpast.nim
+++ b/tests/macros/tdumpast.nim
@@ -48,3 +48,34 @@ macro fun3(): untyped =
     int | float | array | seq | object | ptr | pointer | float32
   doAssert n.repr == "int | float | array | seq | object | ptr | pointer | float32", n.repr
 fun3()
+
+macro fun4() =
+  let n = quote do:
+    (a: 1)
+  doAssert n.repr == "(a: 1)", n.repr
+fun4()
+
+# nkTupleConstr vs nkPar tests:
+block: # lispRepr
+  macro lispRepr2(a: untyped): string = newLit a.lispRepr
+
+  doAssert lispRepr2(()) == """(TupleConstr)"""
+  doAssert lispRepr2((a: 1)) == """(TupleConstr (ExprColonExpr (Ident "a") (IntLit 1)))"""
+  doAssert lispRepr2((a: 1, b: 2)) == """(TupleConstr (ExprColonExpr (Ident "a") (IntLit 1)) (ExprColonExpr (Ident "b") (IntLit 2)))"""
+  doAssert lispRepr2((1,)) == """(TupleConstr (IntLit 1))"""
+  doAssert lispRepr2((1, 2)) == """(TupleConstr (IntLit 1) (IntLit 2))"""
+  doAssert lispRepr2((1, 2, 3.0)) == """(TupleConstr (IntLit 1) (IntLit 2) (FloatLit 3.0))"""
+  doAssert lispRepr2((1)) == """(Par (IntLit 1))"""
+  doAssert lispRepr2((1+2)) == """(Par (Infix (Ident "+") (IntLit 1) (IntLit 2)))"""
+
+block: # repr
+  macro repr2(a: untyped): string = newLit a.repr
+
+  doAssert repr2(()) == "()"
+  doAssert repr2((a: 1)) == "(a: 1)"
+  doAssert repr2((a: 1, b: 2)) == "(a: 1, b: 2)"
+  doAssert repr2((1,)) == "(1,)"
+  doAssert repr2((1, 2)) == "(1, 2)"
+  doAssert repr2((1, 2, 3.0)) == "(1, 2, 3.0)"
+  doAssert repr2((1)) == "(1)"
+  doAssert repr2((1+2)) == "(1 + 2)"
diff --git a/tests/parser/tpostexprblocks.nim b/tests/parser/tpostexprblocks.nim
index bc10a72e2..d272c712f 100644
--- a/tests/parser/tpostexprblocks.nim
+++ b/tests/parser/tpostexprblocks.nim
@@ -464,7 +464,7 @@ StmtList
           DiscardStmt
             Empty
       OfBranch
-        Par
+        TupleConstr
           Ident "a"
           Ident "b"
         StmtList
@@ -476,7 +476,7 @@ StmtList
           DiscardStmt
             Empty
       ElifBranch
-        Par
+        TupleConstr
           Ident "a"
           Ident "b"
         StmtList
@@ -488,7 +488,7 @@ StmtList
           DiscardStmt
             Empty
       ExceptBranch
-        Par
+        TupleConstr
           Ident "a"
           Ident "b"
         StmtList
diff --git a/tests/parser/ttypemodifiers.nim b/tests/parser/ttypemodifiers.nim
index 2c322b44b..9a1ccb1a5 100644
--- a/tests/parser/ttypemodifiers.nim
+++ b/tests/parser/ttypemodifiers.nim
@@ -21,7 +21,7 @@ StmtList
       Ident "PtrTuple"
       Empty
       PtrTy
-        Par
+        TupleConstr
           Ident "int"
           Ident "string"
     TypeDef
@@ -43,14 +43,14 @@ StmtList
       Ident "RefTupleType"
       Empty
       RefTy
-        Par
+        TupleConstr
           Ident "int"
           Ident "string"
     TypeDef
       Ident "RefTupleVars"
       Empty
       RefTy
-        Par
+        TupleConstr
           Ident "a"
           Ident "b"
     TypeDef
@@ -80,7 +80,7 @@ StmtList
       Empty
       Command
         Ident "static"
-        Par
+        TupleConstr
           Ident "int"
           Ident "string"
     TypeDef
@@ -155,7 +155,7 @@ StmtList
       Empty
       Command
         Ident "type"
-        Par
+        TupleConstr
           Ident "a"
           Ident "b"
     TypeDef
@@ -163,7 +163,7 @@ StmtList
       Empty
       Command
         Ident "type"
-        Par
+        TupleConstr
           Ident "int"
           Ident "string"
     TypeDef
@@ -287,7 +287,7 @@ StmtList
       IdentDefs
         Ident "refTuple2"
         RefTy
-          Par
+          TupleConstr
             Ident "int"
             Ident "string"
         Empty
@@ -524,4 +524,3 @@ dumpTree:
     static:
       staticStmtList1
       staticStmtList2
-

From 8db93fd0a29dbb950cc14327b31d683c3b0d8926 Mon Sep 17 00:00:00 2001
From: Danil Yarantsev 
Date: Tue, 30 Mar 2021 09:04:17 +0300
Subject: [PATCH 0538/2472] Fix entries in nimc advopt (#17576)

---
 doc/advopt.txt | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/doc/advopt.txt b/doc/advopt.txt
index 108c07222..763ac80a6 100644
--- a/doc/advopt.txt
+++ b/doc/advopt.txt
@@ -43,8 +43,9 @@ Advanced options:
   --warning[X]:on|off       turn specific warning X on|off
   --hints:on|off|list       turn all hints on|off or list all available
   --hint[X]:on|off          turn specific hint X on|off
-  --warningAsError[X]:on|off turn specific warning X into an error on|off
-  --hintAsError[X]:on|off    turn specific hint X into an error on|off
+  --warningAsError[X]:on|off 
+                            turn specific warning X into an error on|off
+  --hintAsError[X]:on|off   turn specific hint X into an error on|off
   --styleCheck:off|hint|error
                             produce hints or errors for Nim identifiers that
                             do not adhere to Nim's official style guide
@@ -83,7 +84,8 @@ Advanced options:
                             if path == @default (the default and most useful), will use
                             best match among @pkg,@path.
                             if these are nonexistent, will use project path
-  -b, --backend:c|cpp|js|objc sets backend to use with commands like `nim doc` or `nim r`
+  -b, --backend:c|cpp|js|objc 
+                            sets backend to use with commands like `nim doc` or `nim r`
   --docCmd:cmd              if `cmd == skip`, skips runnableExamples
                             else, runs runnableExamples with given options, e.g.:
                             `--docCmd:"-d:foo --threads:on"`

From a672ec3c9e25159d3482aebfe1d0bb4271910869 Mon Sep 17 00:00:00 2001
From: Clyybber 
Date: Tue, 30 Mar 2021 08:05:37 +0200
Subject: [PATCH 0539/2472] Fix #17299, fix setAffinity for android (#17574)

* Fix #17299

* Comment

* Fix typo
---
 lib/system/threadlocalstorage.nim | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/lib/system/threadlocalstorage.nim b/lib/system/threadlocalstorage.nim
index 922150fff..cbb74c7df 100644
--- a/lib/system/threadlocalstorage.nim
+++ b/lib/system/threadlocalstorage.nim
@@ -194,8 +194,23 @@ else:
   proc cpusetIncl(cpu: cint; s: var CpuSet) {.
     importc: "CPU_SET", header: schedh.}
 
-  proc setAffinity(thread: SysThread; setsize: csize_t; s: var CpuSet) {.
-    importc: "pthread_setaffinity_np", header: pthreadh.}
+  when defined(android):
+    # libc of android doesn't implement pthread_setaffinity_np,
+    # it exposes pthread_gettid_np though, so we can use that in combination
+    # with sched_setaffinity to set the thread affinity.
+    type Pid {.importc: "pid_t", header: "".} = int32 # From posix_other.nim
+
+    proc setAffinityTID(tid: Pid; setsize: csize_t; s: var CpuSet) {.
+      importc: "sched_setaffinity", header: schedh.}
+
+    proc pthread_gettid_np(thread: SysThread): Pid {.
+      importc: "pthread_gettid_np", header: pthreadh.}
+
+    proc setAffinity(thread: SysThread; setsize: csize_t; s: var CpuSet) =
+      setAffinityTID(pthread_gettid_np(thread), setsize, s)
+  else:
+    proc setAffinity(thread: SysThread; setsize: csize_t; s: var CpuSet) {.
+      importc: "pthread_setaffinity_np", header: pthreadh.}
 
 
 const

From 9e88425d7c5e3edfcbddedbe062c5f028789a002 Mon Sep 17 00:00:00 2001
From: Ardek Romak 
Date: Tue, 30 Mar 2021 09:07:45 +0300
Subject: [PATCH 0540/2472] Fix void async in templates (#17562)

* Fix https://github.com/nim-lang/Nim/issues/16159
* Add test for async in template
* Shorten the expression
* Even shorter

Co-authored-by: Clyybber 
---
 lib/pure/asyncmacro.nim          |  5 +++--
 tests/async/tasyncintemplate.nim | 12 ++++++++++++
 2 files changed, 15 insertions(+), 2 deletions(-)
 create mode 100644 tests/async/tasyncintemplate.nim

diff --git a/lib/pure/asyncmacro.nim b/lib/pure/asyncmacro.nim
index eecec4278..336d10aad 100644
--- a/lib/pure/asyncmacro.nim
+++ b/lib/pure/asyncmacro.nim
@@ -184,8 +184,9 @@ proc asyncSingleProc(prc: NimNode): NimNode =
     verifyReturnType(repr(returnType), returnType)
 
   let subtypeIsVoid = returnType.kind == nnkEmpty or
-        (baseType.kind == nnkIdent and returnType[1].eqIdent("void"))
-
+        (baseType.kind in {nnkIdent, nnkSym} and
+         baseType.eqIdent("void"))
+  
   let futureVarIdents = getFutureVarIdents(prc.params)
 
   var outerProcBody = newNimNode(nnkStmtList, prc.body)
diff --git a/tests/async/tasyncintemplate.nim b/tests/async/tasyncintemplate.nim
new file mode 100644
index 000000000..31f4cd95c
--- /dev/null
+++ b/tests/async/tasyncintemplate.nim
@@ -0,0 +1,12 @@
+discard """
+  output: 42
+"""
+
+import asyncdispatch
+
+template foo() =
+  proc temp(): Future[int] {.async.} = return 42
+  proc tempVoid(): Future[void] {.async.} = echo await temp()
+
+foo()
+waitFor tempVoid()

From 40093b4a937d751ef43b5c2ca447677bb858dff4 Mon Sep 17 00:00:00 2001
From: Miran 
Date: Tue, 30 Mar 2021 10:20:58 +0200
Subject: [PATCH 0541/2472] [backport:1.2] Avoid inlining of newObj and
 newObjRC1 calls (#17582)

This is taken from:
https://github.com/status-im/Nim/commit/af69b3ceae16281efd45cbee4ce1bedd14282304

Full original comment:

This is to avoid heavy inlining happening when two allocation calls
would occur shortly after each other.

This inlining would sometimes be accompanied with an optimisation
as the compiler is able to see that cellToUsr ending the first
allocation call is shortly followed by an usrToCell call. The
pointer arithmetic is redundant and the compiler can eliminate it,
leaving only the cell address in a register (and later the stack)
instead of the actual pointer to the user data, as one would expect.

This combined with a GC collect cycle will cause the stack scan to
only notice the cell address, which is of no good due to a usrToCell
in the gcMark call which shifts that address to an adjacent cell.
This means that the actual cell of importance will not get marked
and thus cause a premature collection of that cell. BOOM.
---
 lib/system/gc.nim | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/system/gc.nim b/lib/system/gc.nim
index ae29f3466..229b26f3c 100644
--- a/lib/system/gc.nim
+++ b/lib/system/gc.nim
@@ -482,7 +482,7 @@ proc newObjNoInit(typ: PNimType, size: int): pointer {.compilerRtl.} =
   result = rawNewObj(typ, size, gch)
   when defined(memProfiler): nimProfile(size)
 
-proc newObj(typ: PNimType, size: int): pointer {.compilerRtl.} =
+proc newObj(typ: PNimType, size: int): pointer {.compilerRtl, noinline.} =
   result = rawNewObj(typ, size, gch)
   zeroMem(result, size)
   when defined(memProfiler): nimProfile(size)
@@ -497,7 +497,7 @@ proc newSeq(typ: PNimType, len: int): pointer {.compilerRtl.} =
   when defined(memProfiler): nimProfile(size)
 {.pop.}
 
-proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} =
+proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl, noinline.} =
   # generates a new object and sets its reference counter to 1
   incTypeSize typ, size
   sysAssert(allocInv(gch.region), "newObjRC1 begin")

From 3715fc41d54ae5c139fba79f1f61419e320e8ba5 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Tue, 30 Mar 2021 16:27:24 +0800
Subject: [PATCH 0542/2472] fix #17512 (#17520)

---
 compiler/ccgcalls.nim | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim
index 64b883087..2a93cb5af 100644
--- a/compiler/ccgcalls.nim
+++ b/compiler/ccgcalls.nim
@@ -259,6 +259,10 @@ proc withTmpIfNeeded(p: BProc, a: TLoc, needsTmp: bool): TLoc =
   else:
     result = a
 
+proc literalsNeedsTmp(p: BProc, a: TLoc): TLoc =
+  getTemp(p, a.lode.typ, result, needsInit=false)
+  genAssignment(p, result, a, {})
+
 proc genArgStringToCString(p: BProc, n: PNode, needsTmp: bool): Rope {.inline.} =
   var a: TLoc
   initLocExpr(p, n[0], a)
@@ -273,7 +277,10 @@ proc genArg(p: BProc, n: PNode, param: PSym; call: PNode, needsTmp = false): Rop
     result = openArrayLoc(p, param.typ, n)
   elif ccgIntroducedPtr(p.config, param, call[0].typ[0]):
     initLocExpr(p, n, a)
-    result = addrLoc(p.config, withTmpIfNeeded(p, a, needsTmp))
+    if n.kind in {nkCharLit..nkNilLit}:
+      result = addrLoc(p.config, literalsNeedsTmp(p, a))
+    else:
+      result = addrLoc(p.config, withTmpIfNeeded(p, a, needsTmp))
   elif p.module.compileToCpp and param.typ.kind in {tyVar} and
       n.kind == nkHiddenAddr:
     initLocExprSingleUse(p, n[0], a)

From f5b1e384a411e1418180ea81bb8fc71bf2feaa26 Mon Sep 17 00:00:00 2001
From: Araq 
Date: Tue, 30 Mar 2021 09:17:21 +0200
Subject: [PATCH 0543/2472] parser.nim: simply layout change

---
 compiler/parser.nim | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/compiler/parser.nim b/compiler/parser.nim
index 83ea4cbdc..d103d161a 100644
--- a/compiler/parser.nim
+++ b/compiler/parser.nim
@@ -1990,7 +1990,8 @@ proc parseObject(p: var Parser): PNode =
     result.add(parseObjectPart(p))
 
 proc parseTypeClassParam(p: var Parser): PNode =
-  let modifier = case p.tok.tokType
+  let modifier =
+    case p.tok.tokType
     of tkOut, tkVar: nkVarTy
     of tkPtr: nkPtrTy
     of tkRef: nkRefTy

From af2a7e8cbc3729ac62be64b5683e4a2187160b2f Mon Sep 17 00:00:00 2001
From: rockcavera 
Date: Tue, 30 Mar 2021 07:24:21 -0300
Subject: [PATCH 0544/2472] Rotation right and left refactored to generics.
 Continuation of #16622 (#17578)

* Rotation right and left refactored to generics

Continuation of [16622](https://github.com/nim-lang/Nim/pull/16622)

* add runnableExamples
---
 lib/pure/bitops.nim | 121 ++++++++++++--------------------------------
 1 file changed, 32 insertions(+), 89 deletions(-)

diff --git a/lib/pure/bitops.nim b/lib/pure/bitops.nim
index 4ef190c1b..b5b08985e 100644
--- a/lib/pure/bitops.nim
+++ b/lib/pure/bitops.nim
@@ -771,7 +771,9 @@ func rotr[T: SomeUnsignedInt](value: T, rot: int32): T {.inline.} =
   let rot = rot and mask
   (value shr rot) or (value shl ((-rot) and mask))
 
-func shiftTypeToImpl(size: static int, shift: int): auto {.inline.} =
+func shiftTypeTo(size: static int, shift: int): auto {.inline.} =
+  ## Returns the `shift` for the rotation according to the compiler and the
+  ## `size`.
   when (defined(vcc) and (size in [4, 8])) or defined(gcc) or defined(icl):
     cint(shift)
   elif (defined(vcc) and (size in [1, 2])) or (defined(clang) and size == 1):
@@ -784,118 +786,59 @@ func shiftTypeToImpl(size: static int, shift: int): auto {.inline.} =
     elif size == 8:
       culonglong(shift)
 
-template shiftTypeTo[T](value: T, shift: int): auto =
-  ## Returns the `shift` for the rotation according to the compiler and the size
-  ## of the` value`.
-  shiftTypeToImpl(sizeof(value), shift)
-
-func rotateLeftBits*(value: uint8, shift: range[0..8]): uint8 {.inline.} =
-  ## Left-rotate bits in a 8-bits value.
+func rotateLeftBits*[T: SomeUnsignedInt](value: T, shift: range[0..(sizeof(T) * 8)]): T {.inline.} =
+  ## Left-rotate bits in a `value`.
   runnableExamples:
     doAssert rotateLeftBits(0b0110_1001'u8, 4) == 0b1001_0110'u8
-
-  when nimvm:
-    rotl(value, shift.int32)
-  else:
-    when useBuiltinsRotate:
-      builtin_rotl8(value.cuchar, shiftTypeTo(value, shift)).uint8
-    else:
-      rotl(value, shift.int32)
-
-func rotateLeftBits*(value: uint16, shift: range[0..16]): uint16 {.inline.} =
-  ## Left-rotate bits in a 16-bits value.
-  runnableExamples:
     doAssert rotateLeftBits(0b00111100_11000011'u16, 8) ==
       0b11000011_00111100'u16
-
-  when nimvm:
-    rotl(value, shift.int32)
-  else:
-    when useBuiltinsRotate:
-      builtin_rotl16(value.cushort, shiftTypeTo(value, shift)).uint16
-    else:
-      rotl(value, shift.int32)
-
-func rotateLeftBits*(value: uint32, shift: range[0..32]): uint32 {.inline.} =
-  ## Left-rotate bits in a 32-bits value.
-  runnableExamples:
     doAssert rotateLeftBits(0b0000111111110000_1111000000001111'u32, 16) ==
       0b1111000000001111_0000111111110000'u32
-
-  when nimvm:
-    rotl(value, shift.int32)
-  else:
-    when useBuiltinsRotate:
-      builtin_rotl32(value.cuint, shiftTypeTo(value, shift)).uint32
-    else:
-      rotl(value, shift.int32)
-
-func rotateLeftBits*(value: uint64, shift: range[0..64]): uint64 {.inline.} =
-  ## Left-rotate bits in a 64-bits value.
-  runnableExamples:
     doAssert rotateLeftBits(0b00000000111111111111111100000000_11111111000000000000000011111111'u64, 32) ==
       0b11111111000000000000000011111111_00000000111111111111111100000000'u64
-
   when nimvm:
     rotl(value, shift.int32)
   else:
-    when useBuiltinsRotate and defined(amd64):
-      builtin_rotl64(value.culonglong, shiftTypeTo(value, shift)).uint64
+    when useBuiltinsRotate:
+      const size = sizeof(T)
+      when size == 1:
+        builtin_rotl8(value.cuchar, shiftTypeTo(size, shift)).T
+      elif size == 2:
+        builtin_rotl16(value.cushort, shiftTypeTo(size, shift)).T
+      elif size == 4:
+        builtin_rotl32(value.cuint, shiftTypeTo(size, shift)).T
+      elif size == 8 and arch64:
+        builtin_rotl64(value.culonglong, shiftTypeTo(size, shift)).T
+      else:
+        rotl(value, shift.int32)
     else:
       rotl(value, shift.int32)
 
-func rotateRightBits*(value: uint8, shift: range[0..8]): uint8 {.inline.} =
-  ## Right-rotate bits in a 8-bits value.
+func rotateRightBits*[T: SomeUnsignedInt](value: T, shift: range[0..(sizeof(T) * 8)]): T {.inline.} =
+  ## Right-rotate bits in a `value`.
   runnableExamples:
     doAssert rotateRightBits(0b0110_1001'u8, 4) == 0b1001_0110'u8
-
-  when nimvm:
-    rotr(value, shift.int32)
-  else:
-    when useBuiltinsRotate:
-      builtin_rotr8(value.cuchar, shiftTypeTo(value, shift)).uint8
-    else:
-      rotr(value, shift.int32)
-
-func rotateRightBits*(value: uint16, shift: range[0..16]): uint16 {.inline.} =
-  ## Right-rotate bits in a 16-bits value.
-  runnableExamples:
     doAssert rotateRightBits(0b00111100_11000011'u16, 8) ==
       0b11000011_00111100'u16
-
-  when nimvm:
-    rotr(value, shift.int32)
-  else:
-    when useBuiltinsRotate:
-      builtin_rotr16(value.cushort, shiftTypeTo(value, shift)).uint16
-    else:
-      rotr(value, shift.int32)
-
-func rotateRightBits*(value: uint32, shift: range[0..32]): uint32 {.inline.} =
-  ## Right-rotate bits in a 32-bits value.
-  runnableExamples:
     doAssert rotateRightBits(0b0000111111110000_1111000000001111'u32, 16) ==
       0b1111000000001111_0000111111110000'u32
-
+    doAssert rotateRightBits(0b00000000111111111111111100000000_11111111000000000000000011111111'u64, 32) ==
+      0b11111111000000000000000011111111_00000000111111111111111100000000'u64
   when nimvm:
     rotr(value, shift.int32)
   else:
     when useBuiltinsRotate:
-      builtin_rotr32(value.cuint, shiftTypeTo(value, shift)).uint32
-    else:
-      rotr(value, shift.int32)
-
-func rotateRightBits*(value: uint64, shift: range[0..64]): uint64 {.inline.} =
-  ## Right-rotate bits in a 64-bits value.
-  runnableExamples:
-    doAssert rotateRightBits(0b00000000111111111111111100000000_11111111000000000000000011111111'u64, 32) ==
-      0b11111111000000000000000011111111_00000000111111111111111100000000'u64
-
-  when nimvm:
-    rotr(value, shift.int32)
-  else:
-    when useBuiltinsRotate and defined(amd64):
-      builtin_rotr64(value.culonglong, shiftTypeTo(value, shift)).uint64
+      const size = sizeof(T)
+      when size == 1:
+        builtin_rotr8(value.cuchar, shiftTypeTo(size, shift)).T
+      elif size == 2:
+        builtin_rotr16(value.cushort, shiftTypeTo(size, shift)).T
+      elif size == 4:
+        builtin_rotr32(value.cuint, shiftTypeTo(size, shift)).T
+      elif size == 8 and arch64:
+        builtin_rotr64(value.culonglong, shiftTypeTo(size, shift)).T
+      else:
+        rotr(value, shift.int32)
     else:
       rotr(value, shift.int32)
 

From 5ecbe6737110bd3fb9a85c2582409483fb5494b0 Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Tue, 30 Mar 2021 20:13:18 +0200
Subject: [PATCH 0545/2472] makes DrNim compile again (#17584)

---
 compiler/modulegraphs.nim |  3 ++-
 drnim/drnim.nim           | 29 +++++++++++++++++------------
 2 files changed, 19 insertions(+), 13 deletions(-)

diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim
index e6f5267cd..4430e3baa 100644
--- a/compiler/modulegraphs.nim
+++ b/compiler/modulegraphs.nim
@@ -393,8 +393,9 @@ proc registerModule*(g: ModuleGraph; m: PSym) =
 proc registerModuleById*(g: ModuleGraph; m: FileIndex) =
   registerModule(g, g.packed[int m].module)
 
-proc initOperators(g: ModuleGraph): Operators =
+proc initOperators*(g: ModuleGraph): Operators =
   # These are safe for IC.
+  # Public because it's used by DrNim.
   result.opLe = createMagic(g, "<=", mLeI)
   result.opLt = createMagic(g, "<", mLtI)
   result.opAnd = createMagic(g, "and", mAnd)
diff --git a/drnim/drnim.nim b/drnim/drnim.nim
index 44eab8625..d549e1d5b 100644
--- a/drnim/drnim.nim
+++ b/drnim/drnim.nim
@@ -68,6 +68,7 @@ type
   DrnimContext = ref object
     z3: Z3_context
     graph: ModuleGraph
+    idgen: IdGenerator
     facts: seq[(PNode, VersionScope)]
     varVersions: seq[int] # this maps variable IDs to their current version.
     varSyms: seq[PSym] # mirrors 'varVersions'
@@ -80,6 +81,7 @@ type
 
   DrCon = object
     graph: ModuleGraph
+    idgen: IdGenerator
     mapping: Table[string, Z3_ast]
     canonParameterNames: bool
     assumeUniqueness: bool
@@ -301,12 +303,12 @@ proc nodeToZ3(c: var DrCon; n: PNode; scope: VersionScope; vars: var seq[PNode])
 
 proc nodeToDomain(c: var DrCon; n, q: PNode; opAnd: PSym): PNode =
   assert n.kind == nkInfix
-  let opLe = createMagic(c.graph, "<=", mLeI)
+  let opLe = createMagic(c.graph, c.idgen, "<=", mLeI)
   case $n[0]
   of "..":
     result = buildCall(opAnd, buildCall(opLe, n[1], q), buildCall(opLe, q, n[2]))
   of "..<":
-    let opLt = createMagic(c.graph, "<", mLtI)
+    let opLt = createMagic(c.graph, c.idgen, "<", mLtI)
     result = buildCall(opAnd, buildCall(opLe, n[1], q), buildCall(opLt, q, n[2]))
   else:
     notImplemented(n)
@@ -315,7 +317,7 @@ template quantorToZ3(fn) {.dirty.} =
   template ctx: untyped = c.up.z3
 
   var bound = newSeq[Z3_app](n.len-2)
-  let opAnd = createMagic(c.graph, "and", mAnd)
+  let opAnd = createMagic(c.graph, c.idgen, "and", mAnd)
   var known: PNode
   for i in 1..n.len-2:
     let it = n[i]
@@ -333,7 +335,7 @@ template quantorToZ3(fn) {.dirty.} =
 
   var dummy: seq[PNode]
   assert known != nil
-  let x = nodeToZ3(c, buildCall(createMagic(c.graph, "->", mImplies),
+  let x = nodeToZ3(c, buildCall(createMagic(c.graph, c.idgen, "->", mImplies),
                    known, n[^1]), scope, dummy)
   result = fn(ctx, 0, bound.len.cuint, addr(bound[0]), 0, nil, x)
 
@@ -395,8 +397,8 @@ proc nodeToZ3(c: var DrCon; n: PNode; scope: VersionScope; vars: var seq[PNode])
       else:
         notImplemented(n)
     of mHigh:
-      let addOpr = createMagic(c.graph, "+", mAddI)
-      let lenOpr = createMagic(c.graph, "len", mLengthOpenArray)
+      let addOpr = createMagic(c.graph, c.idgen, "+", mAddI)
+      let lenOpr = createMagic(c.graph, c.idgen, "len", mLengthOpenArray)
       let asLenExpr = addOpr.buildCall(lenOpr.buildCall(n[1]), nkIntLit.newIntNode(-1))
       result = rec asLenExpr
     of mLow:
@@ -577,8 +579,8 @@ proc addRangeInfo(c: var DrCon, n: PNode; scope: VersionScope, res: var seq[Z3_a
         res.add nodeToZ3(c, translateEnsures(ensures, n), scope, dummy)
     return
 
-  let x = newTree(nkInfix, newSymNode createMagic(c.graph, "<=", cmpOp), lowBound, n)
-  let y = newTree(nkInfix, newSymNode createMagic(c.graph, "<=", cmpOp), n, highBound)
+  let x = newTree(nkInfix, newSymNode createMagic(c.graph, c.idgen, "<=", cmpOp), lowBound, n)
+  let y = newTree(nkInfix, newSymNode createMagic(c.graph, c.idgen, "<=", cmpOp), n, highBound)
 
   var dummy: seq[PNode]
   res.add nodeToZ3(c, x, scope, dummy)
@@ -677,6 +679,7 @@ proc proofEngine(ctx: DrnimContext; assumptions: seq[(PNode, VersionScope)];
                  toProve: (PNode, VersionScope)): (bool, string) =
   var c: DrCon
   c.graph = ctx.graph
+  c.idgen = ctx.idgen
   c.assumeUniqueness = assumeUniqueness
   c.up = ctx
   result = proofEngineAux(c, assumptions, toProve)
@@ -738,10 +741,11 @@ proc compatibleProps(graph: ModuleGraph; formal, actual: PType): bool {.nimcall.
 
       var c: DrCon
       c.graph = graph
+      c.idgen = graph.idgen
       c.canonParameterNames = true
       try:
         c.up = DrnimContext(z3: setupZ3(), o: initOperators(graph), graph: graph, owner: nil,
-          opImplies: createMagic(graph, "->", mImplies))
+          opImplies: createMagic(graph, c.idgen, "->", mImplies))
         template zero: untyped = VersionScope(0)
         if not frequires.isEmpty:
           result = not arequires.isEmpty and proofEngineAux(c, @[(frequires, zero)], (arequires, zero))[0]
@@ -847,7 +851,7 @@ proc checkLe(c: DrnimContext, a, b: PNode) =
     of tyChar, tyUInt..tyUInt64: cmpOp = mLeU
     else: discard
 
-  let cmp = newTree(nkInfix, newSymNode createMagic(c.graph, "<=", cmpOp), a, b)
+  let cmp = newTree(nkInfix, newSymNode createMagic(c.graph, c.idgen, "<=", cmpOp), a, b)
   cmp.info = a.info
   discard prove(c, cmp)
 
@@ -1026,7 +1030,7 @@ proc traverseAsgn(c: DrnimContext; n: PNode) =
   proc replaceByOldParams(fact, le: PNode): PNode =
     if guards.sameTree(fact, le):
       result = newNodeIT(nkCall, fact.info, fact.typ)
-      result.add newSymNode createMagic(c.graph, "old", mOld)
+      result.add newSymNode createMagic(c.graph, c.idgen, "old", mOld)
       result.add fact
     else:
       result = shallowCopy(fact)
@@ -1176,8 +1180,9 @@ proc strongSemCheck(graph: ModuleGraph; owner: PSym; n: PNode) =
     c.z3 = setupZ3()
     c.o = initOperators(graph)
     c.graph = graph
+    c.idgen = graph.idgen
     c.owner = owner
-    c.opImplies = createMagic(c.graph, "->", mImplies)
+    c.opImplies = createMagic(c.graph, c.idgen, "->", mImplies)
     try:
       traverse(c, n)
       ensuresCheck(c, owner)

From 72988509ba0859a5bdfb43a78273c2de26c3d6dc Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Tue, 30 Mar 2021 11:14:39 -0700
Subject: [PATCH 0546/2472] add getPort to resolve Port(0) (#17559)

* add getPort to resolve Port(0)

* fixup

* use getPort in examples + tests

* address comments: do not re-export Port
---
 changelog.md                                  |  2 +
 lib/pure/asynchttpserver.nim                  | 42 ++++++++++++-------
 testament/lib/stdtest/netutils.nim            |  3 +-
 .../tasynchttpserver_transferencoding.nim     |  3 +-
 4 files changed, 32 insertions(+), 18 deletions(-)

diff --git a/changelog.md b/changelog.md
index 7452e3182..c234bf174 100644
--- a/changelog.md
+++ b/changelog.md
@@ -71,6 +71,8 @@
 - Added `asyncdispatch.activeDescriptors` that returns the number of currently
   active async event handles/file descriptors.
 
+- Added to `asynchttpserver` `getPort` and `getSocket`.
+
 - `--gc:orc` is now 10% faster than previously for common workloads. If
   you have trouble with its changed behavior, compile with `-d:nimOldOrc`.
 
diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim
index 38be4ceac..ec464682e 100644
--- a/lib/pure/asynchttpserver.nim
+++ b/lib/pure/asynchttpserver.nim
@@ -15,20 +15,20 @@
 ## instead of allowing users to connect directly to this server.
 
 runnableExamples("-r:off"):
-  # This example will create an HTTP server on port 8080. The server will
-  # respond to all requests with a `200 OK` response code and "Hello World"
+  # This example will create an HTTP server on an automatically chosen port.
+  # It will respond to all requests with a `200 OK` response code and "Hello World"
   # as the response body.
   import std/asyncdispatch
   proc main {.async.} =
-    const port = 8080
     var server = newAsyncHttpServer()
     proc cb(req: Request) {.async.} =
       echo (req.reqMethod, req.url, req.headers)
       let headers = {"Content-type": "text/plain; charset=utf-8"}
       await req.respond(Http200, "Hello World", headers.newHttpHeaders())
 
-    echo "test this with: curl localhost:" & $port & "/"
-    server.listen(Port(port))
+    server.listen(Port(0)) # or Port(8080) to hardcode the standard HTTP port.
+    let port = server.getPort
+    echo "test this with: curl localhost:" & $port.uint16 & "/"
     while true:
       if server.shouldAcceptRequest():
         await server.acceptRequest(cb)
@@ -41,6 +41,7 @@ runnableExamples("-r:off"):
 
 import asyncnet, asyncdispatch, parseutils, uri, strutils
 import httpcore
+from nativesockets import getLocalAddr, AF_INET
 import std/private/since
 
 export httpcore except parseHeader
@@ -70,21 +71,32 @@ type
     maxBody: int ## The maximum content-length that will be read for the body.
     maxFDs: int
 
-func getSocket*(a: AsyncHttpServer): AsyncSocket {.since: (1, 5, 1).} =
-  ## Returns the `AsyncHttpServer`s internal `AsyncSocket` instance.
-  ## 
-  ## Useful for identifying what port the AsyncHttpServer is bound to, if it
-  ## was chosen automatically.
+proc getPort*(self: AsyncHttpServer): Port {.since: (1, 5, 1).} =
+  ## Returns the port `self` was bound to.
+  ##
+  ## Useful for identifying what port `self` is bound to, if it
+  ## was chosen automatically, for example via `listen(Port(0))`.
+  runnableExamples:
+    from std/nativesockets import Port
+    let server = newAsyncHttpServer()
+    server.listen(Port(0))
+    assert server.getPort.uint16 > 0
+    server.close()
+  result = getLocalAddr(self.socket.getFd, AF_INET)[1]
+
+func getSocket*(self: AsyncHttpServer): AsyncSocket {.since: (1, 5, 1).} =
+  ## Field accessor.
   runnableExamples:
-    from std/asyncdispatch import Port
     from std/asyncnet import getFd
-    from std/nativesockets import getLocalAddr, AF_INET
+    from std/nativesockets import getLocalAddr, AF_INET, Port
     let server = newAsyncHttpServer()
     server.listen(Port(0)) # Socket is not bound until this point
-    let port = getLocalAddr(server.getSocket.getFd, AF_INET)[1]
-    doAssert uint16(port) > 0
+    # note: a more direct way to get the port is `getPort`.
+    let (laddr, port) = getLocalAddr(server.getSocket.getFd, AF_INET)
+    assert uint16(port) > 0
+    assert laddr == "0.0.0.0"
     server.close()
-  a.socket
+  self.socket
 
 proc newAsyncHttpServer*(reuseAddr = true, reusePort = false,
                          maxBody = 8388608): AsyncHttpServer =
diff --git a/testament/lib/stdtest/netutils.nim b/testament/lib/stdtest/netutils.nim
index eb913a56a..5115390e0 100644
--- a/testament/lib/stdtest/netutils.nim
+++ b/testament/lib/stdtest/netutils.nim
@@ -1,6 +1,7 @@
 import std/[nativesockets, asyncdispatch, os]
 
 proc bindAvailablePort*(handle: SocketHandle, port = Port(0)): Port =
+  ## See also `asynchttpserver.getPort`.
   block:
     var name: Sockaddr_in
     name.sin_family = typeof(name.sin_family)(toInt(AF_INET))
@@ -8,5 +9,5 @@ proc bindAvailablePort*(handle: SocketHandle, port = Port(0)): Port =
     name.sin_addr.s_addr = htonl(INADDR_ANY)
     if bindAddr(handle, cast[ptr SockAddr](addr(name)),
                 sizeof(name).Socklen) < 0'i32:
-      raiseOSError(osLastError())
+      raiseOSError(osLastError(), $port)
   result = getLocalAddr(handle, AF_INET)[1]
diff --git a/tests/stdlib/tasynchttpserver_transferencoding.nim b/tests/stdlib/tasynchttpserver_transferencoding.nim
index 34f3cef11..bfa0e2f69 100644
--- a/tests/stdlib/tasynchttpserver_transferencoding.nim
+++ b/tests/stdlib/tasynchttpserver_transferencoding.nim
@@ -27,10 +27,9 @@ template genTest(input, expected) =
 
   let server = newAsyncHttpServer()
   waitFor runSleepLoop(server)
-  let port = getLocalAddr(server.getSocket.getFd, AF_INET)[1]
   let data = postBegin & input
   var socket = newSocket()
-  socket.connect("127.0.0.1", port)
+  socket.connect("127.0.0.1", server.getPort)
   socket.send(data)
   waitFor sleepAsync(10)
   socket.close()

From 7c09e0c75773a2df6469a2acd94f3090aef83255 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Wed, 31 Mar 2021 02:15:14 +0800
Subject: [PATCH 0547/2472] fix #15617(fix compilation failure on -d:useMalloc
 --gc:none) (#17555)

Co-authored-by: Andreas Rumpf 
Co-authored-by: Timothee Cour 
---
 lib/system/mmdisp.nim    | 10 ++++++++++
 tests/system/tgcnone.nim |  6 ++++++
 2 files changed, 16 insertions(+)
 create mode 100644 tests/system/tgcnone.nim

diff --git a/lib/system/mmdisp.nim b/lib/system/mmdisp.nim
index 96dc86bfd..3317ba627 100644
--- a/lib/system/mmdisp.nim
+++ b/lib/system/mmdisp.nim
@@ -57,6 +57,16 @@ elif defined(gogc):
 elif (defined(nogc) or defined(gcDestructors)) and defined(useMalloc):
   include system / mm / malloc
 
+  when defined(nogc):
+    proc GC_getStatistics(): string = ""
+    proc newObj(typ: PNimType, size: int): pointer {.compilerproc.} =
+      result = alloc0(size)
+
+    proc newSeq(typ: PNimType, len: int): pointer {.compilerproc.} =
+      result = newObj(typ, align(GenericSeqSize, typ.align) + len * typ.base.size)
+      cast[PGenericSeq](result).len = len
+      cast[PGenericSeq](result).reserved = len
+
 elif defined(nogc):
   include system / mm / none
 
diff --git a/tests/system/tgcnone.nim b/tests/system/tgcnone.nim
new file mode 100644
index 000000000..700176d5f
--- /dev/null
+++ b/tests/system/tgcnone.nim
@@ -0,0 +1,6 @@
+discard """
+  matrix: "--gc:none -d:useMalloc"
+"""
+# bug #15617
+let x = 4
+doAssert x == 4

From 8ee0eda84156c8d9074b7a42cdedc209cffdc4a4 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Wed, 31 Mar 2021 01:15:08 -0700
Subject: [PATCH 0548/2472] fix #12282 distinct now does not create erroneous
 copy in VM (#17594)

---
 compiler/vmgen.nim           | 14 ++++++++++++--
 tests/distinct/tdistinct.nim | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+), 2 deletions(-)

diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim
index bd80df219..f6e385976 100644
--- a/compiler/vmgen.nim
+++ b/compiler/vmgen.nim
@@ -833,10 +833,20 @@ proc genAddSubInt(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode) =
   c.genNarrow(n, dest)
 
 proc genConv(c: PCtx; n, arg: PNode; dest: var TDest; opc=opcConv) =
-  if n.typ.kind == arg.typ.kind and arg.typ.kind == tyProc:
-    # don't do anything for lambda lifting conversions:
+  let t2 = n.typ.skipTypes({tyDistinct})
+  let targ2 = arg.typ.skipTypes({tyDistinct})
+
+  proc implicitConv(): bool =
+    if sameType(t2, targ2): return true
+    # xxx consider whether to use t2 and targ2 here
+    if n.typ.kind == arg.typ.kind and arg.typ.kind == tyProc:
+      # don't do anything for lambda lifting conversions:
+      return true
+
+  if implicitConv():
     gen(c, arg, dest)
     return
+
   let tmp = c.genx(arg)
   if dest < 0: dest = c.getTemp(n.typ)
   c.gABC(n, opc, dest, tmp)
diff --git a/tests/distinct/tdistinct.nim b/tests/distinct/tdistinct.nim
index d64f33443..876975a7c 100644
--- a/tests/distinct/tdistinct.nim
+++ b/tests/distinct/tdistinct.nim
@@ -1,4 +1,5 @@
 discard """
+  targets: "c js"
   output: '''
 tdistinct
 25
@@ -138,3 +139,34 @@ block tRequiresInit:
   accept:
     let s = "test"
     doAssert s == "test"
+
+type Foo = distinct string
+
+template main() =
+  # xxx put everything here to test under RT + VM
+  block: # bug #12282
+    block:
+      proc test() =
+        var s: Foo
+        s.string.add('c')
+        doAssert s.string == "c" # was failing
+      test()
+
+    block:
+      proc add(a: var Foo, b: char) {.borrow.}
+      proc test() =
+        var s: Foo
+        s.add('c')
+        doAssert s.string == "c" # was ok
+      test()
+
+    block:
+      proc add(a: var Foo, b: char) {.borrow.}
+      proc test() =
+        var s: string
+        s.Foo.add('c')
+        doAssert s.string == "c" # was failing
+      test()
+
+static: main()
+main()

From 6d7d1e60fea201109bdafce550622b5b49e23323 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Wed, 31 Mar 2021 02:25:26 -0700
Subject: [PATCH 0549/2472] fix #14585, fix #17589: access to static param now
 works (#17590)

---
 compiler/types.nim            |  2 +-
 compiler/vmgen.nim            |  2 +-
 tests/statictypes/tstatic.nim | 56 +++++++++++++++++++++++++++++++++++
 3 files changed, 58 insertions(+), 2 deletions(-)
 create mode 100644 tests/statictypes/tstatic.nim

diff --git a/compiler/types.nim b/compiler/types.nim
index a0d43ec09..1dbbc3261 100644
--- a/compiler/types.nim
+++ b/compiler/types.nim
@@ -67,7 +67,7 @@ const
                     tyAlias, tyInferred, tySink, tyOwned}
   # see also ast.abstractVarRange
   abstractInst* = {tyGenericInst, tyDistinct, tyOrdinal, tyTypeDesc, tyAlias,
-                   tyInferred, tySink, tyOwned}
+                   tyInferred, tySink, tyOwned} # xxx what about tyStatic?
   abstractInstOwned* = abstractInst + {tyOwned}
   skipPtrs* = {tyVar, tyPtr, tyRef, tyGenericInst, tyTypeDesc, tyAlias,
                tyInferred, tySink, tyLent, tyOwned}
diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim
index f6e385976..be26f98b3 100644
--- a/compiler/vmgen.nim
+++ b/compiler/vmgen.nim
@@ -980,7 +980,7 @@ proc genBindSym(c: PCtx; n: PNode; dest: var TDest) =
 
 proc fitsRegister*(t: PType): bool =
   assert t != nil
-  t.skipTypes(abstractInst-{tyTypeDesc}).kind in {
+  t.skipTypes(abstractInst + {tyStatic} - {tyTypeDesc}).kind in {
     tyRange, tyEnum, tyBool, tyInt..tyUInt64, tyChar}
 
 proc ldNullOpcode(t: PType): TOpcode =
diff --git a/tests/statictypes/tstatic.nim b/tests/statictypes/tstatic.nim
new file mode 100644
index 000000000..aaa63534b
--- /dev/null
+++ b/tests/statictypes/tstatic.nim
@@ -0,0 +1,56 @@
+discard """
+  targets: "c cpp js"
+"""
+
+template main() =
+  block: # bug #17589
+    #[
+    # all those gave some variation of the same bug:
+    'intVal' is not accessible using discriminant 'kind' of type 'TFullReg'
+    'floatVal' is not accessible using discriminant 'kind' of type 'TFullReg'
+    ]#
+    block:
+      proc f(a: static uint64): uint64 =
+        a
+      const x = 3'u64
+      static: doAssert f(x) == 3'u64
+      doAssert f(x) == 3'u64
+
+    block:
+      proc f(a: static uint64): uint64 =
+        a
+      const x = 3'u64
+      static: doAssert f(x) == 3'u64
+      doAssert f(x) == 3'u64
+
+    block:
+      proc foo(x: uint8): uint8 = x
+      proc f(a: static uint8): auto = foo(a)
+      const x = 3'u8
+      static: doAssert f(x) == 3'u8
+      doAssert f(x) == 3'u8
+
+    block:
+      template foo2(x: float) =
+        let b = x == 0
+      proc foo(x: float) = foo2(x)
+      proc f(a: static float) = foo(a)
+      const x = 1.0
+      static: f(x)
+
+    block:
+      proc foo(x: int32) =
+        let b = x == 0
+      proc f(a: static int32) = foo(a)
+      static: f(32767) # was working
+      static: f(32768) # was failing because >= int16.high (see isInt16Lit)
+
+  block: # bug #14585
+    const foo_m0ninv = 0x1234'u64
+    proc foo(m0ninv: static uint64) =
+      let b = $m0ninv
+    static:
+      foo(foo_m0ninv)
+
+static: main()
+main()

From b18307f94060d2f672a83d72c557f3856368b73a Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Wed, 31 Mar 2021 02:27:02 -0700
Subject: [PATCH 0550/2472] fix #17572 (#17586)

---
 compiler/vm.nim              |  3 ++-
 tests/misc/tunsignedconv.nim | 16 ++++++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/compiler/vm.nim b/compiler/vm.nim
index baaf0f14e..e3ba2994f 100644
--- a/compiler/vm.nim
+++ b/compiler/vm.nim
@@ -415,7 +415,8 @@ proc opConv(c: PCtx; dest: var TFullReg, src: TFullReg, desttyp, srctyp: PType):
     else:
       internalError(c.config, "cannot convert to string " & desttyp.typeToString)
   else:
-    case skipTypes(desttyp, abstractVarRange).kind
+    let desttyp = skipTypes(desttyp, abstractVarRange)
+    case desttyp.kind
     of tyInt..tyInt64:
       dest.ensureKind(rkInt)
       case skipTypes(srctyp, abstractRange).kind
diff --git a/tests/misc/tunsignedconv.nim b/tests/misc/tunsignedconv.nim
index b9463b5f0..0acb39106 100644
--- a/tests/misc/tunsignedconv.nim
+++ b/tests/misc/tunsignedconv.nim
@@ -79,3 +79,19 @@ except RangeDefect:
   success = true
 
 doAssert success, "conversion should fail at runtime"
+
+template main() =
+  # xxx move all tests under here so it gets tested in CT and RT
+  block: # bug #17572
+    type T = distinct uint64
+    func f(x: uint64): auto =
+      let a = T(x)
+      (x, a.uint64)
+    const x = 1'u64 shl 63 or 7
+    const b = T(x)
+    doAssert b.uint64 == 9223372036854775815'u64
+    doAssert $b.uint64 == "9223372036854775815"
+    doAssert f(x) == (9223372036854775815'u64, 9223372036854775815'u64)
+
+static: main()
+main()

From ece714773491de1757d9afbda6d499de9b610b31 Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Wed, 31 Mar 2021 13:26:44 +0200
Subject: [PATCH 0551/2472] better error messages for .raise effect analysis
 (#17595)

---
 compiler/sempass2.nim | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim
index a1572c85e..87d196fad 100644
--- a/compiler/sempass2.nim
+++ b/compiler/sempass2.nim
@@ -304,8 +304,8 @@ proc addToIntersection(inter: var TIntersection, s: int) =
 proc throws(tracked, n, orig: PNode) =
   if n.typ == nil or n.typ.kind != tyError:
     if orig != nil:
-      let x = copyNode(n)
-      x.info = orig.info
+      let x = copyTree(orig)
+      x.typ = n.typ
       tracked.add x
     else:
       tracked.add n
@@ -329,7 +329,7 @@ proc createTag(g: ModuleGraph; n: PNode): PNode =
   if not n.isNil: result.info = n.info
 
 proc addRaiseEffect(a: PEffects, e, comesFrom: PNode) =
-  assert e.kind != nkRaiseStmt
+  #assert e.kind != nkRaiseStmt
   var aa = a.exc
   for i in a.bottom.. 0: rr = rr.lastSon
+      localError(g.config, r.info, errGenerated, renderTree(rr) & " " & msg & typeToString(r.typ))
       popInfoContext(g.config)
   # hint about unnecessarily listed exception types:
   if hints:

From d3529d0d5afb4d30bd7bdcbc49ff77fd5e67ed39 Mon Sep 17 00:00:00 2001
From: narimiran 
Date: Wed, 31 Mar 2021 13:34:32 +0200
Subject: [PATCH 0552/2472] [ci skip] minor: fix indentation in manual.rst

---
 doc/manual.rst | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/doc/manual.rst b/doc/manual.rst
index fdc744bc7..1fe554437 100644
--- a/doc/manual.rst
+++ b/doc/manual.rst
@@ -6156,12 +6156,12 @@ avoid ambiguity when there are multiple modules with the same path.
 There are two pseudo directories:
 
 1. `std`: The `std` pseudo directory is the abstract location of Nim's standard
-library. For example, the syntax `import std / strutils` is used to unambiguously
-refer to the standard library's `strutils` module.
+    library. For example, the syntax `import std / strutils` is used to unambiguously
+    refer to the standard library's `strutils` module.
 2. `pkg`: The `pkg` pseudo directory is used to unambiguously refer to a Nimble
-package. However, for technical details that lie outside the scope of this document,
-its semantics are: *Use the search path to look for module name but ignore the standard
-library locations*. In other words, it is the opposite of `std`.
+    package. However, for technical details that lie outside the scope of this document,
+    its semantics are: *Use the search path to look for module name but ignore the standard
+    library locations*. In other words, it is the opposite of `std`.
 
 
 From import statement

From 0be7f854ed0327c3c2c75e682eddab2b502538bb Mon Sep 17 00:00:00 2001
From: narimiran 
Date: Wed, 31 Mar 2021 13:45:14 +0200
Subject: [PATCH 0553/2472] [ci skip] correct indentation this time

---
 doc/manual.rst | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/doc/manual.rst b/doc/manual.rst
index 1fe554437..b1aae2669 100644
--- a/doc/manual.rst
+++ b/doc/manual.rst
@@ -6156,12 +6156,12 @@ avoid ambiguity when there are multiple modules with the same path.
 There are two pseudo directories:
 
 1. `std`: The `std` pseudo directory is the abstract location of Nim's standard
-    library. For example, the syntax `import std / strutils` is used to unambiguously
-    refer to the standard library's `strutils` module.
+   library. For example, the syntax `import std / strutils` is used to unambiguously
+   refer to the standard library's `strutils` module.
 2. `pkg`: The `pkg` pseudo directory is used to unambiguously refer to a Nimble
-    package. However, for technical details that lie outside the scope of this document,
-    its semantics are: *Use the search path to look for module name but ignore the standard
-    library locations*. In other words, it is the opposite of `std`.
+   package. However, for technical details that lie outside the scope of this document,
+   its semantics are: *Use the search path to look for module name but ignore the standard
+   library locations*. In other words, it is the opposite of `std`.
 
 
 From import statement

From 65efa727ffec223fd9e9fa8efcff38246f7b9ad4 Mon Sep 17 00:00:00 2001
From: Dominik Picheta 
Date: Wed, 31 Mar 2021 17:20:42 +0100
Subject: [PATCH 0554/2472] Removes asynchttpserver.getSocket. (#17587)

---
 lib/pure/asynchttpserver.nim | 14 --------------
 1 file changed, 14 deletions(-)

diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim
index ec464682e..3982edc54 100644
--- a/lib/pure/asynchttpserver.nim
+++ b/lib/pure/asynchttpserver.nim
@@ -84,20 +84,6 @@ proc getPort*(self: AsyncHttpServer): Port {.since: (1, 5, 1).} =
     server.close()
   result = getLocalAddr(self.socket.getFd, AF_INET)[1]
 
-func getSocket*(self: AsyncHttpServer): AsyncSocket {.since: (1, 5, 1).} =
-  ## Field accessor.
-  runnableExamples:
-    from std/asyncnet import getFd
-    from std/nativesockets import getLocalAddr, AF_INET, Port
-    let server = newAsyncHttpServer()
-    server.listen(Port(0)) # Socket is not bound until this point
-    # note: a more direct way to get the port is `getPort`.
-    let (laddr, port) = getLocalAddr(server.getSocket.getFd, AF_INET)
-    assert uint16(port) > 0
-    assert laddr == "0.0.0.0"
-    server.close()
-  self.socket
-
 proc newAsyncHttpServer*(reuseAddr = true, reusePort = false,
                          maxBody = 8388608): AsyncHttpServer =
   ## Creates a new `AsyncHttpServer` instance.

From 9b67e5c61b50424e66b2a9bc8cc566058c71a223 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Wed, 31 Mar 2021 09:24:39 -0700
Subject: [PATCH 0555/2472] jsonutils: support cstring (including as Table
 key); improve docs (#16062)

* jsonutils: support cstring (including as Table key); improve docs
* changelog
* un-disable a test now that #16061 was fixed
---
 changelog.md                |  2 ++
 lib/std/jsonutils.nim       | 22 +++++++++++++++++-----
 tests/stdlib/thashes.nim    |  1 +
 tests/stdlib/tjsonutils.nim | 13 ++++++++-----
 4 files changed, 28 insertions(+), 10 deletions(-)

diff --git a/changelog.md b/changelog.md
index c234bf174..f6dcee833 100644
--- a/changelog.md
+++ b/changelog.md
@@ -47,6 +47,8 @@
 - Added an overload for the `collect` macro that inferes the container type based
   on the syntax of the last expression. Works with std seqs, tables and sets.
 
+- `jsonutils` now handles `cstring` (including as Table key).
+
 - Added `randState` template that exposes the default random number generator.
   Useful for library authors.
 
diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim
index fa61d79db..a5daa9fb4 100644
--- a/lib/std/jsonutils.nim
+++ b/lib/std/jsonutils.nim
@@ -11,7 +11,7 @@ runnableExamples:
     z1: int8
   let a = (1.5'f32, (b: "b2", a: "a2"), 'x', @[Foo(t: true, z1: -3), nil], [{"name": "John"}.newStringTable])
   let j = a.toJson
-  doAssert j.jsonTo(typeof(a)).toJson == j
+  assert j.jsonTo(typeof(a)).toJson == j
 
 import std/[json,strutils,tables,sets,strtabs,options]
 
@@ -197,6 +197,11 @@ proc fromJson*[T](a: var T, b: JsonNode, opt = Joptions()) =
     else:
       a.distinctBase.fromJson(b)
   elif T is string|SomeNumber: a = to(b,T)
+  elif T is cstring:
+    case b.kind
+    of JNull: a = nil
+    of JString: a = b.str
+    else: checkJson false, $($T, " ", b)
   elif T is JsonNode: a = b
   elif T is ref | ptr:
     if b.kind == JNull: a = nil
@@ -273,9 +278,10 @@ proc toJson*[T](a: T): JsonNode =
   elif T is bool: result = %(a)
   elif T is SomeInteger: result = %a
   elif T is Ordinal: result = %(a.ord)
+  elif T is cstring: (if a == nil: result = newJNull() else: result = % $a)
   else: result = %a
 
-proc fromJsonHook*[K, V](t: var (Table[K, V] | OrderedTable[K, V]),
+proc fromJsonHook*[K: string|cstring, V](t: var (Table[K, V] | OrderedTable[K, V]),
                          jsonNode: JsonNode) =
   ## Enables `fromJson` for `Table` and `OrderedTable` types.
   ## 
@@ -296,21 +302,27 @@ proc fromJsonHook*[K, V](t: var (Table[K, V] | OrderedTable[K, V]),
   for k, v in jsonNode:
     t[k] = jsonTo(v, V)
 
-proc toJsonHook*[K, V](t: (Table[K, V] | OrderedTable[K, V])): JsonNode =
+proc toJsonHook*[K: string|cstring, V](t: (Table[K, V] | OrderedTable[K, V])): JsonNode =
   ## Enables `toJson` for `Table` and `OrderedTable` types.
   ##
   ## See also:
   ## * `fromJsonHook proc<#fromJsonHook,,JsonNode>`_
+  # pending PR #9217 use: toSeq(a) instead of `collect` in `runnableExamples`.
   runnableExamples:
-    import std/[tables, json]
+    import std/[tables, json, sugar]
     let foo = (
       t: [("two", 2)].toTable,
       ot: [("one", 1), ("three", 3)].toOrderedTable)
     assert $toJson(foo) == """{"t":{"two":2},"ot":{"one":1,"three":3}}"""
+    # if keys are not string|cstring, you can use this:
+    let a = {10: "foo", 11: "bar"}.newOrderedTable
+    let a2 = collect: (for k,v in a: (k,v))
+    assert $toJson(a2) == """[[10,"foo"],[11,"bar"]]"""
 
   result = newJObject()
   for k, v in pairs(t):
-    result[k] = toJson(v)
+    # not sure if $k has overhead for string
+    result[(when K is string: k else: $k)] = toJson(v)
 
 proc fromJsonHook*[A](s: var SomeSet[A], jsonNode: JsonNode) =
   ## Enables `fromJson` for `HashSet` and `OrderedSet` types.
diff --git a/tests/stdlib/thashes.nim b/tests/stdlib/thashes.nim
index a4487c8c0..044259f88 100644
--- a/tests/stdlib/thashes.nim
+++ b/tests/stdlib/thashes.nim
@@ -87,6 +87,7 @@ block largeSize: # longer than 4 characters
 
 proc main() =
   doAssert hash(0.0) == hash(0)
+  # bug #16061
   doAssert hash(cstring"abracadabra") == 97309975
   doAssert hash(cstring"abracadabra") == hash("abracadabra")
 
diff --git a/tests/stdlib/tjsonutils.nim b/tests/stdlib/tjsonutils.nim
index 63bf97704..eaf5d68f9 100644
--- a/tests/stdlib/tjsonutils.nim
+++ b/tests/stdlib/tjsonutils.nim
@@ -14,6 +14,7 @@ proc testRoundtrip[T](t: T, expected: string) =
   doAssert t2.toJson == j
 
 import tables, sets, algorithm, sequtils, options, strtabs
+from strutils import contains
 
 type Foo = ref object
   id: int
@@ -40,27 +41,29 @@ template fn() =
     # https://github.com/nim-lang/Nim/issues/12282
     testRoundtrip(Foo(1.5)): """1.5"""
 
-  block:
+  block: # OrderedTable
     testRoundtrip({"z": "Z", "y": "Y"}.toOrderedTable): """{"z":"Z","y":"Y"}"""
+    doAssert toJson({"z": 10, "": 11}.newTable).`$`.contains """"":11""" # allows hash to change
+    testRoundtrip({"z".cstring: 1, "".cstring: 2}.toOrderedTable): """{"z":1,"":2}"""
     testRoundtrip({"z": (f1: 'f'), }.toTable): """{"z":{"f1":102}}"""
 
-  block:
+  block: # StringTable
     testRoundtrip({"name": "John", "city": "Monaco"}.newStringTable): """{"mode":"modeCaseSensitive","table":{"city":"Monaco","name":"John"}}"""
 
   block: # complex example
     let t = {"z": "Z", "y": "Y"}.newStringTable
     type A = ref object
       a1: string
-    let a = (1.1, "fo", 'x', @[10,11], [true, false], [t,newStringTable()], [0'i8,3'i8], -4'i16, (foo: 0.5'f32, bar: A(a1: "abc"), bar2: A.default))
+    let a = (1.1, "fo", 'x', @[10,11], [true, false], [t,newStringTable()], [0'i8,3'i8], -4'i16, (foo: 0.5'f32, bar: A(a1: "abc"), bar2: A.default, cstring1: "foo", cstring2: "", cstring3: cstring(nil)))
     testRoundtrip(a):
-      """[1.1,"fo",120,[10,11],[true,false],[{"mode":"modeCaseSensitive","table":{"y":"Y","z":"Z"}},{"mode":"modeCaseSensitive","table":{}}],[0,3],-4,{"foo":0.5,"bar":{"a1":"abc"},"bar2":null}]"""
+      """[1.1,"fo",120,[10,11],[true,false],[{"mode":"modeCaseSensitive","table":{"y":"Y","z":"Z"}},{"mode":"modeCaseSensitive","table":{}}],[0,3],-4,{"foo":0.5,"bar":{"a1":"abc"},"bar2":null,"cstring1":"foo","cstring2":"","cstring3":null}]"""
 
   block:
     # edge case when user defined `==` doesn't handle `nil` well, e.g.:
     # https://github.com/nim-lang/nimble/blob/63695f490728e3935692c29f3d71944d83bb1e83/src/nimblepkg/version.nim#L105
     testRoundtrip(@[Foo(id: 10), nil]): """[{"id":10},null]"""
 
-  block:
+  block: # enum
     type Foo = enum f1, f2, f3, f4, f5
     type Bar = enum b1, b2, b3, b4
     let a = [f2: b2, f3: b3, f4: b4]

From 3f9c26539de010eaa12a879154f60e4bf656e6e3 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Thu, 1 Apr 2021 02:54:36 +0800
Subject: [PATCH 0556/2472] close #16786 (#17598)

---
 tests/async/tasyncintemplate.nim | 36 +++++++++++++++++++++++++++++++-
 1 file changed, 35 insertions(+), 1 deletion(-)

diff --git a/tests/async/tasyncintemplate.nim b/tests/async/tasyncintemplate.nim
index 31f4cd95c..0f810b5be 100644
--- a/tests/async/tasyncintemplate.nim
+++ b/tests/async/tasyncintemplate.nim
@@ -1,12 +1,46 @@
 discard """
-  output: 42
+  output: '''
+42
+1
+2
+3
+4
+'''
 """
 
 import asyncdispatch
 
+# bug #16159
 template foo() =
   proc temp(): Future[int] {.async.} = return 42
   proc tempVoid(): Future[void] {.async.} = echo await temp()
 
 foo()
 waitFor tempVoid()
+
+
+block: # bug #16786
+  block:
+    proc main(a: int|string)=
+      proc bar(b: int|string) = echo b
+      bar(a)
+    main(1)
+
+  block:
+    proc main(a: int) : Future[void] {.async.} =
+      proc bar(b: int): Future[void] {.async.} = echo b
+      await bar(a)
+    waitFor main(2)
+
+  block:
+    proc main(a: int) : Future[void] {.async.} =
+      proc bar(b: int | string): Future[void] {.async.} = echo b
+      await bar(a)
+    waitFor main(3)
+
+  block:
+    # bug
+    proc main(a: int|string) =
+      proc bar(b: int): Future[void] {.async.} = echo b
+      waitFor bar(a)
+    main(4)

From fe9a37f2a57d0e76984b25c3dfe0fbdc94a30d9e Mon Sep 17 00:00:00 2001
From: flywind 
Date: Thu, 1 Apr 2021 02:55:27 +0800
Subject: [PATCH 0557/2472] close #12684 add testcase (#17599)

---
 tests/compiler/t12684.nim | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 tests/compiler/t12684.nim

diff --git a/tests/compiler/t12684.nim b/tests/compiler/t12684.nim
new file mode 100644
index 000000000..a5f33d2c9
--- /dev/null
+++ b/tests/compiler/t12684.nim
@@ -0,0 +1,7 @@
+discard """
+  cmd: "nim check --hints:off --warnings:off $file"
+  errormsg: "undeclared identifier: 'Undeclared'"
+"""
+
+var x: Undeclared
+import compiler/nimeval

From 924ab3adea30b8ffe590d1dd08cc3ff23f4881de Mon Sep 17 00:00:00 2001
From: flywind 
Date: Thu, 1 Apr 2021 14:05:04 +0800
Subject: [PATCH 0558/2472] fix #17190 `nimscript` now accepts arbitrary file
 extensions for `nim e main.customext` (#17596)

* fix #17190

* cah

* merge

* Update tnimscriptwithnimext.nim

* Update tnimscriptwithmacro.nims

* Apply suggestions from code review

* Delete tnimscriptwithnimext.nim

* Update tests/tools/tnimscriptwithmacro.nims

* fix

* fix

* add a test

* Apply suggestions from code review

Co-authored-by: Timothee Cour 

* Apply suggestions from code review

* Update changelog.md

Co-authored-by: Timothee Cour 

Co-authored-by: Timothee Cour 
Co-authored-by: Andreas Rumpf 
---
 changelog.md                         | 2 ++
 compiler/main.nim                    | 2 --
 compiler/nimconf.nim                 | 7 +++++--
 tests/misc/trunner.nim               | 9 +++++++++
 tests/newconfig/foo2/mfoo2.customext | 2 ++
 5 files changed, 18 insertions(+), 4 deletions(-)
 create mode 100644 tests/newconfig/foo2/mfoo2.customext

diff --git a/changelog.md b/changelog.md
index f6dcee833..900693644 100644
--- a/changelog.md
+++ b/changelog.md
@@ -281,6 +281,8 @@
 - Tuple expressions are now parsed consistently as
   `nnkTupleConstr` node. Will affect macros expecting nodes to be of `nnkPar`.
 
+- `nim e` now accepts arbitrary file extensions for the nimscript file,
+  although `.nims` is still the preferred extension in general.
 
 ## Compiler changes
 
diff --git a/compiler/main.nim b/compiler/main.nim
index b61cdcadb..26a19063f 100644
--- a/compiler/main.nim
+++ b/compiler/main.nim
@@ -342,8 +342,6 @@ proc mainCommand*(graph: ModuleGraph) =
     if conf.projectIsCmd or conf.projectIsStdin: discard
     elif not fileExists(conf.projectFull):
       rawMessage(conf, errGenerated, "NimScript file does not exist: " & conf.projectFull.string)
-    elif not conf.projectFull.string.endsWith(".nims"):
-      rawMessage(conf, errGenerated, "not a NimScript file: " & conf.projectFull.string)
     # main NimScript logic handled in `loadConfigs`.
   of cmdNop: discard
   of cmdJsonscript:
diff --git a/compiler/nimconf.nim b/compiler/nimconf.nim
index 1691e7ccf..b63e5a0ad 100644
--- a/compiler/nimconf.nim
+++ b/compiler/nimconf.nim
@@ -298,11 +298,14 @@ proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen:
     for filename in configFiles:
       # delayed to here so that `hintConf` is honored
       rawMessage(conf, hintConf, filename.string)
-  if scriptIsProj:
+  if conf.cmd == cmdNimscript:
     showHintConf()
     configFiles.setLen 0
   if conf.cmd != cmdIdeTools:
-    runNimScriptIfExists(scriptFile, isMain = true)
+    if conf.cmd == cmdNimscript:
+      runNimScriptIfExists(conf.projectFull, isMain = true)
+    else:
+      runNimScriptIfExists(scriptFile, isMain = true)
   else:
     if not scriptIsProj:
       runNimScriptIfExists(scriptFile, isMain = true)
diff --git a/tests/misc/trunner.nim b/tests/misc/trunner.nim
index a66177f28..1895eeb97 100644
--- a/tests/misc/trunner.nim
+++ b/tests/misc/trunner.nim
@@ -238,6 +238,15 @@ tests/newconfig/bar/mfoo.nims""".splitLines
       expected.add &"Hint: used config file '{b}' [Conf]\n"
     doAssert outp.endsWith expected, outp & "\n" & expected
 
+  block: # mfoo2.customext
+    let filename = testsDir / "newconfig/foo2/mfoo2.customext"
+    let cmd = fmt"{nim} e --hint:conf {filename}"
+    let (outp, exitCode) = execCmdEx(cmd, options = {poStdErrToStdOut})
+    doAssert exitCode == 0
+    var expected = &"Hint: used config file '{filename}' [Conf]\n"
+    doAssert outp.endsWith "123" & "\n" & expected
+
+
   block: # nim --eval
     let opt = "--hints:off"
     check fmt"""{nim} {opt} --eval:"echo defined(nimscript)"""".execCmdEx == ("true\n", 0)
diff --git a/tests/newconfig/foo2/mfoo2.customext b/tests/newconfig/foo2/mfoo2.customext
new file mode 100644
index 000000000..66c8b1d15
--- /dev/null
+++ b/tests/newconfig/foo2/mfoo2.customext
@@ -0,0 +1,2 @@
+doAssert defined(nimscript)
+echo "123"

From a7bb973a24fb73d378230216a6bfcfd2f62e4e78 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Wed, 31 Mar 2021 23:45:55 -0700
Subject: [PATCH 0559/2472] docgen: render pragmas by default except for a
 select list (and fix #9074) (#17054)

---
 compiler/docgen.nim                           | 22 ++---
 compiler/lexer.nim                            |  2 +-
 compiler/renderer.nim                         | 22 ++++-
 .../expected/index.html                       |  2 +-
 .../expected/subdir/subdir_b/utils.html       |  2 +-
 nimdoc/testproject/expected/testproject.html  | 82 +++++++++----------
 6 files changed, 76 insertions(+), 56 deletions(-)

diff --git a/compiler/docgen.nim b/compiler/docgen.nim
index f1a9750b7..67f21e1c4 100644
--- a/compiler/docgen.nim
+++ b/compiler/docgen.nim
@@ -446,22 +446,22 @@ proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var Rope; renderFlags: TRe
               "\\spanIdentifier{$1}", [escLit])
     of tkSpaces, tkInvalid:
       result.add(literal)
-    of tkCurlyDotLe:
+    of tkHideableStart:
       template fun(s) = dispA(d.conf, result, s, "\\spanOther{$1}", [escLit])
       if renderRunnableExamples in renderFlags: fun "$1"
-      else: fun: "" & # This span is required for the JS to work properly
-        """{...}
+      else:
+        # 1st span is required for the JS to work properly
+        fun """
+
+...
 
-
-$1
-""".replace("\n", "")  # Must remove newlines because wrapped in a 
    -    of tkCurlyDotRi:
+""".replace("\n", "")  # Must remove newlines because wrapped in a 
    +    of tkHideableEnd:
       template fun(s) = dispA(d.conf, result, s, "\\spanOther{$1}", [escLit])
       if renderRunnableExamples in renderFlags: fun "$1"
-      else: fun """
-
-$1
-""".replace("\n", "")
+      else: fun ""
+    of tkCurlyDotLe: dispA(d.conf, result, "$1", "\\spanOther{$1}", [escLit])
+    of tkCurlyDotRi: dispA(d.conf, result, "$1", "\\spanOther{$1}", [escLit])
     of tkParLe, tkParRi, tkBracketLe, tkBracketRi, tkCurlyLe, tkCurlyRi,
        tkBracketDotLe, tkBracketDotRi, tkParDotLe,
        tkParDotRi, tkComma, tkSemiColon, tkColon, tkEquals, tkDot, tkDotDot,
diff --git a/compiler/lexer.nim b/compiler/lexer.nim
index cca7765e5..7c7ac4177 100644
--- a/compiler/lexer.nim
+++ b/compiler/lexer.nim
@@ -72,7 +72,7 @@ type
     tkDot = ".", tkDotDot = "..", tkBracketLeColon = "[:",
     tkOpr, tkComment, tkAccent = "`",
     # these are fake tokens used by renderer.nim
-    tkSpaces, tkInfixOpr, tkPrefixOpr, tkPostfixOpr
+    tkSpaces, tkInfixOpr, tkPrefixOpr, tkPostfixOpr, tkHideableStart, tkHideableEnd
 
   TokTypes* = set[TokType]
 
diff --git a/compiler/renderer.nim b/compiler/renderer.nim
index b9ec48185..3d260f350 100644
--- a/compiler/renderer.nim
+++ b/compiler/renderer.nim
@@ -185,7 +185,7 @@ proc dedent(g: var TSrcGen) =
 proc put(g: var TSrcGen, kind: TokType, s: string; sym: PSym = nil) =
   if kind != tkSpaces:
     addPendingNL(g)
-    if s.len > 0:
+    if s.len > 0 or kind in {tkHideableStart, tkHideableEnd}:
       addTok(g, kind, s, sym)
   else:
     g.pendingWhitespace = s.len
@@ -587,13 +587,30 @@ proc putWithSpace(g: var TSrcGen, kind: TokType, s: string) =
   put(g, kind, s)
   put(g, tkSpaces, Space)
 
+proc isHideable(config: ConfigRef, n: PNode): bool =
+  # xxx compare `ident` directly with `getIdent(cache, wRaises)`, but
+  # this requires a `cache`.
+  case n.kind
+  of nkExprColonExpr: result = n[0].kind == nkIdent and n[0].ident.s in ["raises", "tags", "extern"]
+  of nkIdent: result = n.ident.s in ["gcsafe"]
+  else: result = false
+
 proc gcommaAux(g: var TSrcGen, n: PNode, ind: int, start: int = 0,
                theEnd: int = - 1, separator = tkComma) =
+  let inPragma = g.inPragma == 1 # just the top-level
+  var inHideable = false
   for i in start..n.len + theEnd:
     var c = i < n.len + theEnd
     var sublen = lsub(g, n[i]) + ord(c)
     if not fits(g, g.lineLen + sublen) and (ind + sublen < MaxLineLen): optNL(g, ind)
     let oldLen = g.tokens.len
+    if inPragma:
+      if not inHideable and isHideable(g.config, n[i]):
+        inHideable = true
+        put(g, tkHideableStart, "")
+      elif inHideable and not isHideable(g.config, n[i]):
+        inHideable = false
+        put(g, tkHideableEnd, "")
     gsub(g, n[i])
     if c:
       if g.tokens.len > oldLen:
@@ -601,6 +618,9 @@ proc gcommaAux(g: var TSrcGen, n: PNode, ind: int, start: int = 0,
       if shouldRenderComment(g) and hasCom(n[i]):
         gcoms(g)
         optNL(g, ind)
+  if inHideable:
+    put(g, tkHideableEnd, "")
+    inHideable = false
 
 proc gcomma(g: var TSrcGen, n: PNode, c: TContext, start: int = 0,
             theEnd: int = - 1) =
diff --git a/nimdoc/test_out_index_dot_html/expected/index.html b/nimdoc/test_out_index_dot_html/expected/index.html
index db7470050..bac39fd33 100644
--- a/nimdoc/test_out_index_dot_html/expected/index.html
+++ b/nimdoc/test_out_index_dot_html/expected/index.html
@@ -117,7 +117,7 @@ window.addEventListener('DOMContentLoaded', main);
 
    Procs
    
 
    
 
-
    proc foo() {...}{.raises: [], tags: [].}
    
+
    proc foo() {....raises: [], tags: [].}
    
 
    
 
 I do foo
diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html
index cb80f2873..c1ee9c49d 100644
--- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html
+++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html
@@ -177,7 +177,7 @@ window.addEventListener('DOMContentLoaded', main);
 
    Procs
    
 
    
 
-
    proc someType(): SomeType {...}{.raises: [], tags: [].}
    
+
    proc someType(): SomeType {....raises: [], tags: [].}
    
 
    
 
 constructor.
diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html
index 599a904a2..8525048ea 100644
--- a/nimdoc/testproject/expected/testproject.html
+++ b/nimdoc/testproject/expected/testproject.html
@@ -444,7 +444,7 @@ window.addEventListener('DOMContentLoaded', main);
 
    Types
    
 
    
 
-
    A {...}{.inject.} = enum
+
    A {.inject.} = enum
   aA
    
 
    
 
@@ -452,7 +452,7 @@ The enum A.
 
 
    
 
-
    B {...}{.inject.} = enum
+
    B {.inject.} = enum
   bB
    
 
    
 
@@ -542,7 +542,7 @@ This should be visible.
 
 
    
 
-
    proc baz[T](a, b: T): T {...}{.deprecated.}
    
+
    proc baz[T](a, b: T): T {.deprecated.}
    
 
    
   
    
     Deprecated 
@@ -552,7 +552,7 @@ This is deprecated without message.
 
 
    
 
-
    proc buzz[T](a, b: T): T {...}{.deprecated: "since v0.20".}
    
+
    proc buzz[T](a, b: T): T {.deprecated: "since v0.20".}
    
 
    
   
    
     Deprecated: since v0.20
@@ -562,14 +562,14 @@ This is deprecated with a message.
 
 
    
 
-
    func someFunc() {...}{.raises: [], tags: [].}
    
+
    func someFunc() {....raises: [], tags: [].}
    
 
    
 
 My someFunc. Stuff in quotes here. Some link
 
 
    
 
-
    proc fromUtils3() {...}{.raises: [], tags: [].}
    
+
    proc fromUtils3() {....raises: [], tags: [].}
    
 
    
 
 came form utils but should be shown where fromUtilsGen is called
@@ -585,14 +585,14 @@ came form utils but should be shown where 
 
-
    proc z1(): Foo {...}{.raises: [], tags: [].}
    
+
    proc z1(): Foo {....raises: [], tags: [].}
    
 
    
 
 cz1
 
 
    
 
-
    proc z2() {...}{.raises: [], tags: [].}
    
+
    proc z2() {....raises: [], tags: [].}
    
 
    
 
 cz2
@@ -601,49 +601,49 @@ cz2
 
 
    
 
-
    proc z3() {...}{.raises: [], tags: [].}
    
+
    proc z3() {....raises: [], tags: [].}
    
 
    
 
 cz3
 
 
    
 
-
    proc z4() {...}{.raises: [], tags: [].}
    
+
    proc z4() {....raises: [], tags: [].}
    
 
    
 
 cz4
 
 
    
 
-
    proc z5(): int {...}{.raises: [], tags: [].}
    
+
    proc z5(): int {....raises: [], tags: [].}
    
 
    
 
 cz5
 
 
    
 
-
    proc z6(): int {...}{.raises: [], tags: [].}
    
+
    proc z6(): int {....raises: [], tags: [].}
    
 
    
 
 cz6
 
 
    
 
-
    proc z7(): int {...}{.raises: [], tags: [].}
    
+
    proc z7(): int {....raises: [], tags: [].}
    
 
    
 
 cz7
 
 
    
 
-
    proc z8(): int {...}{.raises: [], tags: [].}
    
+
    proc z8(): int {....raises: [], tags: [].}
    
 
    
 
 cz8
 
 
    
 
-
    proc z9() {...}{.raises: [], tags: [].}
    
+
    proc z9() {....raises: [], tags: [].}
    
 
    
 
 
@@ -652,7 +652,7 @@ cz8
 
 
    
 
-
    proc z10() {...}{.raises: [], tags: [].}
    
+
    proc z10() {....raises: [], tags: [].}
    
 
    
 
 
@@ -661,7 +661,7 @@ cz8
 
 
    
 
-
    proc z11() {...}{.raises: [], tags: [].}
    
+
    proc z11() {....raises: [], tags: [].}
    
 
    
 
 
@@ -670,7 +670,7 @@ cz8
 
 
    
 
-
    proc z12(): int {...}{.raises: [], tags: [].}
    
+
    proc z12(): int {....raises: [], tags: [].}
    
 
    
 
 
@@ -679,7 +679,7 @@ cz8
 
 
    
 
-
    proc z13() {...}{.raises: [], tags: [].}
    
+
    proc z13() {....raises: [], tags: [].}
    
 
    
 
 cz13
@@ -688,14 +688,14 @@ cz13
 
 
    
 
-
    proc baz() {...}{.raises: [], tags: [].}
    
+
    proc baz() {....raises: [], tags: [].}
    
 
    
 
 
 
 
    
 
-
    proc z17() {...}{.raises: [], tags: [].}
    
+
    proc z17() {....raises: [], tags: [].}
    
 
    
 
 cz17 rest
@@ -704,7 +704,7 @@ cz17 rest
 
 
    
 
-
    proc p1() {...}{.raises: [], tags: [].}
    
+
    proc p1() {....raises: [], tags: [].}
    
 
    
 
 cp1
@@ -727,7 +727,7 @@ this is a nested doc comment
 
 
    
 
-
    proc addfBug14485() {...}{.raises: [], tags: [].}
    
+
    proc addfBug14485() {....raises: [], tags: [].}
    
 
    
 
 Some proc
@@ -746,23 +746,23 @@ Some proc
 
 
    
 
-
    proc c_printf(frmt: cstring): cint {...}{.importc: "printf", header: "<stdio.h>",
-                                     varargs, discardable.}
    
+
    proc c_printf(frmt: cstring): cint {.importc: "printf", header: "<stdio.h>",
+                                     varargs, discardable.}
    
 
    
 
 the c printf. etc.
 
 
    
 
-
    proc c_nonexistant(frmt: cstring): cint {...}{.importc: "nonexistant",
-    header: "<stdio.h>", varargs, discardable.}
    
+
    proc c_nonexistant(frmt: cstring): cint {.importc: "nonexistant",
+    header: "<stdio.h>", varargs, discardable.}
    
 
    
 
 
 
 
    
 
-
    proc low[T: Ordinal | enum | range](x: T): T {...}{.magic: "Low", noSideEffect.}
    
+
    proc low[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect.}
    
 
    
 
 
    Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.
    
@@ -773,7 +773,7 @@ the c printf. etc.
 
 
    
 
-
    proc low2[T: Ordinal | enum | range](x: T): T {...}{.magic: "Low", noSideEffect.}
    
+
    proc low2[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect.}
    
 
    
 
 
    Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.
    
@@ -786,7 +786,7 @@ the c printf. etc.
 
 
    
 
-
    proc tripleStrLitTest() {...}{.raises: [], tags: [].}
    
+
    proc tripleStrLitTest() {....raises: [], tags: [].}
    
 
    
 
 
@@ -829,22 +829,22 @@ at indent 0
 
 
    
 
-
    proc asyncFun1(): Future[int] {...}{.raises: [Exception, ValueError],
-                                tags: [RootEffect].}
    
+
    proc asyncFun1(): Future[int] {....raises: [Exception, ValueError],
+                                tags: [RootEffect].}
    
 
    
 
 ok1
 
 
    
 
-
    proc asyncFun2(): owned(Future[void]) {...}{.raises: [Exception], tags: [RootEffect].}
    
+
    proc asyncFun2(): owned(Future[void]) {....raises: [Exception], tags: [RootEffect].}
    
 
    
 
 
 
 
    
 
-
    proc asyncFun3(): owned(Future[void]) {...}{.raises: [Exception], tags: [RootEffect].}
    
+
    proc asyncFun3(): owned(Future[void]) {....raises: [Exception], tags: [RootEffect].}
    
 
    
 
 
@@ -853,7 +853,7 @@ ok1
 
 
    
 
-
    proc anything() {...}{.raises: [], tags: [].}
    
+
    proc anything() {....raises: [], tags: [].}
    
 
    
 
 There is no block quote after blank lines at the beginning.
@@ -865,21 +865,21 @@ There is no block quote after blank lines at the beginning.
 
    Methods
    
 
    
 
-
    method method1(self: Moo) {...}{.raises: [], tags: [].}
    
+
    method method1(self: Moo) {....raises: [], tags: [].}
    
 
    
 
 foo1
 
 
    
 
-
    method method2(self: Moo): int {...}{.raises: [], tags: [].}
    
+
    method method2(self: Moo): int {....raises: [], tags: [].}
    
 
    
 
 foo2
 
 
    
 
-
    method method3(self: Moo): int {...}{.raises: [], tags: [].}
    
+
    method method3(self: Moo): int {....raises: [], tags: [].}
    
 
    
 
 foo3
@@ -891,7 +891,7 @@ foo3
 
    Iterators
    
 
    
 
-
    iterator fromUtils1(): int {...}{.raises: [], tags: [].}
    
+
    iterator fromUtils1(): int {....raises: [], tags: [].}
    
 
    
 
 
@@ -902,14 +902,14 @@ foo3
 
 
    
 
-
    iterator iter1(n: int): int {...}{.raises: [], tags: [].}
    
+
    iterator iter1(n: int): int {....raises: [], tags: [].}
    
 
    
 
 foo1
 
 
    
 
-
    iterator iter2(n: int): int {...}{.raises: [], tags: [].}
    
+
    iterator iter2(n: int): int {....raises: [], tags: [].}
    
 
    
 
 foo2

From 6f53579a8ceb7576fb1b31f72c9d584916b4b5d9 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Thu, 1 Apr 2021 14:47:59 +0800
Subject: [PATCH 0560/2472] close #9534 add testcase (#17607)

---
 tests/template/t9534.nim | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 tests/template/t9534.nim

diff --git a/tests/template/t9534.nim b/tests/template/t9534.nim
new file mode 100644
index 000000000..8d66f42a0
--- /dev/null
+++ b/tests/template/t9534.nim
@@ -0,0 +1,21 @@
+discard """
+  targets: "c cpp"
+"""
+
+# bug #9534
+type
+  Object = object
+    data: int
+
+template test() =
+  proc methodName(o: Object): int =
+    var p: pointer
+    doAssert o.data == 521
+    let f {.used.} = cast[proc (o: int): int {.nimcall.}](p)
+    doAssert o.data == 521
+    result = 1314
+
+  var a = Object(data: 521)
+  doAssert methodName(a) == 1314
+
+test()

From a31e601550cc6f9fd9e09b385bfbd869a408cd4b Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Wed, 31 Mar 2021 23:48:44 -0700
Subject: [PATCH 0561/2472] misc fixes: build_all.sh, changelog (#17601)

* fix a bug in build_all.sh: $@ => "$@"

* remove getSocket from changelog following #17587

* remove unused import
---
 build_all.sh               | 2 +-
 changelog.md               | 2 +-
 compiler/errorhandling.nim | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/build_all.sh b/build_all.sh
index e66980e56..c3c6e68b7 100755
--- a/build_all.sh
+++ b/build_all.sh
@@ -42,7 +42,7 @@ build_nim_csources(){
   echo_run cp bin/nim $nim_csources
 }
 
-[ -f $nim_csources ] || echo_run build_nim_csources $@
+[ -f $nim_csources ] || echo_run build_nim_csources "$@"
 
 # Note: if fails, may need to `cd csources && git pull`
 echo_run bin/nim c --skipUserCfg --skipParentCfg koch
diff --git a/changelog.md b/changelog.md
index 900693644..53d30bf36 100644
--- a/changelog.md
+++ b/changelog.md
@@ -73,7 +73,7 @@
 - Added `asyncdispatch.activeDescriptors` that returns the number of currently
   active async event handles/file descriptors.
 
-- Added to `asynchttpserver` `getPort` and `getSocket`.
+- Added `getPort` to `asynchttpserver`.
 
 - `--gc:orc` is now 10% faster than previously for common workloads. If
   you have trouble with its changed behavior, compile with `-d:nimOldOrc`.
diff --git a/compiler/errorhandling.nim b/compiler/errorhandling.nim
index d7092d477..cda7ab3f4 100644
--- a/compiler/errorhandling.nim
+++ b/compiler/errorhandling.nim
@@ -10,7 +10,7 @@
 ## This module contains support code for new-styled error
 ## handling via an `nkError` node kind.
 
-import ast, renderer, options, lineinfos, strutils, types
+import ast, renderer, options, strutils, types
 
 type
   ErrorKind* = enum ## expand as you need.

From f3c504e49a4e24b7fbdcd5c8eec786fa86616233 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Thu, 1 Apr 2021 14:49:10 +0800
Subject: [PATCH 0562/2472] a bit better message (#17606)

---
 compiler/ccgtypes.nim | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim
index 761fd5bf8..9d95e3f67 100644
--- a/compiler/ccgtypes.nim
+++ b/compiler/ccgtypes.nim
@@ -194,8 +194,8 @@ proc mapType(conf: ConfigRef; typ: PType; kind: TSymKind): TCTypeKind =
     result = TCTypeKind(ord(typ.kind) - ord(tyInt) + ord(ctInt))
   of tyStatic:
     if typ.n != nil: result = mapType(conf, lastSon typ, kind)
-    else: doAssert(false, "mapType")
-  else: doAssert(false, "mapType")
+    else: doAssert(false, "mapType: " & $typ.kind)
+  else: doAssert(false, "mapType: " & $typ.kind)
 
 proc mapReturnType(conf: ConfigRef; typ: PType): TCTypeKind =
   #if skipTypes(typ, typedescInst).kind == tyArray: result = ctPtr

From e25a487ed6237d30a461137d0a206e1e1a2f62d9 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Thu, 1 Apr 2021 18:46:47 +0800
Subject: [PATCH 0563/2472] close #6133 add testcase (#17605)

* close #6133 add testcase

* b
---
 tests/system/tstatic.nim | 50 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 50 insertions(+)
 create mode 100644 tests/system/tstatic.nim

diff --git a/tests/system/tstatic.nim b/tests/system/tstatic.nim
new file mode 100644
index 000000000..1f1b9dbe3
--- /dev/null
+++ b/tests/system/tstatic.nim
@@ -0,0 +1,50 @@
+discard """
+  targets: "c cpp js"
+"""
+
+import std/strutils
+
+# bug #6133
+template main() =
+  block:
+    block:
+      proc foo(q: string, a: int): int =
+        result = q.len
+
+      proc foo(q: static[string]): int =
+        result = foo(q, 5)
+
+      doAssert foo("123") == 3
+
+    block:
+      type E = enum A
+
+      if false:
+        var e = A
+        discard $e
+
+      proc foo(a: string): int =
+        len(a) # 16640
+
+      proc foo(a: static[bool]): int {.used.} =
+        discard
+
+      doAssert foo("") == 0
+
+    block:
+      proc foo(a: string): int =
+        len(a)
+
+      proc foo(a: static[bool]): int {.used.} =
+        discard
+
+      doAssert foo("abc") == 3
+
+    block:
+      proc parseInt(f: static[bool]): int {.used.} = discard
+
+      doAssert "123".parseInt == 123
+
+
+static: main()
+main()

From f0587e02e255b1fc1a0d0b21aebcacf37ec43c8f Mon Sep 17 00:00:00 2001
From: konsumlamm <44230978+konsumlamm@users.noreply.github.com>
Date: Thu, 1 Apr 2021 12:47:56 +0200
Subject: [PATCH 0564/2472] Fix #17017 (math edge cases) (#17588)

* Fix #17017

Add more test cases

* USe signbit in example
---
 lib/pure/math.nim      | 18 ++++++++++++-----
 tests/stdlib/tmath.nim | 45 +++++++++++++++++++++++++++++++++++-------
 2 files changed, 51 insertions(+), 12 deletions(-)

diff --git a/lib/pure/math.nim b/lib/pure/math.nim
index 74a98e655..2ef7ee102 100644
--- a/lib/pure/math.nim
+++ b/lib/pure/math.nim
@@ -949,18 +949,25 @@ func frexp*[T: float32|float64](x: T): tuple[frac: T, exp: int] {.inline.} =
     doAssert frexp(8.0) == (0.5, 4)
     doAssert frexp(-8.0) == (-0.5, 4)
     doAssert frexp(0.0) == (0.0, 0)
+
     # special cases:
-    when not defined(windows):
-      doAssert frexp(-0.0) == (-0.0, 0) # signbit preserved for +-0
-      doAssert frexp(Inf).frac == Inf # +- Inf preserved
-      doAssert frexp(NaN).frac.isNaN
+    doAssert frexp(-0.0).frac.signbit # signbit preserved for +-0
+    doAssert frexp(Inf).frac == Inf # +- Inf preserved
+    doAssert frexp(NaN).frac.isNaN
+
   when not defined(js):
     var exp: cint
     result.frac = c_frexp2(x, exp)
     result.exp = exp
   else:
     if x == 0.0:
-      result = (0.0, 0)
+      # reuse signbit implementation
+      let uintBuffer = toBitsImpl(x)
+      if (uintBuffer[1] shr 31) != 0:
+        # x is -0.0
+        result = (-0.0, 0)
+      else:
+        result = (0.0, 0)
     elif x < 0.0:
       result = frexp(-x)
       result.frac = -result.frac
@@ -980,6 +987,7 @@ func frexp*[T: float32|float64](x: T, exponent: var int): T {.inline.} =
     var x: int
     doAssert frexp(5.0, x) == 0.625
     doAssert x == 3
+
   (result, exponent) = frexp(x)
 
 
diff --git a/tests/stdlib/tmath.nim b/tests/stdlib/tmath.nim
index 49b4c82f1..2cf544ddb 100644
--- a/tests/stdlib/tmath.nim
+++ b/tests/stdlib/tmath.nim
@@ -29,10 +29,6 @@ template main() =
       doAssert erf(6.0) > erf(5.0)
       doAssert erfc(6.0) < erfc(5.0)
 
-  when not defined(js) and not defined(windows): # xxx pending bug #17017
-    doAssert gamma(-1.0).isNaN
-
-
   block: # sgn() tests
     doAssert sgn(1'i8) == 1
     doAssert sgn(1'i16) == 1
@@ -46,6 +42,7 @@ template main() =
     doAssert sgn(123.9834'f64) == 1
     doAssert sgn(0'i32) == 0
     doAssert sgn(0'f32) == 0
+    doAssert sgn(-0.0'f64) == 0
     doAssert sgn(NegInf) == -1
     doAssert sgn(Inf) == 1
     doAssert sgn(NaN) == 0
@@ -358,7 +355,7 @@ template main() =
     doAssert almostEqual(prod([1.5, 3.4]), 5.1)
     let x: seq[float] = @[]
     doAssert prod(x) == 1.0
-  
+
   block: # clamp range
     doAssert clamp(10, 1..5) == 5
     doAssert clamp(3, 1..5) == 3
@@ -370,8 +367,42 @@ template main() =
     doAssert a1.clamp(a2..a4) == a2
     doAssert clamp((3, 0), (1, 0) .. (2, 9)) == (2, 9)
 
-  when not defined(windows): # xxx pending bug #17017
-    doAssert sqrt(-1.0).isNaN
+  block: # edge cases
+    doAssert sqrt(-4.0).isNaN
+
+    doAssert ln(0.0) == -Inf
+    doAssert ln(-0.0) == -Inf
+    doAssert ln(-12.0).isNaN
+
+    doAssert log10(0.0) == -Inf
+    doAssert log10(-0.0) == -Inf
+    doAssert log10(-12.0).isNaN
+
+    doAssert log2(0.0) == -Inf
+    doAssert log2(-0.0) == -Inf
+    doAssert log2(-12.0).isNaN
+
+    when nimvm: discard
+    else:
+      doAssert frexp(0.0) == (0.0, 0)
+      doAssert frexp(-0.0) == (-0.0, 0)
+      doAssert classify(frexp(-0.0)[0]) == fcNegZero
+
+    when not defined(js):
+      doAssert gamma(0.0) == Inf
+      doAssert gamma(-0.0) == -Inf
+      doAssert gamma(-1.0).isNaN
+
+      doAssert lgamma(0.0) == Inf
+      doAssert lgamma(-0.0) == Inf
+      doAssert lgamma(-1.0) == Inf
+
+      when nimvm: discard
+      else:
+        var exponent: cint
+        doAssert c_frexp(0.0, exponent) == 0.0
+        doAssert c_frexp(-0.0, exponent) == -0.0
+        doAssert classify(c_frexp(-0.0, exponent)) == fcNegZero
 
 static: main()
 main()

From 2365b5250ec8a8893a376e75ad841d01c327893d Mon Sep 17 00:00:00 2001
From: flywind 
Date: Fri, 2 Apr 2021 12:06:50 +0800
Subject: [PATCH 0565/2472] docs: note regarding emit with backticks (#17608)

---
 doc/manual.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/manual.rst b/doc/manual.rst
index b1aae2669..3c71ece95 100644
--- a/doc/manual.rst
+++ b/doc/manual.rst
@@ -7039,9 +7039,9 @@ Example:
   void fun(){}
   """.}
 
-For backward compatibility, if the argument to the `emit` statement
-is a single string literal, Nim symbols can be referred to via backticks.
-This usage is however deprecated.
+.. note:: For backward compatibility, if the argument to the `emit` statement
+  is a single string literal, Nim symbols can be referred to via backticks.
+  This usage is however deprecated.
 
 For a top-level emit statement, the section where in the generated C/C++ file
 the code should be emitted can be influenced via the

From 792a03b7561195b27f5ad2b9654781cc17b86c75 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Fri, 2 Apr 2021 15:18:40 +0800
Subject: [PATCH 0566/2472] [docs minor] close #16553 (#17612)

Co-authored-by: Timothee Cour 
---
 lib/pure/os.nim | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/lib/pure/os.nim b/lib/pure/os.nim
index d0b3aef1a..0710b8333 100644
--- a/lib/pure/os.nim
+++ b/lib/pure/os.nim
@@ -2266,16 +2266,12 @@ iterator walkDir*(dir: string; relative = false, checkDir = false):
   ##        / fileA2.txt
   ##
   ## and this code:
-  ##
-  ## .. code-block:: Nim
-  ##     for kind, path in walkDir("dirA"):
-  ##       echo(path)
-  ##
-  ## produce this output (but not necessarily in this order!)::
-  ##   dirA/dirB
-  ##   dirA/dirC
-  ##   dirA/fileA1.txt
-  ##   dirA/fileA2.txt
+  runnableExamples("-r:off"):
+    import std/[strutils, sugar]
+    # note: order is not guaranteed
+    # this also works at compile time
+    assert collect(for k in walkDir("dirA"): k.path).join(" ") ==
+                          "dirA/dirB dirA/dirC dirA/fileA2.txt dirA/fileA1.txt"
   ##
   ## See also:
   ## * `walkPattern iterator <#walkPattern.i,string>`_

From 774e66f3d123520c44016ca23fd103538fc3c87d Mon Sep 17 00:00:00 2001
From: flywind 
Date: Fri, 2 Apr 2021 17:56:30 +0800
Subject: [PATCH 0567/2472] close #7875 add testcase (#17611)

* close #7875 add testcase

* fix
---
 tests/macros/t7875.nim | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 tests/macros/t7875.nim

diff --git a/tests/macros/t7875.nim b/tests/macros/t7875.nim
new file mode 100644
index 000000000..7b6e47b86
--- /dev/null
+++ b/tests/macros/t7875.nim
@@ -0,0 +1,22 @@
+discard """
+  nimout: "var mysym`gensym0: MyType[float32]"
+  joinable: false
+"""
+
+import macros
+
+type
+  MyType[T] = object
+
+# this is totally fine
+var mysym: MyType[float32]
+
+macro foobar(): untyped =
+  let floatSym = bindSym"float32"
+
+  result = quote do:
+    var mysym: MyType[`floatSym`]
+
+  echo result.repr
+
+foobar()

From 9abd383a2ae36e50ee23b78654e9d8154db46c84 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Fri, 2 Apr 2021 04:11:03 -0700
Subject: [PATCH 0568/2472] new `genAst` as replacement for `quote do` (#17426)

* new `macros.genAst`: fixes all issues with `quote do`
* add changelog entry
* add workaround for https://github.com/nim-lang/Nim/issues/2465#issuecomment-511076669
* add test for #9607
* add kNoExposeLocalInjects option
* add test case for nested application of genAst
* genAst: automatically call newLit when needed
* allow skipping `{}`: genAst: foo
* add test that shows this fixes #11986
* add examples showing mixin; add examples showing passing types, macros, templates
* move to std/genasts
* improve docs
---
 changelog.md             |   3 +
 lib/core/macros.nim      |   5 +-
 lib/std/genasts.nim      |  87 +++++++++++++
 tests/stdlib/mgenast.nim |  53 ++++++++
 tests/stdlib/tgenast.nim | 269 +++++++++++++++++++++++++++++++++++++++
 5 files changed, 416 insertions(+), 1 deletion(-)
 create mode 100644 lib/std/genasts.nim
 create mode 100644 tests/stdlib/mgenast.nim
 create mode 100644 tests/stdlib/tgenast.nim

diff --git a/changelog.md b/changelog.md
index 53d30bf36..020d8a350 100644
--- a/changelog.md
+++ b/changelog.md
@@ -255,6 +255,9 @@
 - Added `hasClosure` to `std/typetraits`.
 
 
+- Added `genasts.genAst` that avoids the problems inherent with `quote do` and can
+  be used as a replacement.
+
 ## Language changes
 
 - `nimscript` now handles `except Exception as e`.
diff --git a/lib/core/macros.nim b/lib/core/macros.nim
index 795909c6b..49c9a999c 100644
--- a/lib/core/macros.nim
+++ b/lib/core/macros.nim
@@ -539,6 +539,7 @@ proc parseStmt*(s: string): NimNode {.noSideEffect.} =
 
 proc getAst*(macroOrTemplate: untyped): NimNode {.magic: "ExpandToAst", noSideEffect.}
   ## Obtains the AST nodes returned from a macro or template invocation.
+  ## See also `genasts.genAst`.
   ## Example:
   ##
   ## .. code-block:: nim
@@ -559,6 +560,8 @@ proc quote*(bl: typed, op = "``"): NimNode {.magic: "QuoteAst", noSideEffect.} =
   ##
   ## A custom operator interpolation needs accent quoted (``) whenever it resolves
   ## to a symbol.
+  ##
+  ## See also `genasts `_ which avoids some issues with `quote`.
   runnableExamples:
     macro check(ex: untyped) =
       # this is a simplified version of the check macro from the
@@ -1419,7 +1422,7 @@ proc expectIdent*(n: NimNode, name: string) {.since: (1,1).} =
   if not eqIdent(n, name):
     error("Expected identifier to be `" & name & "` here", n)
 
-proc hasArgOfName*(params: NimNode; name: string): bool=
+proc hasArgOfName*(params: NimNode; name: string): bool =
   ## Search `nnkFormalParams` for an argument.
   expectKind(params, nnkFormalParams)
   for i in 1..`_ for details.
+    # Default is unset, to avoid hijacking of uncaptured local symbols by
+    # symbols in caller scope.
+  kNoNewLit,
+    # don't call call newLit automatically in `genAst` capture parameters
+
+macro genAstOpt*(options: static set[GenAstOpt], args: varargs[untyped]): untyped =
+  ## Accepts a list of captured variables `a=b` or `a` and a block and returns the
+  ## AST that represents it. Local `{.inject.}` symbols (e.g. procs) are captured
+  ## unless `kDirtyTemplate in options`.
+  runnableExamples:
+    # This example shows how one could write a simplified version of `unittest.check`.
+    import std/[macros, strutils]
+    macro check2(cond: bool): untyped =
+      assert cond.kind == nnkInfix, "$# not implemented" % $cond.kind
+      result = genAst(cond, s = repr(cond), lhs = cond[1], rhs = cond[2]):
+        # each local symbol we access must be explicitly captured
+        if not cond:
+          doAssert false, "'$#'' failed: lhs: '$#', rhs: '$#'" % [s, $lhs, $rhs]
+    let a = 3
+    check2 a*2 == a+3
+    if false: check2 a*2 < a+1 # would error with: 'a * 2 < a + 1'' failed: lhs: '6', rhs: '4'
+
+  runnableExamples:
+    # This example goes in more details about the capture semantics.
+    macro fun(a: string, b: static bool): untyped =
+      let c = 'z'
+      var d = 11 # implicitly {.gensym.} and needs to be captured for use in `genAst`.
+      proc localFun(): auto = 12 # implicitly {.inject.}, doesn't need to be captured.
+      genAst(a, b, c = true):
+        # `a`, `b` are captured explicitly, `c` is a local definition masking `c = 'z'`.
+        const b2 = b # macro static param `b` is forwarded here as a static param.
+        # `echo d` would give: `var not init` because `d` is not captured.
+        (a & a, b, c, localFun()) # localFun can be called without capture.
+    assert fun("ab", false) == ("abab", false, true, 12)
+
+  let params = newTree(nnkFormalParams, newEmptyNode())
+  let pragmas =
+    if kDirtyTemplate in options:
+      nnkPragma.newTree(ident"dirty")
+    else:
+      newEmptyNode()
+
+  template newLitMaybe(a): untyped =
+    when (a is type) or (typeof(a) is (proc | iterator | func | NimNode)):
+      a # `proc` actually also covers template, macro
+    else: newLit(a)
+
+  # using `_` as workaround, see https://github.com/nim-lang/Nim/issues/2465#issuecomment-511076669
+  let name = genSym(nskTemplate, "_fun")
+  let call = newCall(name)
+  for a in args[0..^2]:
+    var varName: NimNode
+    var varVal: NimNode
+    case a.kind
+    of nnkExprEqExpr:
+      varName = a[0]
+      varVal = a[1]
+    of nnkIdent:
+      varName = a
+      varVal = a
+    else: error("invalid argument kind: " & $a.kind, a)
+    if kNoNewLit notin options: varVal = newCall(bindSym"newLitMaybe", varVal)
+
+    params.add newTree(nnkIdentDefs, varName, newEmptyNode(), newEmptyNode())
+    call.add varVal
+
+  result = newStmtList()
+  result.add nnkTemplateDef.newTree(
+      name,
+      newEmptyNode(),
+      newEmptyNode(),
+      params,
+      pragmas,
+      newEmptyNode(),
+      args[^1])
+  result.add newCall(bindSym"getAst", call)
+
+template genAst*(args: varargs[untyped]): untyped =
+  ## Convenience wrapper around `genAstOpt`.
+  genAstOpt({}, args)
diff --git a/tests/stdlib/mgenast.nim b/tests/stdlib/mgenast.nim
new file mode 100644
index 000000000..2b5381891
--- /dev/null
+++ b/tests/stdlib/mgenast.nim
@@ -0,0 +1,53 @@
+import std/genasts
+import std/macros
+
+# Using a enum instead of, say, int, to make apparent potential bugs related to
+# forgetting converting to NimNode via newLit, see bug #9607
+
+type Foo* = enum kfoo0, kfoo1, kfoo2, kfoo3, kfoo4
+
+proc myLocalPriv(): auto = kfoo1
+proc myLocalPriv2(): auto = kfoo1
+macro bindme2*(): untyped =
+  genAst: myLocalPriv()
+macro bindme3*(): untyped =
+  ## myLocalPriv must be captured explicitly
+  genAstOpt({kDirtyTemplate}, myLocalPriv): myLocalPriv()
+
+macro bindme4*(): untyped =
+  ## calling this won't compile because `myLocalPriv` isn't captured
+  genAstOpt({kDirtyTemplate}): myLocalPriv()
+
+macro bindme5UseExpose*(): untyped =
+  genAst: myLocalPriv2()
+
+macro bindme5UseExposeFalse*(): untyped =
+  genAstOpt({kDirtyTemplate}): myLocalPriv2()
+
+# example from bug #7889
+from std/streams import newStringStream, readData, writeData
+
+macro bindme6UseExpose*(): untyped =
+  genAst:
+    var tst = "sometext"
+    var ss = newStringStream("anothertext")
+    writeData(ss, tst[0].addr, 2)
+    discard readData(ss, tst[0].addr, 2)
+
+macro bindme6UseExposeFalse*(): untyped =
+  ## with `kDirtyTemplate`, requires passing all referenced symbols
+  ## which can be tedious
+  genAstOpt({kDirtyTemplate}, newStringStream, writeData, readData):
+    var tst = "sometext"
+    var ss = newStringStream("anothertext")
+    writeData(ss, tst[0].addr, 2)
+    discard readData(ss, tst[0].addr, 2)
+
+
+proc locafun1(): auto = "in locafun1"
+proc locafun2(): auto = "in locafun2"
+# locafun3 in caller scope only
+macro mixinExample*(): untyped =
+  genAst:
+    mixin locafun1
+    (locafun1(), locafun2(), locafun3())
diff --git a/tests/stdlib/tgenast.nim b/tests/stdlib/tgenast.nim
new file mode 100644
index 000000000..0904b83dd
--- /dev/null
+++ b/tests/stdlib/tgenast.nim
@@ -0,0 +1,269 @@
+# xxx also test on js
+
+import std/genasts
+import std/macros
+from std/strformat import `&`
+import ./mgenast
+
+proc main =
+  block:
+    macro bar(x0: static Foo, x1: Foo, x2: Foo, xignored: Foo): untyped =
+      let s0 = "not captured!"
+      let s1 = "not captured!"
+      let xignoredLocal = kfoo4
+
+      # newLit optional:
+      let x3 = newLit kfoo4
+      let x3b = kfoo4
+
+      result = genAstOpt({kDirtyTemplate}, s1=true, s2="asdf", x0, x1=x1, x2, x3, x3b):
+        doAssert not declared(xignored)
+        doAssert not declared(xignoredLocal)
+        (s1, s2, s0, x0, x1, x2, x3, x3b)
+
+    let s0 = "caller scope!"
+
+    doAssert bar(kfoo1, kfoo2, kfoo3, kfoo4) ==
+      (true, "asdf", "caller scope!", kfoo1, kfoo2, kfoo3, kfoo4, kfoo4)
+
+  block:
+    # doesn't have limitation mentioned in https://github.com/nim-lang/RFCs/issues/122#issue-401636535
+    macro abc(name: untyped): untyped =
+      result = genAst(name):
+        type name = object
+
+    abc(Bar)
+    doAssert Bar.default == Bar()
+
+  block:
+    # backticks parser limitations / ambiguities not are an issue with `genAst`:
+    # (#10326 #9745 are fixed but `quote do` still has underlying ambiguity issue
+    # with backticks)
+    type Foo = object
+      a: int
+
+    macro m1(): untyped =
+      # result = quote do: # Error: undeclared identifier: 'a1'
+      result = genAst:
+        template `a1=`(x: var Foo, val: int) =
+          x.a = val
+
+    m1()
+    var x0: Foo
+    x0.a1 = 10
+    doAssert x0 == Foo(a: 10)
+
+  block:
+    # avoids bug #7375
+    macro fun(b: static[bool], b2: bool): untyped =
+      result = newStmtList()
+    macro foo(c: bool): untyped =
+      var b = false
+      result = genAst(b, c):
+        fun(b, c)
+
+    foo(true)
+
+  block:
+    # avoids bug #7589
+    # since `==` works with genAst, the problem goes away
+    macro foo2(): untyped =
+      # result = quote do: # Error: '==' cannot be passed to a procvar
+      result = genAst:
+        `==`(3,4)
+    doAssert not foo2()
+
+  block:
+    # avoids bug #7726
+    # expressions such as `a.len` are just passed as arguments to `genAst`, and
+    # caller scope is not polluted with definitions such as `let b = newLit a.len`
+    macro foo(): untyped =
+      let a = @[1, 2, 3, 4, 5]
+      result = genAst(a, b = a.len): # shows 2 ways to get a.len
+        (a.len, b)
+    doAssert foo() == (5, 5)
+
+  block:
+    # avoids bug #9607
+    proc fun1(info:LineInfo): string = "bar1"
+    proc fun2(info:int): string = "bar2"
+
+    macro bar2(args: varargs[untyped]): untyped =
+      let info = args.lineInfoObj
+      let fun1 = bindSym"fun1" # optional; we can remove this and also the
+      # capture of fun1, as show in next example
+      result = genAst(info, fun1):
+        (fun1(info), fun2(info.line))
+    doAssert bar2() == ("bar1", "bar2")
+
+    macro bar3(args: varargs[untyped]): untyped =
+      let info = args.lineInfoObj
+      result = genAst(info):
+        (fun1(info), fun2(info.line))
+    doAssert bar3() == ("bar1", "bar2")
+
+    macro bar(args: varargs[untyped]): untyped =
+      let info = args.lineInfoObj
+      let fun1 = bindSym"fun1"
+      let fun2 = bindSym"fun2"
+      result = genAstOpt({kDirtyTemplate}, info):
+        (fun1(info), fun2(info.line))
+    doAssert bar() == ("bar1", "bar2")
+
+  block:
+    # example from bug #7889 works
+    # after changing method call syntax to regular call syntax; this is a
+    # limitation described in bug #7085
+    # note that `quote do` would also work after that change in this example.
+    doAssert bindme2() == kfoo1
+    doAssert bindme3() == kfoo1
+    doAssert not compiles(bindme4()) # correctly gives Error: undeclared identifier: 'myLocalPriv'
+    proc myLocalPriv2(): auto = kfoo2
+    doAssert bindme5UseExpose() == kfoo1
+
+    # example showing hijacking behavior when using `kDirtyTemplate`
+    doAssert bindme5UseExposeFalse() == kfoo2
+      # local `myLocalPriv2` hijacks symbol `mgenast.myLocalPriv2`. In most
+      # use cases this is probably not what macro writer intends as it's
+      # surprising; hence `kDirtyTemplate` is not the default.
+
+    when nimvm: # disabled because `newStringStream` is used
+      discard
+    else:
+      bindme6UseExpose()
+      bindme6UseExposeFalse()
+
+  block:
+    macro mbar(x3: Foo, x3b: static Foo): untyped =
+      var x1=kfoo3
+      var x2=newLit kfoo3
+      var x4=kfoo3
+      var xLocal=kfoo3
+
+      proc funLocal(): auto = kfoo4
+
+      result = genAst(x1, x2, x3, x4):
+        # local x1 overrides remote x1
+        when false:
+          # one advantage of using `kDirtyTemplate` is that these would hold:
+          doAssert not declared xLocal
+          doAssert not compiles(echo xLocal)
+          # however, even without it, we at least correctly generate CT error
+          # if trying to use un-captured symbol; this correctly gives:
+          # Error: internal error: environment misses: xLocal
+          echo xLocal
+
+        proc foo1(): auto =
+          # note that `funLocal` is captured implicitly, according to hygienic
+          # template rules; with `kDirtyTemplate` it would not unless
+          # captured in `genAst` capture list explicitly
+          (a0: xRemote, a1: x1, a2: x2, a3: x3, a4: x4, a5: funLocal())
+
+      return result
+
+    proc main()=
+      var xRemote=kfoo1
+      var x1=kfoo2
+      mbar(kfoo4, kfoo4)
+      doAssert foo1() == (a0: kfoo1, a1: kfoo3, a2: kfoo3, a3: kfoo4, a4: kfoo3, a5: kfoo4)
+
+    main()
+
+  block:
+    # With `kDirtyTemplate`, the example from #8220 works.
+    # See https://nim-lang.github.io/Nim/strformat.html#limitations for
+    # an explanation of why {.dirty.} is needed.
+    macro foo(): untyped =
+      result = genAstOpt({kDirtyTemplate}):
+        let bar = "Hello, World"
+        &"Let's interpolate {bar} in the string"
+    doAssert foo() == "Let's interpolate Hello, World in the string"
+
+
+  block: # nested application of genAst
+    macro createMacro(name, obj, field: untyped): untyped =
+      result = genAst(obj = newDotExpr(obj, field), lit = 10, name, field):
+        # can't reuse `result` here, would clash
+        macro name(arg: untyped): untyped =
+          genAst(arg2=arg): # somehow `arg2` rename is needed
+            (obj, astToStr(field), lit, arg2)
+
+    var x = @[1, 2, 3]
+    createMacro foo, x, len
+    doAssert (foo 20) == (3, "len", 10, 20)
+
+  block: # test with kNoNewLit
+    macro bar(): untyped =
+      let s1 = true
+      template boo(x): untyped =
+        fun(x)
+      result = genAstOpt({kNoNewLit}, s1=newLit(s1), s1b=s1): (s1, s1b)
+    doAssert bar() == (true, 1)
+
+  block: # sanity check: check passing `{}` also works
+    macro bar(): untyped =
+      result = genAstOpt({}, s1=true): s1
+    doAssert bar() == true
+
+  block: # test passing function and type symbols
+    proc z1(): auto = 41
+    type Z4 = type(1'i8)
+    macro bar(Z1: typedesc): untyped =
+      proc z2(): auto = 42
+      proc z3[T](a: T): auto = 43
+      let Z2 = genAst():
+        type(true)
+      let z4 = genAst():
+        proc myfun(): auto = 44
+        myfun
+      type Z3 = type(1'u8)
+      result = genAst(z4, Z1, Z2):
+        # z1, z2, z3, Z3, Z4 are captured automatically
+        # z1, z2, z3 can optionally be specified in capture list
+        (z1(), z2(), z3('a'), z4(), $Z1, $Z2, $Z3, $Z4)
+    type Z1 = type('c')
+    doAssert bar(Z1) == (41, 42, 43, 44, "char", "bool", "uint8", "int8")
+
+  block: # fix bug #11986
+    proc foo(): auto =
+      var s = { 'a', 'b' }
+      # var n = quote do: `s` # would print {97, 98}
+      var n = genAst(s): s
+      n.repr
+    static: doAssert foo() == "{'a', 'b'}"
+
+  block: # also from #11986
+    macro foo(): untyped =
+      var s = { 'a', 'b' }
+      # quote do:
+      #   let t = `s`
+      #   $typeof(t) # set[range 0..65535(int)]
+      genAst(s):
+        let t = s
+        $typeof(t)
+    doAssert foo() == "set[char]"
+
+  block:
+    macro foo(): untyped =
+      type Foo = object
+      template baz2(a: int): untyped = a*10
+      macro baz3(a: int): untyped = newLit 13
+      result = newStmtList()
+
+      result.add genAst(Foo, baz2, baz3) do: # shows you can pass types, templates etc
+        var x: Foo
+        $($typeof(x), baz2(3), baz3(4))
+
+      let ret = genAst() do: # shows you don't have to, since they're inject'd
+        var x: Foo
+        $($typeof(x), baz2(3), baz3(4))
+    doAssert foo() == """("Foo", 30, 13)"""
+
+  block: # illustrates how symbol visiblity can be controlled precisely using `mixin`
+    proc locafun1(): auto = "in locafun1 (caller scope)" # this will be used because of `mixin locafun1` => explicit hijacking is ok
+    proc locafun2(): auto = "in locafun2 (caller scope)" # this won't be used => no hijacking
+    proc locafun3(): auto = "in locafun3 (caller scope)"
+    doAssert mixinExample() == ("in locafun1 (caller scope)", "in locafun2", "in locafun3 (caller scope)")
+
+static: main()
+main()

From 07991d902e6630593ce2db0f34c06bbb2abb336e Mon Sep 17 00:00:00 2001
From: flywind 
Date: Sat, 3 Apr 2021 04:06:29 +0800
Subject: [PATCH 0569/2472] [docs minor] close #17618 (#17620)

* [docs minor] close #17618

* Update lib/pure/times.nim

Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com>

Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com>
---
 lib/pure/times.nim | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/lib/pure/times.nim b/lib/pure/times.nim
index e763b17bb..a92eccc8c 100644
--- a/lib/pure/times.nim
+++ b/lib/pure/times.nim
@@ -313,8 +313,11 @@ type
   Duration* = object ## Represents a fixed duration of time, meaning a duration
                      ## that has constant length independent of the context.
                      ##
-                     ## To create a new `Duration`, use `initDuration proc
+                     ## To create a new `Duration`, use `initDuration
                      ## <#initDuration,int64,int64,int64,int64,int64,int64,int64,int64>`_.
+                     ## Instead of trying to access the private attributes, use
+                     ## `inSeconds <#inSeconds,Duration>`_ for converting to seconds and
+                     ## `inNanoseconds <#inNanoseconds,Duration>`_ for converting to nanoseconds.
     seconds: int64
     nanosecond: NanosecondRange
 
@@ -622,56 +625,56 @@ template convert(dur: Duration, unit: static[FixedTimeUnit]): int64 =
           convert(Nanoseconds, unit, dur.nanosecond)
 
 proc inWeeks*(dur: Duration): int64 =
-  ## Convert the duration to the number of whole weeks.
+  ## Converts the duration to the number of whole weeks.
   runnableExamples:
     let dur = initDuration(days = 8)
     doAssert dur.inWeeks == 1
   dur.convert(Weeks)
 
 proc inDays*(dur: Duration): int64 =
-  ## Convert the duration to the number of whole days.
+  ## Converts the duration to the number of whole days.
   runnableExamples:
     let dur = initDuration(hours = -50)
     doAssert dur.inDays == -2
   dur.convert(Days)
 
 proc inHours*(dur: Duration): int64 =
-  ## Convert the duration to the number of whole hours.
+  ## Converts the duration to the number of whole hours.
   runnableExamples:
     let dur = initDuration(minutes = 60, days = 2)
     doAssert dur.inHours == 49
   dur.convert(Hours)
 
 proc inMinutes*(dur: Duration): int64 =
-  ## Convert the duration to the number of whole minutes.
+  ## Converts the duration to the number of whole minutes.
   runnableExamples:
     let dur = initDuration(hours = 2, seconds = 10)
     doAssert dur.inMinutes == 120
   dur.convert(Minutes)
 
 proc inSeconds*(dur: Duration): int64 =
-  ## Convert the duration to the number of whole seconds.
+  ## Converts the duration to the number of whole seconds.
   runnableExamples:
     let dur = initDuration(hours = 2, milliseconds = 10)
     doAssert dur.inSeconds == 2 * 60 * 60
   dur.convert(Seconds)
 
 proc inMilliseconds*(dur: Duration): int64 =
-  ## Convert the duration to the number of whole milliseconds.
+  ## Converts the duration to the number of whole milliseconds.
   runnableExamples:
     let dur = initDuration(seconds = -2)
     doAssert dur.inMilliseconds == -2000
   dur.convert(Milliseconds)
 
 proc inMicroseconds*(dur: Duration): int64 =
-  ## Convert the duration to the number of whole microseconds.
+  ## Converts the duration to the number of whole microseconds.
   runnableExamples:
     let dur = initDuration(seconds = -2)
     doAssert dur.inMicroseconds == -2000000
   dur.convert(Microseconds)
 
 proc inNanoseconds*(dur: Duration): int64 =
-  ## Convert the duration to the number of whole nanoseconds.
+  ## Converts the duration to the number of whole nanoseconds.
   runnableExamples:
     let dur = initDuration(seconds = -2)
     doAssert dur.inNanoseconds == -2000000000

From e35946f306c31d02869dfc4f191b76893b792700 Mon Sep 17 00:00:00 2001
From: Andrey Makarov 
Date: Fri, 2 Apr 2021 23:11:44 +0300
Subject: [PATCH 0570/2472] enable syntax highlighting for inline code (#17585)

* enable syntax highlighting for inline code

* finish '.. default-role' and preliminary '.. role'

implementation

* more compact check in dirRole

* set :literal: as default role for *.rst

* Update lib/packages/docutils/rst.nim

Co-authored-by: Timothee Cour 

* use whichRole for setting currRoleKind

* Update lib/packages/docutils/rst.nim

Co-authored-by: Timothee Cour 

* rename rnGeneralRole -> rnUnknownRole

Co-authored-by: Timothee Cour 
---
 compiler/docgen.nim                          |   3 +-
 doc/contributing.rst                         |  38 ++++++
 lib/packages/docutils/rst.nim                | 116 +++++++++++++++----
 lib/packages/docutils/rstast.nim             |   9 +-
 lib/packages/docutils/rstgen.nim             |  33 ++++--
 nimdoc/rst2html/expected/rst_examples.html   |   6 +-
 nimdoc/rst2html/source/rst_examples.rst      |   4 +
 nimdoc/testproject/expected/testproject.html |   8 +-
 tests/stdlib/trstgen.nim                     |  73 ++++++++++--
 9 files changed, 239 insertions(+), 51 deletions(-)

diff --git a/compiler/docgen.nim b/compiler/docgen.nim
index 67f21e1c4..37d8955f6 100644
--- a/compiler/docgen.nim
+++ b/compiler/docgen.nim
@@ -192,7 +192,8 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
   result.cache = cache
   result.outDir = conf.outDir.string
   initRstGenerator(result[], (if conf.cmd != cmdRst2tex: outHtml else: outLatex),
-                   conf.configVars, filename.string, {roSupportRawDirective, roSupportMarkdown},
+                   conf.configVars, filename.string,
+                   {roSupportRawDirective, roSupportMarkdown, roNimFile},
                    docgenFindFile, compilerMsgHandler)
 
   if conf.configVars.hasKey("doc.googleAnalytics"):
diff --git a/doc/contributing.rst b/doc/contributing.rst
index 279a4ee94..7ee3aa444 100644
--- a/doc/contributing.rst
+++ b/doc/contributing.rst
@@ -295,6 +295,44 @@ example below) from `Nim Index`_ can be used in doc comment this way:
 
 .. _`Nim Index`: https://nim-lang.org/docs/theindex.html
 
+Inline monospaced text can be input using \`single backticks\` or
+\`\`double backticks\`\`. The former are syntactically highlighted,
+the latter are not.
+To avoid accidental highlighting follow this rule in `*.nim` files:
+
+* use single backticks for fragments of code in Nim and other
+  programming languages, including identifiers, in `*.nim` files.
+
+  For languages other than Nim add a role after final backtick,
+  e.g. for C++ inline highlighting::
+
+    `#include `:cpp:
+
+  For a currently unsupported language add the `:code:` role,
+  like for SQL in this example::
+
+    `SELECT * FROM ;`:code:
+
+* prefer double backticks otherwise:
+
+  * for file names: \`\`os.nim\`\`
+  * for fragments of strings **not** enclosed by `"` and `"` and not
+    related to code, e.g. text of compiler messages
+  * for command line options: \`\`--docInternal\`\`
+  * also when code ends with a standalone ``\`` (otherwise a combination of
+    ``\`` and a final \` would get escaped)
+
+.. Note:: `*.rst` files have `:literal:` as their default role.
+          So for them the rule above is only applicable if the `:nim:` role
+          is set up manually as the default::
+
+            .. role:: nim(code)
+               :language: nim
+            .. default-role:: nim
+
+          The first 2 lines are for other RST implementations,
+          including Github one.
+
 Best practices
 ==============
 
diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim
index d331c2c12..66efa3a3c 100644
--- a/lib/packages/docutils/rst.nim
+++ b/lib/packages/docutils/rst.nim
@@ -78,6 +78,13 @@
 ##
 ## * directives: ``code-block`` [cmp:Sphinx]_, ``title``,
 ##   ``index`` [cmp:Sphinx]_
+## * predefined roles ``:nim:`` (default), ``:c:`` (C programming language),
+##   ``:python:``, ``:yaml:``, ``:java:``, ``:cpp:`` (C++), ``:csharp`` (C#).
+##   That is every language that `highlite `_ supports.
+##   They turn on appropriate syntax highlighting in inline code.
+##
+##   .. Note:: default role for Nim files is ``:nim:``,
+##             for ``*.rst`` it's currently ``:literal:``.
 ##
 ## * ***triple emphasis*** (bold and italic) using \*\*\*
 ## * ``:idx:`` role for \`interpreted text\` to include the link to this
@@ -161,7 +168,9 @@ type
     roSupportSmilies,         ## make the RST parser support smilies like ``:)``
     roSupportRawDirective,    ## support the ``raw`` directive (don't support
                               ## it for sandboxing)
-    roSupportMarkdown         ## support additional features of Markdown
+    roSupportMarkdown,        ## support additional features of Markdown
+    roNimFile                 ## set for Nim files where default interpreted
+                              ## text role should be :nim:
 
   RstParseOptions* = set[RstParseOption]
 
@@ -454,6 +463,8 @@ type
     hTitleCnt: int              # =0 if no title, =1 if only main title,
                                 # =2 if both title and subtitle are present
     hCurLevel: int              # current section level
+    currRole: string            # current interpreted text role
+    currRoleKind: RstNodeKind   # ... and its node kind
     subs: seq[Substitution]     # substitutions
     refs: seq[Substitution]     # references
     anchors: seq[AnchorSubst]   # internal target substitutions
@@ -514,10 +525,36 @@ proc defaultFindFile*(filename: string): string =
   if fileExists(filename): result = filename
   else: result = ""
 
+proc defaultRole(options: RstParseOptions): string =
+  if roNimFile in options: "nim" else: "literal"
+
+# mirror highlite.nim sourceLanguageToStr with substitutions c++ cpp, c# csharp
+const supportedLanguages = ["nim", "yaml", "python", "java", "c",
+                            "cpp", "csharp"]
+
+proc whichRoleAux(sym: string): RstNodeKind =
+  let r = sym.toLowerAscii
+  case r
+  of "idx": result = rnIdx
+  of "literal": result = rnInlineLiteral
+  of "strong": result = rnStrongEmphasis
+  of "emphasis": result = rnEmphasis
+  of "sub", "subscript": result = rnSub
+  of "sup", "superscript": result = rnSup
+  # literal and code are the same in our implementation
+  of "code": result = rnInlineLiteral
+  # c++ currently can be spelled only as cpp, c# only as csharp
+  elif r in supportedLanguages:
+    result = rnInlineCode
+  else:  # unknown role
+    result = rnUnknownRole
+
 proc newSharedState(options: RstParseOptions,
                     findFile: FindFileHandler,
                     msgHandler: MsgHandler): PSharedState =
   new(result)
+  result.currRole = defaultRole(options)
+  result.currRoleKind = whichRoleAux(result.currRole)
   result.subs = @[]
   result.refs = @[]
   result.options = options
@@ -1018,15 +1055,28 @@ proc fixupEmbeddedRef(n, a, b: PRstNode) =
   for i in countup(0, sep - incr): a.add(n.sons[i])
   for i in countup(sep + 1, n.len - 2): b.add(n.sons[i])
 
-proc whichRole(sym: string): RstNodeKind =
-  case sym
-  of "idx": result = rnIdx
-  of "literal": result = rnInlineLiteral
-  of "strong": result = rnStrongEmphasis
-  of "emphasis": result = rnEmphasis
-  of "sub", "subscript": result = rnSub
-  of "sup", "superscript": result = rnSup
-  else: result = rnGeneralRole
+proc whichRole(p: RstParser, sym: string): RstNodeKind =
+  result = whichRoleAux(sym)
+  if result == rnUnknownRole:
+    rstMessage(p, mwUnsupportedLanguage, p.s.currRole)
+
+proc toInlineCode(n: PRstNode, language: string): PRstNode =
+  ## Creates rnInlineCode and attaches `n` contents as code (in 3rd son).
+  result = newRstNode(rnInlineCode)
+  let args = newRstNode(rnDirArg)
+  var lang = language
+  if language == "cpp": lang = "c++"
+  elif language == "csharp": lang = "c#"
+  args.add newLeaf(lang)
+  result.add args
+  result.add PRstNode(nil)
+  var lb = newRstNode(rnLiteralBlock)
+  var s: string
+  for i in n.sons:
+    assert i.kind == rnLeaf
+    s.add i.text
+  lb.add newLeaf(s)
+  result.add lb
 
 proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode =
   var newKind = n.kind
@@ -1052,14 +1102,23 @@ proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode =
     result = newRstNode(newKind, newSons)
   elif match(p, p.idx, ":w:"):
     # a role:
-    newKind = whichRole(nextTok(p).symbol)
-    if newKind == rnGeneralRole:
+    let roleName = nextTok(p).symbol
+    newKind = whichRole(p, roleName)
+    if newKind == rnUnknownRole:
       let newN = newRstNode(rnInner, n.sons)
-      newSons = @[newN, newLeaf(nextTok(p).symbol)]
+      newSons = @[newN, newLeaf(roleName)]
+      result = newRstNode(newKind, newSons)
+    elif newKind == rnInlineCode:
+      result = n.toInlineCode(language=roleName)
+    else:
+      result = newRstNode(newKind, newSons)
     inc p.idx, 3
-    result = newRstNode(newKind, newSons)
-  else:  # no change
-    result = n
+  else:
+    if p.s.currRoleKind == rnInlineCode:
+      result = n.toInlineCode(language=p.s.currRole)
+    else:
+      newKind = p.s.currRoleKind
+      result = newRstNode(newKind, newSons)
 
 proc matchVerbatim(p: RstParser, start: int, expr: string): int =
   result = start
@@ -1315,9 +1374,12 @@ proc parseInline(p: var RstParser, father: PRstNode) =
       parseUntil(p, n, "``", false)
       father.add(n)
     elif match(p, p.idx, ":w:") and p.tok[p.idx+3].symbol == "`":
-      let k = whichRole(nextTok(p).symbol)
-      let n = newRstNode(k)
+      let roleName = nextTok(p).symbol
+      let k = whichRole(p, roleName)
+      var n = newRstNode(k)
       inc p.idx, 3
+      if k == rnInlineCode:
+        n = n.toInlineCode(language=roleName)
       parseUntil(p, n, "`", false) # bug #17260
       father.add(n)
     elif isInlineMarkupStart(p, "`"):
@@ -2421,6 +2483,18 @@ proc dirAdmonition(p: var RstParser, d: string): PRstNode =
 
 proc dirDefaultRole(p: var RstParser): PRstNode =
   result = parseDirective(p, rnDefaultRole, {hasArg}, nil)
+  if result.sons[0].len == 0: p.s.currRole = defaultRole(p.s.options)
+  else:
+    assert result.sons[0].sons[0].kind == rnLeaf
+    p.s.currRole = result.sons[0].sons[0].text
+  p.s.currRoleKind = whichRole(p, p.s.currRole)
+
+proc dirRole(p: var RstParser): PRstNode =
+  result = parseDirective(p, rnDirective, {hasArg, hasOptions}, nil)
+  # just check that language is supported, TODO: real role association
+  let lang = getFieldValue(result, "language").strip
+  if lang != "" and lang notin supportedLanguages:
+    rstMessage(p, mwUnsupportedLanguage, lang)
 
 proc dirRawAux(p: var RstParser, result: var PRstNode, kind: RstNodeKind,
                contentParser: SectionParser) =
@@ -2465,7 +2539,9 @@ proc selectDir(p: var RstParser, d: string): PRstNode =
   of "code-block": result = dirCodeBlock(p, nimExtension = true)
   of "container": result = dirContainer(p)
   of "contents": result = dirContents(p)
-  of "danger", "error": result = dirAdmonition(p, d)
+  of "danger": result = dirAdmonition(p, d)
+  of "default-role": result = dirDefaultRole(p)
+  of "error": result = dirAdmonition(p, d)
   of "figure": result = dirFigure(p)
   of "hint": result = dirAdmonition(p, d)
   of "image": result = dirImage(p)
@@ -2478,10 +2554,10 @@ proc selectDir(p: var RstParser, d: string): PRstNode =
       result = dirRaw(p)
     else:
       rstMessage(p, meInvalidDirective, d)
+  of "role": result = dirRole(p)
   of "tip": result = dirAdmonition(p, d)
   of "title": result = dirTitle(p)
   of "warning": result = dirAdmonition(p, d)
-  of "default-role": result = dirDefaultRole(p)
   else:
     let tok = p.tok[p.idx-2]  # report on directive in ".. directive::"
     rstMessage(p, meInvalidDirective, d, tok.line, tok.col)
diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim
index ff34da2d1..dd456b577 100644
--- a/lib/packages/docutils/rstast.nim
+++ b/lib/packages/docutils/rstast.nim
@@ -55,12 +55,15 @@ type
                               #     * `file#id `_
                               #     * `file#id '_
     rnSubstitutionDef,        # a definition of a substitution
-    rnGeneralRole,            # Inline markup:
+    # Inline markup:
+    rnInlineCode,
+    rnUnknownRole,            # interpreted text with an unknown role
     rnSub, rnSup, rnIdx,
     rnEmphasis,               # "*"
     rnStrongEmphasis,         # "**"
     rnTripleEmphasis,         # "***"
-    rnInterpretedText,        # "`"
+    rnInterpretedText,        # "`" an auxiliary role for parsing that will
+                              # be converted into other kinds like rnInlineCode
     rnInlineLiteral,          # "``"
     rnInlineTarget,           # "_`target`"
     rnSubstitutionReferences, # "|"
@@ -252,7 +255,7 @@ proc renderRstToRst(d: var RenderContext, n: PRstNode, result: var string) =
     result.add(" <")
     renderRstToRst(d, n.sons[1], result)
     result.add(">`_")
-  of rnGeneralRole:
+  of rnUnknownRole:
     result.add('`')
     renderRstToRst(d, n.sons[0],result)
     result.add("`:")
diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim
index 30c7f3080..1a16f590e 100644
--- a/lib/packages/docutils/rstgen.nim
+++ b/lib/packages/docutils/rstgen.nim
@@ -942,7 +942,7 @@ proc parseCodeBlockParams(d: PDoc, n: PRstNode): CodeBlockParams =
   result.init
   if n.isNil:
     return
-  assert n.kind == rnCodeBlock
+  assert n.kind in {rnCodeBlock, rnInlineCode}
   assert(not n.sons[2].isNil)
 
   # Parse the field list for rendering parameters if there are any.
@@ -987,8 +987,8 @@ proc buildLinesHtmlTable(d: PDoc; params: CodeBlockParams, code: string,
       "
    text1
    " & (
       d.config.getOrDefault"doc.listing_button" % id)
 
-proc renderCodeBlock(d: PDoc, n: PRstNode, result: var string) =
-  ## Renders a code block, appending it to `result`.
+proc renderCode(d: PDoc, n: PRstNode, result: var string) =
+  ## Renders a code (code block or inline code), appending it to `result`.
   ##
   ## If the code block uses the ``number-lines`` option, a table will be
   ## generated with two columns, the first being a list of numbers and the
@@ -997,7 +997,7 @@ proc renderCodeBlock(d: PDoc, n: PRstNode, result: var string) =
   ## may also come from the parser through the internal ``default-language``
   ## option to differentiate between a plain code block and Nim's code block
   ## extension.
-  assert n.kind == rnCodeBlock
+  assert n.kind in {rnCodeBlock, rnInlineCode}
   if n.sons[2] == nil: return
   var params = d.parseCodeBlockParams(n)
   var m = n.sons[2].sons[0]
@@ -1006,10 +1006,23 @@ proc renderCodeBlock(d: PDoc, n: PRstNode, result: var string) =
   if params.testCmd.len > 0 and d.onTestSnippet != nil:
     d.onTestSnippet(d, params.filename, params.testCmd, params.status, m.text)
 
-  let (blockStart, blockEnd) = buildLinesHtmlTable(d, params, m.text,
+  var blockStart, blockEnd: string
+  case d.target
+  of outHtml:
+    if n.kind == rnCodeBlock:
+      (blockStart, blockEnd) = buildLinesHtmlTable(d, params, m.text,
                                                    n.anchor.idS)
-  dispA(d.target, result, blockStart,
-        "\\begin{rstpre}\n" & n.anchor.idS & "\n", [])
+    else:  # rnInlineCode
+      blockStart = ""
+      blockEnd = ""
+  of outLatex:
+    if n.kind == rnCodeBlock:
+      blockStart = "\n\n\\begin{rstpre}" & n.anchor.idS & "\n"
+      blockEnd = "\n\\end{rstpre}\n"
+    else:  # rnInlineCode
+      blockStart = "\\texttt{"
+      blockEnd = "}"
+  dispA(d.target, result, blockStart, blockStart, [])
   if params.lang == langNone:
     if len(params.langStr) > 0:
       d.msgHandler(d.filename, 1, 0, mwUnsupportedLanguage, params.langStr)
@@ -1028,7 +1041,7 @@ proc renderCodeBlock(d: PDoc, n: PRstNode, result: var string) =
           esc(d.target, substr(m.text, g.start, g.length+g.start-1)),
           tokenClassToStr[g.kind]])
     deinitGeneralTokenizer(g)
-  dispA(d.target, result, blockEnd, "\n\\end{rstpre}\n")
+  dispA(d.target, result, blockEnd, blockEnd)
 
 proc renderContainer(d: PDoc, n: PRstNode, result: var string) =
   var tmp = ""
@@ -1294,13 +1307,13 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) =
       result.add addNodes(lastSon(n))
 
   of rnImage, rnFigure: renderImage(d, n, result)
-  of rnCodeBlock: renderCodeBlock(d, n, result)
+  of rnCodeBlock, rnInlineCode: renderCode(d, n, result)
   of rnContainer: renderContainer(d, n, result)
   of rnSubstitutionReferences, rnSubstitutionDef:
     renderAux(d, n, "|$1|", "|$1|", result)
   of rnDirective:
     renderAux(d, n, "", "", result)
-  of rnGeneralRole:
+  of rnUnknownRole:
     var tmp0 = ""
     var tmp1 = ""
     renderRstToOut(d, n.sons[0], tmp0)
diff --git a/nimdoc/rst2html/expected/rst_examples.html b/nimdoc/rst2html/expected/rst_examples.html
index 8253289a4..32abc0f80 100644
--- a/nimdoc/rst2html/expected/rst_examples.html
+++ b/nimdoc/rst2html/expected/rst_examples.html
@@ -213,7 +213,7 @@ stmt = IND{>} stmt ^+ IND{=} DED  # list of statements
 
    
 
    Let T's be p's return type. NRVO applies for T if sizeof(T) >= N (where N is implementation dependent), in other words, it applies for "big" structures.
    
 
    Apart from built-in operations like array indexing, memory allocation, etc. the raise statement is the only way to raise an exception.
    
-
    typedesc used as a parameter type also introduces an implicit generic. typedesc has its own set of rules:
    
+
    typedesc used as a parameter type also introduces an implicit generic. typedesc has its own set of rules:
    
 
    The !=, >, >=, in, notin, isnot operators are in fact templates:
    
 
    a > b is transformed into b < a.
    a in b is transformed into contains(b, a).
    notin and isnot have the obvious meanings.
    A template where every parameter is untyped is called an immediate template. For historical reasons templates can be explicitly annotated with an immediate pragma and then these templates do not take part in overloading resolution and the parameters' types are ignored by the compiler. Explicit immediate templates are now deprecated.
    
 
@@ -297,10 +297,10 @@ stmt = IND{>} stmt ^+ IND{=} DED  # list of statements
 
    Introduction
    
 "Der Mensch ist doch ein Augentier -- schöne Dinge wünsch ich mir."
 
    This document is a tutorial for the programming language Nim. This tutorial assumes that you are familiar with basic programming concepts like variables, types, or statements but is kept very basic. The manual contains many more examples of the advanced language features. All code examples in this tutorial, as well as the ones found in the rest of Nim's documentation, follow the Nim style guide.
    
-
    However, this does not work. The problem is that the procedure should not only return, but return and continue after an iteration has finished. This return and continue is called a yield statement. Now the only thing left to do is to replace the proc keyword by iterator and here it is - our first iterator:
    
+
    However, this does not work. The problem is that the procedure should not only return, but return and continue after an iteration has finished. This return and continue is called a yield statement. Now the only thing left to do is to replace the proc keyword by iterator and here it is - our first iterator:
    
 
-
+
    A1 headerA2 | not fooled
    
 
    C1C2 bold
    D1 code \|D2
    D1 code \|D2
    
 
    E1 | text
    
 
    F2 without pipe
    
 
    not in table 
    
diff --git a/nimdoc/rst2html/source/rst_examples.rst b/nimdoc/rst2html/source/rst_examples.rst
index 54f0124c8..7fa20de6c 100644
--- a/nimdoc/rst2html/source/rst_examples.rst
+++ b/nimdoc/rst2html/source/rst_examples.rst
@@ -5,6 +5,10 @@ Not a Nim Manual
 :Authors: Andreas Rumpf, Zahary Karadjov
 :Version: |nimversion|
 
+.. role:: nim(code)
+   :language: nim
+.. default-role:: nim
+
 .. contents::
 
 
diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html
index 8525048ea..ba1791d81 100644
--- a/nimdoc/testproject/expected/testproject.html
+++ b/nimdoc/testproject/expected/testproject.html
@@ -565,14 +565,14 @@ This is deprecated with a message.
 
    func someFunc() {....raises: [], tags: [].}
    
 
    
 
-My someFunc. Stuff in quotes here. Some link
+My someFunc. Stuff in quotes here. Some link
 
 
    
 
 
    proc fromUtils3() {....raises: [], tags: [].}
    
 
    
 
-came form utils but should be shown where fromUtilsGen is called
+came form utils but should be shown where fromUtilsGen is called
 
    Example:
    
 
    discard 1
    
 
@@ -765,7 +765,7 @@ the c printf. etc.
 
    proc low[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect.}
    
 
    
 
-
    Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.
    
+
    Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.
    
 
    See also:
    
 
@@ -776,7 +776,7 @@ the c printf. etc.
 
    proc low2[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect.}
    
 
    
 
-
    Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.
    
+
    Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.
    
 
    See also:
    
 
diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim
index da01c30d2..ad0c27f05 100644
--- a/tests/stdlib/trstgen.nim
+++ b/tests/stdlib/trstgen.nim
@@ -10,7 +10,7 @@ import unittest, strutils, strtabs
 import std/private/miscdollars
 
 proc toHtml(input: string,
-            rstOptions: RstParseOptions = {roSupportMarkdown},
+            rstOptions: RstParseOptions = {roSupportMarkdown, roNimFile},
             error: ref string = nil,
             warnings: ref seq[string] = nil): string =
   ## If `error` is nil then no errors should be generated.
@@ -36,6 +36,11 @@ proc toHtml(input: string,
   except EParseError:
     discard
 
+# inline code tags (for parsing originated from highlite.nim)
+proc id(str: string): string = """"""  & str & ""
+proc op(str: string): string = """"""    & str & ""
+proc pu(str: string): string = """""" & str & ""
+
 suite "YAML syntax highlighting":
   test "Basics":
     let input = """.. code-block:: yaml
@@ -201,14 +206,14 @@ not in table"""
         `|` outside a table cell should render as `\|`
     consistently with markdown, see https://stackoverflow.com/a/66557930/1426932
     ]#
-    doAssert output1 == """
+    check(output1 == """
 
-
+
    A1 headerA2 | not fooled
    
 
    C1C2 bold
    D1 code \|D2
    D1 """ & id"code" & " " & op"\|" & """D2
    
 
    E1 | text
    
 
    F2 without pipe
    
 
    not in table
    
-"""
+""")
     let input2 = """
 | A1 header | A2 |
 | --- | --- |"""
@@ -556,19 +561,66 @@ let x = 1
     doAssert "foo.bar    """
-    check """`foo\`\`bar`""".toHtml == """foo``bar"""
-    check """`foo\`bar`""".toHtml == """foo`bar"""
-    check """`\`bar`""".toHtml == """`bar"""
-    check """`a\b\x\\ar`""".toHtml == """a\b\x\\ar"""
+    check("""`foo.bar`""".toHtml ==
+      """""" &
+      id"foo" & op"." & id"bar" & "")
+    check("""`foo\`\`bar`""".toHtml ==
+      """""" &
+      id"foo" & pu"`" & pu"`" & id"bar" & "")
+    check("""`foo\`bar`""".toHtml ==
+      """""" &
+      id"foo" & pu"`" & id"bar" & "")
+    check("""`\`bar`""".toHtml ==
+      """""" &
+      pu"`" & id"bar" & "")
+    check("""`a\b\x\\ar`""".toHtml ==
+      """""" &
+      id"a" & op"""\""" & id"b" & op"""\""" & id"x" & op"""\\""" & id"ar" &
+      "")
 
   test "inline literal":
     check """``foo.bar``""".toHtml == """foo.bar"""
     check """``foo\bar``""".toHtml == """foo\bar"""
     check """``f\`o\\o\b`ar``""".toHtml == """f\`o\\o\b`ar"""
 
+  test "default-role":
+    # nim(default) -> literal -> nim -> code(=literal)
+    let input = dedent"""
+      Par1 `value1`.
+
+      .. default-role:: literal
+
+      Par2 `value2`.
+
+      .. default-role:: nim
+
+      Par3 `value3`.
+
+      .. default-role:: code
+
+      Par4 `value4`."""
+    let p1 = """Par1 """ & id"value1" & "."
+    let p2 = """
    Par2 value2.
    """
+    let p3 = """
    Par3 """ & id"value3" & ".
    "
+    let p4 = """
    Par4 value4.
    """
+    let expected = p1 & p2 & "\n" & p3 & "\n" & p4 & "\n"
+    check(input.toHtml == expected)
+
+  test "role directive":
+    let input = dedent"""
+      .. role:: y(code)
+         :language: yaml
+
+      .. role:: brainhelp(code)
+         :language: brainhelp
+    """
+    var warnings = new seq[string]
+    let output = input.toHtml(warnings=warnings)
+    check(warnings[].len == 1 and "language 'brainhelp' not supported" in warnings[0])
+
   test "RST comments":
     let input1 = """
+
 Check that comment disappears:
 
 ..
@@ -1341,7 +1393,8 @@ Test1
 
   test "(not) Roles: check escaping 1":
     let expected = """See :subscript:""" &
-                   """some text."""
+                   """""" & id"some" & " " & id"text" &
+                   "."
     check """See \:subscript:`some text`.""".toHtml == expected
     check """See :subscript\:`some text`.""".toHtml == expected
 

From a807233aebcd3759bc3e21b450ed89e1eb6ddace Mon Sep 17 00:00:00 2001
From: flywind 
Date: Sat, 3 Apr 2021 09:26:30 +0800
Subject: [PATCH 0571/2472] fix #17615(runnableExamples silently ignored if
 placed after some code) (#17619)

Co-authored-by: Timothee Cour 
---
 compiler/docgen.nim                          | 33 +++++++++++---------
 lib/experimental/diff.nim                    |  8 ++---
 lib/pure/coro.nim                            |  2 +-
 lib/std/jsbigints.nim                        |  4 +--
 lib/wrappers/openssl.nim                     |  3 +-
 nimdoc/testproject/expected/testproject.html |  4 ++-
 nimdoc/testproject/testproject.nim           |  4 +--
 tests/nimdoc/t17615.nim                      | 11 +++++++
 8 files changed, 41 insertions(+), 28 deletions(-)
 create mode 100644 tests/nimdoc/t17615.nim

diff --git a/compiler/docgen.nim b/compiler/docgen.nim
index 37d8955f6..2f7415241 100644
--- a/compiler/docgen.nim
+++ b/compiler/docgen.nim
@@ -608,20 +608,23 @@ proc getAllRunnableExamplesImpl(d: PDoc; n: PNode, dest: var Rope, state: Runnab
       return rsComment
   of nkCallKinds:
     if isRunnableExamples(n[0]) and
-        n.len >= 2 and n.lastSon.kind == nkStmtList and state in {rsStart, rsComment, rsRunnable}:
-      let (rdoccmd, code) = prepareExample(d, n, topLevel)
-      var msg = "Example:"
-      if rdoccmd.len > 0: msg.add " cmd: " & rdoccmd
-      dispA(d.conf, dest, "\n
    $1
    \n",
-          "\n\\textbf{$1}\n", [msg.rope])
-      inc d.listingCounter
-      let id = $d.listingCounter
-      dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim", ""])
-      var dest2 = ""
-      renderNimCode(dest2, code, isLatex = d.conf.cmd == cmdRst2tex)
-      dest.add dest2
-      dest.add(d.config.getOrDefault"doc.listing_end" % id)
-      return rsRunnable
+        n.len >= 2 and n.lastSon.kind == nkStmtList:
+      if state in {rsStart, rsComment, rsRunnable}:
+        let (rdoccmd, code) = prepareExample(d, n, topLevel)
+        var msg = "Example:"
+        if rdoccmd.len > 0: msg.add " cmd: " & rdoccmd
+        dispA(d.conf, dest, "\n
    $1
    \n",
+            "\n\\textbf{$1}\n", [msg.rope])
+        inc d.listingCounter
+        let id = $d.listingCounter
+        dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim", ""])
+        var dest2 = ""
+        renderNimCode(dest2, code, isLatex = d.conf.cmd == cmdRst2tex)
+        dest.add dest2
+        dest.add(d.config.getOrDefault"doc.listing_end" % id)
+        return rsRunnable
+      else:
+        localError(d.conf, n.info, errUser, "runnableExamples must appear before the first non-comment statement")
   else: discard
   return rsDone
     # change this to `rsStart` if you want to keep generating doc comments
@@ -671,7 +674,7 @@ proc getAllRunnableExamples(d: PDoc, n: PNode, dest: var Rope) =
     else:
       for i in 0..Example:
    
 
    discard 4
    ok5 ok5b
 
    Example:
    
-
    assert true
    in or out?
+
    assert true
    
+
    Example:
    
+
    discard 1
    in or out?
 
 
    
 
diff --git a/nimdoc/testproject/testproject.nim b/nimdoc/testproject/testproject.nim
index 69edb0d23..6b82acb9b 100644
--- a/nimdoc/testproject/testproject.nim
+++ b/nimdoc/testproject/testproject.nim
@@ -330,13 +330,11 @@ when true: # (most) templates
     ## ok5
     ## ok5b
     runnableExamples: assert true
+    runnableExamples: discard 1
 
     ## in or out?
-    # this is an edge case; a newline separate last runnableExamples from 
-    # next doc comment but AST isnt' aware of it; this could change in future
     discard 8
     ## out
-    runnableExamples: discard 1
 
 when true: # issue #14473
   import std/[sequtils]
diff --git a/tests/nimdoc/t17615.nim b/tests/nimdoc/t17615.nim
new file mode 100644
index 000000000..77ae35a15
--- /dev/null
+++ b/tests/nimdoc/t17615.nim
@@ -0,0 +1,11 @@
+discard """
+  cmd: "nim doc -r $file"
+  errormsg: "runnableExamples must appear before the first non-comment statement"
+  line: 10
+"""
+
+func fn*() =
+  ## foo
+  discard
+  runnableExamples:
+    assert true

From 270964c487e5347c61dade25bec903580483dda5 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Fri, 2 Apr 2021 22:15:21 -0700
Subject: [PATCH 0572/2472] implement RFCs/294 ; disallow enum <=> enum
 conversion (#16351)

* fix https://github.com/nim-lang/RFCs/issues/294 ; disallow enum <=> enum conversion
* fix the runnableExamples that was the instigator of this RFC
* legacy -d:nimLegacyConvEnumEnum
* use -d:nimLegacyConvEnumEnum in important_package nimgame2
* add test for enum cast
* improve changelog
* add changelog: Changes affecting backward compatibility
* cleanup changelog
* fix changelog
---
 changelog.md                     | 58 +++++++++++++++++++-------------
 changelogs/changelog_X_XX_X.md   |  4 +++
 compiler/semexprs.nim            |  5 +++
 compiler/sempass2.nim            |  6 ++--
 compiler/vmgen.nim               |  4 +--
 lib/system/comparisons.nim       |  2 +-
 testament/important_packages.nim |  5 +--
 tests/misc/tcast.nim             | 36 +++++++++++++++++---
 tests/misc/tconv.nim             | 33 +++++++++++++++++-
 9 files changed, 115 insertions(+), 38 deletions(-)

diff --git a/changelog.md b/changelog.md
index 020d8a350..43a15e398 100644
--- a/changelog.md
+++ b/changelog.md
@@ -2,6 +2,40 @@
 
 
 
+## Changes affecting backward compatibility
+
+- `repr` now doesn't insert trailing newline; previous behavior was very inconsistent,
+  see #16034. Use `-d:nimLegacyReprWithNewline` for previous behavior.
+
+- An enum now can't be converted to another enum directly, you must use `ord` (or `cast`, but
+  compiler won't help if you misuse it).
+  ```
+  type A = enum a1, a2
+  type B = enum b1, b2
+  doAssert not compiles(a1.B)
+  doAssert compiles(a1.ord.B)
+  ```
+  for a transition period, use `-d:nimLegacyConvEnumEnum`.
+
+- Type mismatch errors now show more context, use `-d:nimLegacyTypeMismatch` for previous
+  behavior.
+
+- `echo` and `debugEcho` will now raise `IOError` if writing to stdout fails.  Previous behavior
+  silently ignored errors.  See #16366.  Use `-d:nimLegacyEchoNoRaise` for previous behavior.
+
+- `math.round` now is rounded "away from zero" in JS backend which is consistent
+  with other backends. See #9125. Use `-d:nimLegacyJsRound` for previous behavior.
+
+- Changed the behavior of `uri.decodeQuery` when there are unencoded `=`
+  characters in the decoded values. Prior versions would raise an error. This is
+  no longer the case to comply with the HTML spec and other languages
+  implementations. Old behavior can be obtained with
+  `-d:nimLegacyParseQueryStrict`. `cgi.decodeData` which uses the same
+  underlying code is also updated the same way.
+
+- In `std/os`, `getHomeDir`, `expandTilde`, `getTempDir`, `getConfigDir` now do not include trailing `DirSep`,
+  unless `-d:nimLegacyHomeDir` is specified (for a transition period).
+
 ## Standard library additions and changes
 
 - Added `sections` iterator in `parsecfg`.
@@ -84,9 +118,6 @@
 
 - Added a simpler to use `io.readChars` overload.
 
-- `repr` now doesn't insert trailing newline; previous behavior was very inconsistent,
-  see #16034. Use `-d:nimLegacyReprWithNewline` for previous behavior.
-
 - Added `**` to jsffi.
 
 - `writeStackTrace` is available in JS backend now.
@@ -109,9 +140,6 @@
 
 - Added `math.isNaN`.
 
-- `echo` and `debugEcho` will now raise `IOError` if writing to stdout fails.  Previous behavior
-  silently ignored errors.  See #16366.  Use `-d:nimLegacyEchoNoRaise` for previous behavior.
-
 - Added `jsbigints` module, arbitrary precision integers for JavaScript target.
 
 - Added `math.copySign`.
@@ -130,18 +158,8 @@
 - Added `posix_utils.osReleaseFile` to get system identification from `os-release` file on Linux and the BSDs.
   https://www.freedesktop.org/software/systemd/man/os-release.html
 
-- `math.round` now is rounded "away from zero" in JS backend which is consistent
-  with other backends. See #9125. Use `-d:nimLegacyJsRound` for previous behavior.
-
 - Added `socketstream` module that wraps sockets in the stream interface
 
-- Changed the behavior of `uri.decodeQuery` when there are unencoded `=`
-  characters in the decoded values. Prior versions would raise an error. This is
-  no longer the case to comply with the HTML spec and other languages
-  implementations. Old behavior can be obtained with
-  `-d:nimLegacyParseQueryStrict`. `cgi.decodeData` which uses the same
-  underlying code is also updated the same way.
-
 - Added `sugar.dumpToString` which improves on `sugar.dump`.
 
 - Added `math.signbit`.
@@ -235,7 +253,6 @@
 
 - Added `jscore.debugger` to [call any available debugging functionality, such as breakpoints.](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/debugger).
 
-
 - Added `std/channels`.
 
 - Added `htmlgen.portal` for [making "SPA style" pages using HTML only](https://web.dev/hands-on-portals).
@@ -243,9 +260,6 @@
 - Added `ZZZ` and `ZZZZ` patterns to `times.nim` `DateTime` parsing, to match time
   zone offsets without colons, e.g. `UTC+7 -> +0700`.
 
-- In `std/os`, `getHomeDir`, `expandTilde`, `getTempDir`, `getConfigDir` now do not include trailing `DirSep`,
-  unless `-d:nimLegacyHomeDir` is specified (for a transition period).
-
 - Added `jsconsole.dir`, `jsconsole.dirxml`, `jsconsole.timeStamp`.
 
 - Added dollar `$` and `len` for `jsre.RegExp`.
@@ -254,7 +268,6 @@
 
 - Added `hasClosure` to `std/typetraits`.
 
-
 - Added `genasts.genAst` that avoids the problems inherent with `quote do` and can
   be used as a replacement.
 
@@ -304,9 +317,6 @@
 
 - VM now supports `addr(mystring[ind])` (index + index assignment)
 
-- Type mismatch errors now show more context, use `-d:nimLegacyTypeMismatch` for previous
-  behavior.
-
 - Added `--hintAsError` with similar semantics as `--warningAsError`.
 
 - TLS: OSX now uses native TLS (`--tlsEmulation:off`), TLS now works with importcpp non-POD types,
diff --git a/changelogs/changelog_X_XX_X.md b/changelogs/changelog_X_XX_X.md
index 48520fd44..77b421f33 100644
--- a/changelogs/changelog_X_XX_X.md
+++ b/changelogs/changelog_X_XX_X.md
@@ -3,10 +3,14 @@
 This is an example file.
 The changes should go to changelog.md!
 
+## Changes affecting backward compatibility
+
+- `foo` now behaves differently, use `-d:nimLegacyFoo` for previous behavior.
 
 ## Standard library additions and changes
 
 - Added `example.exampleProc`.
+
 - Changed `example.foo` to take additional `bar` parameter.
 
 
diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim
index 27b78aa6f..71c9e0295 100644
--- a/compiler/semexprs.nim
+++ b/compiler/semexprs.nim
@@ -163,6 +163,11 @@ proc checkConvertible(c: PContext, targetTyp: PType, src: PNode): TConvStatus =
     result = checkConversionBetweenObjects(d.skipTypes(abstractInst), s.skipTypes(abstractInst), pointers)
   elif (targetBaseTyp.kind in IntegralTypes) and
       (srcBaseTyp.kind in IntegralTypes):
+    if targetTyp.kind == tyEnum and srcBaseTyp.kind == tyEnum:
+      if c.config.isDefined("nimLegacyConvEnumEnum"):
+        message(c.config, src.info, warnUser, "enum to enum conversion is now deprecated")
+      else: result = convNotLegal
+    # `elif` would be incorrect here
     if targetTyp.kind == tyBool:
       discard "convOk"
     elif targetTyp.isOrdinalType:
diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim
index 87d196fad..52998c214 100644
--- a/compiler/sempass2.nim
+++ b/compiler/sempass2.nim
@@ -137,7 +137,7 @@ proc guardGlobal(a: PEffects; n: PNode; guard: PSym) =
   # we allow accesses nevertheless in top level statements for
   # easier initialization:
   #if a.isTopLevel:
-  #  message(n.info, warnUnguardedAccess, renderTree(n))
+  #  message(a.config, n.info, warnUnguardedAccess, renderTree(n))
   #else:
   if not a.isTopLevel:
     localError(a.config, n.info, "unguarded access: " & renderTree(n))
@@ -478,7 +478,7 @@ proc getLockLevel(s: PSym): TLockLevel =
       result = 0.TLockLevel
     else:
       result = UnknownLockLevel
-      #message(s.info, warnUser, "FOR THIS " & s.name.s)
+      #message(??.config, s.info, warnUser, "FOR THIS " & s.name.s)
 
 proc mergeLockLevels(tracked: PEffects, n: PNode, lockLevel: TLockLevel) =
   if lockLevel >= tracked.currLockLevel:
@@ -544,7 +544,7 @@ proc assumeTheWorst(tracked: PEffects; n: PNode; op: PType) =
   let lockLevel = if op.lockLevel == UnspecifiedLockLevel: UnknownLockLevel
                   else: op.lockLevel
   #if lockLevel == UnknownLockLevel:
-  #  message(n.info, warnUser, "had to assume the worst here")
+  #  message(??.config, n.info, warnUser, "had to assume the worst here")
   mergeLockLevels(tracked, n, lockLevel)
 
 proc isOwnedProcVar(n: PNode; owner: PSym): bool =
diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim
index be26f98b3..69d7ec4b3 100644
--- a/compiler/vmgen.nim
+++ b/compiler/vmgen.nim
@@ -1435,7 +1435,7 @@ proc genAddr(c: PCtx, n: PNode, dest: var TDest, flags: TGenFlags) =
       # permanent, so that it's not used for anything else:
       c.prc.slots[tmp].kind = slotTempPerm
       # XXX this is still a hack
-      #message(n.info, warnUser, "suspicious opcode used")
+      #message(c.congig, n.info, warnUser, "suspicious opcode used")
     else:
       gABC(c, n, opcAddrReg, dest, tmp)
     c.freeTemp(tmp)
@@ -1687,7 +1687,7 @@ proc genArrAccessOpcode(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode;
     c.gABC(n, opcNodeToReg, dest, cc)
     c.freeTemp(cc)
   else:
-    #message(n.info, warnUser, "argh")
+    #message(c.config, n.info, warnUser, "argh")
     #echo "FLAGS ", flags, " ", fitsRegister(n.typ), " ", typeToString(n.typ)
     c.gABC(n, opc, dest, a, b)
   c.freeTemp(a)
diff --git a/lib/system/comparisons.nim b/lib/system/comparisons.nim
index 2e5664a95..7122daa20 100644
--- a/lib/system/comparisons.nim
+++ b/lib/system/comparisons.nim
@@ -9,7 +9,7 @@ proc `==`*[Enum: enum](x, y: Enum): bool {.magic: "EqEnum", noSideEffect.} =
         place1, place2 = 3
     var
       e1 = field1
-      e2 = Enum1(place2)
+      e2 = place2.ord.Enum1
     assert e1 == e2
     assert not compiles(e1 == place2) # raises error
 proc `==`*(x, y: pointer): bool {.magic: "EqRef", noSideEffect.} =
diff --git a/testament/important_packages.nim b/testament/important_packages.nim
index 8a8d6ffd0..8f096b71e 100644
--- a/testament/important_packages.nim
+++ b/testament/important_packages.nim
@@ -96,14 +96,15 @@ pkg "nimcrypto", "nim r --path:. tests/testall.nim" # `--path:.` workaround need
 pkg "NimData", "nim c -o:nimdataa src/nimdata.nim"
 pkg "nimes", "nim c src/nimes.nim"
 pkg "nimfp", "nim c -o:nfp -r src/fp.nim"
-pkg "nimgame2", "nim c nimgame2/nimgame.nim", allowFailure = true # XXX Doesn't work with deprecated 'randomize', will create a PR.
+pkg "nimgame2", "nim c -d:nimLegacyConvEnumEnum nimgame2/nimgame.nim", allowFailure = true
+  # XXX Doesn't work with deprecated 'randomize', will create a PR.
 pkg "nimgen", "nim c -o:nimgenn -r src/nimgen/runcfg.nim"
 pkg "nimlsp"
 pkg "nimly", "nim c -r tests/test_readme_example.nim"
 pkg "nimongo", "nimble test_ci", allowFailure = true
 pkg "nimph", "nimble test", "https://github.com/disruptek/nimph", allowFailure = true
 pkg "nimpy", "nim c -r tests/nimfrompy.nim"
-pkg "nimquery"
+pkg "nimquery", allowFailure = true # pending https://github.com/GULPF/nimquery/pull/10
 pkg "nimsl"
 pkg "nimsvg"
 pkg "nimterop", "nimble minitest"
diff --git a/tests/misc/tcast.nim b/tests/misc/tcast.nim
index 454801a2d..6d67b1c52 100644
--- a/tests/misc/tcast.nim
+++ b/tests/misc/tcast.nim
@@ -48,7 +48,7 @@ block:
   var x: ref int = nil
   doAssert cast[int](cast[ptr int](x)) == 0
 
-block:
+block: # cast of nil
   block:
     static:
       let a = cast[pointer](nil)
@@ -71,7 +71,33 @@ block:
     static:
       doAssert cast[RootRef](nil).repr == "nil"
 
-  # Issue #15730, not fixed yet
-  # block:
-  #   static:
-  #     doAssert cast[cstring](nil).repr == "nil"
+  when false: # xxx bug #15730, not fixed yet
+    block:
+      static:
+        doAssert cast[cstring](nil).repr == "nil"
+
+template main() =
+  # xxx move all under here to get tested in VM
+  block: # cast of enum
+    type Koo = enum k1, k2
+    type Goo = enum g1, g2
+    type Boo = enum b1 = -1, b2, b3, b4
+    type Coo = enum c1 = -1i8, c2, c3, c4
+    when nimvm:
+      # xxx: Error: VM does not support 'cast' from tyEnum to tyEnum
+      discard
+    else:
+      doAssert cast[Koo](k2) == k2
+      doAssert cast[Goo](k2) == g2
+      doAssert cast[Goo](k2.ord) == g2
+
+      doAssert b3.ord == 1
+      doAssert cast[Koo](b3) == k2
+      doAssert cast[Boo](k2) == b3
+
+      doAssert c3.ord == 1
+      doAssert cast[Koo](c3) == k2
+      doAssert cast[Coo](k2) == c3
+
+static: main()
+main()
diff --git a/tests/misc/tconv.nim b/tests/misc/tconv.nim
index f7d15b0b5..c93fc57f8 100644
--- a/tests/misc/tconv.nim
+++ b/tests/misc/tconv.nim
@@ -1,5 +1,13 @@
+discard """
+  nimout:'''
+tconv.nim(81, 15) Warning: enum to enum conversion is now deprecated [User]
+'''
+"""
+
 template reject(x) =
-    static: assert(not compiles(x))
+  static: doAssert(not compiles(x))
+template accept(x) =
+  static: doAssert(compiles(x))
 
 reject:
     const x = int8(300)
@@ -55,3 +63,26 @@ block: # issue 3766
     proc r(x: static[R]) =
       echo x
     r 3.R
+
+
+block: # https://github.com/nim-lang/RFCs/issues/294
+  type Koo = enum k1, k2
+  type Goo = enum g1, g2
+
+  accept: Koo(k2)
+  accept: k2.Koo
+  accept: k2.int.Goo
+
+  reject: Goo(k2)
+  reject: k2.Goo
+  reject: k2.string
+
+  {.define(nimLegacyConvEnumEnum).}
+  discard Goo(k2)
+  accept: Goo(k2)
+  accept: k2.Goo
+  reject: k2.string
+  {.undef(nimLegacyConvEnumEnum).}
+
+  reject: Goo(k2)
+  reject: k2.Goo

From 61c1e35181206b4305b6f57662bd9b2524c0fef8 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Sat, 3 Apr 2021 14:17:55 +0800
Subject: [PATCH 0573/2472] close #14806 (#17626)

* close #14806

* tiny
---
 tests/tools/compile/config.nims | 1 +
 tests/tools/compile/readme.md   | 1 +
 tests/tools/compile/tdeps.nim   | 5 +++++
 tests/tools/compile/tdetect.nim | 5 +++++
 4 files changed, 12 insertions(+)
 create mode 100644 tests/tools/compile/config.nims
 create mode 100644 tests/tools/compile/readme.md
 create mode 100644 tests/tools/compile/tdeps.nim
 create mode 100644 tests/tools/compile/tdetect.nim

diff --git a/tests/tools/compile/config.nims b/tests/tools/compile/config.nims
new file mode 100644
index 000000000..a19545668
--- /dev/null
+++ b/tests/tools/compile/config.nims
@@ -0,0 +1 @@
+switch("path", "$lib/../")
\ No newline at end of file
diff --git a/tests/tools/compile/readme.md b/tests/tools/compile/readme.md
new file mode 100644
index 000000000..cb5058e02
--- /dev/null
+++ b/tests/tools/compile/readme.md
@@ -0,0 +1 @@
+Test whether the tools compile.
\ No newline at end of file
diff --git a/tests/tools/compile/tdeps.nim b/tests/tools/compile/tdeps.nim
new file mode 100644
index 000000000..971ca1b8e
--- /dev/null
+++ b/tests/tools/compile/tdeps.nim
@@ -0,0 +1,5 @@
+discard """
+  action: compile
+"""
+
+include tools/deps
diff --git a/tests/tools/compile/tdetect.nim b/tests/tools/compile/tdetect.nim
new file mode 100644
index 000000000..253fc0d39
--- /dev/null
+++ b/tests/tools/compile/tdetect.nim
@@ -0,0 +1,5 @@
+discard """
+  action: compile
+"""
+
+include tools/detect/detect

From fe7a76f62fa1563a76e1dcce18e3ee98bcd5d36a Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Fri, 2 Apr 2021 23:19:17 -0700
Subject: [PATCH 0574/2472] repr: fix rendering of `'big`, `=destroy` etc
 (#17624)

---
 compiler/renderer.nim  | 22 +++++++++++++++++++---
 tests/stdlib/trepr.nim | 18 ++++++++++++++++++
 2 files changed, 37 insertions(+), 3 deletions(-)

diff --git a/compiler/renderer.nim b/compiler/renderer.nim
index 3d260f350..1ca5a41e3 100644
--- a/compiler/renderer.nim
+++ b/compiler/renderer.nim
@@ -51,6 +51,8 @@ type
     config*: ConfigRef
     mangler: seq[PSym]
 
+proc renderTree*(n: PNode, renderFlags: TRenderFlags = {}): string
+
 # We render the source code in a two phases: The first
 # determines how long the subtree will likely be, the second
 # phase appends to a buffer that will be the output.
@@ -1314,9 +1316,23 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
     put(g, tkOpr, "[]")
   of nkAccQuoted:
     put(g, tkAccent, "`")
-    if n.len > 0: gsub(g, n[0])
-    for i in 1.. 0 and tmp[0] in {'a'..'z', 'A'..'Z'}:
+          # handle `=destroy`, `'big'
+          discard
+        else:
+          useSpace = true
+      elif i > 0: useSpace = true
+      if useSpace:  put(g, tkSpaces, Space)
       gsub(g, n[i])
     put(g, tkAccent, "`")
   of nkIfExpr:
diff --git a/tests/stdlib/trepr.nim b/tests/stdlib/trepr.nim
index 357854d67..bca9b9f91 100644
--- a/tests/stdlib/trepr.nim
+++ b/tests/stdlib/trepr.nim
@@ -145,5 +145,23 @@ do:
 do:
   4"""
 
+  block: # bug #17292 (bug 4)
+    let a = deb:
+      proc `=destroy`() = discard
+      proc `'foo`(): int = discard
+      proc `foo bar baz`(): int = discard
+    let a2 = """
+
+proc `=destroy`() =
+  discard
+
+proc `'foo`(): int =
+  discard
+
+proc `foo bar baz`(): int =
+  discard
+"""
+    doAssert a2 == a
+
 static: main()
 main()

From 4a11a04fba99e868c371be09216dc93ae2fe3cd6 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Sat, 3 Apr 2021 07:05:37 -0700
Subject: [PATCH 0575/2472] fix #14850: `repr` now correctly renders `do`
 (#17623)

* fix #14850: `repr` now correctly renders `do`

* add tests

* fix test
---
 compiler/renderer.nim         | 20 +++++++++-----
 tests/errmsgs/twrongcolon.nim |  4 +--
 tests/stdlib/trepr.nim        | 49 +++++++++++++++++++++++++++++++++++
 3 files changed, 64 insertions(+), 9 deletions(-)

diff --git a/compiler/renderer.nim b/compiler/renderer.nim
index 1ca5a41e3..9a867b0a1 100644
--- a/compiler/renderer.nim
+++ b/compiler/renderer.nim
@@ -569,11 +569,11 @@ proc initContext(c: var TContext) =
   c.spacing = 0
   c.flags = {}
 
-proc gsub(g: var TSrcGen, n: PNode, c: TContext)
-proc gsub(g: var TSrcGen, n: PNode) =
+proc gsub(g: var TSrcGen, n: PNode, c: TContext, fromStmtList = false)
+proc gsub(g: var TSrcGen, n: PNode, fromStmtList = false) =
   var c: TContext
   initContext(c)
-  gsub(g, n, c)
+  gsub(g, n, c, fromStmtList = fromStmtList)
 
 proc hasCom(n: PNode): bool =
   result = false
@@ -681,7 +681,7 @@ proc gstmts(g: var TSrcGen, n: PNode, c: TContext, doIndent=true) =
       if n[i].kind in {nkStmtList, nkStmtListExpr, nkStmtListType}:
         gstmts(g, n[i], c, doIndent=false)
       else:
-        gsub(g, n[i])
+        gsub(g, n[i], fromStmtList = true)
       gcoms(g)
     if doIndent: dedent(g)
   else:
@@ -1003,7 +1003,7 @@ proc isCustomLit(n: PNode): bool =
     (n[1].kind == nkIdent and n[1].ident.s.startsWith('\'')) or
     (n[1].kind == nkSym and n[1].sym.name.s.startsWith('\''))
 
-proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
+proc gsub(g: var TSrcGen, n: PNode, c: TContext, fromStmtList = false) =
   if isNil(n): return
   var
     a: TContext
@@ -1040,9 +1040,15 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
         put(g, tkParLe, "(")
         gcomma(g, n, 1, i - 1 - n.len)
         put(g, tkParRi, ")")
-      put(g, tkColon, ":")
+      if fromStmtList:
+        put(g, tkColon, ":")
+      else:
+        put(g, tkSpaces, Space)
+        put(g, tkDo, "do")
+        put(g, tkColon, ":")
       gsub(g, n, i)
-      for j in i+1 ..< n.len:
+      i.inc
+      for j in i ..< n.len:
         optNL(g)
         put(g, tkDo, "do")
         put(g, tkColon, ":")
diff --git a/tests/errmsgs/twrongcolon.nim b/tests/errmsgs/twrongcolon.nim
index e59e37660..f4f996c3e 100644
--- a/tests/errmsgs/twrongcolon.nim
+++ b/tests/errmsgs/twrongcolon.nim
@@ -1,7 +1,7 @@
 discard """
-errormsg: "in expression ':"
+errormsg: "in expression ' do:"
 nimout: '''
-Error: in expression ':
+Error: in expression ' do:
   890': identifier expected, but found ''
 '''
 
diff --git a/tests/stdlib/trepr.nim b/tests/stdlib/trepr.nim
index bca9b9f91..3dcbe9b69 100644
--- a/tests/stdlib/trepr.nim
+++ b/tests/stdlib/trepr.nim
@@ -163,5 +163,54 @@ proc `foo bar baz`(): int =
 """
     doAssert a2 == a
 
+  block: # bug #14850
+    block:
+      let a = deb:
+        template bar(): untyped =
+          foo1:
+            discard
+            4
+          foo2(1):
+            discard
+            4
+          foo3(1):
+            discard
+            4
+          do: 1
+          do: 2
+          x.add foo4
+          x.add: foo5: 3
+          x.add foo6 do: 4
+          a.add(foo7 do:
+            echo "baz"
+            4)
+
+      doAssert a == """
+
+template bar(): untyped =
+  foo1:
+    discard
+    4
+  foo2(1):
+    discard
+    4
+  foo3(1):
+    discard
+    4
+  do:
+    1
+  do:
+    2
+  x.add foo4
+  x.add:
+    foo5:
+      3
+  x.add foo6 do:
+    4
+  a.add(foo7 do:
+    echo "baz"
+    4)
+"""
+
 static: main()
 main()

From 66890c3ad476edc54358d5ab9758df218c5c4229 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Sun, 4 Apr 2021 01:31:33 +0800
Subject: [PATCH 0576/2472] enable some packages (#17629)

---
 testament/important_packages.nim | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/testament/important_packages.nim b/testament/important_packages.nim
index 8f096b71e..620ad3a00 100644
--- a/testament/important_packages.nim
+++ b/testament/important_packages.nim
@@ -49,7 +49,7 @@ pkg "c2nim", "nim c testsuite/tester.nim"
 pkg "cascade"
 pkg "cello"
 pkg "chroma"
-pkg "chronicles", "nim c -o:chr -r chronicles.nim", allowFailure = true # pending https://github.com/status-im/nim-chronos/issues/169
+pkg "chronicles", "nim c -o:chr -r chronicles.nim"
 pkg "chronos", "nim c -r -d:release tests/testall", allowFailure = true # pending https://github.com/nim-lang/Nim/issues/17130
 pkg "cligen", "nim c --path:. -r cligen.nim"
 pkg "combparser", "nimble test --gc:orc"
@@ -72,7 +72,7 @@ pkg "gnuplot", "nim c gnuplot.nim"
 # pkg "gram", "nim c -r --gc:arc --define:danger tests/test.nim", "https://github.com/disruptek/gram"
   # pending https://github.com/nim-lang/Nim/issues/16509
 pkg "hts", "nim c -o:htss src/hts.nim"
-pkg "httpauth", allowFailure = true
+pkg "httpauth"
 pkg "illwill", "nimble examples"
 pkg "inim"
 pkg "itertools", "nim doc src/itertools.nim"
@@ -110,7 +110,7 @@ pkg "nimsvg"
 pkg "nimterop", "nimble minitest"
 pkg "nimwc", "nim c nimwc.nim"
 pkg "nimx", "nim c --threads:on test/main.nim", allowFailure = true
-pkg "nitter", "nim c src/nitter.nim", "https://github.com/zedeus/nitter", allowFailure = true
+pkg "nitter", "nim c src/nitter.nim", "https://github.com/zedeus/nitter"
 pkg "norm", "nim c -r tests/sqlite/trows.nim"
 pkg "npeg", "nimble testarc"
 pkg "numericalnim", "nim c -r tests/test_integrate.nim"

From 97764c8dc91081875f57029ae3f72d3a503f5e45 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Sat, 3 Apr 2021 15:24:01 -0700
Subject: [PATCH 0577/2472] remove un-needed {.push
 hint[ConvFromXtoItselfNotNeeded]: off.} in asyncfutures, asyncmacro (#17631)

---
 lib/pure/asyncfutures.nim | 2 --
 lib/pure/asyncmacro.nim   | 5 +----
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim
index c0c5c3f07..3c36c3453 100644
--- a/lib/pure/asyncfutures.nim
+++ b/lib/pure/asyncfutures.nim
@@ -363,12 +363,10 @@ proc read*[T](future: Future[T] | FutureVar[T]): T =
   ## this function will fail with a `ValueError` exception.
   ##
   ## If the result of the future is an error then that error will be raised.
-  {.push hint[ConvFromXtoItselfNotNeeded]: off.}
   when future is Future[T]:
     let fut = future
   else:
     let fut = Future[T](future)
-  {.pop.}
   if fut.finished:
     if fut.error != nil:
       injectStacktrace(fut)
diff --git a/lib/pure/asyncmacro.nim b/lib/pure/asyncmacro.nim
index 336d10aad..aaa5d9acd 100644
--- a/lib/pure/asyncmacro.nim
+++ b/lib/pure/asyncmacro.nim
@@ -35,14 +35,11 @@ template createCb(retFutureSym, iteratorNameSym,
 
         if next == nil:
           if not retFutureSym.finished:
-            let msg = "Async procedure ($1) yielded `nil`, are you await'ing a " &
-                    "`nil` Future?"
+            let msg = "Async procedure ($1) yielded `nil`, are you await'ing a `nil` Future?"
             raise newException(AssertionDefect, msg % strName)
         else:
           {.gcsafe.}:
-            {.push hint[ConvFromXtoItselfNotNeeded]: off.}
             next.addCallback cast[proc() {.closure, gcsafe.}](identName)
-            {.pop.}
     except:
       futureVarCompletions
       if retFutureSym.finished:

From f02e159b56aaa63713991c0a7f8e7125e91c832e Mon Sep 17 00:00:00 2001
From: Andrey Makarov 
Date: Sun, 4 Apr 2021 21:54:26 +0300
Subject: [PATCH 0578/2472] fix rst option list at EOF (follow-up #17442)
 (#17638)

---
 lib/packages/docutils/rst.nim | 2 +-
 tests/stdlib/trstgen.nim      | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim
index 66efa3a3c..dfa2f12be 100644
--- a/lib/packages/docutils/rst.nim
+++ b/lib/packages/docutils/rst.nim
@@ -2067,7 +2067,7 @@ proc parseOptionList(p: var RstParser): PRstNode =
       c.add(b)
       result.add(c)
     else:
-      dec p.idx  # back to tkIndent
+      if currentTok(p).kind != tkEof: dec p.idx  # back to tkIndent
       break
 
 proc parseDefinitionList(p: var RstParser): PRstNode =
diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim
index ad0c27f05..57dacdf11 100644
--- a/tests/stdlib/trstgen.nim
+++ b/tests/stdlib/trstgen.nim
@@ -1362,6 +1362,7 @@ Test1
           output)
     check("""-doption""" in
           output)
+    check "
    option
    " notin output
 
   test "Option list 3 (double /)":
     let input = dedent """
@@ -1378,6 +1379,7 @@ Test1
           output)
     check("""-doption""" in
           output)
+    check "
    option
    " notin output
 
   test "Roles: subscript prefix/postfix":
     let expected = "See some text."

From 70a30317f7b0a5711e248e0653b50d1c057bd16b Mon Sep 17 00:00:00 2001
From: flywind 
Date: Mon, 5 Apr 2021 04:47:28 +0800
Subject: [PATCH 0579/2472] fix #16693: testament spec nimout too lax (#16698)

Co-authored-by: Timothee Cour 
---
 testament/lib/stdtest/testutils.nim           | 33 +++++++++++++------
 testament/tests/shouldfail/tccodecheck.nim    |  3 +-
 testament/tests/shouldfail/tcolumn.nim        |  7 ++--
 testament/tests/shouldfail/terrormsg.nim      |  7 ++--
 testament/tests/shouldfail/texitcode1.nim     |  3 +-
 testament/tests/shouldfail/tfile.nim          |  5 +--
 testament/tests/shouldfail/tline.nim          |  7 ++--
 testament/tests/shouldfail/tmaxcodesize.nim   |  3 +-
 testament/tests/shouldfail/tnimout.nim        |  5 +--
 testament/tests/shouldfail/toutput.nim        |  7 ++--
 testament/tests/shouldfail/toutputsub.nim     |  3 +-
 testament/tests/shouldfail/treject.nim        |  3 +-
 testament/tests/shouldfail/tsortoutput.nim    | 11 ++++---
 testament/tests/shouldfail/ttimeout.nim       |  1 +
 testament/tests/shouldfail/tvalgrind.nim      |  5 +--
 tests/compilerfeatures/texpandmacro.nim       |  2 +-
 tests/errmsgs/tgcsafety.nim                   |  2 +-
 tests/errmsgs/twrongcolon.nim                 |  2 +-
 tests/exprs/tresultwarning.nim                |  2 +-
 tests/init/tuninit1.nim                       |  2 +-
 tests/objvariant/tcheckedfield1.nim           |  2 +-
 tests/pragmas/thintprocessing.nim             | 18 ++++++++++
 tests/pragmas/twarning_off.nim                |  2 --
 tests/stdlib/tcstring.nim                     |  2 +-
 tests/stdlib/ttestutils.nim                   |  5 +++
 .../varres/tprevent_forloopvar_mutations.nim  |  2 +-
 tests/vm/tcompiletimetable.nim                |  8 +++--
 tests/vm/tmisc_vm.nim                         |  3 +-
 28 files changed, 103 insertions(+), 52 deletions(-)
 create mode 100644 tests/pragmas/thintprocessing.nim

diff --git a/testament/lib/stdtest/testutils.nim b/testament/lib/stdtest/testutils.nim
index 36f951272..58d136696 100644
--- a/testament/lib/stdtest/testutils.nim
+++ b/testament/lib/stdtest/testutils.nim
@@ -1,5 +1,4 @@
 import std/private/miscdollars
-import std/strutils
 from std/os import getEnv
 
 template flakyAssert*(cond: untyped, msg = "", notifySuccess = true) =
@@ -26,15 +25,29 @@ template flakyAssert*(cond: untyped, msg = "", notifySuccess = true) =
     msg2.add $expr & " " & msg
     echo msg2
 
-proc greedyOrderedSubsetLines*(lhs, rhs: string): bool =
-  ## returns true if each stripped line in `lhs` appears in rhs, using a greedy matching.
-  let rhs = rhs.strip
-  var currentPos = 0
-  for line in lhs.strip.splitLines:
-    currentPos = rhs.find(line.strip, currentPos)
-    if currentPos < 0:
-      return false
-  return true
+when not defined(js):
+  import std/strutils
+
+  proc greedyOrderedSubsetLines*(lhs, rhs: string): bool =
+    ## Returns true if each stripped line in `lhs` appears in rhs, using a greedy matching.
+    iterator splitLinesClosure(): string {.closure.} =
+      for line in splitLines(rhs.strip):
+        yield line
+
+    var rhsIter = splitLinesClosure
+    var currentLine = strip(rhsIter())
+
+    for line in lhs.strip.splitLines:
+      let line = line.strip
+      if line.len != 0:
+        while line != currentLine:
+          currentLine = strip(rhsIter())
+          if rhsIter.finished:
+            return false
+
+      if rhsIter.finished:
+        return false
+    return true
 
 template enableRemoteNetworking*: bool =
   ## Allows contolling whether to run some test at a statement-level granularity.
diff --git a/testament/tests/shouldfail/tccodecheck.nim b/testament/tests/shouldfail/tccodecheck.nim
index a8d216a5b..7b5f0cce6 100644
--- a/testament/tests/shouldfail/tccodecheck.nim
+++ b/testament/tests/shouldfail/tccodecheck.nim
@@ -1,5 +1,6 @@
 discard """
-ccodecheck: "baz"
+  targets: "c"
+  ccodecheck: "baz"
 """
 
 proc foo(): void {.exportc: "bar".}=
diff --git a/testament/tests/shouldfail/tcolumn.nim b/testament/tests/shouldfail/tcolumn.nim
index 89482e673..b79ec52a4 100644
--- a/testament/tests/shouldfail/tcolumn.nim
+++ b/testament/tests/shouldfail/tcolumn.nim
@@ -1,7 +1,8 @@
 discard """
-errormsg: "undeclared identifier: 'undeclared'"
-line: 8
-column: 7
+  errormsg: "undeclared identifier: 'undeclared'"
+  targets: "c"
+  line: 9
+  column: 7
 """
 
 # test should fail because the line directive is wrong
diff --git a/testament/tests/shouldfail/terrormsg.nim b/testament/tests/shouldfail/terrormsg.nim
index dbbdf5021..e69035235 100644
--- a/testament/tests/shouldfail/terrormsg.nim
+++ b/testament/tests/shouldfail/terrormsg.nim
@@ -1,7 +1,8 @@
 discard """
-errormsg: "wrong error message"
-line: 8
-column: 6
+  errormsg: "wrong error message"
+  targets: "c"
+  line: 9
+  column: 6
 """
 
 # test should fail because the line directive is wrong
diff --git a/testament/tests/shouldfail/texitcode1.nim b/testament/tests/shouldfail/texitcode1.nim
index 1b38b4f2e..e5e061578 100644
--- a/testament/tests/shouldfail/texitcode1.nim
+++ b/testament/tests/shouldfail/texitcode1.nim
@@ -1,3 +1,4 @@
 discard """
-exitcode: 1
+  targets: "c"
+  exitcode: 1
 """
diff --git a/testament/tests/shouldfail/tfile.nim b/testament/tests/shouldfail/tfile.nim
index 20d4bd1f3..9463882f9 100644
--- a/testament/tests/shouldfail/tfile.nim
+++ b/testament/tests/shouldfail/tfile.nim
@@ -1,6 +1,7 @@
 discard """
-errormsg: "undeclared identifier: 'undefined'"
-file: "notthisfile.nim"
+  targets: "c"
+  errormsg: "undeclared identifier: 'undefined'"
+  file: "notthisfile.nim"
 """
 
 echo undefined
diff --git a/testament/tests/shouldfail/tline.nim b/testament/tests/shouldfail/tline.nim
index f7a09875c..7f7e90896 100644
--- a/testament/tests/shouldfail/tline.nim
+++ b/testament/tests/shouldfail/tline.nim
@@ -1,7 +1,8 @@
 discard """
-errormsg: "undeclared identifier: 'undeclared'"
-line: 9
-column: 6
+  targets: "c"
+  errormsg: "undeclared identifier: 'undeclared'"
+  line: 10
+  column: 6
 """
 
 # test should fail because the line directive is wrong
diff --git a/testament/tests/shouldfail/tmaxcodesize.nim b/testament/tests/shouldfail/tmaxcodesize.nim
index 9879e4181..9e2bd9cfb 100644
--- a/testament/tests/shouldfail/tmaxcodesize.nim
+++ b/testament/tests/shouldfail/tmaxcodesize.nim
@@ -1,5 +1,6 @@
 discard """
-maxcodesize: 1
+  targets: "c"
+  maxcodesize: 1
 """
 
 echo "Hello World"
diff --git a/testament/tests/shouldfail/tnimout.nim b/testament/tests/shouldfail/tnimout.nim
index c0e332053..832f134b0 100644
--- a/testament/tests/shouldfail/tnimout.nim
+++ b/testament/tests/shouldfail/tnimout.nim
@@ -1,6 +1,7 @@
 discard """
-nimout: "Hello World!"
-action: compile
+  targets: "c"
+  nimout: "Hello World!"
+  action: compile
 """
 
 static:
diff --git a/testament/tests/shouldfail/toutput.nim b/testament/tests/shouldfail/toutput.nim
index ac0bc7a46..0fa4d7278 100644
--- a/testament/tests/shouldfail/toutput.nim
+++ b/testament/tests/shouldfail/toutput.nim
@@ -1,7 +1,8 @@
 discard """
-output: '''
-done
-'''
+  targets: "c"
+  output: '''
+  done
+  '''
 """
 
 echo "broken"
diff --git a/testament/tests/shouldfail/toutputsub.nim b/testament/tests/shouldfail/toutputsub.nim
index 7cc51ee8d..b34f3a8f2 100644
--- a/testament/tests/shouldfail/toutputsub.nim
+++ b/testament/tests/shouldfail/toutputsub.nim
@@ -1,5 +1,6 @@
 discard """
-outputsub: "something else"
+  outputsub: "something else"
+  targets: "c"
 """
 
 echo "Hello World!"
diff --git a/testament/tests/shouldfail/treject.nim b/testament/tests/shouldfail/treject.nim
index aaf2b4a63..395dc4251 100644
--- a/testament/tests/shouldfail/treject.nim
+++ b/testament/tests/shouldfail/treject.nim
@@ -1,5 +1,6 @@
 discard """
-action: "reject"
+  action: "reject"
+  targets: "c"
 """
 
 # Because we set action="reject", we expect this line not to compile. But the
diff --git a/testament/tests/shouldfail/tsortoutput.nim b/testament/tests/shouldfail/tsortoutput.nim
index 4ce9ce26d..0c165d21b 100644
--- a/testament/tests/shouldfail/tsortoutput.nim
+++ b/testament/tests/shouldfail/tsortoutput.nim
@@ -1,9 +1,10 @@
 discard """
-sortoutput: true
-output: '''
-2
-1
-'''
+  sortoutput: true
+  targets: "c"
+  output: '''
+  2
+  1
+  '''
 """
 
 # this test should ensure that the output is actually sorted
diff --git a/testament/tests/shouldfail/ttimeout.nim b/testament/tests/shouldfail/ttimeout.nim
index fd3e1a598..8ffd71aaa 100644
--- a/testament/tests/shouldfail/ttimeout.nim
+++ b/testament/tests/shouldfail/ttimeout.nim
@@ -1,5 +1,6 @@
 discard """
   timeout: "0.1"
+  targets: "c"
 """
 
 import os
diff --git a/testament/tests/shouldfail/tvalgrind.nim b/testament/tests/shouldfail/tvalgrind.nim
index 4f699fd3b..5502705b3 100644
--- a/testament/tests/shouldfail/tvalgrind.nim
+++ b/testament/tests/shouldfail/tvalgrind.nim
@@ -1,6 +1,7 @@
 discard """
-valgrind: true
-cmd: "nim $target --gc:arc -d:useMalloc $options $file"
+  valgrind: true
+  targets: "c"
+  cmd: "nim $target --gc:arc -d:useMalloc $options $file"
 """
 
 # this is the same check used by testament/specs.nim whether or not valgrind
diff --git a/tests/compilerfeatures/texpandmacro.nim b/tests/compilerfeatures/texpandmacro.nim
index 76b0263ae..fea8b571f 100644
--- a/tests/compilerfeatures/texpandmacro.nim
+++ b/tests/compilerfeatures/texpandmacro.nim
@@ -1,6 +1,6 @@
 discard """
   cmd: "nim c --expandMacro:foo $file"
-  nimout: '''Hint: expanded macro:
+  nimout: '''texpandmacro.nim(17, 1) Hint: expanded macro:
 echo ["injected echo"]
 var x = 4 [ExpandMacro]
 '''
diff --git a/tests/errmsgs/tgcsafety.nim b/tests/errmsgs/tgcsafety.nim
index 77515b74f..09ef92e75 100644
--- a/tests/errmsgs/tgcsafety.nim
+++ b/tests/errmsgs/tgcsafety.nim
@@ -2,7 +2,7 @@ discard """
 cmd: "nim check $file"
 errormsg: "type mismatch: got .}>"
 nimout: '''
-type mismatch: got .}>
+tgcsafety.nim(30, 18) Error: type mismatch: got .}>
 but expected one of:
 proc serve(server: AsyncHttpServer; port: Port;
            callback: proc (request: Request): Future[void] {.closure, gcsafe.};
diff --git a/tests/errmsgs/twrongcolon.nim b/tests/errmsgs/twrongcolon.nim
index f4f996c3e..20063cbc4 100644
--- a/tests/errmsgs/twrongcolon.nim
+++ b/tests/errmsgs/twrongcolon.nim
@@ -1,7 +1,7 @@
 discard """
 errormsg: "in expression ' do:"
 nimout: '''
-Error: in expression ' do:
+twrongcolon.nim(11, 12) Error: in expression ' do:
   890': identifier expected, but found ''
 '''
 
diff --git a/tests/exprs/tresultwarning.nim b/tests/exprs/tresultwarning.nim
index 32934408e..28dabfdb1 100644
--- a/tests/exprs/tresultwarning.nim
+++ b/tests/exprs/tresultwarning.nim
@@ -1,5 +1,5 @@
 discard """
-  nimout: "Special variable 'result' is shadowed. [ResultShadowed]"
+  nimout: "tresultwarning.nim(6, 7) Warning: Special variable 'result' is shadowed. [ResultShadowed]"
 """
 
 proc test(): string =
diff --git a/tests/init/tuninit1.nim b/tests/init/tuninit1.nim
index fe91733ff..b281bcf89 100644
--- a/tests/init/tuninit1.nim
+++ b/tests/init/tuninit1.nim
@@ -1,5 +1,5 @@
 discard """
-  nimout: "Warning: use explicit initialization of 'y' for clarity [Uninit]"
+  nimout: "tuninit1.nim(35, 11) Warning: use explicit initialization of 'y' for clarity [Uninit]"
   line:34
   action: compile
 """
diff --git a/tests/objvariant/tcheckedfield1.nim b/tests/objvariant/tcheckedfield1.nim
index 69b099f24..e1a2e60e6 100644
--- a/tests/objvariant/tcheckedfield1.nim
+++ b/tests/objvariant/tcheckedfield1.nim
@@ -1,5 +1,5 @@
 discard """
-  nimout: "Warning: cannot prove that field 'x.s' is accessible [ProveField]"
+  nimout: "tcheckedfield1.nim(40, 6) Warning: cannot prove that field 'x.s' is accessible [ProveField]"
   line:51
   action: run
   output: "abc abc"
diff --git a/tests/pragmas/thintprocessing.nim b/tests/pragmas/thintprocessing.nim
new file mode 100644
index 000000000..c608bc6e4
--- /dev/null
+++ b/tests/pragmas/thintprocessing.nim
@@ -0,0 +1,18 @@
+discard """
+  disabled: windows
+  matrix: "--hint:processing"
+  nimout: '''
+compile start
+..
+warn_module.nim(6, 6) Hint: 'test' is declared but not used [XDeclaredButNotUsed]
+compile end
+'''
+"""
+
+static:
+  echo "compile start"
+
+import warn_module
+
+static:
+  echo "compile end"
diff --git a/tests/pragmas/twarning_off.nim b/tests/pragmas/twarning_off.nim
index bada2999b..ccf07b9c4 100644
--- a/tests/pragmas/twarning_off.nim
+++ b/tests/pragmas/twarning_off.nim
@@ -1,8 +1,6 @@
 discard """
-  matrix: "--hint:processing"
   nimout: '''
 compile start
-..
 warn_module.nim(6, 6) Hint: 'test' is declared but not used [XDeclaredButNotUsed]
 compile end
 '''
diff --git a/tests/stdlib/tcstring.nim b/tests/stdlib/tcstring.nim
index 98da5d5c4..04a26b53c 100644
--- a/tests/stdlib/tcstring.nim
+++ b/tests/stdlib/tcstring.nim
@@ -1,6 +1,6 @@
 discard """
   targets: "c cpp js"
-  matrix: "; --gc:arc"
+  matrix: "--gc:refc; --gc:arc"
 """
 
 from std/sugar import collect
diff --git a/tests/stdlib/ttestutils.nim b/tests/stdlib/ttestutils.nim
index 1a50d311b..7e39c9ae3 100644
--- a/tests/stdlib/ttestutils.nim
+++ b/tests/stdlib/ttestutils.nim
@@ -4,3 +4,8 @@ block: # greedyOrderedSubsetLines
   doAssert greedyOrderedSubsetLines("a1\na3", "a0\na1\na2\na3\na4")
   doAssert not greedyOrderedSubsetLines("a3\na1", "a0\na1\na2\na3\na4") # out of order
   doAssert not greedyOrderedSubsetLines("a1\na5", "a0\na1\na2\na3\na4") # a5 not in lhs
+
+  doAssert not greedyOrderedSubsetLines("a1\na5", "a0\na1\na2\na3\na4\nprefix:a5")
+  doAssert not greedyOrderedSubsetLines("a1\na5", "a0\na1\na2\na3\na4\na5:suffix")
+  doAssert not greedyOrderedSubsetLines("a5", "a0\na1\na2\na3\na4\nprefix:a5")
+  doAssert not greedyOrderedSubsetLines("a5", "a0\na1\na2\na3\na4\na5:suffix")
diff --git a/tests/varres/tprevent_forloopvar_mutations.nim b/tests/varres/tprevent_forloopvar_mutations.nim
index 15f31d8a3..045dc7cbb 100644
--- a/tests/varres/tprevent_forloopvar_mutations.nim
+++ b/tests/varres/tprevent_forloopvar_mutations.nim
@@ -1,7 +1,7 @@
 discard """
   errormsg: "type mismatch: got "
   line: 17
-  nimout: '''type mismatch: got 
+  nimout: '''tprevent_forloopvar_mutations.nim(17, 7) Error: type mismatch: got 
 but expected one of:
 proc inc[T: Ordinal](x: var T; y = 1)
   first type mismatch at position: 1
diff --git a/tests/vm/tcompiletimetable.nim b/tests/vm/tcompiletimetable.nim
index ece2ddfe9..1db490f1a 100644
--- a/tests/vm/tcompiletimetable.nim
+++ b/tests/vm/tcompiletimetable.nim
@@ -1,12 +1,16 @@
 discard """
-  nimout: '''2
+  nimout: '''
+2
 3
 4:2
 Got Hi
 Got Hey
+'''
+  output:'''
 a
 b
-c'''
+c
+'''
 """
 
 # bug #404
diff --git a/tests/vm/tmisc_vm.nim b/tests/vm/tmisc_vm.nim
index 2d3e30c5e..bbf618622 100644
--- a/tests/vm/tmisc_vm.nim
+++ b/tests/vm/tmisc_vm.nim
@@ -19,6 +19,7 @@ foo4
 (a: 0, b: 0)
 '''
 """
+import std/sets
 
 #bug #1009
 type
@@ -95,8 +96,6 @@ static: simpleTryFinally()
 
 # bug #10981
 
-import sets
-
 proc main =
   for i in 0..<15:
     var someSets = @[initHashSet[int]()]

From ca405167ed79a6591104de62de6bed077f0698f5 Mon Sep 17 00:00:00 2001
From: Andrey Makarov 
Date: Mon, 5 Apr 2021 09:33:06 +0300
Subject: [PATCH 0580/2472] fix :number-lines: regression (#17639)

---
 lib/packages/docutils/rstgen.nim |  2 +-
 tests/stdlib/trstgen.nim         | 10 ++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim
index 1a16f590e..f0a2604ff 100644
--- a/lib/packages/docutils/rstgen.nim
+++ b/lib/packages/docutils/rstgen.nim
@@ -980,7 +980,7 @@ proc buildLinesHtmlTable(d: PDoc; params: CodeBlockParams, code: string,
     result.beginTable.add($line & "\n")
     line.inc
     codeLines.dec
-  result.beginTable.add("" & (
+  result.beginTable.add("
    " & ( d.config.getOrDefault"doc.listing_start" % [id, sourceLanguageToStr[params.lang], idStr])) result.endTable = (d.config.getOrDefault"doc.listing_end" % id) & diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index 57dacdf11..99b8d8db5 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -1089,6 +1089,16 @@ Test1 let output0 = input0.toHtml doAssert "

    Paragraph1

    " in output0 + test "Nim code-block :number-lines:": + let input = dedent """ + .. code-block:: nim + :number-lines: 55 + + x + y + """ + check "
    55\n56\n
    " in input.toHtml + test "RST admonitions": # check that all admonitions are implemented let input0 = dedent """ From b9c94f22aa6f3b43962f1a1f2a9b27e9cba42069 Mon Sep 17 00:00:00 2001 From: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Date: Mon, 5 Apr 2021 09:42:18 +0200 Subject: [PATCH 0581/2472] Improve the typeinfo module (#17625) Co-authored-by: Timothee Cour --- lib/core/typeinfo.nim | 251 ++++++++++++++++++++++-------------------- 1 file changed, 129 insertions(+), 122 deletions(-) diff --git a/lib/core/typeinfo.nim b/lib/core/typeinfo.nim index 1c21e10bf..196ac8fe1 100644 --- a/lib/core/typeinfo.nim +++ b/lib/core/typeinfo.nim @@ -9,16 +9,29 @@ ## This module implements an interface to Nim's `runtime type information`:idx: ## (`RTTI`:idx:). See the `marshal `_ module for an example of -## what this module allows you to do. +## what this allows you to do. ## -## Note that even though `Any` and its operations hide the nasty low level -## details from its clients, it remains inherently unsafe! Also, Nim's -## runtime type information will evolve and may eventually be deprecated. -## As an alternative approach to programmatically understanding and -## manipulating types, consider using the `macros `_ package to -## work with the types' AST representation at compile time. See, for example, -## the `getTypeImpl proc`_. As an alternative -## approach to storing arbitrary types at runtime, consider using generics. +## ..note:: Even though `Any` and its operations hide the nasty low level +## details from its users, it remains inherently unsafe! Also, Nim's +## runtime type information will evolve and may eventually be deprecated. +## As an alternative approach to programmatically understanding and +## manipulating types, consider using the `macros `_ module to +## work with the types' AST representation at compile time. See for example +## the `getTypeImpl proc `_. As an alternative +## approach to storing arbitrary types at runtime, consider using generics. + +runnableExamples: + var x: Any + + var i = 42 + x = i.toAny + assert x.kind == akInt + assert x.getInt == 42 + + var s = @[1, 2, 3] + x = s.toAny + assert x.kind == akSequence + assert x.len == 3 {.push hints: off.} @@ -28,44 +41,45 @@ include "system/hti.nim" {.pop.} type - AnyKind* = enum ## what kind of `any` it is - akNone = 0, ## invalid any - akBool = 1, ## any represents a `bool` - akChar = 2, ## any represents a `char` - akEnum = 14, ## any represents an enum - akArray = 16, ## any represents an array - akObject = 17, ## any represents an object - akTuple = 18, ## any represents a tuple - akSet = 19, ## any represents a set - akRange = 20, ## any represents a range - akPtr = 21, ## any represents a ptr - akRef = 22, ## any represents a ref - akSequence = 24, ## any represents a sequence - akProc = 25, ## any represents a proc - akPointer = 26, ## any represents a pointer - akString = 28, ## any represents a string - akCString = 29, ## any represents a cstring - akInt = 31, ## any represents an int - akInt8 = 32, ## any represents an int8 - akInt16 = 33, ## any represents an int16 - akInt32 = 34, ## any represents an int32 - akInt64 = 35, ## any represents an int64 - akFloat = 36, ## any represents a float - akFloat32 = 37, ## any represents a float32 - akFloat64 = 38, ## any represents a float64 - akFloat128 = 39, ## any represents a float128 - akUInt = 40, ## any represents an unsigned int - akUInt8 = 41, ## any represents an unsigned int8 - akUInt16 = 42, ## any represents an unsigned in16 - akUInt32 = 43, ## any represents an unsigned int32 - akUInt64 = 44, ## any represents an unsigned int64 + AnyKind* = enum ## The kind of `Any`. + akNone = 0, ## invalid + akBool = 1, ## bool + akChar = 2, ## char + akEnum = 14, ## enum + akArray = 16, ## array + akObject = 17, ## object + akTuple = 18, ## tuple + akSet = 19, ## set + akRange = 20, ## range + akPtr = 21, ## ptr + akRef = 22, ## ref + akSequence = 24, ## sequence + akProc = 25, ## proc + akPointer = 26, ## pointer + akString = 28, ## string + akCString = 29, ## cstring + akInt = 31, ## int + akInt8 = 32, ## int8 + akInt16 = 33, ## int16 + akInt32 = 34, ## int32 + akInt64 = 35, ## int64 + akFloat = 36, ## float + akFloat32 = 37, ## float32 + akFloat64 = 38, ## float64 + akFloat128 = 39, ## float128 + akUInt = 40, ## uint + akUInt8 = 41, ## uint8 + akUInt16 = 42, ## uin16 + akUInt32 = 43, ## uint32 + akUInt64 = 44, ## uint64 # akOpt = 44+18 ## the builtin 'opt' type. - Any* = object ## can represent any nim value; NOTE: the wrapped - ## value can be modified with its wrapper! This means - ## that `Any` keeps a non-traced pointer to its - ## wrapped value and **must not** live longer than - ## its wrapped value. + Any* = object + ## A type that can represent any nim value. + ## + ## .. danger:: The wrapped value can be modified with its wrapper! This means + ## that `Any` keeps a non-traced pointer to its wrapped value and + ## **must not** live longer than its wrapped value. value: pointer when defined(js): rawType: PNimType @@ -84,9 +98,9 @@ when not defined(gcDestructors): PGenSeq = ptr TGenericSeq when defined(gogc): - const GenericSeqSize = (3 * sizeof(int)) + const GenericSeqSize = 3 * sizeof(int) else: - const GenericSeqSize = (2 * sizeof(int)) + const GenericSeqSize = 2 * sizeof(int) else: include system/seqs_v2_reimpl @@ -103,8 +117,7 @@ when not defined(js): proc genericAssign(dest, src: pointer, mt: PNimType) {.importCompilerProc.} when not defined(gcDestructors): - proc genericShallowAssign(dest, src: pointer, mt: PNimType) {. - importCompilerProc.} + proc genericShallowAssign(dest, src: pointer, mt: PNimType) {.importCompilerProc.} proc incrSeq(seq: PGenSeq, elemSize, elemAlign: int): PGenSeq {.importCompilerProc.} proc newObj(typ: PNimType, size: int): pointer {.importCompilerProc.} proc newSeq(typ: PNimType, len: int): pointer {.importCompilerProc.} @@ -120,7 +133,7 @@ template `+!!`(a, b): untyped = cast[pointer](cast[ByteAddress](a) + b) proc getDiscriminant(aa: pointer, n: ptr TNimNode): int = assert(n.kind == nkCase) var d: int - var a = cast[ByteAddress](aa) + let a = cast[ByteAddress](aa) case n.typ.size of 1: d = ze(cast[ptr int8](a +% n.offset)[]) of 2: d = ze(cast[ptr int16](a +% n.offset)[]) @@ -130,7 +143,7 @@ proc getDiscriminant(aa: pointer, n: ptr TNimNode): int = return d proc selectBranch(aa: pointer, n: ptr TNimNode): ptr TNimNode = - var discr = getDiscriminant(aa, n) + let discr = getDiscriminant(aa, n) if discr <% n.len: result = n.sons[discr] if result == nil: result = n.sons[n.len] @@ -144,17 +157,17 @@ proc newAny(value: pointer, rawType: PNimType): Any {.inline.} = when declared(system.VarSlot): proc toAny*(x: VarSlot): Any {.inline.} = - ## Constructs a `Any` object from a variable slot `x`. + ## Constructs an `Any` object from a variable slot `x`. ## This captures `x`'s address, so `x` can be modified with its - ## `Any` wrapper! The client needs to ensure that the wrapper + ## `Any` wrapper! The caller needs to ensure that the wrapper ## **does not** live longer than `x`! ## This is provided for easier reflection capabilities of a debugger. result.value = x.address result.rawType = x.typ proc toAny*[T](x: var T): Any {.inline.} = - ## Constructs a `Any` object from `x`. This captures `x`'s address, so - ## `x` can be modified with its `Any` wrapper! The client needs to ensure + ## Constructs an `Any` object from `x`. This captures `x`'s address, so + ## `x` can be modified with its `Any` wrapper! The caller needs to ensure ## that the wrapper **does not** live longer than `x`! newAny(addr(x), cast[PNimType](getTypeInfo(x))) @@ -167,12 +180,12 @@ proc size*(x: Any): int {.inline.} = result = x.rawType.size proc baseTypeKind*(x: Any): AnyKind {.inline.} = - ## Gets the base type's kind; `akNone` is returned if `x` has no base type. + ## Gets the base type's kind. If `x` has no base type, `akNone` is returned. if x.rawType.base != nil: result = AnyKind(ord(x.rawType.base.kind)) proc baseTypeSize*(x: Any): int {.inline.} = - ## Returns the size of `x`'s basetype. + ## Returns the size of `x`'s base type. If `x` has no base type, 0 is returned. if x.rawType.base != nil: result = x.rawType.base.size @@ -301,33 +314,32 @@ proc len*(x: Any): int = proc base*(x: Any): Any = - ## Returns base Any (useful for inherited object types). + ## Returns the base type of `x` (useful for inherited object types). result.rawType = x.rawType.base result.value = x.value proc isNil*(x: Any): bool = - ## `isNil` for an any `x` that represents a cstring, proc or + ## `isNil` for an `x` that represents a cstring, proc or ## some pointer type. assert x.rawType.kind in {tyCString, tyRef, tyPtr, tyPointer, tyProc} result = isNil(cast[ppointer](x.value)[]) -const - pointerLike = when defined(gcDestructors): {tyCString, tyRef, tyPtr, tyPointer, tyProc} - else: {tyString, tyCString, tyRef, tyPtr, tyPointer, - tySequence, tyProc} +const pointerLike = + when defined(gcDestructors): {tyCString, tyRef, tyPtr, tyPointer, tyProc} + else: {tyString, tyCString, tyRef, tyPtr, tyPointer, tySequence, tyProc} proc getPointer*(x: Any): pointer = ## Retrieves the pointer value out of `x`. `x` needs to be of kind ## `akString`, `akCString`, `akProc`, `akRef`, `akPtr`, - ## `akPointer`, `akSequence`. + ## `akPointer` or `akSequence`. assert x.rawType.kind in pointerLike result = cast[ppointer](x.value)[] proc setPointer*(x: Any, y: pointer) = ## Sets the pointer value of `x`. `x` needs to be of kind ## `akString`, `akCString`, `akProc`, `akRef`, `akPtr`, - ## `akPointer`, `akSequence`. + ## `akPointer` or `akSequence`. assert x.rawType.kind in pointerLike if y != nil and x.rawType.kind != tyPointer: genericAssign(x.value, y, x.rawType) @@ -349,15 +361,15 @@ proc fieldsAux(p: pointer, n: ptr TNimNode, if m != nil: fieldsAux(p, m, ret) iterator fields*(x: Any): tuple[name: string, any: Any] = - ## Iterates over every active field of the any `x` that represents an object + ## Iterates over every active field of `x`. `x` needs to represent an object ## or a tuple. assert x.rawType.kind in {tyTuple, tyObject} - var p = x.value + let p = x.value var t = x.rawType # XXX BUG: does not work yet, however is questionable anyway when false: if x.rawType.kind == tyObject: t = cast[ptr PNimType](x.value)[] - var ret: seq[tuple[name: cstring, any: Any]] = @[] + var ret: seq[tuple[name: cstring, any: Any]] if t.kind == tyObject: while true: fieldsAux(p, t.node, ret) @@ -368,8 +380,7 @@ iterator fields*(x: Any): tuple[name: string, any: Any] = for name, any in items(ret): yield ($name, any) -proc getFieldNode(p: pointer, n: ptr TNimNode, - name: cstring): ptr TNimNode = +proc getFieldNode(p: pointer, n: ptr TNimNode, name: cstring): ptr TNimNode = case n.kind of nkNone: assert(false) of nkSlot: @@ -383,17 +394,17 @@ proc getFieldNode(p: pointer, n: ptr TNimNode, if cmpNimIdentifier(n.name, name) == 0: result = n else: - var m = selectBranch(p, n) + let m = selectBranch(p, n) if m != nil: result = getFieldNode(p, m, name) proc `[]=`*(x: Any, fieldName: string, value: Any) = - ## Sets a field of `x`; `x` represents an object or a tuple. + ## Sets a field of `x`. `x` needs to represent an object or a tuple. var t = x.rawType # XXX BUG: does not work yet, however is questionable anyway when false: if x.rawType.kind == tyObject: t = cast[ptr PNimType](x.value)[] assert x.rawType.kind in {tyTuple, tyObject} - var n = getFieldNode(x.value, t.node, fieldName) + let n = getFieldNode(x.value, t.node, fieldName) if n != nil: assert n.typ == value.rawType genericAssign(x.value +!! n.offset, value.value, value.rawType) @@ -401,13 +412,13 @@ proc `[]=`*(x: Any, fieldName: string, value: Any) = raise newException(ValueError, "invalid field name: " & fieldName) proc `[]`*(x: Any, fieldName: string): Any = - ## Gets a field of `x`; `x` represents an object or a tuple. + ## Gets a field of `x`. `x` needs to represent an object or a tuple. var t = x.rawType # XXX BUG: does not work yet, however is questionable anyway when false: if x.rawType.kind == tyObject: t = cast[ptr PNimType](x.value)[] assert x.rawType.kind in {tyTuple, tyObject} - var n = getFieldNode(x.value, t.node, fieldName) + let n = getFieldNode(x.value, t.node, fieldName) if n != nil: result.value = x.value +!! n.offset result.rawType = n.typ @@ -417,39 +428,39 @@ proc `[]`*(x: Any, fieldName: string): Any = raise newException(ValueError, "invalid field name: " & fieldName) proc `[]`*(x: Any): Any = - ## Dereference operation for the any `x` that represents a ptr or a ref. + ## Dereference operator for `Any`. `x` needs to represent a ptr or a ref. assert x.rawType.kind in {tyRef, tyPtr} result.value = cast[ppointer](x.value)[] result.rawType = x.rawType.base proc `[]=`*(x, y: Any) = - ## Dereference operation for the any `x` that represents a ptr or a ref. + ## Dereference operator for `Any`. `x` needs to represent a ptr or a ref. assert x.rawType.kind in {tyRef, tyPtr} assert y.rawType == x.rawType.base genericAssign(cast[ppointer](x.value)[], y.value, y.rawType) proc getInt*(x: Any): int = - ## Retrieves the int value out of `x`. `x` needs to represent an int. + ## Retrieves the `int` value out of `x`. `x` needs to represent an `int`. assert skipRange(x.rawType).kind == tyInt result = cast[ptr int](x.value)[] proc getInt8*(x: Any): int8 = - ## Retrieves the int8 value out of `x`. `x` needs to represent an int8. + ## Retrieves the `int8` value out of `x`. `x` needs to represent an `int8`. assert skipRange(x.rawType).kind == tyInt8 result = cast[ptr int8](x.value)[] proc getInt16*(x: Any): int16 = - ## Retrieves the int16 value out of `x`. `x` needs to represent an int16. + ## Retrieves the `int16` value out of `x`. `x` needs to represent an `int16`. assert skipRange(x.rawType).kind == tyInt16 result = cast[ptr int16](x.value)[] proc getInt32*(x: Any): int32 = - ## Retrieves the int32 value out of `x`. `x` needs to represent an int32. + ## Retrieves the `int32` value out of `x`. `x` needs to represent an `int32`. assert skipRange(x.rawType).kind == tyInt32 result = cast[ptr int32](x.value)[] proc getInt64*(x: Any): int64 = - ## Retrieves the int64 value out of `x`. `x` needs to represent an int64. + ## Retrieves the `int64` value out of `x`. `x` needs to represent an `int64`. assert skipRange(x.rawType).kind == tyInt64 result = cast[ptr int64](x.value)[] @@ -457,7 +468,7 @@ proc getBiggestInt*(x: Any): BiggestInt = ## Retrieves the integer value out of `x`. `x` needs to represent ## some integer, a bool, a char, an enum or a small enough bit set. ## The value might be sign-extended to `BiggestInt`. - var t = skipRange(x.rawType) + let t = skipRange(x.rawType) case t.kind of tyInt: result = BiggestInt(cast[ptr int](x.value)[]) of tyInt8: result = BiggestInt(cast[ptr int8](x.value)[]) @@ -482,7 +493,7 @@ proc getBiggestInt*(x: Any): BiggestInt = proc setBiggestInt*(x: Any, y: BiggestInt) = ## Sets the integer value of `x`. `x` needs to represent ## some integer, a bool, a char, an enum or a small enough bit set. - var t = skipRange(x.rawType) + let t = skipRange(x.rawType) case t.kind of tyInt: cast[ptr int](x.value)[] = int(y) of tyInt8: cast[ptr int8](x.value)[] = int8(y) @@ -505,38 +516,34 @@ proc setBiggestInt*(x: Any, y: BiggestInt) = else: assert false proc getUInt*(x: Any): uint = - ## Retrieves the uint value out of `x`, `x` needs to represent an uint. + ## Retrieves the `uint` value out of `x`. `x` needs to represent a `uint`. assert skipRange(x.rawType).kind == tyUInt result = cast[ptr uint](x.value)[] proc getUInt8*(x: Any): uint8 = - ## Retrieves the uint8 value out of `x`, `x` needs to represent an - ## uint8. + ## Retrieves the `uint8` value out of `x`. `x` needs to represent a `uint8`. assert skipRange(x.rawType).kind == tyUInt8 result = cast[ptr uint8](x.value)[] proc getUInt16*(x: Any): uint16 = - ## Retrieves the uint16 value out of `x`, `x` needs to represent an - ## uint16. + ## Retrieves the `uint16` value out of `x`. `x` needs to represent a `uint16`. assert skipRange(x.rawType).kind == tyUInt16 result = cast[ptr uint16](x.value)[] proc getUInt32*(x: Any): uint32 = - ## Retrieves the uint32 value out of `x`, `x` needs to represent an - ## uint32. + ## Retrieves the `uint32` value out of `x`. `x` needs to represent a `uint32`. assert skipRange(x.rawType).kind == tyUInt32 result = cast[ptr uint32](x.value)[] proc getUInt64*(x: Any): uint64 = - ## Retrieves the uint64 value out of `x`, `x` needs to represent an - ## uint64. + ## Retrieves the `uint64` value out of `x`. `x` needs to represent a `uint64`. assert skipRange(x.rawType).kind == tyUInt64 result = cast[ptr uint64](x.value)[] proc getBiggestUint*(x: Any): uint64 = ## Retrieves the unsigned integer value out of `x`. `x` needs to ## represent an unsigned integer. - var t = skipRange(x.rawType) + let t = skipRange(x.rawType) case t.kind of tyUInt: result = uint64(cast[ptr uint](x.value)[]) of tyUInt8: result = uint64(cast[ptr uint8](x.value)[]) @@ -546,9 +553,9 @@ proc getBiggestUint*(x: Any): uint64 = else: assert false proc setBiggestUint*(x: Any; y: uint64) = - ## Sets the unsigned integer value of `c`. `c` needs to represent an + ## Sets the unsigned integer value of `x`. `x` needs to represent an ## unsigned integer. - var t = skipRange(x.rawType) + let t = skipRange(x.rawType) case t.kind: of tyUInt: cast[ptr uint](x.value)[] = uint(y) of tyUInt8: cast[ptr uint8](x.value)[] = uint8(y) @@ -558,14 +565,14 @@ proc setBiggestUint*(x: Any; y: uint64) = else: assert false proc getChar*(x: Any): char = - ## Retrieves the char value out of `x`. `x` needs to represent a char. - var t = skipRange(x.rawType) + ## Retrieves the `char` value out of `x`. `x` needs to represent a `char`. + let t = skipRange(x.rawType) assert t.kind == tyChar result = cast[ptr char](x.value)[] proc getBool*(x: Any): bool = - ## Retrieves the bool value out of `x`. `x` needs to represent a bool. - var t = skipRange(x.rawType) + ## Retrieves the `bool` value out of `x`. `x` needs to represent a `bool`. + let t = skipRange(x.rawType) assert t.kind == tyBool result = cast[ptr bool](x.value)[] @@ -579,10 +586,10 @@ proc getEnumOrdinal*(x: Any, name: string): int = ## Gets the enum field ordinal from `name`. `x` needs to represent an enum ## but is only used to access the type information. In case of an error ## `low(int)` is returned. - var typ = skipRange(x.rawType) + let typ = skipRange(x.rawType) assert typ.kind == tyEnum - var n = typ.node - var s = n.sons + let n = typ.node + let s = n.sons for i in 0 .. n.len-1: if cmpNimIdentifier($s[i].name, name) == 0: if ntfEnumHole notin typ.flags: @@ -595,16 +602,16 @@ proc getEnumField*(x: Any, ordinalValue: int): string = ## Gets the enum field name as a string. `x` needs to represent an enum ## but is only used to access the type information. The field name of ## `ordinalValue` is returned. - var typ = skipRange(x.rawType) + let typ = skipRange(x.rawType) assert typ.kind == tyEnum - var e = ordinalValue + let e = ordinalValue if ntfEnumHole notin typ.flags: if e <% typ.node.len: return $typ.node.sons[e].name else: # ugh we need a slow linear search: - var n = typ.node - var s = n.sons + let n = typ.node + let s = n.sons for i in 0 .. n.len-1: if s[i].offset == e: return $s[i].name result = $e @@ -614,17 +621,17 @@ proc getEnumField*(x: Any): string = result = getEnumField(x, getBiggestInt(x).int) proc getFloat*(x: Any): float = - ## Retrieves the float value out of `x`. `x` needs to represent an float. + ## Retrieves the `float` value out of `x`. `x` needs to represent a `float`. assert skipRange(x.rawType).kind == tyFloat result = cast[ptr float](x.value)[] proc getFloat32*(x: Any): float32 = - ## Retrieves the float32 value out of `x`. `x` needs to represent an float32. + ## Retrieves the `float32` value out of `x`. `x` needs to represent a `float32`. assert skipRange(x.rawType).kind == tyFloat32 result = cast[ptr float32](x.value)[] proc getFloat64*(x: Any): float64 = - ## Retrieves the float64 value out of `x`. `x` needs to represent an float64. + ## Retrieves the `float64` value out of `x`. `x` needs to represent a `float64`. assert skipRange(x.rawType).kind == tyFloat64 result = cast[ptr float64](x.value)[] @@ -647,7 +654,7 @@ proc setBiggestFloat*(x: Any, y: BiggestFloat) = else: assert false proc getString*(x: Any): string = - ## Retrieves the string value out of `x`. `x` needs to represent a string. + ## Retrieves the `string` value out of `x`. `x` needs to represent a `string`. assert x.rawType.kind == tyString when defined(gcDestructors): result = cast[ptr string](x.value)[] @@ -656,12 +663,12 @@ proc getString*(x: Any): string = result = cast[ptr string](x.value)[] proc setString*(x: Any, y: string) = - ## Sets the string value of `x`. `x` needs to represent a string. + ## Sets the `string` value of `x`. `x` needs to represent a `string`. assert x.rawType.kind == tyString cast[ptr string](x.value)[] = y # also correct for gcDestructors proc getCString*(x: Any): cstring = - ## Retrieves the cstring value out of `x`. `x` needs to represent a cstring. + ## Retrieves the `cstring` value out of `x`. `x` needs to represent a `cstring`. assert x.rawType.kind == tyCString result = cast[ptr cstring](x.value)[] @@ -672,10 +679,10 @@ proc assign*(x, y: Any) = genericAssign(x.value, y.value, y.rawType) iterator elements*(x: Any): int = - ## Iterates over every element of `x` that represents a Nim bitset. + ## Iterates over every element of `x`. `x` needs to represent a `set`. assert x.rawType.kind == tySet - var typ = x.rawType - var p = x.value + let typ = x.rawType + let p = x.value # "typ.slots.len" field is for sets the "first" field var u: int64 case typ.size @@ -684,22 +691,22 @@ iterator elements*(x: Any): int = of 4: u = ze64(cast[ptr int32](p)[]) of 8: u = cast[ptr int64](p)[] else: - var a = cast[pbyteArray](p) + let a = cast[pbyteArray](p) for i in 0 .. typ.size*8-1: if (ze(a[i div 8]) and (1 shl (i mod 8))) != 0: - yield i+typ.node.len + yield i + typ.node.len if typ.size <= 8: for i in 0..sizeof(int64)*8-1: if (u and (1'i64 shl int64(i))) != 0'i64: - yield i+typ.node.len + yield i + typ.node.len proc inclSetElement*(x: Any, elem: int) = ## Includes an element `elem` in `x`. `x` needs to represent a Nim bitset. assert x.rawType.kind == tySet - var typ = x.rawType - var p = x.value + let typ = x.rawType + let p = x.value # "typ.slots.len" field is for sets the "first" field - var e = elem - typ.node.len + let e = elem - typ.node.len case typ.size of 1: var a = cast[ptr int8](p) From 526157917598bc603f5e2296f86946b1ba7f2e65 Mon Sep 17 00:00:00 2001 From: Saem Ghani Date: Mon, 5 Apr 2021 02:37:28 -0700 Subject: [PATCH 0582/2472] [nim-gdb] Fixed enums and flag output [ci skip] (#17634) Debugger works for enums again. Additionally, flags work better than before. Reworked object printer as well, but the approach needs much more work or has to be replaced all together. This is mostly to save the work and myself or someone else can revisit it. --- .../untestable/gdb/gdb_pretty_printer_test.py | 5 +- .../gdb/gdb_pretty_printer_test_program.nim | 32 +++ tools/nim-gdb.py | 238 +++++++++++++----- 3 files changed, 204 insertions(+), 71 deletions(-) diff --git a/tests/untestable/gdb/gdb_pretty_printer_test.py b/tests/untestable/gdb/gdb_pretty_printer_test.py index 5b34bcb3d..8f0f88e85 100644 --- a/tests/untestable/gdb/gdb_pretty_printer_test.py +++ b/tests/untestable/gdb/gdb_pretty_printer_test.py @@ -25,6 +25,7 @@ outputs = [ 'seq(3, 3) = {"one", "two", "three"}', 'Table(3, 64) = {[4] = "four", [5] = "five", [6] = "six"}', 'Table(3, 8) = {["two"] = 2, ["three"] = 3, ["one"] = 1}', + '{a = 1, b = "some string"}' ] for i, expected in enumerate(outputs): @@ -32,7 +33,7 @@ for i, expected in enumerate(outputs): gdb.flush() functionSymbol = gdb.selected_frame().block().function - assert functionSymbol.line == 21 + assert functionSymbol.line == 41, str(functionSymbol.line) if i == 6: # myArray is passed as pointer to int to myDebug. I look up myArray up in the stack @@ -47,6 +48,6 @@ for i, expected in enumerate(outputs): output = str(raw) - assert output == expected, output + " != " + expected + assert output == expected, "{0} : output: ({1}) != expected: ({2})".format(i, output, expected) gdb.write(f"passed\n", gdb.STDLOG) gdb.execute("continue") diff --git a/tests/untestable/gdb/gdb_pretty_printer_test_program.nim b/tests/untestable/gdb/gdb_pretty_printer_test_program.nim index c376ef89a..d2acdd282 100644 --- a/tests/untestable/gdb/gdb_pretty_printer_test_program.nim +++ b/tests/untestable/gdb/gdb_pretty_printer_test_program.nim @@ -14,7 +14,27 @@ type moTwo, moThree, moFoure, + + MyObj = object + a*: int + b*: string + + # MyVariant = ref object + # id*: int + # case kind*: MyEnum + # of meOne: mInt*: int + # of meTwo, meThree: discard + # of meFour: + # moInt*: int + # babies*: seq[MyVariant] + # after: float + # MyIntVariant = ref object + # stuff*: int + # case myKind*: range[0..32766] + # of 0: mFloat*: float + # of 2: mString*: string + # else: mBabies*: seq[MyIntVariant] var counter = 0 @@ -74,6 +94,18 @@ proc testProc(): void = var myOtherTable = {"one": 1, "two": 2, "three": 3}.toTable myDebug(myOtherTable) #14 + var obj = MyObj(a: 1, b: "some string") + myDebug(obj) #15 + + # var varObj = MyVariant(id: 13, kind: meFour, moInt: 94, + # babies: @[MyVariant(id: 18, kind: meOne, mInt: 7, after: 1.0), + # MyVariant(id: 21, kind: meThree, after: 2.0)], + # after: 3.0) + # myDebug(varObj) #16 + + # var varObjInt = MyIntVariant(stuff: 5, myKind: 2, mString: "this is my sweet string") + # myDebug(varObjInt) #17 + echo(counter) diff --git a/tools/nim-gdb.py b/tools/nim-gdb.py index 8143b94d5..f35b9a033 100644 --- a/tools/nim-gdb.py +++ b/tools/nim-gdb.py @@ -1,6 +1,7 @@ import gdb import re import sys +import traceback # some feedback that the nim runtime support is loading, isn't a bad # thing at all. @@ -13,14 +14,14 @@ def printErrorOnce(id, message): global errorSet if id not in errorSet: errorSet.add(id) - gdb.write(message, gdb.STDERR) + gdb.write("printErrorOnce: " + message, gdb.STDERR) ################################################################################ ##### Type pretty printers ################################################################################ -type_hash_regex = re.compile("^\w*_([A-Za-z0-9]*)$") +type_hash_regex = re.compile("^([A-Za-z0-9]*)_([A-Za-z0-9]*)_+([A-Za-z0-9]*)$") def getNimRti(type_name): """ Return a ``gdb.Value`` object for the Nim Runtime Information of ``type_name``. """ @@ -28,13 +29,20 @@ def getNimRti(type_name): # Get static const TNimType variable. This should be available for # every non trivial Nim type. m = type_hash_regex.match(type_name) + lookups = [ + "NTI" + m.group(2).lower() + "__" + m.group(3) + "_", + "NTI" + "__" + m.group(3) + "_", + "NTI" + m.group(2).replace("colon", "58").lower() + "__" + m.group(3) + "_" + ] if m: - try: - return gdb.parse_and_eval("NTI__" + m.group(1) + "_") - except: - return None + for l in lookups: + try: + return gdb.parse_and_eval(l) + except: + pass + None -def getNameFromNimRti(rti_val): +def getNameFromNimRti(rti): """ Return name (or None) given a Nim RTI ``gdb.Value`` """ try: # sometimes there isn't a name field -- example enums @@ -192,6 +200,7 @@ class NimStringEqFunction (gdb.Function): NimStringEqFunction() + ################################################################################ ##### GDB Command, equivalent of Nim's $ operator ################################################################################ @@ -315,22 +324,6 @@ class NimStringPrinter: else: return "" -# class NimStringPrinter: -# pattern = re.compile(r'^NimStringDesc$') - -# def __init__(self, val): -# self.val = val - -# def display_hint(self): -# return 'string' - -# def to_string(self): -# if self.val: -# l = int(self.val['Sup']['len']) -# return self.val['data'].lazy_string(encoding="utf-8", length=l) -# else: -# return "" - class NimRopePrinter: pattern = re.compile(r'^tyObject_RopeObj__([A-Za-z0-9]*) \*$') @@ -372,8 +365,8 @@ def reprEnum(e, typ): e = int(e) n = typ["node"] flags = int(typ["flags"]) - # 1 << 2 is {ntfEnumHole} - if ((1 << 2) & flags) == 0: + # 1 << 6 is {ntfEnumHole} + if ((1 << 6) & flags) == 0: o = e - int(n["sons"][0]["offset"]) if o >= 0 and 0 < int(n["len"]): return n["sons"][o]["name"].string("utf-8", "ignore") @@ -386,19 +379,26 @@ def reprEnum(e, typ): return str(e) + " (invalid data!)" +def enumNti(typeNimName, idString): + typeInfoName = "NTI" + typeNimName.lower() + "__" + idString + "_" + nti = gdb.lookup_global_symbol(typeInfoName) + if nti is None: + typeInfoName = "NTI" + "__" + idString + "_" + nti = gdb.lookup_global_symbol(typeInfoName) + return (typeInfoName, nti) + class NimEnumPrinter: - pattern = re.compile(r'^tyEnum_(\w*)__([A-Za-z0-9]*)$') + pattern = re.compile(r'^tyEnum_([A-Za-z0-9]+)__([A-Za-z0-9]*)$') def __init__(self, val): self.val = val typeName = self.val.type.name match = self.pattern.match(typeName) self.typeNimName = match.group(1) - typeInfoName = "NTI__" + match.group(2) + "_" - self.nti = gdb.lookup_global_symbol(typeInfoName) + typeInfoName, self.nti = enumNti(self.typeNimName, match.group(2)) if self.nti is None: - printErrorOnce(typeInfoName, f"NimEnumPrinter: lookup global symbol '{typeInfoName}' failed for {typeName}.\n") + printErrorOnce(typeInfoName, f"NimEnumPrinter: lookup global symbol: '{typeInfoName}' failed for {typeName}.\n") def to_string(self): if self.nti: @@ -414,18 +414,17 @@ class NimSetPrinter: ## the set printer is limited to sets that fit in an integer. Other ## sets are compiled to `NU8 *` (ptr uint8) and are invisible to ## gdb (currently). - pattern = re.compile(r'^tySet_tyEnum_(\w*)_([A-Za-z0-9]*)$') + pattern = re.compile(r'^tySet_tyEnum_([A-Za-z0-9]+)__([A-Za-z0-9]*)$') def __init__(self, val): self.val = val - match = self.pattern.match(self.val.type.name) - self.typeNimName = match.group(1) - - typeInfoName = "NTI__" + match.group(2) + "_" - self.nti = gdb.lookup_global_symbol(typeInfoName) + typeName = self.val.type.name + match = self.pattern.match(typeName) + self.typeNimName = match.group(1) + typeInfoName, self.nti = enumNti(self.typeNimName, match.group(2)) if self.nti is None: - printErrorOnce(typeInfoName, "NimSetPrinter: lookup global symbol '"+ typeInfoName +" failed for " + self.val.type.name + ".\n") + printErrorOnce(typeInfoName, f"NimSetPrinter: lookup global symbol: '{typeInfoName}' failed for {typeName}.\n") def to_string(self): if self.nti: @@ -563,7 +562,7 @@ class NimStringTablePrinter: def children(self): if self.val: - data = NimSeqPrinter(self.val['data'].dereference()) + data = NimSeqPrinter(self.val['data'].referenced_value()) for idxStr, entry in data.children(): if int(entry['Field0']) != 0: yield (idxStr + ".Field0", entry['Field0']) @@ -603,55 +602,156 @@ class NimTablePrinter: # this is untested, therefore disabled # class NimObjectPrinter: -# pattern = re.compile(r'^tyObject_.*$') +# pattern = re.compile(r'^tyObject_([A-Za-z0-9]+)__(_?[A-Za-z0-9]*)(:? \*)?$') # def __init__(self, val): # self.val = val +# self.valType = None +# self.valTypeNimName = None # def display_hint(self): # return 'object' +# def _determineValType(self): +# if self.valType is None: +# vt = self.val.type +# if vt.name is None: +# target = vt.target() +# self.valType = target.pointer() +# self.fields = target.fields() +# self.valTypeName = target.name +# self.isPointer = True +# else: +# self.valType = vt +# self.fields = vt.fields() +# self.valTypeName = vt.name +# self.isPointer = False + # def to_string(self): -# return str(self.val.type) +# if self.valTypeNimName is None: +# self._determineValType() +# match = self.pattern.match(self.valTypeName) +# self.valTypeNimName = match.group(1) + +# return self.valTypeNimName # def children(self): -# if not self.val: -# yield "object", "" -# raise StopIteration +# self._determineValType() +# if self.isPointer and int(self.val) == 0: +# return +# self.baseVal = self.val.referenced_value() if self.isPointer else self.val -# for (i, field) in enumerate(self.val.type.fields()): -# if field.type.code == gdb.TYPE_CODE_UNION: -# yield _union_field -# else: -# yield (field.name, self.val[field]) +# for c in self.handleFields(self.baseVal, getNimRti(self.valTypeName)): +# yield c + +# def handleFields(self, currVal, rti, fields = None): +# rtiSons = None +# discField = (0, None) +# seenSup = False +# if fields is None: +# fields = self.fields +# try: # XXX: remove try after finished debugging this method +# for (i, field) in enumerate(fields): +# if field.name == "Sup": # inherited data +# seenSup = True +# baseRef = rti['base'] +# if baseRef: +# baseRti = baseRef.referenced_value() +# baseVal = currVal['Sup'] +# baseValType = baseVal.type +# if baseValType.name is None: +# baseValType = baseValType.target().pointer() +# baseValFields = baseValType.target().fields() +# else: +# baseValFields = baseValType.fields() + +# for c in self.handleFields(baseVal, baseRti, baseValFields): +# yield c +# else: +# if field.type.code == gdb.TYPE_CODE_UNION: +# # if not rtiSons: +# rtiNode = rti['node'].referenced_value() +# rtiSons = rtiNode['sons'] -# def _union_field(self, i, field): -# rti = getNimRti(self.val.type.name) -# if rti is None: -# return (field.name, "UNION field can't be displayed without RTI") +# if not rtiSons and int(rtiNode['len']) == 0 and str(rtiNode['name']) != "0x0": +# rtiSons = [rti['node']] # sons are dereferenced by the consumer + +# if not rtiSons: +# printErrorOnce(self.valTypeName, f"NimObjectPrinter: UNION field can't be displayed without RTI {self.valTypeName}, using fallback.\n") +# # yield (field.name, self.baseVal[field]) # XXX: this fallback seems wrong +# return # XXX: this should probably continue instead? -# node_sons = rti['node'].dereference()['sons'] -# prev_field = self.val.type.fields()[i - 1] +# if int(rtiNode['len']) != 0 and str(rtiNode['name']) != "0x0": +# gdb.write(f"wtf IT HAPPENED {self.valTypeName}\n", gdb.STDERR) -# descriminant_node = None -# for i in range(int(node['len'])): -# son = node_sons[i].dereference() -# if son['name'].string("utf-8", "ignore") == str(prev_field.name): -# descriminant_node = son -# break -# if descriminant_node is None: -# raise ValueError("Can't find union descriminant field in object RTI") +# discNode = rtiSons[discField[0]].referenced_value() +# if not discNode: +# raise ValueError("Can't find union discriminant field in object RTI") + +# discNodeLen = int(discNode['len']) +# discFieldVal = int(currVal[discField[1].name]) -# if descriminant_node is None: raise ValueError("Can't find union field in object RTI") -# union_node = descriminant_node['sons'][int(self.val[prev_field])].dereference() -# union_val = self.val[field] +# unionNodeRef = None +# if discFieldVal < discNodeLen: +# unionNodeRef = discNode['sons'][discFieldVal] +# if not unionNodeRef: +# unionNodeRef = discNode['sons'][discNodeLen] -# for f1 in union_val.type.fields(): -# for f2 in union_val[f1].type.fields(): -# if str(f2.name) == union_node['name'].string("utf-8", "ignore"): -# return (str(f2.name), union_val[f1][f2]) +# if not unionNodeRef: +# printErrorOnce(self.valTypeName + "no union node", f"wtf is up with sons {self.valTypeName} {unionNodeRef} {rtiNode['offset']} {discNode} {discFieldVal} {discNodeLen} {discField[1].name} {field.name} {field.type}\n") +# continue -# raise ValueError("RTI is absent or incomplete, can't find union definition in RTI") +# unionNode = unionNodeRef.referenced_value() + +# fieldName = "" if field.name == None else field.name.lower() +# unionNodeName = "" if not unionNode['name'] else unionNode['name'].string("utf-8", "ignore") +# if not unionNodeName or unionNodeName.lower() != fieldName: +# unionFieldName = f"_{discField[1].name.lower()}_{int(rti['node'].referenced_value()['len'])}" +# gdb.write(f"wtf i: {i} union: {unionFieldName} field: {fieldName} type: {field.type.name} tag: {field.type.tag}\n", gdb.STDERR) +# else: +# unionFieldName = unionNodeName + +# if discNodeLen == 0: +# yield (unionFieldName, currVal[unionFieldName]) +# else: +# unionNodeLen = int(unionNode['len']) +# if unionNodeLen > 0: +# for u in range(unionNodeLen): +# un = unionNode['sons'][u].referenced_value()['name'].string("utf-8", "ignore") +# yield (un, currVal[unionFieldName][un]) +# else: +# yield(unionNodeName, currVal[unionFieldName]) +# else: +# discIndex = i - 1 if seenSup else i +# discField = (discIndex, field) # discriminant field is the last normal field +# yield (field.name, currVal[field.name]) +# except GeneratorExit: +# raise +# except: +# gdb.write(f"wtf {self.valTypeName} {i} fn: {field.name} df: {discField} rti: {rti} rtiNode: {rti['node'].referenced_value()} rtiSons: {rtiSons} {sys.exc_info()} {traceback.format_tb(sys.exc_info()[2], limit = 10)}\n", gdb.STDERR) +# gdb.write(f"wtf {self.valTypeName} {i} {field.name}\n", gdb.STDERR) + +# # seenSup = False +# # for (i, field) in enumerate(fields): +# # # if field.name: +# # # val = currVal[field.name] +# # # else: +# # # val = None +# # rtiNode = rti['node'].referenced_value() +# # rtiLen = int(rtiNode['len']) +# # if int(rtiNode['len']) > 0: +# # sons = rtiNode['sons'] +# # elif int(rti['len']) == 0 and str(rti['name']) != "0x0": +# # sons = [rti['node']] # sons are dereferenced by the consumer +# # sonsIdx = i - 1 if seenSup else i +# # s = sons[sonsIdx].referenced_value() +# # addr = int(currVal.address) +# # off = addr + int(rtiNode['offset']) +# # seenSup = seenSup or field.name == "Sup" + +# # gdb.write(f"wtf: i: {i} sonsIdx: {sonsIdx} field: {field.name} rtiLen: {rtiLen} rti: {rti} rtiNode: {rtiNode} isUnion: {field.type.code == gdb.TYPE_CODE_UNION} s: {s}\n", gdb.STDERR) + +# raise ################################################################################ From 28de32c8b274d5eb55dd4a777abbcfe1945861d5 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 5 Apr 2021 12:19:17 -0700 Subject: [PATCH 0583/2472] document caveats of quit (#17648) --- lib/system.nim | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/system.nim b/lib/system.nim index ba27da0c7..bd9ae2762 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1185,10 +1185,6 @@ when defined(nimdoc): ## ## Before stopping the program the "exit procedures" are called in the ## opposite order they were added with `addExitProc `_. - ## `quit` never returns and ignores any exception that may have been raised - ## by the quit procedures. It does *not* call the garbage collector to free - ## all the memory, unless a quit procedure calls `GC_fullCollect - ## <#GC_fullCollect>`_. ## ## The proc `quit(QuitSuccess)` is called implicitly when your nim ## program finishes without incident for platforms where this is the @@ -1199,6 +1195,14 @@ when defined(nimdoc): ## have any compile time effect. If you need to stop the compiler inside a ## macro, use the `error `_ or `fatal ## `_ pragmas. + ## + ## .. danger:: In almost all cases, in particular in library code, prefer + ## alternatives, e.g. `doAssert false` or raise a `Defect`. + ## `quit` bypasses regular control flow in particular `defer`, + ## `try`, `catch`, `finally` and `destructors`, and exceptions that may have been + ## raised by an `addExitProc` proc, as well as cleanup code in other threads. + ## It does *not* call the garbage collector to free all the memory, + ## unless an `addExitProc` proc calls `GC_fullCollect <#GC_fullCollect>`_. elif defined(genode): include genode/env From aa4f18eff0344c4de828c1e22e440c0734113086 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 6 Apr 2021 02:11:30 -0700 Subject: [PATCH 0584/2472] items(array)+friends: remove a RT comparison (#17650) --- lib/system/iterators.nim | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/system/iterators.nim b/lib/system/iterators.nim index d8af4eed5..86d64d0c7 100644 --- a/lib/system/iterators.nim +++ b/lib/system/iterators.nim @@ -29,8 +29,8 @@ iterator mitems*[T](a: var openArray[T]): var T {.inline.} = iterator items*[IX, T](a: array[IX, T]): T {.inline.} = ## Iterates over each item of `a`. - var i = low(IX) - if i <= high(IX): + when a.len > 0: + var i = low(IX) while true: yield a[i] if i >= high(IX): break @@ -38,8 +38,8 @@ iterator items*[IX, T](a: array[IX, T]): T {.inline.} = iterator mitems*[IX, T](a: var array[IX, T]): var T {.inline.} = ## Iterates over each item of `a` so that you can modify the yielded value. - var i = low(IX) - if i <= high(IX): + when a.len > 0: + var i = low(IX) while true: yield a[i] if i >= high(IX): break @@ -146,8 +146,8 @@ iterator mpairs*[T](a: var openArray[T]): tuple[key: int, val: var T]{.inline.} iterator pairs*[IX, T](a: array[IX, T]): tuple[key: IX, val: T] {.inline.} = ## Iterates over each item of `a`. Yields `(index, a[index])` pairs. - var i = low(IX) - if i <= high(IX): + when a.len > 0: + var i = low(IX) while true: yield (i, a[i]) if i >= high(IX): break @@ -156,8 +156,8 @@ iterator pairs*[IX, T](a: array[IX, T]): tuple[key: IX, val: T] {.inline.} = iterator mpairs*[IX, T](a: var array[IX, T]): tuple[key: IX, val: var T] {.inline.} = ## Iterates over each item of `a`. Yields `(index, a[index])` pairs. ## `a[index]` can be modified. - var i = low(IX) - if i <= high(IX): + when a.len > 0: + var i = low(IX) while true: yield (i, a[i]) if i >= high(IX): break From 7bfb9f00022513c2d0f4f93799c07be7e9ffa1e6 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 6 Apr 2021 22:20:01 +0800 Subject: [PATCH 0585/2472] close #17636 (#17643) --- tests/concepts/texplain.nim | 86 +++++++++---------- tests/concepts/twrapconcept.nim | 3 +- .../tdont_return_unowned_from_owned.nim | 4 +- tests/effects/tfuncs_cannot_mutate.nim | 4 +- tests/effects/tfuncs_cannot_mutate_simple.nim | 3 +- tests/errmsgs/t6608.nim | 3 +- tests/errmsgs/tconceptconstraint.nim | 1 - tests/errmsgs/tgenericconstraint.nim | 1 - tests/errmsgs/twrong_at_operator.nim | 3 +- tests/errmsgs/twrongcolon.nim | 3 +- tests/init/tuninit1.nim | 3 +- tests/misc/tnoop.nim | 3 +- tests/objvariant/tcheckedfield1.nim | 3 +- tests/parallel/tgc_unsafe2.nim | 7 +- .../varres/tprevent_forloopvar_mutations.nim | 3 +- tests/views/tcannot_borrow.nim | 5 +- 16 files changed, 60 insertions(+), 75 deletions(-) diff --git a/tests/concepts/texplain.nim b/tests/concepts/texplain.nim index de3d540cd..49eb8eb6b 100644 --- a/tests/concepts/texplain.nim +++ b/tests/concepts/texplain.nim @@ -1,45 +1,45 @@ discard """ cmd: "nim c --verbosity:0 --colors:off $file" nimout: ''' -texplain.nim(164, 10) Hint: Non-matching candidates for e(y) +texplain.nim(162, 10) Hint: Non-matching candidates for e(y) proc e(i: int): int first type mismatch at position: 1 required type for i: int but expression 'y' is of type: MatchingType -texplain.nim(167, 7) Hint: Non-matching candidates for e(10) +texplain.nim(165, 7) Hint: Non-matching candidates for e(10) proc e(o: ExplainedConcept): int first type mismatch at position: 1 required type for o: ExplainedConcept but expression '10' is of type: int literal(10) -texplain.nim(130, 6) ExplainedConcept: undeclared field: 'foo' -texplain.nim(130, 6) ExplainedConcept: undeclared field: '.' -texplain.nim(130, 6) ExplainedConcept: expression '.' cannot be called -texplain.nim(130, 6) ExplainedConcept: expression '' has no type (or is ambiguous) -texplain.nim(130, 5) ExplainedConcept: concept predicate failed -texplain.nim(131, 6) ExplainedConcept: undeclared field: 'bar' -texplain.nim(131, 6) ExplainedConcept: undeclared field: '.' -texplain.nim(131, 6) ExplainedConcept: expression '.' cannot be called -texplain.nim(131, 6) ExplainedConcept: expression '' has no type (or is ambiguous) -texplain.nim(130, 5) ExplainedConcept: concept predicate failed +texplain.nim(128, 6) ExplainedConcept: undeclared field: 'foo' +texplain.nim(128, 6) ExplainedConcept: undeclared field: '.' +texplain.nim(128, 6) ExplainedConcept: expression '.' cannot be called +texplain.nim(128, 6) ExplainedConcept: expression '' has no type (or is ambiguous) +texplain.nim(128, 5) ExplainedConcept: concept predicate failed +texplain.nim(129, 6) ExplainedConcept: undeclared field: 'bar' +texplain.nim(129, 6) ExplainedConcept: undeclared field: '.' +texplain.nim(129, 6) ExplainedConcept: expression '.' cannot be called +texplain.nim(129, 6) ExplainedConcept: expression '' has no type (or is ambiguous) +texplain.nim(128, 5) ExplainedConcept: concept predicate failed -texplain.nim(170, 10) Hint: Non-matching candidates for e(10) +texplain.nim(168, 10) Hint: Non-matching candidates for e(10) proc e(o: ExplainedConcept): int first type mismatch at position: 1 required type for o: ExplainedConcept but expression '10' is of type: int literal(10) -texplain.nim(130, 6) ExplainedConcept: undeclared field: 'foo' -texplain.nim(130, 6) ExplainedConcept: undeclared field: '.' -texplain.nim(130, 6) ExplainedConcept: expression '.' cannot be called -texplain.nim(130, 6) ExplainedConcept: expression '' has no type (or is ambiguous) -texplain.nim(130, 5) ExplainedConcept: concept predicate failed -texplain.nim(131, 6) ExplainedConcept: undeclared field: 'bar' -texplain.nim(131, 6) ExplainedConcept: undeclared field: '.' -texplain.nim(131, 6) ExplainedConcept: expression '.' cannot be called -texplain.nim(131, 6) ExplainedConcept: expression '' has no type (or is ambiguous) -texplain.nim(130, 5) ExplainedConcept: concept predicate failed +texplain.nim(128, 6) ExplainedConcept: undeclared field: 'foo' +texplain.nim(128, 6) ExplainedConcept: undeclared field: '.' +texplain.nim(128, 6) ExplainedConcept: expression '.' cannot be called +texplain.nim(128, 6) ExplainedConcept: expression '' has no type (or is ambiguous) +texplain.nim(128, 5) ExplainedConcept: concept predicate failed +texplain.nim(129, 6) ExplainedConcept: undeclared field: 'bar' +texplain.nim(129, 6) ExplainedConcept: undeclared field: '.' +texplain.nim(129, 6) ExplainedConcept: expression '.' cannot be called +texplain.nim(129, 6) ExplainedConcept: expression '' has no type (or is ambiguous) +texplain.nim(128, 5) ExplainedConcept: concept predicate failed -texplain.nim(174, 20) Error: type mismatch: got +texplain.nim(172, 20) Error: type mismatch: got but expected one of: proc e(i: int): int first type mismatch at position: 1 @@ -49,11 +49,11 @@ proc e(o: ExplainedConcept): int first type mismatch at position: 1 required type for o: ExplainedConcept but expression 'n' is of type: NonMatchingType -texplain.nim(174, 9) template/generic instantiation of `assert` from here -texplain.nim(130, 5) ExplainedConcept: concept predicate failed +texplain.nim(172, 9) template/generic instantiation of `assert` from here +texplain.nim(128, 5) ExplainedConcept: concept predicate failed expression: e(n) -texplain.nim(175, 20) Error: type mismatch: got +texplain.nim(173, 20) Error: type mismatch: got but expected one of: proc r(i: string): int first type mismatch at position: 1 @@ -63,15 +63,15 @@ proc r(o: RegularConcept): int first type mismatch at position: 1 required type for o: RegularConcept but expression 'n' is of type: NonMatchingType -texplain.nim(175, 9) template/generic instantiation of `assert` from here -texplain.nim(134, 5) RegularConcept: concept predicate failed +texplain.nim(173, 9) template/generic instantiation of `assert` from here +texplain.nim(132, 5) RegularConcept: concept predicate failed proc r[T](a: SomeNumber; b: T; c: auto) first type mismatch at position: 1 required type for a: SomeNumber but expression 'n' is of type: NonMatchingType expression: r(n) -texplain.nim(176, 20) Hint: Non-matching candidates for r(y) +texplain.nim(174, 20) Hint: Non-matching candidates for r(y) proc r(i: string): int first type mismatch at position: 1 required type for i: string @@ -81,28 +81,26 @@ proc r[T](a: SomeNumber; b: T; c: auto) required type for a: SomeNumber but expression 'y' is of type: MatchingType -texplain.nim(184, 2) Error: type mismatch: got +texplain.nim(182, 2) Error: type mismatch: got but expected one of: proc f(o: NestedConcept) first type mismatch at position: 1 required type for o: NestedConcept but expression 'y' is of type: MatchingType -texplain.nim(134, 6) RegularConcept: undeclared field: 'foo' -texplain.nim(134, 6) RegularConcept: undeclared field: '.' -texplain.nim(134, 6) RegularConcept: expression '.' cannot be called -texplain.nim(134, 6) RegularConcept: expression '' has no type (or is ambiguous) -texplain.nim(134, 5) RegularConcept: concept predicate failed -texplain.nim(135, 6) RegularConcept: undeclared field: 'bar' -texplain.nim(135, 6) RegularConcept: undeclared field: '.' -texplain.nim(135, 6) RegularConcept: expression '.' cannot be called -texplain.nim(135, 6) RegularConcept: expression '' has no type (or is ambiguous) -texplain.nim(134, 5) RegularConcept: concept predicate failed -texplain.nim(138, 5) NestedConcept: concept predicate failed +texplain.nim(132, 6) RegularConcept: undeclared field: 'foo' +texplain.nim(132, 6) RegularConcept: undeclared field: '.' +texplain.nim(132, 6) RegularConcept: expression '.' cannot be called +texplain.nim(132, 6) RegularConcept: expression '' has no type (or is ambiguous) +texplain.nim(132, 5) RegularConcept: concept predicate failed +texplain.nim(133, 6) RegularConcept: undeclared field: 'bar' +texplain.nim(133, 6) RegularConcept: undeclared field: '.' +texplain.nim(133, 6) RegularConcept: expression '.' cannot be called +texplain.nim(133, 6) RegularConcept: expression '' has no type (or is ambiguous) +texplain.nim(132, 5) RegularConcept: concept predicate failed +texplain.nim(136, 5) NestedConcept: concept predicate failed expression: f(y)''' errormsg: "type mismatch: got " - line: 184 - """ diff --git a/tests/concepts/twrapconcept.nim b/tests/concepts/twrapconcept.nim index 377b63afe..c3dea2ff9 100644 --- a/tests/concepts/twrapconcept.nim +++ b/tests/concepts/twrapconcept.nim @@ -1,7 +1,6 @@ discard """ errormsg: "type mismatch: got " - line: 21 - nimout: "twrapconcept.nim(11, 5) Foo: concept predicate failed" + nimout: "twrapconcept.nim(10, 5) Foo: concept predicate failed" """ # https://github.com/nim-lang/Nim/issues/5127 diff --git a/tests/destructor/tdont_return_unowned_from_owned.nim b/tests/destructor/tdont_return_unowned_from_owned.nim index a726960c6..d27626dea 100644 --- a/tests/destructor/tdont_return_unowned_from_owned.nim +++ b/tests/destructor/tdont_return_unowned_from_owned.nim @@ -1,6 +1,7 @@ discard """ cmd: "nim check --newruntime --hints:off $file" - nimout: '''tdont_return_unowned_from_owned.nim(36, 10) Error: cannot return an owned pointer as an unowned pointer; use 'owned(Obj)' as the return type + nimout: ''' +tdont_return_unowned_from_owned.nim(36, 10) Error: cannot return an owned pointer as an unowned pointer; use 'owned(Obj)' as the return type tdont_return_unowned_from_owned.nim(39, 10) Error: cannot return an owned pointer as an unowned pointer; use 'owned(Obj)' as the return type tdont_return_unowned_from_owned.nim(42, 6) Error: type mismatch: got but expected one of: @@ -16,7 +17,6 @@ tdont_return_unowned_from_owned.nim(51, 13) Error: assignment produces a danglin tdont_return_unowned_from_owned.nim(55, 10) Error: cannot return an owned pointer as an unowned pointer; use 'owned(RootRef)' as the return type ''' errormsg: "cannot return an owned pointer as an unowned pointer; use 'owned(RootRef)' as the return type" - line: 55 """ diff --git a/tests/effects/tfuncs_cannot_mutate.nim b/tests/effects/tfuncs_cannot_mutate.nim index 4768af2b3..8784cbbb1 100644 --- a/tests/effects/tfuncs_cannot_mutate.nim +++ b/tests/effects/tfuncs_cannot_mutate.nim @@ -2,8 +2,8 @@ discard """ errormsg: "'mutate' can have side effects" nimout: '''an object reachable from 'n' is potentially mutated tfuncs_cannot_mutate.nim(39, 15) the mutation is here -tfuncs_cannot_mutate.nim(37, 7) is the statement that connected the mutation to the parameter''' - line: 33 +tfuncs_cannot_mutate.nim(37, 7) is the statement that connected the mutation to the parameter +''' """ {.experimental: "strictFuncs".} diff --git a/tests/effects/tfuncs_cannot_mutate_simple.nim b/tests/effects/tfuncs_cannot_mutate_simple.nim index 9de20d1ec..a94a8d746 100644 --- a/tests/effects/tfuncs_cannot_mutate_simple.nim +++ b/tests/effects/tfuncs_cannot_mutate_simple.nim @@ -1,8 +1,7 @@ discard """ errormsg: "'edit' can have side effects" nimout: '''an object reachable from 'x' is potentially mutated -tfuncs_cannot_mutate_simple.nim(17, 4) the mutation is here''' - line: 16 +tfuncs_cannot_mutate_simple.nim(16, 4) the mutation is here''' """ {.experimental: "strictFuncs".} diff --git a/tests/errmsgs/t6608.nim b/tests/errmsgs/t6608.nim index 3d82f3cd8..88cbf42fd 100644 --- a/tests/errmsgs/t6608.nim +++ b/tests/errmsgs/t6608.nim @@ -1,10 +1,9 @@ discard """ cmd: "nim c --hints:off $file" errormsg: "type mismatch: got <>" - nimout: '''t6608.nim(14, 4) Error: type mismatch: got <> + nimout: '''t6608.nim(13, 4) Error: type mismatch: got <> but expected one of: AcceptCB = proc (s: string){.closure.}''' - line: 14 """ type diff --git a/tests/errmsgs/tconceptconstraint.nim b/tests/errmsgs/tconceptconstraint.nim index b1977acbc..066ec5bdb 100644 --- a/tests/errmsgs/tconceptconstraint.nim +++ b/tests/errmsgs/tconceptconstraint.nim @@ -1,6 +1,5 @@ discard """ errormsg: "cannot instantiate B" - line: 20 nimout: ''' got: but expected: diff --git a/tests/errmsgs/tgenericconstraint.nim b/tests/errmsgs/tgenericconstraint.nim index b7272e787..b51fb3488 100644 --- a/tests/errmsgs/tgenericconstraint.nim +++ b/tests/errmsgs/tgenericconstraint.nim @@ -1,6 +1,5 @@ discard """ errormsg: "cannot instantiate B" - line: 14 nimout: ''' got: but expected: diff --git a/tests/errmsgs/twrong_at_operator.nim b/tests/errmsgs/twrong_at_operator.nim index d7bfca415..438186f01 100644 --- a/tests/errmsgs/twrong_at_operator.nim +++ b/tests/errmsgs/twrong_at_operator.nim @@ -1,8 +1,7 @@ discard """ errormsg: "type mismatch: got " -line: 22 nimout: ''' -twrong_at_operator.nim(22, 30) Error: type mismatch: got +twrong_at_operator.nim(21, 30) Error: type mismatch: got but expected one of: proc `@`[IDX, T](a: sink array[IDX, T]): seq[T] first type mismatch at position: 1 diff --git a/tests/errmsgs/twrongcolon.nim b/tests/errmsgs/twrongcolon.nim index 20063cbc4..06e802eb7 100644 --- a/tests/errmsgs/twrongcolon.nim +++ b/tests/errmsgs/twrongcolon.nim @@ -1,11 +1,10 @@ discard """ errormsg: "in expression ' do:" nimout: ''' -twrongcolon.nim(11, 12) Error: in expression ' do: +twrongcolon.nim(10, 12) Error: in expression ' do: 890': identifier expected, but found '' ''' -line: 11 """ var n: int : 890 diff --git a/tests/init/tuninit1.nim b/tests/init/tuninit1.nim index b281bcf89..9a4161a30 100644 --- a/tests/init/tuninit1.nim +++ b/tests/init/tuninit1.nim @@ -1,6 +1,5 @@ discard """ - nimout: "tuninit1.nim(35, 11) Warning: use explicit initialization of 'y' for clarity [Uninit]" - line:34 + nimout: "tuninit1.nim(34, 11) Warning: use explicit initialization of 'y' for clarity [Uninit]" action: compile """ diff --git a/tests/misc/tnoop.nim b/tests/misc/tnoop.nim index 7f5bb1045..f55f2441a 100644 --- a/tests/misc/tnoop.nim +++ b/tests/misc/tnoop.nim @@ -1,9 +1,8 @@ discard """ nimout: ''' - found 'a' [var declared in tnoop.nim(11, 3)] + found 'a' [var declared in tnoop.nim(10, 3)] ''' file: "tnoop.nim" - line: 13 errormsg: "attempting to call routine: 'a'" """ diff --git a/tests/objvariant/tcheckedfield1.nim b/tests/objvariant/tcheckedfield1.nim index e1a2e60e6..4a6c49f66 100644 --- a/tests/objvariant/tcheckedfield1.nim +++ b/tests/objvariant/tcheckedfield1.nim @@ -1,6 +1,5 @@ discard """ - nimout: "tcheckedfield1.nim(40, 6) Warning: cannot prove that field 'x.s' is accessible [ProveField]" - line:51 + nimout: "tcheckedfield1.nim(39, 6) Warning: cannot prove that field 'x.s' is accessible [ProveField]" action: run output: "abc abc" """ diff --git a/tests/parallel/tgc_unsafe2.nim b/tests/parallel/tgc_unsafe2.nim index 40af728fb..7d98dafcb 100644 --- a/tests/parallel/tgc_unsafe2.nim +++ b/tests/parallel/tgc_unsafe2.nim @@ -1,9 +1,8 @@ discard """ errormsg: "'consumer' is not GC-safe as it calls 'track'" - line: 28 - nimout: '''tgc_unsafe2.nim(22, 6) Warning: 'trick' is not GC-safe as it accesses 'global' which is a global using GC'ed memory [GcUnsafe2] -tgc_unsafe2.nim(26, 6) Warning: 'track' is not GC-safe as it calls 'trick' [GcUnsafe2] -tgc_unsafe2.nim(28, 6) Error: 'consumer' is not GC-safe as it calls 'track' + nimout: '''tgc_unsafe2.nim(21, 6) Warning: 'trick' is not GC-safe as it accesses 'global' which is a global using GC'ed memory [GcUnsafe2] +tgc_unsafe2.nim(25, 6) Warning: 'track' is not GC-safe as it calls 'trick' [GcUnsafe2] +tgc_unsafe2.nim(27, 6) Error: 'consumer' is not GC-safe as it calls 'track' ''' """ diff --git a/tests/varres/tprevent_forloopvar_mutations.nim b/tests/varres/tprevent_forloopvar_mutations.nim index 045dc7cbb..96bb208c2 100644 --- a/tests/varres/tprevent_forloopvar_mutations.nim +++ b/tests/varres/tprevent_forloopvar_mutations.nim @@ -1,7 +1,6 @@ discard """ errormsg: "type mismatch: got " - line: 17 - nimout: '''tprevent_forloopvar_mutations.nim(17, 7) Error: type mismatch: got + nimout: '''tprevent_forloopvar_mutations.nim(16, 7) Error: type mismatch: got but expected one of: proc inc[T: Ordinal](x: var T; y = 1) first type mismatch at position: 1 diff --git a/tests/views/tcannot_borrow.nim b/tests/views/tcannot_borrow.nim index d1c194b25..0b8793159 100644 --- a/tests/views/tcannot_borrow.nim +++ b/tests/views/tcannot_borrow.nim @@ -1,8 +1,7 @@ discard """ errormsg: "cannot borrow" - nimout: '''tcannot_borrow.nim(21, 7) Error: cannot borrow meh; what it borrows from is potentially mutated -tcannot_borrow.nim(22, 3) the mutation is here''' - line: 21 + nimout: '''tcannot_borrow.nim(20, 7) Error: cannot borrow meh; what it borrows from is potentially mutated +tcannot_borrow.nim(21, 3) the mutation is here''' """ From c7b4639460dd377ee937e3468df9b7041744302f Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 6 Apr 2021 22:21:58 +0800 Subject: [PATCH 0586/2472] ref #14873 (#17644) * ref #14873 * comment * Update lib/core/locks.nim --- lib/core/locks.nim | 5 +++++ tests/stdlib/tlocks.nim | 6 +++--- tests/stdlib/uselocks.nim | 4 ++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/lib/core/locks.nim b/lib/core/locks.nim index bddd6d864..1540e2bcd 100644 --- a/lib/core/locks.nim +++ b/lib/core/locks.nim @@ -28,6 +28,11 @@ type {.push stackTrace: off.} + +proc `$`*(lock: Lock): string = + # workaround bug #14873 + result = "()" + proc initLock*(lock: var Lock) {.inline.} = ## Initializes the given lock. when not defined(js): diff --git a/tests/stdlib/tlocks.nim b/tests/stdlib/tlocks.nim index e567cfde8..0815c5d01 100644 --- a/tests/stdlib/tlocks.nim +++ b/tests/stdlib/tlocks.nim @@ -1,10 +1,10 @@ discard """ - output: '''3''' - cmd: "nim $target --threads:on $options $file" + targets: "c cpp js" + matrix: "--threads:on" """ #bug #6049 import uselocks var m = createMyType[int]() -echo $m.use() +doAssert m.use() == 3 diff --git a/tests/stdlib/uselocks.nim b/tests/stdlib/uselocks.nim index cde9641b2..e9d23f9d9 100644 --- a/tests/stdlib/uselocks.nim +++ b/tests/stdlib/uselocks.nim @@ -9,3 +9,7 @@ proc createMyType*[T]: MyType[T] = proc use* (m: var MyType): int = withLock m.lock: result = 3 + +block: + var l: Lock + doAssert $l == "()" From e406e28738cfaebd6e0ff4642daa55c9114bd68a Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 7 Apr 2021 00:01:54 +0800 Subject: [PATCH 0587/2472] fix #16898 #17621 (#17628) * fix #16898 * fix #17621 * Update compiler/semtypes.nim --- compiler/semtypes.nim | 6 ++++-- compiler/typeallowed.nim | 3 ++- tests/lent/t16898.nim | 31 +++++++++++++++++++++++++++++++ tests/lent/t17621.nim | 15 +++++++++++++++ 4 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 tests/lent/t16898.nim create mode 100644 tests/lent/t17621.nim diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 0fce7b417..b34dc36e3 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -10,7 +10,7 @@ # this module does the semantic checking of type declarations # included from sem.nim -import math +import std/math const errStringOrIdentNodeExpected = "string or ident node expected" @@ -1171,6 +1171,7 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode, else: discard proc semParamType(c: PContext, n: PNode, constraint: var PNode): PType = + ## Semchecks the type of parameters. if n.kind == nkCurlyExpr: result = semTypeNode(c, n[0], nil) constraint = semNodeKindConstraints(n, c.config, 1) @@ -1226,10 +1227,11 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode, if hasType: typ = semParamType(c, a[^2], constraint) # TODO: Disallow typed/untyped in procs in the compiler/stdlib - if kind == skProc and (typ.kind == tyTyped or typ.kind == tyUntyped): + if kind in {skProc, skFunc} and (typ.kind == tyTyped or typ.kind == tyUntyped): if not isMagic(getCurrOwner(c)): localError(c.config, a[^2].info, "'" & typ.sym.name.s & "' is only allowed in templates and macros or magic procs") + if hasDefault: def = a[^1] block determineType: diff --git a/compiler/typeallowed.nim b/compiler/typeallowed.nim index db3735359..9075d9e38 100644 --- a/compiler/typeallowed.nim +++ b/compiler/typeallowed.nim @@ -57,7 +57,8 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, of tyVar, tyLent: if kind in {skProc, skFunc, skConst} and (views notin c.features): result = t - elif t.kind == tyLent and kind != skResult and (views notin c.features): + elif t.kind == tyLent and ((kind != skResult and views notin c.features) or + kind == skParam): # lent can't be used as parameters. result = t else: var t2 = skipTypes(t[0], abstractInst-{tyTypeDesc, tySink}) diff --git a/tests/lent/t16898.nim b/tests/lent/t16898.nim new file mode 100644 index 000000000..ea7c934ef --- /dev/null +++ b/tests/lent/t16898.nim @@ -0,0 +1,31 @@ +discard """ + errormsg: "invalid type: 'lent QuadraticExt' in this context: 'proc (r: var QuadraticExt, a: lent QuadraticExt, b: lent QuadraticExt){.noSideEffect, gcsafe, locks: 0.}' for proc" +""" + +# bug #16898 +type + Fp[N: static int, T] = object + big: array[N, T] + +type + QuadraticExt* = concept x + ## Quadratic Extension concept (like complex) + type BaseField = auto + x.c0 is BaseField + x.c1 is BaseField + +{.experimental:"views".} + +func prod(r: var QuadraticExt, a, b: lent QuadraticExt) = + discard + +type + Fp2[N: static int, T] = object + c0, c1: Fp[N, T] + +# This should be passed by reference, +# but concepts do not respect the 24 bytes rule +# or `byref` pragma. +var r, a, b: Fp2[6, uint64] + +prod(r, a, b) diff --git a/tests/lent/t17621.nim b/tests/lent/t17621.nim new file mode 100644 index 000000000..e324a963e --- /dev/null +++ b/tests/lent/t17621.nim @@ -0,0 +1,15 @@ +discard """ + errormsg: "invalid type: 'lent Test' in this context: 'proc (self: lent Test)' for proc" +""" + +# bug #17621 +{.experimental: "views".} + +type Test = ref object + foo: int + +proc modify(self: lent Test) = + self.foo += 1 + +let test = Test(foo: 12) +modify(test) From 020c0a33447287c4d42e67754bae75d01b6160a0 Mon Sep 17 00:00:00 2001 From: Miran Date: Tue, 6 Apr 2021 18:52:12 +0200 Subject: [PATCH 0588/2472] don't run one example on 32-bit machines (#17655) This example seems to break our 32-bit nightlies builds. This is just a temporary solution (TM) until we figure out a better one. --- lib/pure/math.nim | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/pure/math.nim b/lib/pure/math.nim index 2ef7ee102..03e5040ae 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -951,9 +951,10 @@ func frexp*[T: float32|float64](x: T): tuple[frac: T, exp: int] {.inline.} = doAssert frexp(0.0) == (0.0, 0) # special cases: - doAssert frexp(-0.0).frac.signbit # signbit preserved for +-0 - doAssert frexp(Inf).frac == Inf # +- Inf preserved - doAssert frexp(NaN).frac.isNaN + when sizeof(int) == 8: + doAssert frexp(-0.0).frac.signbit # signbit preserved for +-0 + doAssert frexp(Inf).frac == Inf # +- Inf preserved + doAssert frexp(NaN).frac.isNaN when not defined(js): var exp: cint From 6ab58168661b27778dd6bd64bedf70a7889f5aa0 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 6 Apr 2021 13:01:34 -0700 Subject: [PATCH 0589/2472] add compiler/debugutils.nim to help debugging compiler (#17652) * add compiler/debugutils.nim * address comments --- compiler/debugutils.nim | 19 +++++++++++++++++++ compiler/options.nim | 23 ++++++++++++++++------- 2 files changed, 35 insertions(+), 7 deletions(-) create mode 100644 compiler/debugutils.nim diff --git a/compiler/debugutils.nim b/compiler/debugutils.nim new file mode 100644 index 000000000..235c100b3 --- /dev/null +++ b/compiler/debugutils.nim @@ -0,0 +1,19 @@ +##[ +Utilities to help with debugging nim compiler. + +Experimental API, subject to change. +]## + +import options + +var conf0: ConfigRef + +proc onNewConfigRef*(conf: ConfigRef) {.inline.} = + ## Caches `conf`, which can be retrieved with `getConfigRef`. + ## This avoids having to forward `conf` all the way down the call chain to + ## procs that need it during a debugging session. + conf0 = conf + +proc getConfigRef*(): ConfigRef = + ## nil, if -d:nimDebugUtils wasn't specified + result = conf0 diff --git a/compiler/options.nim b/compiler/options.nim index 2c8d43733..0bdb51159 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -446,6 +446,10 @@ proc newProfileData(): ProfileData = const foreignPackageNotesDefault* = { hintProcessing, warnUnknownMagic, hintQuitCalled, hintExecuting, hintUser, warnUser} +proc isDefined*(conf: ConfigRef; symbol: string): bool + +import debugutils + proc newConfigRef*(): ConfigRef = result = ConfigRef( selectedGC: gcRefc, @@ -504,16 +508,21 @@ proc newConfigRef*(): ConfigRef = # enable colors by default on terminals if terminal.isatty(stderr): incl(result.globalOptions, optUseColors) + when defined(nimDebugUtils): + onNewConfigRef(result) proc newPartialConfigRef*(): ConfigRef = ## create a new ConfigRef that is only good enough for error reporting. - result = ConfigRef( - selectedGC: gcRefc, - verbosity: 1, - options: DefaultOptions, - globalOptions: DefaultGlobalOptions, - foreignPackageNotes: foreignPackageNotesDefault, - notes: NotesVerbosity[1], mainPackageNotes: NotesVerbosity[1]) + when defined(nimDebugUtils): + result = getConfigRef() + else: + result = ConfigRef( + selectedGC: gcRefc, + verbosity: 1, + options: DefaultOptions, + globalOptions: DefaultGlobalOptions, + foreignPackageNotes: foreignPackageNotesDefault, + notes: NotesVerbosity[1], mainPackageNotes: NotesVerbosity[1]) proc cppDefine*(c: ConfigRef; define: string) = c.cppDefines.incl define From 34c1c631bb61e8a7dde923c73a2eecbd9d96345a Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 7 Apr 2021 04:08:59 +0800 Subject: [PATCH 0590/2472] close #4451 (#17627) --- doc/estp.rst | 8 ++++++++ lib/pure/nimprof.nim | 3 ++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/doc/estp.rst b/doc/estp.rst index 8146562b6..fa9977d44 100644 --- a/doc/estp.rst +++ b/doc/estp.rst @@ -21,6 +21,14 @@ You can in fact look at `nimprof`'s source code to see how to implement your own profiler. The setting `--profiler:on` defines the conditional symbol `profiler`. +You can use `when compileOption("profiler")` to make the switch seamless. +If `profiler` is `off`, your program runs normally. Otherwise your program +is profiled. + +```nim +when compileOption("profiler"): + import nimprof +``` After your program has finished the profiler will create a file `profile_results.txt` containing the profiling results. diff --git a/lib/pure/nimprof.nim b/lib/pure/nimprof.nim index 721ae35c3..3b1f703e3 100644 --- a/lib/pure/nimprof.nim +++ b/lib/pure/nimprof.nim @@ -9,7 +9,8 @@ ## Profiling support for Nim. This is an embedded profiler that requires ## `--profiler:on`. You only need to import this module to get a profiling -## report at program exit. +## report at program exit. See `Embedded Stack Trace Profiler `_ +## for usage. when not defined(profiler) and not defined(memProfiler): {.error: "Profiling support is turned off! Enable profiling by passing `--profiler:on --stackTrace:on` to the compiler (see the Nim Compiler User Guide for more options).".} From 444658fe5817808d71c06f1b5c272c1132d89bfb Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 7 Apr 2021 07:46:45 +0200 Subject: [PATCH 0591/2472] fixes #17656 (#17657) --- compiler/semexprs.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 71c9e0295..c0aa168ef 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -2962,7 +2962,8 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = of nkMixinStmt: discard of nkBindStmt: if c.p != nil: - c.p.localBindStmts.add n + if n.len > 0 and n[0].kind == nkSym: + c.p.localBindStmts.add n else: localError(c.config, n.info, "invalid context for 'bind' statement: " & renderTree(n, {renderNoComments})) From d3702a4dc3f4226ae601f503bc3f2244f9e1bdce Mon Sep 17 00:00:00 2001 From: rockcavera Date: Wed, 7 Apr 2021 02:52:14 -0300 Subject: [PATCH 0592/2472] removing warning imported and not used debugutils (#17660) --- compiler/options.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/options.nim b/compiler/options.nim index 0bdb51159..7408ce675 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -448,7 +448,8 @@ const foreignPackageNotesDefault* = { proc isDefined*(conf: ConfigRef; symbol: string): bool -import debugutils +when defined(nimDebugUtils): + import debugutils proc newConfigRef*(): ConfigRef = result = ConfigRef( From 939df8cbdaf8b65966c60b4e368ba317cf4d194f Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 7 Apr 2021 13:40:57 +0200 Subject: [PATCH 0593/2472] ensure the avr example keeps compiling (#17663) * ensure the avr example keeps compiling * Update tests/avr/thello.nim * now compiles properly --- lib/system/embedded.nim | 4 ++++ tests/avr/nim.cfg | 12 ++++++++++++ tests/avr/panicoverride.nim | 13 +++++++++++++ tests/avr/thello.nim | 6 ++++++ 4 files changed, 35 insertions(+) create mode 100644 tests/avr/nim.cfg create mode 100644 tests/avr/panicoverride.nim create mode 100644 tests/avr/thello.nim diff --git a/lib/system/embedded.nim b/lib/system/embedded.nim index 258558c3f..e0b053c7b 100644 --- a/lib/system/embedded.nim +++ b/lib/system/embedded.nim @@ -34,6 +34,10 @@ proc quitOrDebug() {.noreturn, importc: "abort", header: "", nodecl.} proc raiseException(e: ref Exception, ename: cstring) {.compilerRtl.} = sysFatal(ReraiseDefect, "exception handling is not available") +proc raiseExceptionEx(e: sink(ref Exception), ename, procname, filename: cstring, + line: int) {.compilerRtl.} = + sysFatal(ReraiseDefect, "exception handling is not available") + proc reraiseException() {.compilerRtl.} = sysFatal(ReraiseDefect, "no exception to reraise") diff --git a/tests/avr/nim.cfg b/tests/avr/nim.cfg new file mode 100644 index 000000000..d6eba8eda --- /dev/null +++ b/tests/avr/nim.cfg @@ -0,0 +1,12 @@ +avr.standalone.gcc.path = "/usr/bin" +avr.standalone.gcc.exe = "avr-gcc" +avr.standalone.gcc.linkerexe = "avr-gcc" +passC = "-Os" +passC = "-DF_CPU=16000000UL" +passC = "-mmcu=atmega328p" +passL = "-mmcu=atmega328p" +passC = "-flto" +passL = "-flto" +cpu = "avr" +deadCodeElim = "on" +gc = "arc" diff --git a/tests/avr/panicoverride.nim b/tests/avr/panicoverride.nim new file mode 100644 index 000000000..770933ddd --- /dev/null +++ b/tests/avr/panicoverride.nim @@ -0,0 +1,13 @@ +proc printf(frmt: cstring) {.varargs, importc, header: "", cdecl.} +proc exit(code: int) {.importc, header: "", cdecl.} + +{.push stack_trace: off, profiler:off.} + +proc rawoutput(s: string) = + printf("%s\n", s) + +proc panic(s: string) = + rawoutput(s) + exit(1) + +{.pop.} diff --git a/tests/avr/thello.nim b/tests/avr/thello.nim new file mode 100644 index 000000000..a0191815c --- /dev/null +++ b/tests/avr/thello.nim @@ -0,0 +1,6 @@ +discard """ + cmd: "nim c --compileOnly --os:standalone --exceptions:quirky -d:noSignalHandler -d:danger $file" + action: "compile" +""" + +echo "hi" From 834d5641e88b144c1cf94fa13f567b0a2acda977 Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 7 Apr 2021 20:17:48 +0800 Subject: [PATCH 0594/2472] use sink and lent in deques (#17661) * use sink and lent in deques * Update lib/pure/collections/deques.nim Co-authored-by: Andreas Rumpf Co-authored-by: Andreas Rumpf --- lib/pure/collections/deques.nim | 20 ++++++------- tests/stdlib/tdeques.nim | 53 +++++++++++++++++++++++++++++++-- 2 files changed, 61 insertions(+), 12 deletions(-) diff --git a/lib/pure/collections/deques.nim b/lib/pure/collections/deques.nim index c8bbb1cc6..d2d53d993 100644 --- a/lib/pure/collections/deques.nim +++ b/lib/pure/collections/deques.nim @@ -50,7 +50,7 @@ runnableExamples: import std/private/since -import math +import std/math type Deque*[T] = object @@ -120,7 +120,7 @@ template xBoundsCheck(deq, i) = raise newException(IndexDefect, "Out of bounds: " & $i & " < 0") -proc `[]`*[T](deq: Deque[T], i: Natural): T {.inline.} = +proc `[]`*[T](deq: Deque[T], i: Natural): lent T {.inline.} = ## Accesses the `i`-th element of `deq`. runnableExamples: let a = [10, 20, 30, 40, 50].toDeque @@ -142,7 +142,7 @@ proc `[]`*[T](deq: var Deque[T], i: Natural): var T {.inline.} = xBoundsCheck(deq, i) return deq.data[(deq.head + i) and deq.mask] -proc `[]=`*[T](deq: var Deque[T], i: Natural, val: T) {.inline.} = +proc `[]=`*[T](deq: var Deque[T], i: Natural, val: sink T) {.inline.} = ## Sets the `i`-th element of `deq` to `val`. runnableExamples: var a = [10, 20, 30, 40, 50].toDeque @@ -154,7 +154,7 @@ proc `[]=`*[T](deq: var Deque[T], i: Natural, val: T) {.inline.} = xBoundsCheck(deq, i) deq.data[(deq.head + i) and deq.mask] = val -proc `[]`*[T](deq: Deque[T], i: BackwardsIndex): T {.inline.} = +proc `[]`*[T](deq: Deque[T], i: BackwardsIndex): lent T {.inline.} = ## Accesses the backwards indexed `i`-th element. ## ## `deq[^1]` is the last element. @@ -180,7 +180,7 @@ proc `[]`*[T](deq: var Deque[T], i: BackwardsIndex): var T {.inline.} = xBoundsCheck(deq, deq.len - int(i)) return deq[deq.len - int(i)] -proc `[]=`*[T](deq: var Deque[T], i: BackwardsIndex, x: T) {.inline.} = +proc `[]=`*[T](deq: var Deque[T], i: BackwardsIndex, x: sink T) {.inline.} = ## Sets the backwards indexed `i`-th element of `deq` to `x`. ## ## `deq[^1]` is the last element. @@ -194,7 +194,7 @@ proc `[]=`*[T](deq: var Deque[T], i: BackwardsIndex, x: T) {.inline.} = xBoundsCheck(deq, deq.len - int(i)) deq[deq.len - int(i)] = x -iterator items*[T](deq: Deque[T]): T = +iterator items*[T](deq: Deque[T]): lent T = ## Yields every element of `deq`. ## ## **See also:** @@ -270,7 +270,7 @@ proc expandIfNeeded[T](deq: var Deque[T]) = deq.tail = deq.count deq.head = 0 -proc addFirst*[T](deq: var Deque[T], item: T) = +proc addFirst*[T](deq: var Deque[T], item: sink T) = ## Adds an `item` to the beginning of `deq`. ## ## **See also:** @@ -286,7 +286,7 @@ proc addFirst*[T](deq: var Deque[T], item: T) = deq.head = (deq.head - 1) and deq.mask deq.data[deq.head] = item -proc addLast*[T](deq: var Deque[T], item: T) = +proc addLast*[T](deq: var Deque[T], item: sink T) = ## Adds an `item` to the end of `deq`. ## ## **See also:** @@ -302,7 +302,7 @@ proc addLast*[T](deq: var Deque[T], item: T) = deq.data[deq.tail] = item deq.tail = (deq.tail + 1) and deq.mask -proc peekFirst*[T](deq: Deque[T]): T {.inline.} = +proc peekFirst*[T](deq: Deque[T]): lent T {.inline.} = ## Returns the first element of `deq`, but does not remove it from the deque. ## ## **See also:** @@ -317,7 +317,7 @@ proc peekFirst*[T](deq: Deque[T]): T {.inline.} = emptyCheck(deq) result = deq.data[deq.head] -proc peekLast*[T](deq: Deque[T]): T {.inline.} = +proc peekLast*[T](deq: Deque[T]): lent T {.inline.} = ## Returns the last element of `deq`, but does not remove it from the deque. ## ## **See also:** diff --git a/tests/stdlib/tdeques.nim b/tests/stdlib/tdeques.nim index 99208d4cf..03e951fce 100644 --- a/tests/stdlib/tdeques.nim +++ b/tests/stdlib/tdeques.nim @@ -1,8 +1,11 @@ discard """ - targets: "c js" + matrix: "--gc:refc; --gc:orc" + targets: "c cpp js" """ import std/deques +from std/sequtils import toSeq + block: proc index(self: Deque[int], idx: Natural): int = @@ -125,7 +128,7 @@ block: foo(1, 5) foo(3, 2) -import sets +import std/sets block t13310: proc main() = @@ -137,3 +140,49 @@ block t13310: static: main() + + +proc main() = + block: + let a = [10, 20, 30].toDeque + doAssert toSeq(a.pairs) == @[(0, 10), (1, 20), (2, 30)] + + block: + let q = [7, 9].toDeque + doAssert 7 in q + doAssert q.contains(7) + doAssert 8 notin q + + block: + let a = [10, 20, 30, 40, 50].toDeque + doAssert $a == "[10, 20, 30, 40, 50]" + doAssert a.peekFirst == 10 + doAssert len(a) == 5 + + block: + let a = [10, 20, 30, 40, 50].toDeque + doAssert $a == "[10, 20, 30, 40, 50]" + doAssert a.peekLast == 50 + doAssert len(a) == 5 + + block: + var a = [10, 20, 30, 40, 50].toDeque + doAssert $a == "[10, 20, 30, 40, 50]" + doAssert a.popFirst == 10 + doAssert $a == "[20, 30, 40, 50]" + + block: + var a = [10, 20, 30, 40, 50].toDeque + doAssert $a == "[10, 20, 30, 40, 50]" + doAssert a.popLast == 50 + doAssert $a == "[10, 20, 30, 40]" + + block: + var a = [10, 20, 30, 40, 50].toDeque + doAssert $a == "[10, 20, 30, 40, 50]" + clear(a) + doAssert len(a) == 0 + + +static: main() +main() From a7d6ba45260d9fee3bdc2761fa67bcd114edac7f Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 8 Apr 2021 01:54:28 +0800 Subject: [PATCH 0595/2472] minor typos (#17668) --- compiler/semparallel.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/semparallel.nim b/compiler/semparallel.nim index b5b0be91b..f696f6a0a 100644 --- a/compiler/semparallel.nim +++ b/compiler/semparallel.nim @@ -460,10 +460,10 @@ proc transformSpawn(g: ModuleGraph; idgen: IdGenerator; owner: PSym; n, barrier: result = n proc checkArgs(a: var AnalysisCtx; n: PNode) = - discard "too implement" + discard "to implement" proc generateAliasChecks(a: AnalysisCtx; result: PNode) = - discard "too implement" + discard "to implement" proc liftParallel*(g: ModuleGraph; idgen: IdGenerator; owner: PSym; n: PNode): PNode = # this needs to be called after the 'for' loop elimination From bd7807de4c7f8474b029f36b4d3527f7a03de6e4 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 7 Apr 2021 23:26:53 +0200 Subject: [PATCH 0596/2472] fixes #17647 (#17667) --- doc/hcr.rst | 2 +- lib/nimhcr.nim | 8 ++++---- lib/system/gc.nim | 20 ++++++++++---------- lib/system/gc_common.nim | 4 ++-- lib/system/gc_ms.nim | 10 +++++----- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/doc/hcr.rst b/doc/hcr.rst index 7e9d71da3..fd4ba34d4 100644 --- a/doc/hcr.rst +++ b/doc/hcr.rst @@ -192,7 +192,7 @@ compiled with the `-d:useNimRtl` option and they will depend on both the `nimrtl` library and the `nimhcr` library which implements the hot code reloading run-time. Both libraries can be found in the `lib` folder of Nim and can be compiled into dynamic libraries to satisfy -runtime demands of the example code above. An example of compiling +runtime demands of the example code above. An example of compiling `nimhcr.nim` and `nimrtl.nim` when the source dir of Nim is installed with choosenim follows. diff --git a/lib/nimhcr.nim b/lib/nimhcr.nim index 42e3266ae..2846f931b 100644 --- a/lib/nimhcr.nim +++ b/lib/nimhcr.nim @@ -209,7 +209,7 @@ when not defined(js) and (defined(hotcodereloading) or else: "so" type HcrProcGetter* = proc (libHandle: pointer, procName: cstring): pointer {.nimcall.} - HcrGcMarkerProc = proc () {.nimcall.} + HcrGcMarkerProc = proc () {.nimcall, raises: [].} HcrModuleInitializer* = proc () {.nimcall.} when defined(createNimHcr): @@ -606,7 +606,7 @@ when defined(createNimHcr): proc hcrGeneration*(): int {.nimhcr.} = generation - proc hcrMarkGlobals*() {.nimhcr, nimcall, gcsafe.} = + proc hcrMarkGlobals*() {.compilerproc, exportc, dynlib, nimcall, gcsafe.} = # This is gcsafe, because it will be registered # only in the GC of the main thread. {.gcsafe.}: @@ -648,10 +648,10 @@ elif defined(hotcodereloading) or defined(testNimHcr): proc hcrAddEventHandler*(isBefore: bool, cb: proc ()) {.nimhcr.} - proc hcrMarkGlobals*() {.nimhcr, nimcall, gcsafe.} + proc hcrMarkGlobals*() {.raises: [], nimhcr, nimcall, gcsafe.} when declared(nimRegisterGlobalMarker): - nimRegisterGlobalMarker(hcrMarkGlobals) + nimRegisterGlobalMarker(cast[GlobalMarkerProc](hcrMarkGlobals)) else: proc hcrHasModuleChanged*(moduleHash: string): bool = diff --git a/lib/system/gc.nim b/lib/system/gc.nim index 229b26f3c..039a627ba 100644 --- a/lib/system/gc.nim +++ b/lib/system/gc.nim @@ -94,7 +94,7 @@ type waZctDecRef, waPush #, waDebug - Finalizer {.compilerproc.} = proc (self: pointer) {.nimcall, benign.} + Finalizer {.compilerproc.} = proc (self: pointer) {.nimcall, benign, raises: [].} # A ref type can have a finalizer that is called before the object's # storage is freed. @@ -219,11 +219,11 @@ template gcTrace(cell, state: untyped) = when traceGC: traceCell(cell, state) # forward declarations: -proc collectCT(gch: var GcHeap) {.benign.} -proc isOnStack(p: pointer): bool {.noinline, benign.} -proc forAllChildren(cell: PCell, op: WalkOp) {.benign.} -proc doOperation(p: pointer, op: WalkOp) {.benign.} -proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) {.benign.} +proc collectCT(gch: var GcHeap) {.benign, raises: [].} +proc isOnStack(p: pointer): bool {.noinline, benign, raises: [].} +proc forAllChildren(cell: PCell, op: WalkOp) {.benign, raises: [].} +proc doOperation(p: pointer, op: WalkOp) {.benign, raises: [].} +proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) {.benign, raises: [].} # we need the prototype here for debugging purposes proc incRef(c: PCell) {.inline.} = @@ -636,7 +636,7 @@ proc markS(gch: var GcHeap, c: PCell) = if not containsOrIncl(gch.marked, d): forAllChildren(d, waMarkPrecise) -proc markGlobals(gch: var GcHeap) = +proc markGlobals(gch: var GcHeap) {.raises: [].} = if gch.gcThreadId == 0: for i in 0 .. globalMarkersLen-1: globalMarkers[i]() for i in 0 .. threadLocalMarkersLen-1: threadLocalMarkers[i]() @@ -691,9 +691,9 @@ proc doOperation(p: pointer, op: WalkOp) = proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} = doOperation(d, WalkOp(op)) -proc collectZCT(gch: var GcHeap): bool {.benign.} +proc collectZCT(gch: var GcHeap): bool {.benign, raises: [].} -proc collectCycles(gch: var GcHeap) = +proc collectCycles(gch: var GcHeap) {.raises: [].} = when hasThreadSupport: for c in gch.toDispose: nimGCunref(c) @@ -800,7 +800,7 @@ proc unmarkStackAndRegisters(gch: var GcHeap) = decRef(d[i]) gch.decStack.len = 0 -proc collectCTBody(gch: var GcHeap) = +proc collectCTBody(gch: var GcHeap) {.raises: [].} = when withRealTime: let t0 = getticks() sysAssert(allocInv(gch.region), "collectCT: begin") diff --git a/lib/system/gc_common.nim b/lib/system/gc_common.nim index 7596de6fb..ea8857ece 100644 --- a/lib/system/gc_common.nim +++ b/lib/system/gc_common.nim @@ -415,7 +415,7 @@ else: # end of non-portable code # ---------------------------------------------------------------------------- -proc prepareDealloc(cell: PCell) = +proc prepareDealloc(cell: PCell) {.raises: [].} = when declared(useMarkForDebug): when useMarkForDebug: gcAssert(cell notin gch.marked, "Cell still alive!") @@ -458,7 +458,7 @@ proc deallocHeap*(runFinalizers = true; allowGcAfterwards = true) = initGC() type - GlobalMarkerProc = proc () {.nimcall, benign.} + GlobalMarkerProc = proc () {.nimcall, benign, raises: [].} var globalMarkersLen {.exportc.}: int globalMarkers {.exportc.}: array[0..3499, GlobalMarkerProc] diff --git a/lib/system/gc_ms.nim b/lib/system/gc_ms.nim index 852f5d7aa..b4221f7fd 100644 --- a/lib/system/gc_ms.nim +++ b/lib/system/gc_ms.nim @@ -36,7 +36,7 @@ type # local waMarkPrecise # fast precise marking - Finalizer {.compilerproc.} = proc (self: pointer) {.nimcall, benign.} + Finalizer {.compilerproc.} = proc (self: pointer) {.nimcall, benign, raises: [].} # A ref type can have a finalizer that is called before the object's # storage is freed. @@ -115,10 +115,10 @@ when BitsPerPage mod (sizeof(int)*8) != 0: {.error: "(BitsPerPage mod BitsPerUnit) should be zero!".} # forward declarations: -proc collectCT(gch: var GcHeap; size: int) {.benign.} -proc forAllChildren(cell: PCell, op: WalkOp) {.benign.} -proc doOperation(p: pointer, op: WalkOp) {.benign.} -proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) {.benign.} +proc collectCT(gch: var GcHeap; size: int) {.benign, raises: [].} +proc forAllChildren(cell: PCell, op: WalkOp) {.benign, raises: [].} +proc doOperation(p: pointer, op: WalkOp) {.benign, raises: [].} +proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) {.benign, raises: [].} # we need the prototype here for debugging purposes when defined(nimGcRefLeak): From 643dbc743bf6436cdc659ae57e2841e125aaa9de Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 8 Apr 2021 00:54:47 +0200 Subject: [PATCH 0597/2472] use strstr for a faster find implementation (#17672) * use strstr for a faster find implementation * stress the -d:release and -d:danger switches --- doc/tut1.rst | 7 ++++++- lib/pure/strutils.nim | 29 ++++++++++++++++++++++++++--- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/doc/tut1.rst b/doc/tut1.rst index d2d6f8fe7..cca4e8c0e 100644 --- a/doc/tut1.rst +++ b/doc/tut1.rst @@ -67,6 +67,11 @@ aiming for your debugging pleasure. With `-d:release` some checks are `turned off and optimizations are turned on `_. +For benchmarking or production code, use the `-d:release` switch. +For comparing the performance with unsafe languages like C, use the `-d:danger` switch +in order to get meaningful, comparable results. Otherwise Nim might be handicapped +by checks that are **not even available** for C. + Though it should be pretty obvious what the program does, I will explain the syntax: statements which are not indented are executed when the program starts. Indentation is Nim's way of grouping statements. Indentation is @@ -626,7 +631,7 @@ Some terminology: in the example `question` is called a (formal) *parameter*, Result variable --------------- A procedure that returns a value has an implicit `result` variable declared -that represents the return value. A `return` statement with no expression is +that represents the return value. A `return` statement with no expression is shorthand for `return result`. The `result` value is always returned automatically at the end of a procedure if there is no `return` statement at the exit. diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 3680b11de..bfd53a3a0 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -1826,6 +1826,9 @@ func find*(a: SkipTable, s, sub: string, start: Natural = 0, last = 0): int {. when not (defined(js) or defined(nimdoc) or defined(nimscript)): func c_memchr(cstr: pointer, c: char, n: csize_t): pointer {. importc: "memchr", header: "".} + func c_strstr(haystack, needle: cstring): cstring {. + importc: "strstr", header: "".} + const hasCStringBuiltin = true else: const hasCStringBuiltin = false @@ -1889,9 +1892,29 @@ func find*(s, sub: string, start: Natural = 0, last = 0): int {.rtl, ## * `replace func<#replace,string,string,string>`_ if sub.len > s.len: return -1 if sub.len == 1: return find(s, sub[0], start, last) - var a {.noinit.}: SkipTable - initSkipTable(a, sub) - result = find(a, s, sub, start, last) + + template useSkipTable {.dirty.} = + var a {.noinit.}: SkipTable + initSkipTable(a, sub) + result = find(a, s, sub, start, last) + + when not hasCStringBuiltin: + useSkipTable() + else: + when nimvm: + useSkipTable() + else: + when hasCStringBuiltin: + if last == 0 and s.len > start: + let found = c_strstr(s[start].unsafeAddr, sub) + if not found.isNil: + result = cast[ByteAddress](found) -% cast[ByteAddress](s.cstring) + else: + result = -1 + else: + useSkipTable() + else: + useSkipTable() func rfind*(s: string, sub: char, start: Natural = 0, last = -1): int {.rtl, extern: "nsuRFindChar".} = From 1b03accac78632897065ecfef40a6e40d8dfe328 Mon Sep 17 00:00:00 2001 From: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Date: Thu, 8 Apr 2021 14:08:58 +0200 Subject: [PATCH 0598/2472] Improve endians module (#17674) Extend documentation Add runnableExamples --- lib/pure/endians.nim | 49 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/lib/pure/endians.nim b/lib/pure/endians.nim index 29fde1c80..a0dc97c1d 100644 --- a/lib/pure/endians.nim +++ b/lib/pure/endians.nim @@ -10,6 +10,11 @@ ## This module contains helpers that deal with different byte orders ## (`endian`:idx:). ## +## Endianess is the order of bytes of a value in memory. Big-endian means that +## the most significant byte is stored at the smallest memory address, +## while little endian means that the least-significant byte is stored +## at the smallest address. See also https://en.wikipedia.org/wiki/Endianness. +## ## Unstable API. when defined(gcc) or defined(llvm_gcc) or defined(clang): @@ -47,7 +52,7 @@ else: when useBuiltinSwap: template swapOpImpl(T: typedesc, op: untyped) = - ## We have to use `copyMem` here instead of a simple deference because they + ## We have to use `copyMem` here instead of a simple dereference because they ## may point to a unaligned address. A sufficiently smart compiler _should_ ## be able to elide them when they're not necessary. var tmp: T @@ -56,18 +61,40 @@ when useBuiltinSwap: copyMem(outp, addr tmp, sizeof(T)) proc swapEndian64*(outp, inp: pointer) {.inline, noSideEffect.} = + ## Copies `inp` to `outp`, reversing the byte order. + ## Both buffers are supposed to contain at least 8 bytes. + runnableExamples: + var a = [1'u8, 2, 3, 4, 5, 6, 7, 8] + var b: array[8, uint8] + swapEndian64(addr b, addr a) + assert b == [8'u8, 7, 6, 5, 4, 3, 2, 1] + swapOpImpl(uint64, builtin_bswap64) proc swapEndian32*(outp, inp: pointer) {.inline, noSideEffect.} = + ## Copies `inp` to `outp`, reversing the byte order. + ## Both buffers are supposed to contain at least 4 bytes. + runnableExamples: + var a = [1'u8, 2, 3, 4] + var b: array[4, uint8] + swapEndian32(addr b, addr a) + assert b == [4'u8, 3, 2, 1] + swapOpImpl(uint32, builtin_bswap32) proc swapEndian16*(outp, inp: pointer) {.inline, noSideEffect.} = + ## Copies `inp` to `outp`, reversing the byte order. + ## Both buffers are supposed to contain at least 2 bytes. + runnableExamples: + var a = [1'u8, 2] + var b: array[2, uint8] + swapEndian16(addr b, addr a) + assert b == [2'u8, 1] + swapOpImpl(uint16, builtin_bswap16) else: proc swapEndian64*(outp, inp: pointer) = - ## copies `inp` to `outp` swapping bytes. Both buffers are supposed to - ## contain at least 8 bytes. var i = cast[cstring](inp) var o = cast[cstring](outp) o[0] = i[7] @@ -80,8 +107,6 @@ else: o[7] = i[0] proc swapEndian32*(outp, inp: pointer) = - ## copies `inp` to `outp` swapping bytes. Both buffers are supposed to - ## contain at least 4 bytes. var i = cast[cstring](inp) var o = cast[cstring](outp) o[0] = i[3] @@ -90,8 +115,6 @@ else: o[3] = i[0] proc swapEndian16*(outp, inp: pointer) = - ## copies `inp` to `outp` swapping bytes. Both buffers are supposed to - ## contain at least 2 bytes. var i = cast[cstring](inp) var o = cast[cstring](outp) o[0] = i[1] @@ -106,8 +129,20 @@ when system.cpuEndian == bigEndian: proc bigEndian16*(outp, inp: pointer) {.inline.} = copyMem(outp, inp, 2) else: proc littleEndian64*(outp, inp: pointer) {.inline.} = copyMem(outp, inp, 8) + ## Copies `inp` to `outp`, storing it in 64-bit little-endian order. + ## Both buffers are supposed to contain at least 8 bytes. proc littleEndian32*(outp, inp: pointer) {.inline.} = copyMem(outp, inp, 4) + ## Copies `inp` to `outp`, storing it in 32-bit little-endian order. + ## Both buffers are supposed to contain at least 4 bytes. proc littleEndian16*(outp, inp: pointer){.inline.} = copyMem(outp, inp, 2) + ## Copies `inp` to `outp`, storing it in 16-bit little-endian order. + ## Both buffers are supposed to contain at least 2 bytes. proc bigEndian64*(outp, inp: pointer) {.inline.} = swapEndian64(outp, inp) + ## Copies `inp` to `outp`, storing it in 64-bit big-endian order. + ## Both buffers are supposed to contain at least 8 bytes. proc bigEndian32*(outp, inp: pointer) {.inline.} = swapEndian32(outp, inp) + ## Copies `inp` to `outp`, storing it in 32-bit big-endian order. + ## Both buffers are supposed to contain at least 4 bytes. proc bigEndian16*(outp, inp: pointer) {.inline.} = swapEndian16(outp, inp) + ## Copies `inp` to `outp`, storing it in 16-bit big-endian order. + ## Both buffers are supposed to contain at least 2 bytes. From 4d3f9d3536e84d756f15b708177f289417dca3d2 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 8 Apr 2021 17:46:51 +0200 Subject: [PATCH 0599/2472] manual: minor improvements (#17677) * manual: minor improvements * Update doc/manual.rst Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- doc/manual.rst | 94 +++++++++++++++++++++++++------------------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/doc/manual.rst b/doc/manual.rst index 3c71ece95..aad8034c8 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -222,7 +222,7 @@ comment: `Documentation comments`:idx: are comments that start with two `##`. Documentation comments are tokens; they are only allowed at certain places in -the input file as they belong to the syntax tree! +the input file as they belong to the syntax tree. Multiline comments @@ -315,8 +315,8 @@ it was not case-sensitive and underscores were ignored and there was not even a distinction between `foo` and `Foo`. -Stropping ---------- +Keywords as identifiers +----------------------- If a keyword is enclosed in backticks it loses its keyword property and becomes an ordinary identifier. @@ -326,12 +326,12 @@ Examples var `var` = "Hello Stropping" .. code-block:: nim - type Type = object - `int`: int + type Obj = object + `type`: int - let `object` = Type(`int`: 9) - assert `object` is Type - assert `object`.`int` == 9 + let `object` = Obj(`type`: 9) + assert `object` is Obj + assert `object`.`type` == 9 var `var` = 42 let `let` = 8 @@ -442,7 +442,7 @@ Terminal symbols in the grammar: `GENERALIZED_STR_LIT`, The construct `identifier"string literal"` (without whitespace between the identifier and the opening quotation mark) is a generalized raw string literal. It is a shortcut for the construct -`identifier(r"string literal")`, so it denotes a procedure call with a +`identifier(r"string literal")`, so it denotes a routine call with a raw string literal as its only argument. Generalized raw string literals are especially convenient for embedding mini languages directly into Nim (for example regular expressions). @@ -457,8 +457,8 @@ Character literals Character literals are enclosed in single quotes `''` and can contain the same escape sequences as strings - with one exception: the platform dependent `newline`:idx: (``\p``) -is not allowed as it may be wider than one character (often it is the pair -CR/LF for example). Here are the valid `escape sequences`:idx: for character +is not allowed as it may be wider than one character (it can be the pair +CR/LF). Here are the valid `escape sequences`:idx: for character literals: ================== =================================================== @@ -482,19 +482,21 @@ literals: exactly two hex digits are allowed ================== =================================================== -A character is not a Unicode character but a single byte. The reason for this -is efficiency: for the overwhelming majority of use-cases, the resulting -programs will still handle UTF-8 properly as UTF-8 was specially designed for -this. Another reason is that Nim can thus support `array[char, int]` or -`set[char]` efficiently as many algorithms rely on this feature. The `Rune` -type is used for Unicode characters, it can represent any Unicode character. +A character is not a Unicode character but a single byte. + +Rationale: It enables the efficient support of `array[char, int]` or +`set[char]`. + +The `Rune` type can represent any Unicode character. `Rune` is declared in the `unicode module `_. A character literal that does not end in ``'`` is interpreted as ``'`` if there is a preceeding backtick token. There must be no whitespace between the preceeding -backtick token and the character literal. This special rule ensures that a declaration -like ``proc `'customLiteral`(s: string)`` is valid. See also -`Custom Numeric Literals <#custom-numeric-literals>`_. +backtick token and the character literal. This special case ensures that a declaration +like ``proc `'customLiteral`(s: string)`` is valid. ``proc `'customLiteral`(s: string)`` +is the same as ``proc `'\''customLiteral`(s: string)``. + +See also `Custom Numeric Literals <#custom-numeric-literals>`_. Numeric Literals @@ -908,37 +910,37 @@ constant expressions that may be surprising to those coming from other statically typed languages. For example, the following code echoes the beginning of the Fibonacci series **at compile-time**. (This is a demonstration of flexibility in defining constants, not a recommended style for solving this -problem!) +problem.) .. code-block:: nim :test: "nim c $1" import std/strformat - var fib_n {.compileTime.}: int - var fib_prev {.compileTime.}: int - var fib_prev_prev {.compileTime.}: int + var fibN {.compileTime.}: int + var fibPrev {.compileTime.}: int + var fibPrevPrev {.compileTime.}: int - proc next_fib(): int = - result = if fib_n < 2: - fib_n + proc nextFib(): int = + result = if fibN < 2: + fibN else: - fib_prev_prev + fib_prev - inc(fib_n) - fib_prev_prev = fib_prev - fib_prev = result + fibPrevPrev + fibPrev + inc(fibN) + fibPrevPrev = fibPrev + fibPrev = result - const f0 = next_fib() - const f1 = next_fib() + const f0 = nextFib() + const f1 = nextFib() - const display_fib = block: - const f2 = next_fib() + const displayFib = block: + const f2 = nextFib() var result = fmt"Fibonacci sequence: {f0}, {f1}, {f2}" for i in 3..12: - add(result, fmt", {next_fib()}") + add(result, fmt", {nextFib()}") result static: - echo display_fib + echo displayFib Restrictions on Compile-Time Execution @@ -1013,7 +1015,9 @@ intXX Literals of these types have the suffix 'iXX. `uint` - the generic `unsigned integer`:idx: type; its size is platform-dependent and has the same size as a pointer. An integer literal with the type suffix `'u` is of this type. + the generic `unsigned integer`:idx: type; its size is platform-dependent and + has the same size as a pointer. An integer literal with the type + suffix `'u` is of this type. uintXX additional unsigned integer types of XX bits use this naming scheme @@ -1199,12 +1203,8 @@ Character type -------------- The character type is named `char` in Nim. Its size is one byte. Thus it cannot represent a UTF-8 character, but a part of it. -The reason for this is efficiency: for the overwhelming majority of use-cases, -the resulting programs will still handle UTF-8 properly as UTF-8 was especially -designed for this. -Another reason is that Nim can support `array[char, int]` or -`set[char]` efficiently as many algorithms rely on this feature. The -`Rune` type is used for Unicode characters, it can represent any Unicode + +The `Rune` type is used for Unicode characters, it can represent any Unicode character. `Rune` is declared in the `unicode module `_. @@ -1232,7 +1232,7 @@ Now the following holds:: # Also allowed: ord(Direction.west) == 3 -Thus, north < east < south < west. The comparison operators can be used +The implied order is: north < east < south < west. The comparison operators can be used with enumeration types. Instead of `north` etc, the enum value can also be qualified with the enum type that it resides in, `Direction.north`. @@ -1248,7 +1248,7 @@ An explicit ordered enum can have *holes*: TokenType = enum a = 2, b = 4, c = 89 # holes are valid -However, it is then not ordinal anymore, so it is not possible to use these +However, it is then not ordinal anymore, so it is impossible to use these enums as an index type for arrays. The procedures `inc`, `dec`, `succ` and `pred` are not available for them either. @@ -7885,4 +7885,4 @@ Threads and exceptions The interaction between threads and exceptions is simple: A *handled* exception in one thread cannot affect any other thread. However, an *unhandled* exception -in one thread terminates the whole *process*! +in one thread terminates the whole *process*. From 42687457b079c16f69b22ff2354590780bd4cf45 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Thu, 8 Apr 2021 20:00:14 +0300 Subject: [PATCH 0600/2472] further progress on rst roles & directives (fix #17646) (#17659) * further progress on rst roles & dir-s (fix #17646) * fix documents according to the messages * fix bug 17 from #17340 --- doc/manual_experimental.rst | 2 +- lib/experimental/diff.nim | 4 +- lib/packages/docutils/highlite.nim | 4 +- lib/packages/docutils/rst.nim | 110 ++++++++++++++++++++--------- lib/pure/asyncdispatch.nim | 2 +- lib/pure/asyncftpclient.nim | 6 +- lib/pure/asynchttpserver.nim | 2 +- lib/pure/asyncnet.nim | 2 +- lib/pure/base64.nim | 8 +-- lib/pure/dynlib.nim | 2 +- lib/system/iterators.nim | 2 +- tests/stdlib/trstgen.nim | 36 ++++++++++ 12 files changed, 129 insertions(+), 51 deletions(-) diff --git a/doc/manual_experimental.rst b/doc/manual_experimental.rst index fc46a2a14..51037b509 100644 --- a/doc/manual_experimental.rst +++ b/doc/manual_experimental.rst @@ -330,7 +330,7 @@ Routines with the same type signature can be called differently if a parameter has different names. This does not need an `experimental` switch, but is an unstable feature. -.. code-block::nim +.. code-block:: Nim proc foo(x: int) = echo "Using x: ", x proc foo(y: int) = diff --git a/lib/experimental/diff.nim b/lib/experimental/diff.nim index 3462a0fa4..ba4e8ad64 100644 --- a/lib/experimental/diff.nim +++ b/lib/experimental/diff.nim @@ -12,14 +12,14 @@ ## ## A basic example of `diffInt` on 2 arrays of integers: ## -## .. code::nim +## .. code:: Nim ## ## import experimental/diff ## echo diffInt([0, 1, 2, 3, 4, 5, 6, 7, 8], [-1, 1, 2, 3, 4, 5, 666, 7, 42]) ## ## Another short example of `diffText` to diff strings: ## -## .. code::nim +## .. code:: Nim ## ## import experimental/diff ## # 2 samples of text for testing (from "The Call of Cthulhu" by Lovecraft) diff --git a/lib/packages/docutils/highlite.nim b/lib/packages/docutils/highlite.nim index 2655a7ea3..94cb2ebde 100644 --- a/lib/packages/docutils/highlite.nim +++ b/lib/packages/docutils/highlite.nim @@ -13,7 +13,7 @@ ## ## You can use this to build your own syntax highlighting, check this example: ## -## .. code::nim +## .. code:: Nim ## let code = """for x in $int.high: echo x.ord mod 2 == 0""" ## var toknizr: GeneralTokenizer ## initGeneralTokenizer(toknizr, code) @@ -34,7 +34,7 @@ ## ## The proc `getSourceLanguage` can get the language `enum` from a string: ## -## .. code::nim +## .. code:: Nim ## for l in ["C", "c++", "jAvA", "Nim", "c#"]: echo getSourceLanguage(l) ## diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index dfa2f12be..a8bc04a1a 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -912,6 +912,38 @@ template newLeaf(s: string): PRstNode = newRstLeaf(s) proc newLeaf(p: var RstParser): PRstNode = result = newLeaf(currentTok(p).symbol) +proc validRefnamePunct(x: string): bool = + ## https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#reference-names + x.len == 1 and x[0] in {'-', '_', '.', ':', '+'} + +func getRefnameIdx(p: RstParser, startIdx: int): int = + ## Gets last token index of a refname ("word" in RST terminology): + ## + ## reference names are single words consisting of alphanumerics plus + ## isolated (no two adjacent) internal hyphens, underscores, periods, + ## colons and plus signs; no whitespace or other characters are allowed. + ## + ## Refnames are used for: + ## - reference names + ## - role names + ## - directive names + ## - footnote labels + ## + # TODO: use this func in all other relevant places + var j = startIdx + if p.tok[j].kind == tkWord: + inc j + while p.tok[j].kind == tkPunct and validRefnamePunct(p.tok[j].symbol) and + p.tok[j+1].kind == tkWord: + inc j, 2 + result = j - 1 + +func getRefname(p: RstParser, startIdx: int): (string, int) = + let lastIdx = getRefnameIdx(p, startIdx) + result[1] = lastIdx + for j in startIdx..lastIdx: + result[0].add p.tok[j].symbol + proc getReferenceName(p: var RstParser, endStr: string): PRstNode = var res = newRstNode(rnInner) while true: @@ -1011,7 +1043,10 @@ proc match(p: RstParser, start: int, expr: string): bool = var last = expr.len - 1 while i <= last: case expr[i] - of 'w': result = p.tok[j].kind == tkWord + of 'w': + let lastIdx = getRefnameIdx(p, j) + result = lastIdx >= j + if result: j = lastIdx of ' ': result = p.tok[j].kind == tkWhite of 'i': result = p.tok[j].kind == tkIndent of 'I': result = p.tok[j].kind in {tkIndent, tkEof} @@ -1058,7 +1093,7 @@ proc fixupEmbeddedRef(n, a, b: PRstNode) = proc whichRole(p: RstParser, sym: string): RstNodeKind = result = whichRoleAux(sym) if result == rnUnknownRole: - rstMessage(p, mwUnsupportedLanguage, p.s.currRole) + rstMessage(p, mwUnsupportedLanguage, sym) proc toInlineCode(n: PRstNode, language: string): PRstNode = ## Creates rnInlineCode and attaches `n` contents as code (in 3rd son). @@ -1078,6 +1113,11 @@ proc toInlineCode(n: PRstNode, language: string): PRstNode = lb.add newLeaf(s) result.add lb +proc toUnknownRole(n: PRstNode, roleName: string): PRstNode = + let newN = newRstNode(rnInner, n.sons) + let newSons = @[newN, newLeaf(roleName)] + result = newRstNode(rnUnknownRole, newSons) + proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode = var newKind = n.kind var newSons = n.sons @@ -1102,17 +1142,15 @@ proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode = result = newRstNode(newKind, newSons) elif match(p, p.idx, ":w:"): # a role: - let roleName = nextTok(p).symbol + let (roleName, lastIdx) = getRefname(p, p.idx+1) newKind = whichRole(p, roleName) if newKind == rnUnknownRole: - let newN = newRstNode(rnInner, n.sons) - newSons = @[newN, newLeaf(roleName)] - result = newRstNode(newKind, newSons) + result = n.toUnknownRole(roleName) elif newKind == rnInlineCode: result = n.toInlineCode(language=roleName) else: result = newRstNode(newKind, newSons) - inc p.idx, 3 + p.idx = lastIdx + 2 else: if p.s.currRoleKind == rnInlineCode: result = n.toInlineCode(language=p.s.currRole) @@ -1139,10 +1177,6 @@ proc parseSmiley(p: var RstParser): PRstNode = result.text = val return -proc validRefnamePunct(x: string): bool = - ## https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#reference-names - x.len == 1 and x[0] in {'-', '_', '.', ':', '+'} - proc isUrl(p: RstParser, i: int): bool = result = p.tok[i+1].symbol == ":" and p.tok[i+2].symbol == "//" and p.tok[i+3].kind == tkWord and @@ -1373,14 +1407,18 @@ proc parseInline(p: var RstParser, father: PRstNode) = var n = newRstNode(rnInlineLiteral) parseUntil(p, n, "``", false) father.add(n) - elif match(p, p.idx, ":w:") and p.tok[p.idx+3].symbol == "`": - let roleName = nextTok(p).symbol + elif match(p, p.idx, ":w:") and + (var lastIdx = getRefnameIdx(p, p.idx + 1); + p.tok[lastIdx+2].symbol == "`"): + let (roleName, _) = getRefname(p, p.idx+1) let k = whichRole(p, roleName) var n = newRstNode(k) - inc p.idx, 3 + p.idx = lastIdx + 2 if k == rnInlineCode: n = n.toInlineCode(language=roleName) parseUntil(p, n, "`", false) # bug #17260 + if k == rnUnknownRole: + n = n.toUnknownRole(roleName) father.add(n) elif isInlineMarkupStart(p, "`"): var n = newRstNode(rnInterpretedText) @@ -1438,25 +1476,28 @@ proc parseInline(p: var RstParser, father: PRstNode) = else: discard proc getDirective(p: var RstParser): string = - if currentTok(p).kind == tkWhite and nextTok(p).kind == tkWord: - var j = p.idx - inc p.idx - result = currentTok(p).symbol - inc p.idx - while currentTok(p).kind in {tkWord, tkPunct, tkAdornment, tkOther}: - if currentTok(p).symbol == "::": break - result.add(currentTok(p).symbol) - inc p.idx - if currentTok(p).kind == tkWhite: inc p.idx - if currentTok(p).symbol == "::": - inc p.idx - if currentTok(p).kind == tkWhite: inc p.idx - else: - p.idx = j # set back - result = "" # error - else: - result = "" - result = result.toLowerAscii() + result = "" + if currentTok(p).kind == tkWhite: + let (name, lastIdx) = getRefname(p, p.idx + 1) + let afterIdx = lastIdx + 1 + if name.len > 0: + if p.tok[afterIdx].symbol == "::": + result = name + p.idx = afterIdx + 1 + if currentTok(p).kind == tkWhite: + inc p.idx + elif currentTok(p).kind != tkIndent: + rstMessage(p, mwRstStyle, + "whitespace or newline expected after directive " & name) + result = result.toLowerAscii() + elif p.tok[afterIdx].symbol == ":": + rstMessage(p, mwRstStyle, + "double colon :: may be missing at end of '" & name & "'", + p.tok[afterIdx].line, p.tok[afterIdx].col) + elif p.tok[afterIdx].kind == tkPunct and p.tok[afterIdx].symbol[0] == ':': + rstMessage(p, mwRstStyle, + "too many colons for a directive (should be ::)", + p.tok[afterIdx].line, p.tok[afterIdx].col) proc parseComment(p: var RstParser): PRstNode = case currentTok(p).kind @@ -1711,7 +1752,8 @@ proc whichSection(p: RstParser): RstNodeKind = return rnCodeBlock elif currentTok(p).symbol == "::": return rnLiteralBlock - elif currentTok(p).symbol == ".." and predNL(p): + elif currentTok(p).symbol == ".." and predNL(p) and + nextTok(p).kind in {tkWhite, tkIndent}: return rnDirective case currentTok(p).kind of tkAdornment: diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 55a20270d..3866ebd24 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -42,7 +42,7 @@ ## ## Code to read some data from a socket may look something like this: ## -## .. code-block::nim +## .. code-block:: Nim ## var future = socket.recv(100) ## future.addCallback( ## proc () = diff --git a/lib/pure/asyncftpclient.nim b/lib/pure/asyncftpclient.nim index 0ee45785d..056d65564 100644 --- a/lib/pure/asyncftpclient.nim +++ b/lib/pure/asyncftpclient.nim @@ -21,7 +21,7 @@ ## In order to begin any sort of transfer of files you must first ## connect to an FTP server. You can do so with the `connect` procedure. ## -## .. code-block::nim +## .. code-block:: Nim ## import std/[asyncdispatch, asyncftpclient] ## proc main() {.async.} = ## var ftp = newAsyncFtpClient("example.com", user = "test", pass = "test") @@ -41,7 +41,7 @@ ## working directory before you do so with the `pwd` procedure, you can also ## instead specify an absolute path. ## -## .. code-block::nim +## .. code-block:: Nim ## import std/[asyncdispatch, asyncftpclient] ## proc main() {.async.} = ## var ftp = newAsyncFtpClient("example.com", user = "test", pass = "test") @@ -62,7 +62,7 @@ ## Procs that take an `onProgressChanged` callback will call this every ## `progressInterval` milliseconds. ## -## .. code-block::nim +## .. code-block:: Nim ## import std/[asyncdispatch, asyncftpclient] ## ## proc onProgressChanged(total, progress: BiggestInt, diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index 3982edc54..fd0cad5ca 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -108,7 +108,7 @@ proc respond*(req: Request, code: HttpCode, content: string, ## ## Example: ## - ## .. code-block::nim + ## .. code-block:: Nim ## import std/json ## proc handler(req: Request) {.async.} = ## if req.url.path == "/hello-world": diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim index 7aeb99777..7ef40a128 100644 --- a/lib/pure/asyncnet.nim +++ b/lib/pure/asyncnet.nim @@ -65,7 +65,7 @@ ## ## The following example demonstrates a simple chat server. ## -## .. code-block::nim +## .. code-block:: Nim ## ## import std/[asyncnet, asyncdispatch] ## diff --git a/lib/pure/base64.nim b/lib/pure/base64.nim index bf196b54d..d5c1fda7a 100644 --- a/lib/pure/base64.nim +++ b/lib/pure/base64.nim @@ -23,14 +23,14 @@ ## Encoding data ## ------------- ## -## .. code-block::nim +## .. code-block:: Nim ## import std/base64 ## let encoded = encode("Hello World") ## assert encoded == "SGVsbG8gV29ybGQ=" ## ## Apart from strings you can also encode lists of integers or characters: ## -## .. code-block::nim +## .. code-block:: Nim ## import std/base64 ## let encodedInts = encode([1,2,3]) ## assert encodedInts == "AQID" @@ -41,7 +41,7 @@ ## Decoding data ## ------------- ## -## .. code-block::nim +## .. code-block:: Nim ## import std/base64 ## let decoded = decode("SGVsbG8gV29ybGQ=") ## assert decoded == "Hello World" @@ -49,7 +49,7 @@ ## URL Safe Base64 ## --------------- ## -## .. code-block::nim +## .. code-block:: Nim ## import std/base64 ## doAssert encode("c\xf7>", safe = true) == "Y_c-" ## doAssert encode("c\xf7>", safe = false) == "Y/c+" diff --git a/lib/pure/dynlib.nim b/lib/pure/dynlib.nim index 3e1d84729..a1a94072c 100644 --- a/lib/pure/dynlib.nim +++ b/lib/pure/dynlib.nim @@ -22,7 +22,7 @@ ## If the library fails to load or the function 'greet' is not found, ## it quits with a failure error code. ## -## .. code-block::nim +## .. code-block:: Nim ## ## import std/dynlib ## diff --git a/lib/system/iterators.nim b/lib/system/iterators.nim index 86d64d0c7..0f79970b8 100644 --- a/lib/system/iterators.nim +++ b/lib/system/iterators.nim @@ -304,7 +304,7 @@ iterator fieldPairs*[T: tuple|object](x: T): tuple[key: string, val: RootObj] {. ## picking the appropriate code to a secondary proc which you overload for ## each field type and pass the `value` to. ## - ## .. warning::: This really transforms the 'for' and unrolls the loop. The + ## .. warning:: This really transforms the 'for' and unrolls the loop. The ## current implementation also has a bug that affects symbol binding in the ## loop body. runnableExamples: diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index 99b8d8db5..b403d96c6 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -136,6 +136,25 @@ suite "YAML syntax highlighting": not numbers: [ 42e, 0023, +32.37, 8 ball] }""" + test "Directives: warnings": + let input = dedent""" + .. non-existant-warning: Paragraph. + + .. another.wrong:warning::: Paragraph. + """ + var warnings = new seq[string] + let output = input.toHtml(warnings=warnings) + check output == "" + doAssert warnings[].len == 2 + check "(1, 24) Warning: RST style:" in warnings[0] + check "double colon :: may be missing at end of 'non-existant-warning'" in warnings[0] + check "(3, 25) Warning: RST style:" in warnings[1] + check "RST style: too many colons for a directive (should be ::)" in warnings[1] + + test "not a directive": + let input = "..warning:: I am not a warning." + check input.toHtml == input + test "Anchors, Aliases, Tags": let input = """.. code-block:: yaml --- !!map @@ -1403,6 +1422,23 @@ Test1 check """`3`:sup:\ He is an isotope of helium.""".toHtml == expected check """`3`:superscript:\ He is an isotope of helium.""".toHtml == expected + test "Roles: warnings": + let input = dedent""" + See function :py:func:`spam`. + + See also `egg`:py:class:. + """ + var warnings = new seq[string] + let output = input.toHtml(warnings=warnings) + doAssert warnings[].len == 2 + check "(1, 14) Warning: " in warnings[0] + check "language 'py:func' not supported" in warnings[0] + check "(3, 15) Warning: " in warnings[1] + check "language 'py:class' not supported" in warnings[1] + check("""

    See function spam.

    """ & "\n" & + """

    See also egg.

    """ & "\n" == + output) + test "(not) Roles: check escaping 1": let expected = """See :subscript:""" & """""" & id"some" & " " & id"text" & From 9f0f4775524d6167655e089a864cd9fdbd10c806 Mon Sep 17 00:00:00 2001 From: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Date: Thu, 8 Apr 2021 19:41:43 +0200 Subject: [PATCH 0601/2472] Update `sysrand` documentation (#17676) Co-authored-by: Andreas Rumpf --- doc/lib.rst | 8 ++++---- lib/std/sysrand.nim | 32 ++++++++++++++++---------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/doc/lib.rst b/doc/lib.rst index 15b4f70d6..f09fe16b2 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -272,18 +272,18 @@ Math libraries * `random `_ Fast and tiny random number generator. -* `sysrand `_ - Cryptographically secure pseudorandom number generator. - * `rationals `_ This module implements rational numbers and relevant mathematical operations. * `stats `_ - Statistical analysis + Statistical analysis. * `sums `_ Accurate summation functions. +* `sysrand `_ + Cryptographically secure pseudorandom number generator. + Internet Protocols and Support ------------------------------ diff --git a/lib/std/sysrand.nim b/lib/std/sysrand.nim index c9908143c..4a2cdd10b 100644 --- a/lib/std/sysrand.nim +++ b/lib/std/sysrand.nim @@ -12,21 +12,21 @@ ## therefore may not be secure. ## ## `std/sysrand` generates random numbers from a secure source provided by the operating system. -## It is also called Cryptographically secure pseudorandom number generator. -## It should be unpredictable enough for cryptographic applications, +## It is a cryptographically secure pseudorandom number generator +## and should be unpredictable enough for cryptographic applications, ## though its exact quality depends on the OS implementation. ## -## | Targets | Implementation| -## | :--- | ----: | -## | Windows | `BCryptGenRandom`_ | -## | Linux | `getrandom`_ | -## | MacOSX | `getentropy`_ | -## | IOS | `SecRandomCopyBytes`_ | -## | OpenBSD | `getentropy openbsd`_ | -## | FreeBSD | `getrandom freebsd`_ | -## | JS(Web Browser) | `getRandomValues`_ | -## | Nodejs | `randomFillSync`_ | -## | Other Unix platforms | `/dev/urandom`_ | +## | Targets | Implementation | +## | :--- | ----: | +## | Windows | `BCryptGenRandom`_ | +## | Linux | `getrandom`_ | +## | MacOSX | `getentropy`_ | +## | iOS | `SecRandomCopyBytes`_ | +## | OpenBSD | `getentropy openbsd`_ | +## | FreeBSD | `getrandom freebsd`_ | +## | JS (Web Browser) | `getRandomValues`_ | +## | Node.js | `randomFillSync`_ | +## | Other Unix platforms | `/dev/urandom`_ | ## ## .. _BCryptGenRandom: https://docs.microsoft.com/en-us/windows/win32/api/bcrypt/nf-bcrypt-bcryptgenrandom ## .. _getrandom: https://man7.org/linux/man-pages/man2/getrandom.2.html @@ -195,7 +195,7 @@ elif defined(linux): elif defined(openbsd): proc getentropy(p: pointer, size: cint): cint {.importc: "getentropy", header: "".} - # fills a buffer with high-quality entropy, + # Fills a buffer with high-quality entropy, # which can be used as input for process-context pseudorandom generators like `arc4random`. # The maximum buffer size permitted is 256 bytes. @@ -227,7 +227,7 @@ elif defined(ios): proc secRandomCopyBytes( rnd: SecRandomRef, count: csize_t, bytes: pointer - ): cint {.importc: "SecRandomCopyBytes", header: "".} + ): cint {.importc: "SecRandomCopyBytes", header: "".} ## https://developer.apple.com/documentation/security/1399291-secrandomcopybytes template urandomImpl(result: var int, dest: var openArray[byte]) = @@ -292,7 +292,7 @@ proc urandom*(dest: var openArray[byte]): bool = ## If the call succeeds, returns `true`. ## ## If `dest` is empty, `urandom` immediately returns success, - ## without calling underlying operating system api. + ## without calling the underlying operating system API. ## ## .. warning:: The code hasn't been audited by cryptography experts and ## is provided as-is without guarantees. Use at your own risks. For production From 6ec55ebf48d2b3889cd1391fdd630e65a01c0c91 Mon Sep 17 00:00:00 2001 From: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Date: Thu, 8 Apr 2021 20:29:11 +0200 Subject: [PATCH 0602/2472] Fix rst typo (#17671) --- lib/core/typeinfo.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/core/typeinfo.nim b/lib/core/typeinfo.nim index 196ac8fe1..08d567351 100644 --- a/lib/core/typeinfo.nim +++ b/lib/core/typeinfo.nim @@ -11,7 +11,7 @@ ## (`RTTI`:idx:). See the `marshal `_ module for an example of ## what this allows you to do. ## -## ..note:: Even though `Any` and its operations hide the nasty low level +## .. note:: Even though `Any` and its operations hide the nasty low level ## details from its users, it remains inherently unsafe! Also, Nim's ## runtime type information will evolve and may eventually be deprecated. ## As an alternative approach to programmatically understanding and From 2ac45a4794c9d4cc99ef3b154102fd185d35714e Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 9 Apr 2021 00:18:07 +0200 Subject: [PATCH 0603/2472] undocument unsupported advanced options (#17678) --- doc/advopt.txt | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/doc/advopt.txt b/doc/advopt.txt index 763ac80a6..4842d090e 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -43,7 +43,7 @@ Advanced options: --warning[X]:on|off turn specific warning X on|off --hints:on|off|list turn all hints on|off or list all available --hint[X]:on|off turn specific hint X on|off - --warningAsError[X]:on|off + --warningAsError[X]:on|off turn specific warning X into an error on|off --hintAsError[X]:on|off turn specific hint X into an error on|off --styleCheck:off|hint|error @@ -84,7 +84,7 @@ Advanced options: if path == @default (the default and most useful), will use best match among @pkg,@path. if these are nonexistent, will use project path - -b, --backend:c|cpp|js|objc + -b, --backend:c|cpp|js|objc sets backend to use with commands like `nim doc` or `nim r` --docCmd:cmd if `cmd == skip`, skips runnableExamples else, runs runnableExamples with given options, e.g.: @@ -95,26 +95,22 @@ Advanced options: --lineDir:on|off generation of #line directive on|off --embedsrc:on|off embeds the original source code as comments in the generated output - --threadanalysis:on|off turn thread analysis on|off --tlsEmulation:on|off turn thread local storage emulation on|off --implicitStatic:on|off turn implicit compile time evaluation on|off --trmacros:on|off turn term rewriting macros on|off --multimethods:on|off turn multi-methods on|off - --memTracker:on|off turn memory tracker on|off --hotCodeReloading:on|off turn support for hot code reloading on|off --excessiveStackTrace:on|off stack traces use full file paths --stackTraceMsgs:on|off enable user defined stack frame msgs via `setFrameMsg` - --seqsv2:on|off use the new string/seq implementation based on - destructors --skipCfg:on|off do not read the nim installation's configuration file --skipUserCfg:on|off do not read the user's configuration file --skipParentCfg:on|off do not read the parent dirs' configuration files --skipProjCfg:on|off do not read the project's configuration file --gc:refc|arc|orc|markAndSweep|boehm|go|none|regions select the GC to use; default is 'refc' - --exceptions:setjmp|cpp|goto + --exceptions:setjmp|cpp|goto|quirky select the exception handling implementation --index:on|off turn index file generation on|off --putenv:key=value set an environment variable @@ -149,11 +145,8 @@ Advanced options: --legacy:$2 enable obsolete/legacy language feature --useVersion:1.0|1.2 emulate Nim version X of the Nim compiler, for testing - --profiler:on|off enable profiling; requires `import nimprof`, and - works better with `--stackTrace:on` - see also https://nim-lang.github.io/Nim/estp.html - --benchmarkVM:on|off enable benchmarking of VM code with cpuTime() - --profileVM:on|off enable compile time VM profiler - --sinkInference:on|off en-/disable sink parameter inference (default: on) + --benchmarkVM:on|off turn benchmarking of VM code with cpuTime() on|off + --profileVM:on|off turn compile time VM profiler on|off + --sinkInference:on|off turn sink parameter inference on|off (default: on) --panics:on|off turn panics into process terminations (default: off) --deepcopy:on|off enable 'system.deepCopy' for ``--gc:arc|orc`` From e4b64eee894ddbf41935d3b8aeed1448cd2b5e46 Mon Sep 17 00:00:00 2001 From: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Date: Fri, 9 Apr 2021 00:40:19 +0200 Subject: [PATCH 0604/2472] Fix small typos (#17680) --- doc/intern.rst | 2 +- lib/js/jsconsole.nim | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/intern.rst b/doc/intern.rst index 1e18d975c..3b1a48b41 100644 --- a/doc/intern.rst +++ b/doc/intern.rst @@ -113,7 +113,7 @@ You can import them directly for debugging: The compiler's architecture =========================== -Nim uses the classic compiler architecture: A lexer/scanner feds tokens to a +Nim uses the classic compiler architecture: A lexer/scanner feeds tokens to a parser. The parser builds a syntax tree that is used by the code generators. This syntax tree is the interface between the parser and the code generator. It is essential to understand most of the compiler's code. diff --git a/lib/js/jsconsole.nim b/lib/js/jsconsole.nim index bf43adddd..e74127334 100644 --- a/lib/js/jsconsole.nim +++ b/lib/js/jsconsole.nim @@ -118,6 +118,7 @@ since (1, 5): func timeStamp*(console: Console; label: cstring) {.importcpp.} ## https://developer.mozilla.org/en-US/docs/Web/API/Console/timeStamp + ## ## ..warning:: non-standard From 13b958eb45a3569acaa18f9381d28b8f37d42057 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 9 Apr 2021 06:59:25 +0200 Subject: [PATCH 0605/2472] IC: added basic test case for methods (#17679) * IC: added basic test case for methods * IC: better methods test --- compiler/ic/cbackend.nim | 35 ++++++++++++++++++++++++++++------- compiler/ic/ic.nim | 2 +- tests/ic/mbaseobj.nim | 7 +++++++ tests/ic/tmethods.nim | 28 ++++++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 8 deletions(-) create mode 100644 tests/ic/mbaseobj.nim create mode 100644 tests/ic/tmethods.nim diff --git a/compiler/ic/cbackend.nim b/compiler/ic/cbackend.nim index 9282ac466..2f3910b32 100644 --- a/compiler/ic/cbackend.nim +++ b/compiler/ic/cbackend.nim @@ -30,12 +30,16 @@ proc unpackTree(g: ModuleGraph; thisModule: int; var decoder = initPackedDecoder(g.config, g.cache) result = loadNodes(decoder, g.packed, thisModule, tree, n) -proc generateCodeForModule(g: ModuleGraph; m: var LoadedModule; alive: var AliveSyms) = +proc setupBackendModule(g: ModuleGraph; m: var LoadedModule) = if g.backend == nil: g.backend = cgendata.newModuleList(g) - + assert g.backend != nil var bmod = cgen.newModule(BModuleList(g.backend), m.module, g.config) bmod.idgen = idgenFromLoadedModule(m) + +proc generateCodeForModule(g: ModuleGraph; m: var LoadedModule; alive: var AliveSyms) = + var bmod = BModuleList(g.backend).modules[m.module.position] + assert bmod != nil bmod.flags.incl useAliveDataFromDce bmod.alive = move alive[m.module.position] @@ -118,6 +122,27 @@ proc generateCode*(g: ModuleGraph) = for i in 0..high(g.packed): echo i, " is of status ", g.packed[i].status, " ", toFullPath(g.config, FileIndex(i)) + # First pass: Setup all the backend modules for all the modules that have + # changed: + for i in 0..high(g.packed): + # case statement here to enforce exhaustive checks. + case g.packed[i].status + of undefined: + discard "nothing to do" + of loading, stored: + assert false + of storing, outdated: + setupBackendModule(g, g.packed[i]) + of loaded: + # Even though this module didn't change, DCE might trigger a change. + # Consider this case: Module A uses symbol S from B and B does not use + # S itself. A is then edited not to use S either. Thus we have to + # recompile B in order to remove S from the final result. + if aliveSymsChanged(g.config, g.packed[i].module.position, alive): + g.packed[i].loadedButAliveSetChanged = true + setupBackendModule(g, g.packed[i]) + + # Second pass: Code generation. for i in 0..high(g.packed): # case statement here to enforce exhaustive checks. case g.packed[i].status @@ -130,11 +155,7 @@ proc generateCode*(g: ModuleGraph) = closeRodFile(g, g.packed[i].module) storeAliveSyms(g.config, g.packed[i].module.position, alive) of loaded: - # Even though this module didn't change, DCE might trigger a change. - # Consider this case: Module A uses symbol S from B and B does not use - # S itself. A is then edited not to use S either. Thus we have to - # recompile B in order to remove S from the final result. - if aliveSymsChanged(g.config, g.packed[i].module.position, alive): + if g.packed[i].loadedButAliveSetChanged: generateCodeForModule(g, g.packed[i], alive) else: addFileToLink(g.config, g.packed[i].module) diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim index b5fed7d4e..46bb63e07 100644 --- a/compiler/ic/ic.nim +++ b/compiler/ic/ic.nim @@ -666,7 +666,7 @@ type LoadedModule* = object status*: ModuleStatus - symsInit, typesInit: bool + symsInit, typesInit, loadedButAliveSetChanged*: bool fromDisk*: PackedModule syms: seq[PSym] # indexed by itemId types: seq[PType] diff --git a/tests/ic/mbaseobj.nim b/tests/ic/mbaseobj.nim new file mode 100644 index 000000000..0f4e4a90d --- /dev/null +++ b/tests/ic/mbaseobj.nim @@ -0,0 +1,7 @@ + +type + Base* = ref object of RootObj + s*: string + +method m*(b: Base) {.base.} = + echo "Base ", b.s diff --git a/tests/ic/tmethods.nim b/tests/ic/tmethods.nim new file mode 100644 index 000000000..251f26889 --- /dev/null +++ b/tests/ic/tmethods.nim @@ -0,0 +1,28 @@ +discard """ + output: '''Base abc''' +""" + +import mbaseobj + +var c = Base(s: "abc") +m c + +#!EDIT!# + +discard """ + output: '''Base abc +Inherited abc''' +""" + +import mbaseobj + +type + Inherited = ref object of Base + +method m(i: Inherited) = + procCall m(Base i) + echo "Inherited ", i.s + +var c = Inherited(s: "abc") +m c + From 826ad2f36f34694277f624933049de2be40e0703 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 9 Apr 2021 00:09:04 -0500 Subject: [PATCH 0606/2472] nep1: guidelines for getters and setters (#17645) * nep1: guidelines for getters and setters * address a comment --- doc/nep1.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/nep1.rst b/doc/nep1.rst index bdf8e0eab..8687725bb 100644 --- a/doc/nep1.rst +++ b/doc/nep1.rst @@ -301,3 +301,12 @@ Miscellaneous let a = """foo bar """ + +- A getter API for a private field `foo` should preferably be named `foo`, not `getFoo`. + A getter-like API should preferably be named `getFoo`, not `foo` if: + * the API has side effects + * or the cost is not `O(1)` + For in between cases, there is no clear guideline. + +- Likewise with a setter API, replacing `foo` with `foo=` and `getFoo` with `setFoo` + in the above text. From cce1b24b1cc3860ff503668761d2798b1b1d1ef0 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 9 Apr 2021 01:00:13 -0500 Subject: [PATCH 0607/2472] ast: add getPIdent (#17684) * ast: add getPIdent * fixup --- compiler/ast.nim | 8 ++++++++ compiler/renderer.nim | 20 +++++++++----------- compiler/reorder.nim | 7 ++----- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 7d13956e9..fffe08cb7 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -1054,6 +1054,14 @@ const defaultAlignment = -1 defaultOffset = -1 +proc getPIdent*(a: PNode): PIdent {.inline.} = + ## Returns underlying `PIdent` for `{nkSym, nkIdent}`, or `nil`. + # xxx consider whether also returning the 1st ident for {nkOpenSymChoice, nkClosedSymChoice} + # which may simplify code. + case a.kind + of nkSym: a.sym.name + of nkIdent: a.ident + else: nil proc getnimblePkg*(a: PSym): PSym = result = a diff --git a/compiler/renderer.nim b/compiler/renderer.nim index 9a867b0a1..78e2fe20e 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -964,14 +964,12 @@ proc skipHiddenNodes(n: PNode): PNode = else: break proc accentedName(g: var TSrcGen, n: PNode) = + # This is for cases where ident should've really been a `nkAccQuoted`, e.g. `:tmp` + # or if user writes a macro with `ident":foo"`. It's unclear whether these should be legal. const backticksNeeded = OpChars + {'[', '{', '\''} if n == nil: return - let isOperator = - if n.kind == nkIdent and n.ident.s.len > 0 and n.ident.s[0] in backticksNeeded: true - elif n.kind == nkSym and n.sym.name.s.len > 0 and n.sym.name.s[0] in backticksNeeded: true - else: false - - if isOperator: + let ident = n.getPIdent + if ident != nil and ident.s[0] in backticksNeeded: put(g, tkAccent, "`") gident(g, n) put(g, tkAccent, "`") @@ -999,9 +997,9 @@ proc infixArgument(g: var TSrcGen, n: PNode, i: int) = put(g, tkParRi, ")") proc isCustomLit(n: PNode): bool = - n.len == 2 and n[0].kind == nkRStrLit and - (n[1].kind == nkIdent and n[1].ident.s.startsWith('\'')) or - (n[1].kind == nkSym and n[1].sym.name.s.startsWith('\'')) + if n.len == 2 and n[0].kind == nkRStrLit: + let ident = n[1].getPIdent + result = ident != nil and ident.s.startsWith('\'') proc gsub(g: var TSrcGen, n: PNode, c: TContext, fromStmtList = false) = if isNil(n): return @@ -1234,8 +1232,8 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext, fromStmtList = false) = else: gsub(g, n, 0) put(g, tkDot, ".") - if n.len > 1: - accentedName(g, n[1]) + assert n.len == 2, $n.len + accentedName(g, n[1]) of nkBind: putWithSpace(g, tkBind, "bind") gsub(g, n, 0) diff --git a/compiler/reorder.nim b/compiler/reorder.nim index 4ffe4ccf8..65c1fb9ee 100644 --- a/compiler/reorder.nim +++ b/compiler/reorder.nim @@ -36,11 +36,8 @@ proc newDepN(id: int, pnode: PNode): DepN = proc accQuoted(cache: IdentCache; n: PNode): PIdent = var id = "" for i in 0.. Date: Fri, 9 Apr 2021 02:37:10 -0500 Subject: [PATCH 0608/2472] make repr handle setters `foo=` (#17683) --- compiler/renderer.nim | 13 ++++++++----- tests/stdlib/trepr.nim | 9 +++++++++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/compiler/renderer.nim b/compiler/renderer.nim index 78e2fe20e..b3b0adc01 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -1327,13 +1327,16 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext, fromStmtList = false) = of nkIdent: n.ident.s of nkSym: n.sym.name.s else: "" + proc isAlpha(n: PNode): bool = + if n.kind in {nkIdent, nkSym}: + let tmp = n.getStrVal + result = tmp.len > 0 and tmp[0] in {'a'..'z', 'A'..'Z'} var useSpace = false if i == 1 and n[0].kind == nkIdent and n[0].ident.s in ["=", "'"]: - let tmp = n[1].getStrVal - if tmp.len > 0 and tmp[0] in {'a'..'z', 'A'..'Z'}: - # handle `=destroy`, `'big' - discard - else: + if not n[1].isAlpha: # handle `=destroy`, `'big' + useSpace = true + elif i == 1 and n[1].kind == nkIdent and n[1].ident.s == "=": + if not n[0].isAlpha: # handle setters, e.g. `foo=` useSpace = true elif i > 0: useSpace = true if useSpace: put(g, tkSpaces, Space) diff --git a/tests/stdlib/trepr.nim b/tests/stdlib/trepr.nim index 3dcbe9b69..f72bbb34e 100644 --- a/tests/stdlib/trepr.nim +++ b/tests/stdlib/trepr.nim @@ -163,6 +163,15 @@ proc `foo bar baz`(): int = """ doAssert a2 == a + block: # setters: `foo=` + let a = deb: + proc `foo=`() = discard + doAssert a == """ + +proc `foo=`() = + discard +""" + block: # bug #14850 block: let a = deb: From f25243140baa00db4a663d453a262e41f8d9073f Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 9 Apr 2021 17:51:51 +0800 Subject: [PATCH 0609/2472] add std/tasks (#17447) --- changelog.md | 2 + lib/std/tasks.nim | 272 +++++++++++++++++++++ tests/stdlib/ttasks.nim | 506 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 780 insertions(+) create mode 100644 lib/std/tasks.nim create mode 100644 tests/stdlib/ttasks.nim diff --git a/changelog.md b/changelog.md index 43a15e398..a2879833f 100644 --- a/changelog.md +++ b/changelog.md @@ -264,6 +264,8 @@ - Added dollar `$` and `len` for `jsre.RegExp`. +- Added `std/tasks`. + - Added `hasDataBuffered` to `asyncnet`. - Added `hasClosure` to `std/typetraits`. diff --git a/lib/std/tasks.nim b/lib/std/tasks.nim new file mode 100644 index 000000000..6b7f86ce4 --- /dev/null +++ b/lib/std/tasks.nim @@ -0,0 +1,272 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2021 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module provides basic primitives for creating parallel programs. +## A `Task` should be only owned by a single Thread, it cannot be shared by threads. + +import std/[macros, isolation, typetraits] +import system/ansi_c + +export isolation + + +when compileOption("threads"): + from std/effecttraits import isGcSafe + + +# +# proc hello(a: int, b: string) = +# echo $a & b +# +# let literal = "Nim" +# let t = toTask(hello(521, literal)) +# +# +# is roughly converted to +# +# type +# ScratchObj_369098780 = object +# a: int +# b: string +# +# let scratch_369098762 = cast[ptr ScratchObj_369098780](c_calloc(csize_t 1, +# csize_t sizeof(ScratchObj_369098780))) +# if scratch_369098762.isNil: +# raise newException(OutOfMemDefect, "Could not allocate memory") +# block: +# var isolate_369098776 = isolate(521) +# scratch_369098762.a = extract(isolate_369098776) +# var isolate_369098778 = isolate(literal) +# scratch_369098762.b = extract(isolate_369098778) +# proc hello_369098781(args`gensym3: pointer) {.nimcall.} = +# let objTemp_369098775 = cast[ptr ScratchObj_369098780](args`gensym3) +# let :tmp_369098777 = objTemp_369098775.a +# let :tmp_369098779 = objTemp_369098775.b +# hello(a = :tmp_369098777, b = :tmp_369098779) +# +# proc destroyScratch_369098782(args`gensym3: pointer) {.nimcall.} = +# let obj_369098783 = cast[ptr ScratchObj_369098780](args`gensym3) +# =destroy(obj_369098783[]) +# let t = Task(callback: hello_369098781, args: scratch_369098762, destroy: destroyScratch_369098782) +# + + +type + Task* = object ## `Task` contains the callback and its arguments. + callback: proc (args: pointer) {.nimcall.} + args: pointer + destroy: proc (args: pointer) {.nimcall.} + + +proc `=copy`*(x: var Task, y: Task) {.error.} + +proc `=destroy`*(t: var Task) {.inline.} = + ## Frees the resources allocated for a `Task`. + if t.args != nil: + if t.destroy != nil: + t.destroy(t.args) + c_free(t.args) + +proc invoke*(task: Task) {.inline.} = + ## Invokes the `task`. + task.callback(task.args) + +template checkIsolate(scratchAssignList: seq[NimNode], procParam, scratchDotExpr: NimNode) = + # block: + # var isoTempA = isolate(521) + # scratch.a = extract(isolateA) + # var isoTempB = isolate(literal) + # scratch.b = extract(isolateB) + let isolatedTemp = genSym(nskTemp, "isoTemp") + scratchAssignList.add newVarStmt(isolatedTemp, newCall(newidentNode("isolate"), procParam)) + scratchAssignList.add newAssignment(scratchDotExpr, + newcall(newIdentNode("extract"), isolatedTemp)) + +template addAllNode(assignParam: NimNode, procParam: NimNode) = + let scratchDotExpr = newDotExpr(scratchIdent, formalParams[i][0]) + + checkIsolate(scratchAssignList, procParam, scratchDotExpr) + + let tempNode = genSym(kind = nskTemp, ident = formalParams[i][0].strVal) + callNode.add nnkExprEqExpr.newTree(formalParams[i][0], tempNode) + tempAssignList.add newLetStmt(tempNode, newDotExpr(objTemp, formalParams[i][0])) + scratchRecList.add newIdentDefs(newIdentNode(formalParams[i][0].strVal), assignParam) + +macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkCallStrLit}): Task = + ## Converts the call and its arguments to `Task`. + runnableExamples("--gc:orc"): + proc hello(a: int) = echo a + + let b = toTask hello(13) + assert b is Task + + doAssert getTypeInst(e).typeKind == ntyVoid + + when compileOption("threads"): + if not isGcSafe(e[0]): + error("'toTask' takes a GC safe call expression") + + if hasClosure(e[0]): + error("closure call is not allowed") + + if e.len > 1: + let scratchIdent = genSym(kind = nskTemp, ident = "scratch") + let impl = e[0].getTypeInst + + when defined(nimTasksDebug): + echo impl.treeRepr + echo e.treeRepr + let formalParams = impl[0] + + var + scratchRecList = newNimNode(nnkRecList) + scratchAssignList: seq[NimNode] + tempAssignList: seq[NimNode] + callNode: seq[NimNode] + + let + objTemp = genSym(nskTemp, ident = "objTemp") + + for i in 1 ..< formalParams.len: + var param = formalParams[i][1] + + if param.kind == nnkBracketExpr and param[0].eqIdent("sink"): + param = param[0] + + if param.typeKind in {ntyExpr, ntyStmt}: + error("'toTask'ed function cannot have a 'typed' or 'untyped' parameter") + + case param.kind + of nnkVarTy: + error("'toTask'ed function cannot have a 'var' parameter") + of nnkBracketExpr: + if param[0].typeKind == ntyTypeDesc: + callNode.add nnkExprEqExpr.newTree(formalParams[i][0], e[i]) + elif param[0].typeKind in {ntyVarargs, ntyOpenArray}: + if param[1].typeKind in {ntyExpr, ntyStmt}: + error("'toTask'ed function cannot have a 'typed' or 'untyped' parameter") + let + seqType = nnkBracketExpr.newTree(newIdentNode("seq"), param[1]) + seqCallNode = newcall("@", e[i]) + addAllNode(seqType, seqCallNode) + else: + addAllNode(param, e[i]) + of nnkBracket, nnkObjConstr: + # passing by static parameters + # so we pass them directly instead of passing by scratchObj + callNode.add nnkExprEqExpr.newTree(formalParams[i][0], e[i]) + of nnkSym, nnkPtrTy: + addAllNode(param, e[i]) + of nnkCharLit..nnkNilLit: + callNode.add nnkExprEqExpr.newTree(formalParams[i][0], e[i]) + else: + error("not supported type kinds") + + let scratchObjType = genSym(kind = nskType, ident = "ScratchObj") + let scratchObj = nnkTypeSection.newTree( + nnkTypeDef.newTree( + scratchObjType, + newEmptyNode(), + nnkObjectTy.newTree( + newEmptyNode(), + newEmptyNode(), + scratchRecList + ) + ) + ) + + + let scratchObjPtrType = quote do: + cast[ptr `scratchObjType`](c_calloc(csize_t 1, csize_t sizeof(`scratchObjType`))) + + let scratchLetSection = newLetStmt( + scratchIdent, + scratchObjPtrType + ) + + let scratchCheck = quote do: + if `scratchIdent`.isNil: + raise newException(OutOfMemDefect, "Could not allocate memory") + + var stmtList = newStmtList() + stmtList.add(scratchObj) + stmtList.add(scratchLetSection) + stmtList.add(scratchCheck) + stmtList.add(nnkBlockStmt.newTree(newEmptyNode(), newStmtList(scratchAssignList))) + + var functionStmtList = newStmtList() + let funcCall = newCall(e[0], callNode) + functionStmtList.add tempAssignList + functionStmtList.add funcCall + + let funcName = genSym(nskProc, e[0].strVal) + let destroyName = genSym(nskProc, "destroyScratch") + let objTemp2 = genSym(ident = "obj") + let tempNode = quote("@") do: + `=destroy`(@objTemp2[]) + + result = quote do: + `stmtList` + + proc `funcName`(args: pointer) {.nimcall.} = + let `objTemp` = cast[ptr `scratchObjType`](args) + `functionStmtList` + + proc `destroyName`(args: pointer) {.nimcall.} = + let `objTemp2` = cast[ptr `scratchObjType`](args) + `tempNode` + + Task(callback: `funcName`, args: `scratchIdent`, destroy: `destroyName`) + else: + let funcCall = newCall(e[0]) + let funcName = genSym(nskProc, e[0].strVal) + + result = quote do: + proc `funcName`(args: pointer) {.nimcall.} = + `funcCall` + + Task(callback: `funcName`, args: nil) + + when defined(nimTasksDebug): + echo result.repr + +runnableExamples("--gc:orc"): + block: + var num = 0 + proc hello(a: int) = inc num, a + + let b = toTask hello(13) + b.invoke() + assert num == 13 + # A task can be invoked multiple times + b.invoke() + assert num == 26 + + block: + type + Runnable = ref object + data: int + + var data: int + proc hello(a: Runnable) {.nimcall.} = + a.data += 2 + data = a.data + + + when false: + # the parameters of call must be isolated. + let x = Runnable(data: 12) + let b = toTask hello(x) # error ----> expression cannot be isolated: x + b.invoke() + + let b = toTask(hello(Runnable(data: 12))) + b.invoke() + assert data == 14 + b.invoke() + assert data == 16 diff --git a/tests/stdlib/ttasks.nim b/tests/stdlib/ttasks.nim new file mode 100644 index 000000000..75fed9f9b --- /dev/null +++ b/tests/stdlib/ttasks.nim @@ -0,0 +1,506 @@ +discard """ + targets: "c cpp" + matrix: "--gc:orc" +""" + +import std/[tasks, strformat] + +block: + var s = "" + proc `+`(x: int, y: string) = + s.add $x & y + + let literal = "Nim" + let t = toTask(521 + literal) + t.invoke() + + doAssert s == "521Nim" + +block: + var s = "" + proc `!`(x: int) = + s.add $x + + let t = toTask !12 + t.invoke() + + doAssert s == "12" + + +block: + block: + var called = 0 + proc hello(x: static range[1 .. 5]) = + called += x + + let b = toTask hello(3) + b.invoke() + doAssert called == 3 + b.invoke() + doAssert called == 6 + + block: + var called = 0 + proc hello(x: range[1 .. 5]) = + called += x + + let b = toTask hello(3) + b.invoke() + doAssert called == 3 + b.invoke() + doAssert called == 6 + + block: + var called = 0 + proc hello(x: 1 .. 5) = + called += x + + let b = toTask hello(3) + b.invoke() + doAssert called == 3 + b.invoke() + doAssert called == 6 + + block: + var temp = "" + proc hello(a: int or seq[string]) = + when a is seq[string]: + for s in a: + temp.add s + else: + temp.addInt a + + let x = @["1", "2", "3", "4"] + let b = toTask hello(x) + b.invoke() + doAssert temp == "1234" + b.invoke() + doAssert temp == "12341234" + + + block: + var temp = "" + + proc hello(a: int or string) = + when a is string: + temp.add a + + let x = "!2" + + let b = toTask hello(x) + b.invoke() + doAssert temp == x + + block: + var temp = "" + proc hello(a: int or string) = + when a is string: + temp.add a + + let x = "!2" + let b = toTask hello(x) + b.invoke() + doAssert temp == x + + block: + var x = 0 + proc hello(typ: typedesc) = + x += typ(12) + + let b = toTask hello(int) + b.invoke() + doAssert x == 12 + + block: + var temp = "" + proc hello(a: int or seq[string]) = + when a is seq[string]: + for s in a: + temp.add s + + let x = @["1", "2", "3", "4"] + let b = toTask hello(x) + b.invoke() + doAssert temp == "1234" + + block: + var temp = "" + proc hello(a: int | string) = + when a is string: + temp.add a + + let x = "!2" + let b = toTask hello(x) + b.invoke() + doAssert temp == x + + block: + var x = 0 + proc hello(a: int | string) = + when a is int: + x = a + + let b = toTask hello(12) + b.invoke() + doAssert x == 12 + + block: + var a1: seq[int] + var a2 = 0 + proc hello(c: seq[int], a: int) = + a1 = c + a2 = a + + let x = 12 + var y = @[1, 3, 1, 4, 5, x, 1] + let b = toTask hello(y, 12) + b.invoke() + + doAssert a1 == y + doAssert a2 == x + + block: + var a1: seq[int] + var a2 = 0 + proc hello(c: seq[int], a: int) = + a1 = c + a2 = a + var x = 2 + let b = toTask hello(@[1, 3, 1, 4, 5, x, 1], 12) + b.invoke() + + doAssert a1 == @[1, 3, 1, 4, 5, x, 1] + doAssert a2 == 12 + + block: + var a1: array[7, int] + var a2 = 0 + proc hello(c: array[7, int], a: int) = + a1 = c + a2 = a + + let b = toTask hello([1, 3, 1, 4, 5, 2, 1], 12) + b.invoke() + + doAssert a1 == [1, 3, 1, 4, 5, 2, 1] + doAssert a2 == 12 + + block: + var a1: seq[int] + var a2 = 0 + proc hello(c: seq[int], a: int) = + a1 = c + a2 = a + + let b = toTask hello(@[1, 3, 1, 4, 5, 2, 1], 12) + b.invoke() + + doAssert a1 == @[1, 3, 1, 4, 5, 2, 1] + doAssert a2 == 12 + + block: + var a1: seq[int] + var a2 = 0 + proc hello(a: int, c: seq[int]) = + a1 = c + a2 = a + + let b = toTask hello(8, @[1, 3, 1, 4, 5, 2, 1]) + b.invoke() + + doAssert a1 == @[1, 3, 1, 4, 5, 2, 1] + doAssert a2 == 8 + + let c = toTask 8.hello(@[1, 3, 1, 4, 5, 2, 1]) + c.invoke() + + doAssert a1 == @[1, 3, 1, 4, 5, 2, 1] + doAssert a2 == 8 + + block: + var a1: seq[seq[int]] + var a2: int + proc hello(a: int, c: openArray[seq[int]]) = + a1 = @c + a2 = a + + let b = toTask hello(8, @[@[3], @[4], @[5], @[6], @[12], @[7]]) + b.invoke() + + doAssert a1 == @[@[3], @[4], @[5], @[6], @[12], @[7]] + doAssert a2 == 8 + + block: + var a1: seq[int] + var a2: int + proc hello(a: int, c: openArray[int]) = + a1 = @c + a2 = a + + let b = toTask hello(8, @[3, 4, 5, 6, 12, 7]) + b.invoke() + + doAssert a1 == @[3, 4, 5, 6, 12, 7] + doAssert a2 == 8 + + block: + var a1: seq[int] + var a2: int + proc hello(a: int, c: static varargs[int]) = + a1 = @c + a2 = a + + let b = toTask hello(8, @[3, 4, 5, 6, 12, 7]) + b.invoke() + + doAssert a1 == @[3, 4, 5, 6, 12, 7] + doAssert a2 == 8 + + block: + var a1: seq[int] + var a2: int + proc hello(a: int, c: static varargs[int]) = + a1 = @c + a2 = a + + let b = toTask hello(8, [3, 4, 5, 6, 12, 7]) + b.invoke() + + doAssert a1 == @[3, 4, 5, 6, 12, 7] + doAssert a2 == 8 + + block: + var a1: seq[int] + var a2: int + proc hello(a: int, c: varargs[int]) = + a1 = @c + a2 = a + + let x = 12 + let b = toTask hello(8, 3, 4, 5, 6, x, 7) + b.invoke() + + doAssert a1 == @[3, 4, 5, 6, 12, 7] + doAssert a2 == 8 + + block: + var x = 12 + + proc hello(x: ptr int) = + x[] += 12 + + let b = toTask hello(addr x) + b.invoke() + + doAssert x == 24 + + let c = toTask x.addr.hello + invoke(c) + + doAssert x == 36 + block: + type + Test = ref object + id: int + + var x = 0 + proc hello(a: int, c: static Test) = + x += a + x += c.id + + let b = toTask hello(8, Test(id: 12)) + b.invoke() + + doAssert x == 20 + + block: + type + Test = object + id: int + + var x = 0 + proc hello(a: int, c: static Test) = + x += a + x += c.id + + let b = toTask hello(8, Test(id: 12)) + b.invoke() + doAssert x == 20 + + block: + var x = 0 + proc hello(a: int, c: static seq[int]) = + x += a + for i in c: + x += i + + let b = toTask hello(8, @[3, 4, 5, 6, 12, 7]) + b.invoke() + doAssert x == 45 + + block: + var x = 0 + proc hello(a: int, c: static array[5, int]) = + x += a + for i in c: + x += i + + let b = toTask hello(8, [3, 4, 5, 6, 12]) + b.invoke() + doAssert x == 38 + + block: + var aVal = 0 + var cVal = "" + + proc hello(a: int, c: static string) = + aVal += a + cVal.add c + + var x = 1314 + let b = toTask hello(x, "hello") + b.invoke() + + doAssert aVal == x + doAssert cVal == "hello" + + block: + var aVal = "" + + proc hello(a: static string) = + aVal.add a + let b = toTask hello("hello") + b.invoke() + + doAssert aVal == "hello" + + block: + var aVal = 0 + var cVal = "" + + proc hello(a: static int, c: static string) = + aVal += a + cVal.add c + let b = toTask hello(8, "hello") + b.invoke() + + doAssert aVal == 8 + doAssert cVal == "hello" + + block: + var aVal = 0 + var cVal = 0 + + proc hello(a: static int, c: int) = + aVal += a + cVal += c + + let b = toTask hello(c = 0, a = 8) + b.invoke() + + doAssert aVal == 8 + doAssert cVal == 0 + + block: + var aVal = 0 + var cVal = 0 + + proc hello(a: int, c: static int) = + aVal += a + cVal += c + + let b = toTask hello(c = 0, a = 8) + b.invoke() + + doAssert aVal == 8 + doAssert cVal == 0 + + block: + var aVal = 0 + var cVal = 0 + + proc hello(a: static int, c: static int) = + aVal += a + cVal += c + + let b = toTask hello(0, 8) + b.invoke() + + doAssert aVal == 0 + doAssert cVal == 8 + + block: + var temp = "" + proc hello(x: int, y: seq[string], d = 134) = + temp = fmt"{x=} {y=} {d=}" + + + proc main() = + var x = @["23456"] + let t = toTask hello(2233, x) + t.invoke() + + doAssert temp == """x=2233 y=@["23456"] d=134""" + + main() + + + block: + var temp = "" + proc hello(x: int, y: seq[string], d = 134) = + temp.add fmt"{x=} {y=} {d=}" + + proc ok() = + temp = "ok" + + proc main() = + var x = @["23456"] + let t = toTask hello(2233, x) + t.invoke() + t.invoke() + + doAssert temp == """x=2233 y=@["23456"] d=134x=2233 y=@["23456"] d=134""" + + main() + + var x = @["4"] + let m = toTask hello(2233, x, 7) + m.invoke() + + doAssert temp == """x=2233 y=@["23456"] d=134x=2233 y=@["23456"] d=134x=2233 y=@["4"] d=7""" + + let n = toTask ok() + n.invoke() + + doAssert temp == "ok" + + block: + var called = 0 + block: + proc hello() = + inc called + + let a = toTask hello() + invoke(a) + + doAssert called == 1 + + block: + proc hello(a: int) = + inc called, a + + let b = toTask hello(13) + let c = toTask hello(a = 14) + b.invoke() + c.invoke() + + doAssert called == 28 + + block: + proc hello(a: int, c: int) = + inc called, a + + let b = toTask hello(c = 0, a = 8) + b.invoke() + + doAssert called == 36 From 86a1dcf92864f50fb1ab076cf58e19a3da81d301 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 9 Apr 2021 15:36:06 +0200 Subject: [PATCH 0610/2472] hashes: Made the runnableExample easier to understand (#17689) --- lib/pure/hashes.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/hashes.nim b/lib/pure/hashes.nim index c526c976f..3339adba2 100644 --- a/lib/pure/hashes.nim +++ b/lib/pure/hashes.nim @@ -21,7 +21,7 @@ runnableExamples: foo: int bar: string - iterator items(x: Something): int = + iterator items(x: Something): Hash = yield hash(x.foo) yield hash(x.bar) From 8aa5991beaa01f25a90d4115ed2495c92221f603 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Fri, 9 Apr 2021 16:29:10 +0200 Subject: [PATCH 0611/2472] Genode platform fixes (#17521) * Genode: move dyncall failures to runtime Do not use the "error" pragma to warn that dynamic library loading is not implemented, print a message at runtime and exit. * Genode: use stricter dataspace type in page allocator * Genode: remove compiler configuration from nim.cfg Self-hosting Nim is not supported on Genode and defining the cross-compilation environment can be done externally. * Genode: use new mutex API * Genode: call nim_component_construct as a C procedure * Genode: implement echo for NimStringV2 --- compiler/ccgexprs.nim | 14 ++++++++++---- compiler/cgen.nim | 2 +- config/nim.cfg | 25 +------------------------ lib/genode/alloc.nim | 14 +++++++------- lib/genode_cpp/syslocks.h | 20 +++++++++----------- lib/pure/dynlib.nim | 26 ++++++++++++++++++++++++++ lib/system/dyncalls.nim | 12 ++++++------ 7 files changed, 60 insertions(+), 53 deletions(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index ea09b3400..3872ed360 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1103,15 +1103,21 @@ proc genEcho(p: BProc, n: PNode) = # echo directly to the Genode LOG session var args: Rope = nil var a: TLoc - for it in n.sons: + for i, it in n.sons: if it.skipConv.kind == nkNilLit: args.add(", \"\"") - else: + elif n.len != 0: initLocExpr(p, it, a) - args.add(ropecg(p.module, ", Genode::Cstring($1->data, $1->len)", [rdLoc(a)])) + if i > 0: + args.add(", ") + case detectStrVersion(p.module) + of 2: + args.add(ropecg(p.module, "Genode::Cstring($1.p->data, $1.len)", [a.rdLoc])) + else: + args.add(ropecg(p.module, "Genode::Cstring($1->data, $1->len)", [a.rdLoc])) p.module.includeHeader("") p.module.includeHeader("") - linefmt(p, cpsStmts, """Genode::log(""$1);$n""", [args]) + linefmt(p, cpsStmts, """Genode::log($1);$n""", [args]) else: if n.len == 0: linefmt(p, cpsStmts, "#echoBinSafe(NIM_NIL, $1);$n", [n.len]) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 7dfab6a42..5d6de2a03 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1418,7 +1418,7 @@ proc genMainProc(m: BModule) = GenodeNimMain = "extern Genode::Env *nim_runtime_env;$N" & - "extern void nim_component_construct(Genode::Env*);$N$N" & + "extern \"C\" void nim_component_construct(Genode::Env*);$N$N" & NimMainBody ComponentConstruct = diff --git a/config/nim.cfg b/config/nim.cfg index 39e6c002b..d3ab75825 100644 --- a/config/nim.cfg +++ b/config/nim.cfg @@ -114,7 +114,7 @@ nimblepath="$home/.nimble/pkgs/" clang.options.linker = "-Wl,--as-needed -lnetwork" clang.cpp.options.linker = "-Wl,--as-needed -lnetwork" tcc.options.linker = "-Wl,--as-needed -lnetwork" - @else: + @elif not genode: # -fopenmp gcc.options.linker = "-ldl" gcc.cpp.options.linker = "-ldl" @@ -289,29 +289,6 @@ vcc.cpp.options.size = "/O1" # Configuration for the Tiny C Compiler: tcc.options.always = "-w" -# Configuration for the Genode toolchain -@if genode: - noCppExceptions # avoid std C++ - tlsEmulation:on # no TLS segment register magic - @if i386 or amd64: - gcc.exe = "genode-x86-gcc" - gcc.cpp.exe = "genode-x86-g++" - gcc.cpp.linkerexe = "genode-x86-ld" - @elif arm: - gcc.exe = "genode-arm-gcc" - gcc.cpp.exe = "genode-arm-g++" - gcc.cpp.linkerexe = "genode-arm-ld" - @elif arm64: - gcc.exe = "genode-aarch64-gcc" - gcc.cpp.exe = "genode-aarch64-g++" - gcc.cpp.linkerexe = "genode-aarch64-ld" - @elif riscv64: - gcc.exe = "genode-riscv-gcc" - gcc.cpp.exe = "genode-riscv-g++" - gcc.cpp.linkerexe = "genode-riscv-ld" - @end -@end - @if arm or arm64: --define:nimEmulateOverflowChecks @end diff --git a/lib/genode/alloc.nim b/lib/genode/alloc.nim index a21a3ad7b..3ddd3074b 100644 --- a/lib/genode/alloc.nim +++ b/lib/genode/alloc.nim @@ -17,18 +17,18 @@ when not defined(genode): when not declared(GenodeEnv): include genode/env -type DataspaceCapability {. - importcpp: "Genode::Dataspace_capability", pure.} = object +type RamDataspaceCapability {. + importcpp: "Genode::Ram_dataspace_capability", pure.} = object type Map = object attachment: pointer size: int - ds: DataspaceCapability + ds: RamDataspaceCapability SlabMeta = object next: ptr MapSlab - ds: DataspaceCapability + ds: RamDataspaceCapability MapSlab = object meta: SlabMeta @@ -45,11 +45,11 @@ proc capsAvail(env: GenodeEnv): int {. ## Return the number of available capabilities. ## Each dataspace allocation consumes a capability. -proc allocDataspace(env: GenodeEnv; size: int): DataspaceCapability {. +proc allocDataspace(env: GenodeEnv; size: int): RamDataspaceCapability {. importcpp: "#->pd().alloc(@)".} ## Allocate a dataspace and its capability. -proc attachDataspace(env: GenodeEnv; ds: DataspaceCapability): pointer {. +proc attachDataspace(env: GenodeEnv; ds: RamDataspaceCapability): pointer {. importcpp: "#->rm().attach(@)".} ## Attach a dataspace into the component address-space. @@ -57,7 +57,7 @@ proc detachAddress(env: GenodeEnv; p: pointer) {. importcpp: "#->rm().detach(@)".} ## Detach a dataspace from the component address-space. -proc freeDataspace(env: GenodeEnv; ds: DataspaceCapability) {. +proc freeDataspace(env: GenodeEnv; ds: RamDataspaceCapability) {. importcpp: "#->pd().free(@)".} ## Free a dataspace. diff --git a/lib/genode_cpp/syslocks.h b/lib/genode_cpp/syslocks.h index 8ba39abc2..c50180903 100644 --- a/lib/genode_cpp/syslocks.h +++ b/lib/genode_cpp/syslocks.h @@ -13,7 +13,7 @@ /* Genode includes */ #include -#include +#include namespace Nim { struct SysLock; @@ -22,15 +22,14 @@ namespace Nim { struct Nim::SysLock { - Genode::Lock _lock_a, _lock_b; + Genode::Mutex _mutex_a, _mutex_b; bool _locked; void acquireSys() { - _lock_a.lock(); + Genode::Mutex::Guard guard(_mutex_a); _locked = true; - _lock_a.unlock(); - _lock_b.lock(); + _mutex_b.acquire(); } bool tryAcquireSys() @@ -38,23 +37,22 @@ struct Nim::SysLock if (_locked) return false; - _lock_a.lock(); + Genode::Mutex::Guard guard(_mutex_a); + if (_locked) { - _lock_a.unlock(); return false; } else { _locked = true; - _lock_b.lock(); - _lock_a.unlock(); + _mutex_b.acquire(); return true; } } void releaseSys() { + Genode::Mutex::Guard guard(_mutex_a); _locked = false; - _lock_a.unlock(); - _lock_b.unlock(); + _mutex_b.release(); } }; diff --git a/lib/pure/dynlib.nim b/lib/pure/dynlib.nim index a1a94072c..3c1466322 100644 --- a/lib/pure/dynlib.nim +++ b/lib/pure/dynlib.nim @@ -150,6 +150,32 @@ elif defined(nintendoswitch): proc symAddr(lib: LibHandle, name: cstring): pointer = raise newException(OSError, "symAddr not implemented on Nintendo Switch!") +elif defined(genode): + # + # ========================================================================= + # Not implemented for Genode without POSIX. Raise an error if called. + # ========================================================================= + # + + template raiseErr(prc: string) = + raise newException(OSError, prc & " not implemented, compile with POSIX suport") + + proc dlclose(lib: LibHandle) = + raiseErr(OSError, "dlclose") + proc dlopen(path: cstring, mode: int): LibHandle = + raiseErr(OSError, "dlopen") + proc dlsym(lib: LibHandle, name: cstring): pointer = + raiseErr(OSError, "dlsym") + proc loadLib(path: string, global_symbols = false): LibHandle = + raiseErr(OSError, "loadLib") + proc loadLib(): LibHandle = + raiseErr(OSError, "loadLib") + proc unloadLib(lib: LibHandle) = + raiseErr(OSError, "unloadLib") + proc symAddr(lib: LibHandle, name: cstring): pointer = + raiseErr(OSError, "symAddr") + + elif defined(windows) or defined(dos): # # ======================================================================= diff --git a/lib/system/dyncalls.nim b/lib/system/dyncalls.nim index b0f326bb5..c14fcf31e 100644 --- a/lib/system/dyncalls.nim +++ b/lib/system/dyncalls.nim @@ -167,14 +167,14 @@ elif defined(windows) or defined(dos): elif defined(genode): - proc nimUnloadLibrary(lib: LibHandle) {. - error: "nimUnloadLibrary not implemented".} + proc nimUnloadLibrary(lib: LibHandle) = + raiseAssert("nimUnloadLibrary not implemented") - proc nimLoadLibrary(path: string): LibHandle {. - error: "nimLoadLibrary not implemented".} + proc nimLoadLibrary(path: string): LibHandle = + raiseAssert("nimLoadLibrary not implemented") - proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr {. - error: "nimGetProcAddr not implemented".} + proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr = + raiseAssert("nimGetProcAddr not implemented") elif defined(nintendoswitch) or defined(freertos): proc nimUnloadLibrary(lib: LibHandle) = From 1822ed384abdbbb5d385b1ad24e23e66936dbf74 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 9 Apr 2021 16:41:49 +0200 Subject: [PATCH 0612/2472] IC: added converter test case (#17688) * IC: added converter test case * IC: make converter example work --- compiler/importer.nim | 9 ++++++--- compiler/modulegraphs.nim | 4 ++++ tests/ic/mdefconverter.nim | 2 ++ tests/ic/tconverter.nim | 18 ++++++++++++++++++ 4 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 tests/ic/mdefconverter.nim create mode 100644 tests/ic/tconverter.nim diff --git a/compiler/importer.nim b/compiler/importer.nim index cb529795a..f8ee0d483 100644 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -163,14 +163,17 @@ proc addImport(c: PContext; im: sink ImportedModule) = c.imports.add im template addUnnamedIt(c: PContext, fromMod: PSym; filter: untyped) {.dirty.} = - for it in c.graph.ifaces[fromMod.position].converters: + for it in mitems c.graph.ifaces[fromMod.position].converters: if filter: + loadPackedSym(c.graph, it) addConverter(c, it) - for it in c.graph.ifaces[fromMod.position].patterns: + for it in mitems c.graph.ifaces[fromMod.position].patterns: if filter: + loadPackedSym(c.graph, it) addPattern(c, it) - for it in c.graph.ifaces[fromMod.position].pureEnums: + for it in mitems c.graph.ifaces[fromMod.position].pureEnums: if filter: + loadPackedSym(c.graph, it) importPureEnumFields(c, it.sym, it.sym.typ) proc importAllSymbolsExcept(c: PContext, fromMod: PSym, exceptSet: IntSet) = diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index 4430e3baa..02c745018 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -326,6 +326,10 @@ proc loadCompilerProc*(g: ModuleGraph; name: string): PSym = strTableAdd(g.compilerprocs, result) return result +proc loadPackedSym*(g: ModuleGraph; s: var LazySym) = + if s.sym == nil: + s.sym = loadSymFromId(g.config, g.cache, g.packed, s.id.module, s.id.packed) + proc `$`*(u: SigHash): string = toBase64a(cast[cstring](unsafeAddr u), sizeof(u)) diff --git a/tests/ic/mdefconverter.nim b/tests/ic/mdefconverter.nim new file mode 100644 index 000000000..d0a23f801 --- /dev/null +++ b/tests/ic/mdefconverter.nim @@ -0,0 +1,2 @@ + +converter toBool*(x: int): bool = x != 0 diff --git a/tests/ic/tconverter.nim b/tests/ic/tconverter.nim new file mode 100644 index 000000000..aecdf4b48 --- /dev/null +++ b/tests/ic/tconverter.nim @@ -0,0 +1,18 @@ +discard """ + output: "yes" +""" + +import mdefconverter + +echo "yes" + +#!EDIT!# + +discard """ + output: "converted int to bool" +""" + +import mdefconverter + +if 4: + echo "converted int to bool" From 08262206d35a2f171f09681a1abe47485bd79e88 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 10 Apr 2021 01:55:39 -0500 Subject: [PATCH 0613/2472] refs #15667 improve invalid indentation errors, report when & where `=` could be missing (#16397) * refs #15667 improve invalid indentation errors * also show line info where = is missing * add test * add more tests --- compiler/lexer.nim | 1 + compiler/parser.nim | 19 +++++++++++--- tests/parser/t15667.nim | 58 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 4 deletions(-) create mode 100644 tests/parser/t15667.nim diff --git a/compiler/lexer.nim b/compiler/lexer.nim index 7c7ac4177..59b983060 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -163,6 +163,7 @@ proc prettyTok*(tok: Token): string = else: $tok proc printTok*(conf: ConfigRef; tok: Token) = + # xxx factor with toLocation msgWriteln(conf, $tok.line & ":" & $tok.col & "\t" & $tok.tokType & " " & $tok) proc initToken*(L: var Token) = diff --git a/compiler/parser.nim b/compiler/parser.nim index d103d161a..fd9d06de5 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -44,6 +44,9 @@ type hasProgress: bool # some while loop requires progress ensurance lex*: Lexer # The lexer that is used for parsing tok*: Token # The current token + lineStartPrevious*: int + lineNumberPrevious*: int + bufposPrevious*: int inPragma*: int # Pragma level inSemiStmtList*: int emptyNode: PNode @@ -77,7 +80,7 @@ proc eat*(p: var Parser, tokType: TokType) proc skipInd*(p: var Parser) proc optPar*(p: var Parser) proc optInd*(p: var Parser, n: PNode) -proc indAndComment*(p: var Parser, n: PNode) +proc indAndComment*(p: var Parser, n: PNode, maybeMissEquals = false) proc setBaseFlags*(n: PNode, base: NumericalBase) proc parseSymbol*(p: var Parser, mode = smNormal): PNode proc parseTry(p: var Parser; isExpr: bool): PNode @@ -100,6 +103,9 @@ template prettySection(body) = proc getTok(p: var Parser) = ## Get the next token from the parser's lexer, and store it in the parser's ## `tok` member. + p.lineNumberPrevious = p.lex.lineNumber + p.lineStartPrevious = p.lex.lineStart + p.bufposPrevious = p.lex.bufpos rawGetTok(p.lex, p.tok) p.hasProgress = true when defined(nimpretty): @@ -223,9 +229,13 @@ proc parLineInfo(p: Parser): TLineInfo = ## Retrieve the line information associated with the parser's current state. result = getLineInfo(p.lex, p.tok) -proc indAndComment(p: var Parser, n: PNode) = +proc indAndComment(p: var Parser, n: PNode, maybeMissEquals = false) = if p.tok.indent > p.currInd: if p.tok.tokType == tkComment: rawSkipComment(p, n) + elif maybeMissEquals: + let col = p.bufposPrevious - p.lineStartPrevious + var info = newLineInfo(p.lex.fileIdx, p.lineNumberPrevious, col) + parMessage(p, "invalid indentation, maybe you forgot a '=' at $1 ?" % [p.lex.config$info]) else: parMessage(p, errInvalidIndentation) else: skipComment(p, n) @@ -1773,13 +1783,14 @@ proc parseRoutine(p: var Parser, kind: TNodeKind): PNode = else: result.add(p.emptyNode) # empty exception tracking: result.add(p.emptyNode) - if p.tok.tokType == tkEquals and p.validInd: + let maybeMissEquals = p.tok.tokType != tkEquals + if (not maybeMissEquals) and p.validInd: getTok(p) skipComment(p, result) result.add(parseStmt(p)) else: result.add(p.emptyNode) - indAndComment(p, result) + indAndComment(p, result, maybeMissEquals) proc newCommentStmt(p: var Parser): PNode = #| commentStmt = COMMENT diff --git a/tests/parser/t15667.nim b/tests/parser/t15667.nim new file mode 100644 index 000000000..d15178789 --- /dev/null +++ b/tests/parser/t15667.nim @@ -0,0 +1,58 @@ +discard """ + cmd: "nim check $options $file" + action: "reject" + nimout: ''' +t15667.nim(23, 5) Error: invalid indentation, maybe you forgot a '=' at t15667.nim(22, 13) ? +t15667.nim(28, 5) Error: invalid indentation, maybe you forgot a '=' at t15667.nim(26, 13) ? +t15667.nim(33, 5) Error: invalid indentation, maybe you forgot a '=' at t15667.nim(31, 25) ? +t15667.nim(42, 5) Error: invalid indentation, maybe you forgot a '=' at t15667.nim(38, 12) ? +t15667.nim(56, 5) Error: invalid indentation, maybe you forgot a '=' at t15667.nim(55, 13) ? +''' +""" + + + + + + + + +# line 20 +block: + proc fn1() + discard + +block: + proc fn2() + # + discard + +block: + proc fn3() {.exportc.} + # + discard + +block: # complex example + proc asdfasdfsd() {. exportc, + inline + .} # foo + #[ + bar + ]# + discard + +block: # xxx this doesn't work yet (only a bare `invalid indentation` error) + proc fn5() + ## + discard + +block: # ditto + proc fn6*() + ## foo bar + runnableExamples: discard + +block: + proc fn8() + runnableExamples: + discard + discard From 2150cd1826f11c74ce780dc0aaecedbed094230d Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Sat, 10 Apr 2021 10:49:01 +0300 Subject: [PATCH 0614/2472] restyle RST option lists (#17637) * WIP: restyle RST option lists * apply similar style to Latex * fix tests * minor visual tweaks * update tests * remove leftover comments --- config/nimdoc.tex.cfg | 11 ++++++- doc/nimdoc.css | 27 +++++++++++++++++ lib/packages/docutils/rst.nim | 2 ++ lib/packages/docutils/rstast.nim | 4 +-- lib/packages/docutils/rstgen.nim | 18 +++++++---- nimdoc/testproject/expected/nimdoc.out.css | 27 +++++++++++++++++ tests/stdlib/trstgen.nim | 35 ++++++++++++++-------- 7 files changed, 103 insertions(+), 21 deletions(-) diff --git a/config/nimdoc.tex.cfg b/config/nimdoc.tex.cfg index 307b280cc..69266f85d 100644 --- a/config/nimdoc.tex.cfg +++ b/config/nimdoc.tex.cfg @@ -50,7 +50,7 @@ doc.file = """ \usepackage{fancyvrb, courier} \usepackage{tabularx} \usepackage{hyperref} -\usepackage{enumitem} % for enumList and rstfootnote +\usepackage{enumitem} % for option list, enumList, and rstfootnote \usepackage{xcolor} \usepackage[tikz]{mdframed} @@ -77,6 +77,15 @@ bottomline=false} \newenvironment{rstpre}{\VerbatimEnvironment\begingroup\begin{Verbatim}[fontsize=\footnotesize , commandchars=\\\{\}]}{\end{Verbatim}\endgroup} \newenvironment{rstfootnote}{\begin{description}[labelindent=1em,leftmargin=1em,labelwidth=2.6em]}{\end{description}} +\ifdim\linewidth<30em + \def\rstoptleftmargin{0.4\linewidth} + \def\rstoptlabelwidth{0.35\linewidth} +\else + \def\rstoptleftmargin{12em} + \def\rstoptlabelwidth{10.5em} +\fi +\newenvironment{rstoptlist}{% +\begin{description}[font=\sffamily\bfseries,style=nextline,leftmargin=\rstoptleftmargin,labelwidth=\rstoptlabelwidth]}{\end{description}} % to pack tabularx into a new environment, special syntax is needed :-( \newenvironment{rsttab}[1]{\tabularx{\linewidth}{#1}}{\endtabularx} diff --git a/doc/nimdoc.css b/doc/nimdoc.css index db9a7ce97..ced791d16 100644 --- a/doc/nimdoc.css +++ b/doc/nimdoc.css @@ -511,6 +511,33 @@ div.footnote-label { min-width: 1.7em; } +div.option-list { + border: 0.1em solid var(--border); +} +div.option-list-item { + padding-left: 12em; + padding-right: 0; + padding-bottom: 0.3em; + padding-top: 0.3em; +} +div.odd { + background-color: var(--secondary-background); +} +div.option-list-label { + margin-left: -11.5em; + margin-right: 0em; + min-width: 11.5em; + font-weight: bolder; + display: inline-block; + vertical-align: top; +} +div.option-list-description { + width: calc(100% - 1em); + padding-left: 1em; + padding-right: 0; + display: inline-block; +} + blockquote { font-size: 0.9em; font-style: italic; diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index a8bc04a1a..c2385d517 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -2085,6 +2085,7 @@ proc parseBulletList(p: var RstParser): PRstNode = proc parseOptionList(p: var RstParser): PRstNode = result = newRstNodeA(p, rnOptionList) let col = currentTok(p).col + var order = 1 while true: if currentTok(p).col == col and isOptionList(p): var a = newRstNode(rnOptionGroup) @@ -2107,6 +2108,7 @@ proc parseOptionList(p: var RstParser): PRstNode = if currentTok(p).kind == tkIndent: inc p.idx c.add(a) c.add(b) + c.order = order; inc order result.add(c) else: if currentTok(p).kind != tkEof: dec p.idx # back to tkIndent diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim index dd456b577..394cc2698 100644 --- a/lib/packages/docutils/rstast.nim +++ b/lib/packages/docutils/rstast.nim @@ -90,7 +90,7 @@ type level*: int ## level of headings starting from 1 (main ## chapter) to larger ones (minor sub-sections) ## level=0 means it's document title or subtitle - of rnFootnote, rnCitation, rnFootnoteRef: + of rnFootnote, rnCitation, rnFootnoteRef, rnOptionListItem: order*: int ## footnote order (for auto-symbol footnotes and ## auto-numbered ones without a label) else: @@ -368,7 +368,7 @@ proc renderRstToStr*(node: PRstNode, indent=0): string = result.add txt of rnHeadline, rnOverline, rnMarkdownHeadline: result.add "\tlevel=" & $node.level - of rnFootnote, rnCitation, rnFootnoteRef: + of rnFootnote, rnCitation, rnFootnoteRef, rnOptionListItem: result.add (if node.order == 0: "" else: "\torder=" & $node.order) else: discard diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index f0a2604ff..1b9334a77 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -1165,7 +1165,7 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = renderAux(d, n, "$1\n", "\\begin{description}\n$2\n$1\\end{description}\n", result) of rnDefItem: renderAux(d, n, result) - of rnDefName: renderAux(d, n, "$1\n", "$2\\item[$1] ", result) + of rnDefName: renderAux(d, n, "$1\n", "$2\\item[$1]\\ ", result) of rnDefBody: renderAux(d, n, "$1\n", "$2\n$1\n", result) of rnFieldList: var tmp = "" @@ -1189,14 +1189,20 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = of rnIndex: renderRstToOut(d, n.sons[2], result) of rnOptionList: - renderAux(d, n, "$1", - "\\begin{description}\n$2\n$1\\end{description}\n", result) + renderAux(d, n, "$1
    ", + "\\begin{rstoptlist}$2\n$1\\end{rstoptlist}", result) of rnOptionListItem: - renderAux(d, n, "$1\n", "$1", result) + var addclass = if n.order mod 2 == 1: " odd" else: "" + renderAux(d, n, + "
    $1
    \n", + "$1", result) of rnOptionGroup: - renderAux(d, n, "$1", "\\item[$1]", result) + renderAux(d, n, + "
    $1
    ", + "\\item[$1]", result) of rnDescription: - renderAux(d, n, "$1\n", " $1\n", result) + renderAux(d, n, "
    $1
    ", + " $1\n", result) of rnOption, rnOptionString, rnOptionArgument: doAssert false, "renderRstToOut" of rnLiteralBlock: diff --git a/nimdoc/testproject/expected/nimdoc.out.css b/nimdoc/testproject/expected/nimdoc.out.css index db9a7ce97..ced791d16 100644 --- a/nimdoc/testproject/expected/nimdoc.out.css +++ b/nimdoc/testproject/expected/nimdoc.out.css @@ -511,6 +511,33 @@ div.footnote-label { min-width: 1.7em; } +div.option-list { + border: 0.1em solid var(--border); +} +div.option-list-item { + padding-left: 12em; + padding-right: 0; + padding-bottom: 0.3em; + padding-top: 0.3em; +} +div.odd { + background-color: var(--secondary-background); +} +div.option-list-label { + margin-left: -11.5em; + margin-right: 0em; + min-width: 11.5em; + font-weight: bolder; + display: inline-block; + vertical-align: top; +} +div.option-list-description { + width: calc(100% - 1em); + padding-left: 1em; + padding-right: 0; + display: inline-block; +} + blockquote { font-size: 0.9em; font-style: italic; diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index b403d96c6..ed5d72226 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -1255,7 +1255,7 @@ Test1 doAssert "
    ") == 2) - check(output.count("-mdesc""" in output) - check("""-nvery long desc""" in + check(output.count("
    -m
    """ & + """
    desc
    """ in + output) + check("""
    -n
    """ & + """
    very long desc
    """ in output) test "Option lists 2": @@ -1385,11 +1388,15 @@ Test1 -d option""" let output = input.toHtml check(output.count("-mdesc""" in output) - check("""-nvery long desc""" in + check output.count("
    -m
    """ & + """
    desc
    """ in output) - check("""-doption""" in + check("""
    -n
    """ & + """
    very long desc
    """ in + output) + check("""
    -d
    """ & + """
    option
    """ in output) check "

    option

    " notin output @@ -1402,11 +1409,15 @@ Test1 -d option""" let output = input.toHtml check(output.count("compilecompile1""" in output) - check("""docdoc1 cont""" in + check output.count("
    compile
    """ & + """
    compile1
    """ in output) - check("""-doption""" in + check("""
    doc
    """ & + """
    doc1 cont
    """ in + output) + check("""
    -d
    """ & + """
    option
    """ in output) check "

    option

    " notin output From 3aaec0647b6c924ea1da18e8427f13289ebab980 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Sun, 11 Apr 2021 11:23:08 +0300 Subject: [PATCH 0615/2472] turn on syntax highlighting in Manual & Tutorial (#17692) * turn on syntax highlighting in Manual & Tutorial * avoid highlighting of "method" * use relative path * 2 more changes --- doc/manual.rst | 192 ++++++++++++++++++------------------ doc/manual/var_t_return.rst | 1 + doc/rstcommon.rst | 6 ++ doc/sets_fragment.txt | 38 +++---- doc/tut1.rst | 34 +++---- doc/tut2.rst | 11 ++- doc/tut3.rst | 4 +- 7 files changed, 149 insertions(+), 137 deletions(-) create mode 100644 doc/rstcommon.rst diff --git a/doc/manual.rst b/doc/manual.rst index aad8034c8..1325d1c5f 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -1,5 +1,3 @@ -.. default-role:: code - ========== Nim Manual ========== @@ -7,6 +5,8 @@ Nim Manual :Authors: Andreas Rumpf, Zahary Karadjov :Version: |nimversion| +.. default-role:: code +.. include:: rstcommon.rst .. contents:: @@ -127,9 +127,9 @@ compiler may instead choose to allow the program to die with a fatal error. echo "invalid index" The current implementation allows to switch between these different behaviors -via `--panics:on|off`. When panics are turned on, the program dies with a +via ``--panics:on|off``. When panics are turned on, the program dies with a panic, if they are turned off the runtime errors are turned into -exceptions. The benefit of `--panics:on` is that it produces smaller binary +exceptions. The benefit of ``--panics:on`` is that it produces smaller binary code and the compiler has more freedom to optimize the code. An `unchecked runtime error`:idx: is an error that is not guaranteed to be @@ -490,7 +490,7 @@ Rationale: It enables the efficient support of `array[char, int]` or The `Rune` type can represent any Unicode character. `Rune` is declared in the `unicode module `_. -A character literal that does not end in ``'`` is interpreted as ``'`` if there +A character literal that does not end in `'` is interpreted as `'` if there is a preceeding backtick token. There must be no whitespace between the preceeding backtick token and the character literal. This special case ensures that a declaration like ``proc `'customLiteral`(s: string)`` is valid. ``proc `'customLiteral`(s: string)`` @@ -752,7 +752,7 @@ Otherwise, precedence is determined by the first character. Precedence level Operators First character Terminal symbol ================ ======================================================= ================== =============== 10 (highest) `$ ^` OP10 - 9 `* / div mod shl shr %` ``* % \ /`` OP9 + 9 `* / div mod shl shr %` `* % \ /` OP9 8 `+ -` `+ - ~ |` OP8 7 `&` `&` OP7 6 `..` `.` OP6 @@ -1008,7 +1008,7 @@ These integer types are pre-defined: literal that has no type suffix is of this type if it is in the range `low(int32)..high(int32)` otherwise the literal's type is `int64`. -intXX +`int`\ XX additional signed integer types of XX bits use this naming scheme (example: int16 is a 16-bit wide integer). The current implementation supports `int8`, `int16`, `int32`, `int64`. @@ -1019,7 +1019,7 @@ intXX has the same size as a pointer. An integer literal with the type suffix `'u` is of this type. -uintXX +`uint`\ XX additional unsigned integer types of XX bits use this naming scheme (example: uint16 is a 16-bit wide unsigned integer). The current implementation supports `uint8`, `uint16`, `uint32`, @@ -1039,22 +1039,22 @@ the `%` suffix as convention: ====================== ====================================================== operation meaning ====================== ====================================================== -`a +% b` unsigned integer addition -`a -% b` unsigned integer subtraction -`a *% b` unsigned integer multiplication -`a /% b` unsigned integer division -`a %% b` unsigned integer modulo operation -`a <% b` treat `a` and `b` as unsigned and compare -`a <=% b` treat `a` and `b` as unsigned and compare -`ze(a)` extends the bits of `a` with zeros until it has the +`a +% b` unsigned integer addition +`a -% b` unsigned integer subtraction +`a *% b` unsigned integer multiplication +`a /% b` unsigned integer division +`a %% b` unsigned integer modulo operation +`a <% b` treat `a` and `b` as unsigned and compare +`a <=% b` treat `a` and `b` as unsigned and compare +`ze(a)` extends the bits of `a` with zeros until it has the width of the `int` type -`toU8(a)` treats `a` as unsigned and converts it to an +`toU8(a)` treats `a` as unsigned and converts it to an unsigned integer of 8 bits (but still the `int8` type) -`toU16(a)` treats `a` as unsigned and converts it to an +`toU16(a)` treats `a` as unsigned and converts it to an unsigned integer of 16 bits (but still the `int16` type) -`toU32(a)` treats `a` as unsigned and converts it to an +`toU32(a)` treats `a` as unsigned and converts it to an unsigned integer of 32 bits (but still the `int32` type) ====================== ====================================================== @@ -1117,9 +1117,9 @@ The following floating-point types are pre-defined: but now it is always mapped to `float64`. This type should be used in general. -floatXX +`float`\ XX an implementation may define additional floating-point types of XX bits using - this naming scheme (example: float64 is a 64-bit wide float). The current + this naming scheme (example: `float64` is a 64-bit wide float). The current implementation supports `float32` and `float64`. Literals of these types have the suffix 'fXX. @@ -1459,7 +1459,7 @@ The notation `x[i]` can be used to access the i-th element of `x`. Arrays are always bounds checked (statically or at runtime). These checks can be disabled via pragmas or invoking the compiler with the -`--boundChecks:off` command-line switch. +``--boundChecks:off`` command-line switch. An array constructor can have explicit indexes for readability: @@ -1946,7 +1946,8 @@ memory manually: dealloc(d) Without the `reset` call the memory allocated for the `d.s` string would -never be freed. The example also demonstrates two important features for low-level programming: the `sizeof` proc returns the size of a type or value +never be freed. The example also demonstrates two important features for +low-level programming: the `sizeof` proc returns the size of a type or value in bytes. The `cast` operator can circumvent the type system: the compiler is forced to treat the result of the `alloc0` call (which returns an untyped pointer) as if it would have the type `ptr Data`. Casting should only be @@ -2045,7 +2046,7 @@ Nim supports these `calling conventions`:idx:\: C++ class member functions on the x86 architecture. `syscall`:idx: - The syscall convention is the same as `__syscall` in C. It is used for + The syscall convention is the same as `__syscall`:c: in C. It is used for interrupts. `noconv`:idx: @@ -3217,7 +3218,7 @@ Assembler statement The direct embedding of assembler code into Nim code is supported by the unsafe `asm` statement. Identifiers in the assembler code that refer to Nim identifiers shall be enclosed in a special character which can be -specified in the statement's pragmas. The default special character is ``'`'``: +specified in the statement's pragmas. The default special character is `'\`'`: .. code-block:: nim {.push stackTrace:off.} @@ -3261,7 +3262,7 @@ Instead of: Using statement --------------- -The using statement provides syntactic convenience in modules where +The `using` statement provides syntactic convenience in modules where the same parameter names and types are used over and over. Instead of: .. code-block:: nim @@ -3301,8 +3302,8 @@ that are explicitly typed is possible and requires a semicolon between them. If expression ------------- -An `if expression` is almost like an if statement, but it is an expression. -This feature is similar to `ternary operators` in other languages. +An `if` expression is almost like an if statement, but it is an expression. +This feature is similar to *ternary operators* in other languages. Example: .. code-block:: nim @@ -3314,12 +3315,12 @@ required. `Elif` parts are also allowed. When expression --------------- -Just like an `if expression`, but corresponding to the when statement. +Just like an `if` expression, but corresponding to the `when` statement. Case expression --------------- -The `case expression` is again very similar to the case statement: +The `case` expression is again very similar to the case statement: .. code-block:: nim var favoriteFood = case animal @@ -3337,7 +3338,7 @@ the last expression as the result value. Block expression ---------------- -A `block expression` is almost like a block statement, but it is an expression +A `block` expression is almost like a block statement, but it is an expression that uses the last expression under the block as the value. It is similar to the statement list expression, but the statement list expression does not open a new block scope. @@ -3434,7 +3435,7 @@ Type casts should not be confused with *type conversions,* as mentioned in the prior section. Unlike type conversions, a type cast cannot change the underlying bit pattern of the data being casted (aside from that the size of the target type may differ from the source type). Casting resembles *type punning* in other -languages or C++'s `reinterpret_cast` and `bit_cast` features. +languages or C++'s `reinterpret_cast`:cpp: and `bit_cast`:cpp: features. The addr operator ----------------- @@ -3562,7 +3563,7 @@ the operator's position within an expression.) There is no way to declare postfix operators: all postfix operators are built-in and handled by the grammar explicitly. -Any operator can be called like an ordinary proc with the '`opr`' +Any operator can be called like an ordinary proc with the \`opr\` notation. (Thus an operator can have more than two parameters): .. code-block:: nim @@ -3596,9 +3597,9 @@ current module: Method call syntax ------------------ -For object-oriented programming, the syntax `obj.method(args)` can be used -instead of `method(obj, args)`. The parentheses can be omitted if there are no -remaining arguments: `obj.len` (instead of `len(obj)`). +For object-oriented programming, the syntax `obj.methodName(args)` can be used +instead of `methodName(obj, args)`. The parentheses can be omitted if +there are no remaining arguments: `obj.len` (instead of `len(obj)`). This method call syntax is not restricted to objects, it can be used to supply any type of first argument for procedures: @@ -3813,7 +3814,7 @@ simplicity (they require specialized semantic checking):: Thus they act more like keywords than like ordinary identifiers; unlike a keyword however, a redefinition may `shadow`:idx: the definition in -the `system` module. From this list the following should not be written in dot +the system_ module. From this list the following should not be written in dot notation `x.f` since `x` cannot be type-checked before it gets passed to `f`:: @@ -4059,7 +4060,7 @@ Multi-methods -------------- **Note:** Starting from Nim 0.20, to use multi-methods one must explicitly pass -`--multimethods:on` when compiling. +``--multimethods:on`` when compiling. In a multi-method, all parameters that have an object type are used for the dispatching: @@ -4768,7 +4769,7 @@ And so is: The reason for this is that `DivByZeroDefect` inherits from `Defect` and -with `--panics:on` Defects become unrecoverable errors. +with ``--panics:on`` Defects become unrecoverable errors. (Since version 1.4 of the language.) @@ -5567,7 +5568,7 @@ However, this means that the method call syntax is not available for **Note**: The Nim compiler prior to version 1 was more lenient about this -requirement. Use the `--useVersion:0.19` switch for a transition period. +requirement. Use the ``--useVersion:0.19`` switch for a transition period. @@ -6022,7 +6023,7 @@ A module may gain access to symbols of another module by the `import`:idx: statement. `Recursive module dependencies`:idx: are allowed, but are slightly subtle. Only top-level symbols that are marked with an asterisk (`*`) are exported. A valid module name can only be a valid Nim identifier (and thus its -filename is `identifier.nim`). +filename is ``identifier.nim``). The algorithm for compiling modules is: @@ -6484,7 +6485,7 @@ asmNoStackFrame pragma A proc can be marked with the `asmNoStackFrame` pragma to tell the compiler it should not generate a stack frame for the proc. There are also no exit statements like `return result;` generated and the generated C function is -declared as `__declspec(naked)` or `__attribute__((naked))` (depending on +declared as `__declspec(naked)`:c: or `__attribute__((naked))`:c: (depending on the used C compiler). **Note**: This pragma should only be used by procs which consist solely of @@ -6879,7 +6880,7 @@ This pragma has no effect on the JS backend. Volatile pragma --------------- The `volatile` pragma is for variables only. It declares the variable as -`volatile`, whatever that means in C/C++ (its semantics are not well defined +`volatile`:c:, whatever that means in C/C++ (its semantics are not well defined in C/C++). **Note**: This pragma will not exist for the LLVM backend. @@ -6906,7 +6907,7 @@ Header pragma ------------- The `header` pragma is very similar to the `nodecl` pragma: It can be applied to almost any symbol and specifies that it should not be declared -and instead, the generated code should contain an `#include`: +and instead, the generated code should contain an `#include`:c:\: .. code-block:: Nim type @@ -6915,8 +6916,8 @@ and instead, the generated code should contain an `#include`: The `header` pragma always expects a string constant. The string constant contains the header file: As usual for C, a system header file is enclosed -in angle brackets: `<>`. If no angle brackets are given, Nim -encloses the header file in `""` in the generated C code. +in angle brackets: `<>`:c:. If no angle brackets are given, Nim +encloses the header file in `""`:c: in the generated C code. **Note**: This will not work for the LLVM backend. @@ -6924,7 +6925,7 @@ encloses the header file in `""` in the generated C code. IncompleteStruct pragma ----------------------- The `incompleteStruct` pragma tells the compiler to not use the -underlying C `struct` in a `sizeof` expression: +underlying C `struct`:c: in a `sizeof` expression: .. code-block:: Nim type @@ -6941,8 +6942,8 @@ with the project: {.compile: "myfile.cpp".} **Note**: Nim computes a SHA1 checksum and only recompiles the file if it -has changed. One can use the `-f` command-line option to force the recompilation -of the file. +has changed. One can use the ``-f`` command-line option to force +the recompilation of the file. Since 1.4 the `compile` pragma is also available with this syntax: @@ -6964,7 +6965,7 @@ The `link` pragma can be used to link an additional file with the project: PassC pragma ------------ The `passc` pragma can be used to pass additional parameters to the C -compiler like one would using the command-line switch `--passc`: +compiler like one would using the command-line switch ``--passc``: .. code-block:: Nim {.passc: "-Wall -Werror".} @@ -6992,7 +6993,7 @@ the pragma resides in: PassL pragma ------------ The `passL` pragma can be used to pass additional parameters to the linker -like one would be using the command-line switch `--passL`: +like one would be using the command-line switch ``--passL``: .. code-block:: Nim {.passL: "-lSDLmain -lSDL".} @@ -7029,7 +7030,7 @@ Example: embedsC() `nimbase.h` defines `NIM_EXTERNC` C macro that can be used for -`extern "C"` code to work with both `nim c` and `nim cpp`, e.g.: +`extern "C"`:cpp: code to work with both `nim c` and `nim cpp`, e.g.: .. code-block:: Nim proc foobar() {.importc:"$1".} @@ -7044,8 +7045,8 @@ Example: This usage is however deprecated. For a top-level emit statement, the section where in the generated C/C++ file -the code should be emitted can be influenced via the -prefixes `/*TYPESECTION*/` or `/*VARSECTION*/` or `/*INCLUDESECTION*/`: +the code should be emitted can be influenced via the prefixes +`/*TYPESECTION*/`:c: or `/*VARSECTION*/`:c: or `/*INCLUDESECTION*/`:c:\: .. code-block:: Nim {.emit: """/*TYPESECTION*/ @@ -7075,7 +7076,7 @@ Similar to the `importc pragma for C <#foreign-function-interface-importc-pragma>`_, the `importcpp` pragma can be used to import `C++`:idx: methods or C++ symbols in general. The generated code then uses the C++ method calling -syntax: `obj->method(arg)`. In combination with the `header` and `emit` +syntax: `obj->method(arg)`:cpp:. In combination with the `header` and `emit` pragmas this allows *sloppy* interfacing with libraries written in C++: .. code-block:: Nim @@ -7105,16 +7106,16 @@ pragmas this allows *sloppy* interfacing with libraries written in C++: proc run(device: IrrlichtDevice): bool {. header: irr, importcpp: "#.run(@)".} -The compiler needs to be told to generate C++ (command `cpp`) for +The compiler needs to be told to generate C++ (command ``cpp``) for this to work. The conditional symbol `cpp` is defined when the compiler emits C++ code. Namespaces ~~~~~~~~~~ -The *sloppy interfacing* example uses `.emit` to produce `using namespace` +The *sloppy interfacing* example uses `.emit` to produce `using namespace`:cpp: declarations. It is usually much better to instead refer to the imported name -via the `namespace::identifier` notation: +via the `namespace::identifier`:cpp: notation: .. code-block:: nim type @@ -7126,7 +7127,8 @@ Importcpp for enums ~~~~~~~~~~~~~~~~~~~ When `importcpp` is applied to an enum type the numerical enum values are -annotated with the C++ enum type, like in this example: `((TheCppEnum)(3))`. +annotated with the C++ enum type, like in this example: +`((TheCppEnum)(3))`:cpp:. (This turned out to be the simplest way to implement it.) @@ -7136,10 +7138,11 @@ Importcpp for procs Note that the `importcpp` variant for procs uses a somewhat cryptic pattern language for maximum flexibility: -- A hash `#` symbol is replaced by the first or next argument. -- A dot following the hash `#.` indicates that the call should use C++'s dot +- A hash ``#`` symbol is replaced by the first or next argument. +- A dot following the hash ``#.`` indicates that the call should use C++'s dot or arrow notation. -- An at symbol `@` is replaced by the remaining arguments, separated by commas. +- An at symbol ``@`` is replaced by the remaining arguments, + separated by commas. For example: @@ -7155,7 +7158,7 @@ Produces: As a special rule to keep backward compatibility with older versions of the `importcpp` pragma, if there is no special pattern -character (any of `# ' @`) at all, C++'s +character (any of ``# ' @``) at all, C++'s dot or arrow notation is assumed, so the above example can also be written as: .. code-block:: nim @@ -7169,11 +7172,11 @@ capabilities: proc dictLookup(a: Dict, k: Key): Value {.importcpp: "#[#]".} -- An apostrophe `'` followed by an integer `i` in the range 0..9 +- An apostrophe ``'`` followed by an integer ``i`` in the range 0..9 is replaced by the i'th parameter *type*. The 0th position is the result type. This can be used to pass types to C++ function templates. Between - the `'` and the digit, an asterisk can be used to get to the base type - of the type. (So it "takes away a star" from the type; `T*` becomes `T`.) + the ``'`` and the digit, an asterisk can be used to get to the base type + of the type. (So it "takes away a star" from the type; `T*`:c: becomes `T`.) Two stars can be used to get to the element type of the element type etc. For example: @@ -7191,12 +7194,12 @@ Produces: x = SystemManager::getSubsystem() -- `#@` is a special case to support a `cnew` operation. It is required so +- ``#@`` is a special case to support a `cnew` operation. It is required so that the call expression is inlined directly, without going through a temporary location. This is only required to circumvent a limitation of the current code generator. -For example C++'s `new` operator can be "imported" like this: +For example C++'s `new`:cpp: operator can be "imported" like this: .. code-block:: nim proc cnew*[T](x: T): ptr T {.importcpp: "(new '*0#@)", nodecl.} @@ -7211,7 +7214,7 @@ Produces: .. code-block:: C x = new Foo(3, 4) -However, depending on the use case `new Foo` can also be wrapped like this +However, depending on the use case `new Foo`:cpp: can also be wrapped like this instead: .. code-block:: nim @@ -7224,7 +7227,8 @@ Wrapping constructors ~~~~~~~~~~~~~~~~~~~~~ Sometimes a C++ class has a private copy constructor and so code like -`Class c = Class(1,2);` must not be generated but instead `Class c(1,2);`. +`Class c = Class(1,2);`:cpp: must not be generated but instead +`Class c(1,2);`:cpp:. For this purpose the Nim proc that wraps a C++ constructor needs to be annotated with the `constructor`:idx: pragma. This pragma also helps to generate faster C++ code since construction then doesn't invoke the copy constructor: @@ -7298,7 +7302,7 @@ ImportJs pragma Similar to the `importcpp pragma for C++ <#implementation-specific-pragmas-importcpp-pragma>`_, the `importjs` pragma can be used to import Javascript methods or symbols in general. The generated code then uses the Javascript method -calling syntax: `obj.method(arg)`. +calling syntax: ``obj.method(arg)``. ImportObjC pragma @@ -7306,7 +7310,7 @@ ImportObjC pragma Similar to the `importc pragma for C <#foreign-function-interface-importc-pragma>`_, the `importobjc` pragma can be used to import `Objective C`:idx: methods. The generated code then uses the -Objective C method calling syntax: `[obj method param1: arg]`. +Objective C method calling syntax: ``[obj method param1: arg]``. In addition with the `header` and `emit` pragmas this allows *sloppy* interfacing with libraries written in Objective C: @@ -7346,8 +7350,8 @@ allows *sloppy* interfacing with libraries written in Objective C: g.greet(12, 34) g.free() -The compiler needs to be told to generate Objective C (command `objc`) for -this to work. The conditional symbol `objc` is defined when the compiler +The compiler needs to be told to generate Objective C (command ``objc``) for +this to work. The conditional symbol ``objc`` is defined when the compiler emits Objective C code. @@ -7392,7 +7396,7 @@ will generate this code: The `.cppNonPod` pragma should be used for non-POD `importcpp` types so that they work properly (in particular regarding constructor and destructor) for -`.threadvar` variables. This requires `--tlsEmulation:off`. +`.threadvar` variables. This requires ``--tlsEmulation:off``. .. code-block:: nim type Foo {.cppNonPod, importcpp, header: "funs.h".} = object @@ -7436,12 +7440,12 @@ pragma description :: nim c -d:FooBar=42 foobar.nim -In the above example, providing the -d flag causes the symbol +In the above example, providing the ``-d`` flag causes the symbol `FooBar` to be overwritten at compile-time, printing out 42. If the -`-d:FooBar=42` were to be omitted, the default value of 5 would be +``-d:FooBar=42`` were to be omitted, the default value of 5 would be used. To see if a value was provided, `defined(FooBar)` can be used. -The syntax `-d:flag` is actually just a shortcut for `-d:flag=true`. +The syntax ``-d:flag`` is actually just a shortcut for ``-d:flag=true``. User-defined pragmas ==================== @@ -7604,7 +7608,7 @@ spelled*: proc printf(formatstr: cstring) {.header: "", importc: "printf", varargs.} When `importc` is applied to a `let` statement it can omit its value which -will then be expected to come from C. This can be used to import a C `const`: +will then be expected to come from C. This can be used to import a C `const`:c:\: .. code-block:: {.emit: "const int cconst = 42;".} @@ -7625,8 +7629,8 @@ is not set to C, other pragmas are available: .. code-block:: Nim proc p(s: cstring) {.importc: "prefix$1".} -In the example, the external name of `p` is set to `prefixp`. Only `$1` -is available and a literal dollar sign must be written as `$$`. +In the example, the external name of `p` is set to `prefixp`. Only ``$1`` +is available and a literal dollar sign must be written as ``$$``. Exportc pragma @@ -7648,8 +7652,8 @@ The string literal passed to `exportc` can be a format string: proc p(s: string) {.exportc: "prefix$1".} = echo s -In the example, the external name of `p` is set to `prefixp`. Only `$1` -is available and a literal dollar sign must be written as `$$`. +In the example, the external name of `p` is set to `prefixp`. Only ``$1`` +is available and a literal dollar sign must be written as ``$$``. If the symbol should also be exported to a dynamic library, the `dynlib` pragma should be used in addition to the `exportc` pragma. See @@ -7665,8 +7669,8 @@ mangling. The string literal passed to `extern` can be a format string: proc p(s: string) {.extern: "prefix$1".} = echo s -In the example, the external name of `p` is set to `prefixp`. Only `$1` -is available and a literal dollar sign must be written as `$$`. +In the example, the external name of `p` is set to `prefixp`. Only ``$1`` +is available and a literal dollar sign must be written as ``$$``. Bycopy pragma @@ -7704,8 +7708,8 @@ strings automatically: Union pragma ------------ The `union` pragma can be applied to any `object` type. It means all -of the object's fields are overlaid in memory. This produces a `union` -instead of a `struct` in the generated C/C++ code. The object declaration +of the object's fields are overlaid in memory. This produces a `union`:c: +instead of a `struct`:c: in the generated C/C++ code. The object declaration then must not use inheritance or any GC'ed memory but this is currently not checked. @@ -7728,7 +7732,7 @@ a static error. Usage with inheritance should be defined and documented. Dynlib pragma for import ------------------------ With the `dynlib` pragma, a procedure or a variable can be imported from -a dynamic library (`.dll` files for Windows, `lib*.so` files for UNIX). +a dynamic library (``.dll`` files for Windows, ``lib*.so`` files for UNIX). The non-optional argument has to be the name of the dynamic library: .. code-block:: Nim @@ -7771,14 +7775,14 @@ string expressions in general: proc myImport(s: cstring) {.cdecl, importc, dynlib: getDllName().} -**Note**: Patterns like `libtcl(|8.5|8.4).so` are only supported in constant +**Note**: Patterns like ``libtcl(|8.5|8.4).so`` are only supported in constant strings, because they are precompiled. **Note**: Passing variables to the `dynlib` pragma will fail at runtime because of order of initialization problems. **Note**: A `dynlib` import can be overridden with -the `--dynlibOverride:name` command-line option. The +the ``--dynlibOverride:name`` command-line option. The `Compiler User Guide `_ contains further information. @@ -7793,15 +7797,15 @@ conjunction with the `exportc` pragma: proc exportme(): int {.cdecl, exportc, dynlib.} This is only useful if the program is compiled as a dynamic library via the -`--app:lib` command-line option. +``--app:lib`` command-line option. Threads ======= -To enable thread support the `--threads:on` command-line switch needs to -be used. The `system` module then contains several threading primitives. +To enable thread support the ``--threads:on`` command-line switch needs to +be used. The system_ module then contains several threading primitives. See the `threads `_ and `channels `_ modules for the low-level thread API. There are also high-level parallelism constructs available. See `spawn `_ for @@ -7842,7 +7846,7 @@ any of its parameters contain a `ref` or `closure` type. This enforces the *no heap sharing restriction*. Routines that are imported from C are always assumed to be `gcsafe`. -To disable the GC-safety checking the `--threadAnalysis:off` command-line +To disable the GC-safety checking the ``--threadAnalysis:off`` command-line switch can be used. This is a temporary workaround to ease the porting effort from old code to the new threading model. diff --git a/doc/manual/var_t_return.rst b/doc/manual/var_t_return.rst index e34993e3e..f5c5bc4c0 100644 --- a/doc/manual/var_t_return.rst +++ b/doc/manual/var_t_return.rst @@ -1,4 +1,5 @@ .. default-role:: code +.. include:: ../rstcommon.rst Memory safety for returning by `var T` is ensured by a simple borrowing rule: If `result` does not refer to a location pointing to the heap diff --git a/doc/rstcommon.rst b/doc/rstcommon.rst new file mode 100644 index 000000000..c1503fa96 --- /dev/null +++ b/doc/rstcommon.rst @@ -0,0 +1,6 @@ + +.. role:: nim(code) + :language: nim + +.. default-role:: nim + diff --git a/doc/sets_fragment.txt b/doc/sets_fragment.txt index 8436190a0..84e2fe8c1 100644 --- a/doc/sets_fragment.txt +++ b/doc/sets_fragment.txt @@ -1,13 +1,13 @@ The set type models the mathematical notion of a set. The set's basetype can only be an ordinal type of a certain size, namely: -* ``int8``-``int16`` -* ``uint8``/``byte``-``uint16`` -* ``char`` -* ``enum`` +* `int8`-`int16` +* `uint8`/`byte`-`uint16` +* `char` +* `enum` or equivalent. For signed integers the set's base type is defined to be in the -range ``0 .. MaxSetElements-1`` where ``MaxSetElements`` is currently always +range `0 .. MaxSetElements-1` where `MaxSetElements` is currently always 2^16. The reason is that sets are implemented as high performance bit vectors. @@ -17,7 +17,7 @@ Attempting to declare a set with a larger type will result in an error: var s: set[int64] # Error: set is too large -Sets can be constructed via the set constructor: ``{}`` is the empty set. The +Sets can be constructed via the set constructor: `{}` is the empty set. The empty set is type compatible with any concrete set type. The constructor can also be used to include elements (and ranges of elements): @@ -35,18 +35,18 @@ These operations are supported by sets: ================== ======================================================== operation meaning ================== ======================================================== -``A + B`` union of two sets -``A * B`` intersection of two sets -``A - B`` difference of two sets (A without B's elements) -``A == B`` set equality -``A <= B`` subset relation (A is subset of B or equal to B) -``A < B`` strict subset relation (A is a proper subset of B) -``e in A`` set membership (A contains element e) -``e notin A`` A does not contain element e -``contains(A, e)`` A contains element e -``card(A)`` the cardinality of A (number of elements in A) -``incl(A, elem)`` same as ``A = A + {elem}`` -``excl(A, elem)`` same as ``A = A - {elem}`` +`A + B` union of two sets +`A * B` intersection of two sets +`A - B` difference of two sets (A without B's elements) +`A == B` set equality +`A <= B` subset relation (A is subset of B or equal to B) +`A < B` strict subset relation (A is a proper subset of B) +`e in A` set membership (A contains element e) +`e notin A` A does not contain element e +`contains(A, e)` A contains element e +`card(A)` the cardinality of A (number of elements in A) +`incl(A, elem)` same as `A = A + {elem}` +`excl(A, elem)` same as `A = A - {elem}` ================== ======================================================== Bit fields @@ -54,7 +54,7 @@ Bit fields Sets are often used to define a type for the *flags* of a procedure. This is a cleaner (and type safe) solution than defining integer -constants that have to be ``or``'ed together. +constants that have to be `or`'ed together. Enum, sets and casting can be used together as in: diff --git a/doc/tut1.rst b/doc/tut1.rst index cca4e8c0e..8d80a0520 100644 --- a/doc/tut1.rst +++ b/doc/tut1.rst @@ -1,5 +1,3 @@ -.. default-role:: code - ===================== Nim Tutorial (Part I) ===================== @@ -7,6 +5,8 @@ Nim Tutorial (Part I) :Author: Andreas Rumpf :Version: |nimversion| +.. default-role:: code +.. include:: rstcommon.rst .. contents:: Introduction @@ -48,7 +48,7 @@ Save this code to the file "greetings.nim". Now compile and run it:: nim compile --run greetings.nim -With the `--run` `switch `_ Nim +With the ``--run`` `switch `_ Nim executes the file automatically after compilation. You can give your program command-line arguments by appending them after the filename:: @@ -63,12 +63,12 @@ To compile a release version use:: nim c -d:release greetings.nim By default, the Nim compiler generates a large number of runtime checks -aiming for your debugging pleasure. With `-d:release` some checks are +aiming for your debugging pleasure. With ``-d:release`` some checks are `turned off and optimizations are turned on `_. -For benchmarking or production code, use the `-d:release` switch. -For comparing the performance with unsafe languages like C, use the `-d:danger` switch +For benchmarking or production code, use the ``-d:release`` switch. +For comparing the performance with unsafe languages like C, use the ``-d:danger`` switch in order to get meaningful, comparable results. Otherwise Nim might be handicapped by checks that are **not even available** for C. @@ -535,7 +535,7 @@ differences: that belong to the first condition that evaluates to `true`. The `when` statement is useful for writing platform-specific code, similar to -the `#ifdef` construct in the C programming language. +the `#ifdef`:c: construct in the C programming language. Statements and indentation @@ -596,7 +596,7 @@ Procedures To define new commands like `echo `_ and `readLine `_ in the examples, the concept of a -`procedure` is needed. (Some languages call them *methods* or *functions*.) +*procedure* is needed. (Some languages call them *methods* or *functions*.) In Nim new procedures are defined with the `proc` keyword: .. code-block:: nim @@ -802,7 +802,7 @@ Operators --------- The Nim library makes heavy use of overloading - one reason for this is that each operator like `+` is just an overloaded proc. The parser lets you -use operators in `infix notation` (`a + b`) or `prefix notation` (`+ a`). +use operators in *infix notation* (`a + b`) or *prefix notation* (`+ a`). An infix operator always receives two arguments, a prefix operator always one. (Postfix operators are not possible, because this would be ambiguous: does `a @ @ b` mean `(a) @ (@b)` or `(a@) @ (b)`? It always means @@ -810,7 +810,7 @@ An infix operator always receives two arguments, a prefix operator always one. Apart from a few built-in keyword operators such as `and`, `or`, `not`, operators always consist of these characters: -``+ - * \ / < > = @ $ ~ & % ! ? ^ . |`` +`+ - * \ / < > = @ $ ~ & % ! ? ^ . |` User-defined operators are allowed. Nothing stops you from defining your own `@!?+~` operator, but doing so may reduce readability. @@ -952,7 +952,7 @@ evaluation. For example: Characters ---------- -The `character type` is called `char`. Its size is always one byte, so +The *character type* is called `char`. Its size is always one byte, so it cannot represent most UTF-8 characters, but it *can* represent one of the bytes that makes up a multi-byte UTF-8 character. The reason for this is efficiency: for the overwhelming majority of use-cases, @@ -1220,7 +1220,7 @@ Arrays can be constructed using `[]`: The notation `x[i]` is used to access the i-th element of `x`. Array access is always bounds checked (at compile-time or at runtime). These checks can be disabled via pragmas or invoking the compiler with the -`--bound_checks:off` command line switch. +``--bound_checks:off`` command line switch. Arrays are value types, like any other Nim type. The assignment operator copies the whole array contents. @@ -1620,11 +1620,11 @@ variables! For example: path = "usr/local/nimc.html" (dir, name, ext) = splitFile(path) baddir, badname, badext = splitFile(path) - echo dir # outputs `usr/local` - echo name # outputs `nimc` - echo ext # outputs `.html` + echo dir # outputs "usr/local" + echo name # outputs "nimc" + echo ext # outputs ".html" # All the following output the same line: - # `(dir: usr/local, name: nimc, ext: .html)` + # "(dir: usr/local, name: nimc, ext: .html)" echo baddir echo badname echo badext @@ -1711,7 +1711,7 @@ Examples are provided in the `manual `_. Modules ======= -Nim supports splitting a program into pieces with a module concept. +Nim supports splitting a program into pieces with a *module* concept. Each module is in its own file. Modules enable `information hiding`:idx: and `separate compilation`:idx:. A module may gain access to the symbols of another module by using the `import`:idx: statement. Only top-level symbols that are marked diff --git a/doc/tut2.rst b/doc/tut2.rst index 3aef6bb0f..2624b2690 100644 --- a/doc/tut2.rst +++ b/doc/tut2.rst @@ -1,5 +1,3 @@ -.. default-role:: code - ====================== Nim Tutorial (Part II) ====================== @@ -7,6 +5,8 @@ Nim Tutorial (Part II) :Author: Andreas Rumpf :Version: |nimversion| +.. default-role:: code +.. include:: rstcommon.rst .. contents:: @@ -175,7 +175,8 @@ Method call syntax ------------------ There is a syntactic sugar for calling routines: -The syntax `obj.method(args)` can be used instead of `method(obj, args)`. +The syntax `obj.methodName(args)` can be used +instead of `methodName(obj, args)`. If there are no remaining arguments, the parentheses can be omitted: `obj.len` (instead of `len(obj)`). @@ -296,7 +297,7 @@ because it makes more sense for them to use static binding, but `eval` is a method because it requires dynamic binding. **Note:** Starting from Nim 0.20, to use multi-methods one must explicitly pass -`--multimethods:on` when compiling. +``--multimethods:on`` when compiling. In a multi-method all parameters that have an object type are used for the dispatching: @@ -459,7 +460,7 @@ If you want to add the `{.raises.}` pragma to existing code, the compiler can also help you. You can add the `{.effects.}` pragma statement to your proc and the compiler will output all inferred effects up to that point (exception tracking is part of Nim's effect system). Another more roundabout way to -find out the list of exceptions raised by a proc is to use the Nim `doc` +find out the list of exceptions raised by a proc is to use the Nim ``doc`` command which generates documentation for a whole module and decorates all procs with the list of raised exceptions. You can read more about Nim's `effect system and related pragmas in the manual `_. diff --git a/doc/tut3.rst b/doc/tut3.rst index 358a9b45e..6b5a3b5ca 100644 --- a/doc/tut3.rst +++ b/doc/tut3.rst @@ -1,5 +1,3 @@ -.. default-role:: code - ======================= Nim Tutorial (Part III) ======================= @@ -7,6 +5,8 @@ Nim Tutorial (Part III) :Author: Arne Döring :Version: |nimversion| +.. default-role:: code +.. include:: rstcommon.rst .. contents:: From a5b30c94c2dcd0f17b37685d9ce96eee959ba554 Mon Sep 17 00:00:00 2001 From: shirleyquirk <31934565+shirleyquirk@users.noreply.github.com> Date: Sun, 11 Apr 2021 12:07:23 +0100 Subject: [PATCH 0616/2472] [feature] add arbitrary code execution to strformat (#17694) * changed parser to ignore ':' within parens * Update strformat.nim * Update lib/pure/strformat.nim Co-authored-by: flywind Co-authored-by: Andreas Rumpf --- lib/pure/strformat.nim | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim index 7ab359038..c232c8a46 100644 --- a/lib/pure/strformat.nim +++ b/lib/pure/strformat.nim @@ -74,6 +74,20 @@ runnableExamples: assert fmt"{123.456:13e}" == " 1.234560e+02" ##[ +# Expressions +]## +runnableExamples: + let x = 3.14 + assert fmt"{(if x!=0: 1.0/x else: 0):.5}" == "0.31847" + assert fmt"""{(block: + var res: string + for i in 1..15: + res.add (if i mod 15 == 0: "FizzBuzz" + elif i mod 5 == 0: "Buzz" + elif i mod 3 == 0: "Fizz" + else: $i) & " " + res)}""" == "1 2 Fizz 4 Buzz Fizz 7 8 Fizz Buzz 11 Fizz 13 14 FizzBuzz " +##[ # Debugging strings `fmt"{expr=}"` expands to `fmt"expr={expr}"` namely the text of the expression, @@ -557,7 +571,7 @@ proc strformatImpl(pattern: NimNode; openChar, closeChar: char): NimNode = # XXX: https://github.com/nim-lang/Nim/issues/8405 # When compiling with -d:useNimRtl, certain procs such as `count` from the strutils # module are not accessible at compile-time: - let expectedGrowth = when defined(useNimRtl): 0 else: count(f, '{') * 10 + let expectedGrowth = when defined(useNimRtl): 0 else: count(f, openChar) * 10 result.add newVarStmt(res, newCall(bindSym"newStringOfCap", newLit(f.len + expectedGrowth))) var strlit = "" @@ -573,8 +587,12 @@ proc strformatImpl(pattern: NimNode; openChar, closeChar: char): NimNode = strlit = "" var subexpr = "" - while i < f.len and f[i] != closeChar and f[i] != ':': - if f[i] == '=': + var inParens = 0 + while i < f.len and f[i] != closeChar and (f[i] != ':' or inParens!=0): + case f[i] + of '(': inc inParens + of ')': dec inParens + of '=': let start = i inc i i += f.skipWhitespace(i) @@ -582,10 +600,11 @@ proc strformatImpl(pattern: NimNode; openChar, closeChar: char): NimNode = result.add newCall(bindSym"add", res, newLit(subexpr & f[start ..< i])) else: subexpr.add f[start ..< i] - else: - subexpr.add f[i] - inc i - + continue + else: discard + subexpr.add f[i] + inc i + var x: NimNode try: x = parseExpr(subexpr) @@ -608,11 +627,11 @@ proc strformatImpl(pattern: NimNode; openChar, closeChar: char): NimNode = doAssert false, "invalid format string: missing '}'" result.add newCall(formatSym, res, x, newLit(options)) elif f[i] == closeChar: - if f[i+1] == closeChar: + if i Date: Sun, 11 Apr 2021 07:25:41 -0500 Subject: [PATCH 0617/2472] iterable[T] (#17196) * fix failing test toSeq in manual which now works * changelog * reject proc fn(a: iterable) * add iterable to spec * remove MCS/UFCS limitation that now works --- changelog.md | 5 + compiler/ast.nim | 6 +- compiler/condsyms.nim | 1 + compiler/jsgen.nim | 2 +- compiler/liftdestructors.nim | 2 +- compiler/linter.nim | 2 +- compiler/semcall.nim | 10 +- compiler/semdata.nim | 2 +- compiler/semexprs.nim | 9 +- compiler/semtypes.nim | 13 +++ compiler/sigmatch.nim | 31 ++++-- compiler/typeallowed.nim | 3 + compiler/types.nim | 11 ++- compiler/vmdeps.nim | 1 + doc/manual.rst | 43 +++++---- lib/system.nim | 4 + testament/lib/stdtest/testutils.nim | 6 ++ tests/types/titerable.nim | 143 ++++++++++++++++++++++++++++ 18 files changed, 254 insertions(+), 40 deletions(-) create mode 100644 tests/types/titerable.nim diff --git a/changelog.md b/changelog.md index a2879833f..2f1d2d816 100644 --- a/changelog.md +++ b/changelog.md @@ -302,6 +302,10 @@ - `nim e` now accepts arbitrary file extensions for the nimscript file, although `.nims` is still the preferred extension in general. +- Added `iterable[T]` type class to match called iterators, which enables writing: + `template fn(a: iterable)` instead of `template fn(a: untyped)` + + ## Compiler changes - Added `--declaredlocs` to show symbol declaration location in messages. @@ -341,6 +345,7 @@ - `--hint:CC` now goes to stderr (like all other hints) instead of stdout. + ## Tool changes - The rst parser now supports markdown table syntax. diff --git a/compiler/ast.nim b/compiler/ast.nim index fffe08cb7..155af6375 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -440,10 +440,12 @@ type tyVoid # now different from tyEmpty, hurray! + tyIterable static: # remind us when TTypeKind stops to fit in a single 64-bit word - assert TTypeKind.high.ord <= 63 + # assert TTypeKind.high.ord <= 63 + discard const tyPureObject* = tyTuple @@ -664,7 +666,7 @@ type mDefault, mUnown, mIsolate, mAccessEnv, mReset, mArray, mOpenArray, mRange, mSet, mSeq, mVarargs, mRef, mPtr, mVar, mDistinct, mVoid, mTuple, - mOrdinal, + mOrdinal, mIterableType, mInt, mInt8, mInt16, mInt32, mInt64, mUInt, mUInt8, mUInt16, mUInt32, mUInt64, mFloat, mFloat32, mFloat64, mFloat128, diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index 9ba97a8ed..bdecd7e53 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -132,3 +132,4 @@ proc initDefines*(symbols: StringTableRef) = defineSymbol("nimHasSpellSuggest") defineSymbol("nimHasCustomLiterals") defineSymbol("nimHasUnifiedTuple") + defineSymbol("nimHasIterable") diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 3fc7708bf..ca716b1c0 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -211,7 +211,7 @@ proc mapType(typ: PType): TJSTypeKind = else: result = etyNone of tyProc: result = etyProc of tyCString: result = etyString - of tyConcept: doAssert false + of tyConcept, tyIterable: doAssert false proc mapType(p: PProc; typ: PType): TJSTypeKind = result = mapType(typ) diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim index bb24fed34..436665827 100644 --- a/compiler/liftdestructors.nim +++ b/compiler/liftdestructors.nim @@ -908,7 +908,7 @@ proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode) = of tyOrdinal, tyRange, tyInferred, tyGenericInst, tyAlias, tySink: fillBody(c, lastSon(t), body, x, y) - of tyConcept: doAssert false + of tyConcept, tyIterable: doAssert false proc produceSymDistinctType(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp; info: TLineInfo; diff --git a/compiler/linter.nim b/compiler/linter.nim index 9af4f468e..83dd84699 100644 --- a/compiler/linter.nim +++ b/compiler/linter.nim @@ -40,7 +40,7 @@ proc beautifyName(s: string, k: TSymKind): string = "pointer", "float", "csize", "csize_t", "cdouble", "cchar", "cschar", "cshort", "cu", "nil", "typedesc", "auto", "any", "range", "openarray", "varargs", "set", "cfloat", "ref", "ptr", - "untyped", "typed", "static", "sink", "lent", "type", "owned"]: + "untyped", "typed", "static", "sink", "lent", "type", "owned", "iterable"]: result.add s[i] else: result.add toUpperAscii(s[i]) diff --git a/compiler/semcall.nim b/compiler/semcall.nim index 99979c382..8f29bdf32 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -62,7 +62,7 @@ proc pickBestCandidate(c: PContext, headSymbol: PNode, best, alt: var TCandidate, errors: var CandidateErrors, diagnosticsFlag: bool, - errorsEnabled: bool) = + errorsEnabled: bool, flags: TExprFlags) = var o: TOverloadIter var sym = initOverloadIter(o, c, headSymbol) var scope = o.lastOverloadScope @@ -95,7 +95,11 @@ proc pickBestCandidate(c: PContext, headSymbol: PNode, matches(c, n, orig, z) if z.state == csMatch: # little hack so that iterators are preferred over everything else: - if sym.kind == skIterator: inc(z.exactMatches, 200) + if sym.kind == skIterator: + if not (efWantIterator notin flags and efWantIterable in flags): + inc(z.exactMatches, 200) + else: + dec(z.exactMatches, 200) case best.state of csEmpty, csNoMatch: best = z of csMatch: @@ -356,7 +360,7 @@ proc resolveOverloads(c: PContext, n, orig: PNode, template pickBest(headSymbol) = pickBestCandidate(c, headSymbol, n, orig, initialBinding, filter, result, alt, errors, efExplain in flags, - errorsEnabled) + errorsEnabled, flags) pickBest(f) let overloadsState = result.state diff --git a/compiler/semdata.nim b/compiler/semdata.nim index c655047e2..51b2cdea1 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -54,7 +54,7 @@ type inst*: PInstantiation TExprFlag* = enum - efLValue, efWantIterator, efInTypeof, + efLValue, efWantIterator, efWantIterable, efInTypeof, efNeedStatic, # Use this in contexts where a static value is mandatory efPreferStatic, diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index c0aa168ef..256c008c9 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -853,7 +853,7 @@ proc semStaticExpr(c: PContext, n: PNode): PNode = proc semOverloadedCallAnalyseEffects(c: PContext, n: PNode, nOrig: PNode, flags: TExprFlags): PNode = - if flags*{efInTypeof, efWantIterator} != {}: + if flags*{efInTypeof, efWantIterator, efWantIterable} != {}: # consider: 'for x in pReturningArray()' --> we don't want the restriction # to 'skIterator' anymore; skIterator is preferred in sigmatch already # for typeof support. @@ -877,6 +877,11 @@ proc semOverloadedCallAnalyseEffects(c: PContext, n: PNode, nOrig: PNode, # error correction, prevents endless for loop elimination in transf. # See bug #2051: result[0] = newSymNode(errorSym(c, n)) + elif callee.kind == skIterator: + if efWantIterable in flags: + let typ = newTypeS(tyIterable, c) + rawAddSon(typ, result.typ) + result.typ = typ proc semObjConstr(c: PContext, n: PNode, flags: TExprFlags): PNode @@ -1364,7 +1369,7 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode = onUse(n[1].info, s) return - n[0] = semExprWithType(c, n[0], flags+{efDetermineType}) + n[0] = semExprWithType(c, n[0], flags+{efDetermineType, efWantIterable}) #restoreOldStyleType(n[0]) var i = considerQuotedIdent(c, n[1], n) var ty = n[0].typ diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index b34dc36e3..34a90a3f2 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -356,6 +356,15 @@ proc semArray(c: PContext, n: PNode, prev: PType): PType = localError(c.config, n.info, errArrayExpectsTwoTypeParams) result = newOrPrevType(tyError, prev, c) +proc semIterableType(c: PContext, n: PNode, prev: PType): PType = + result = newOrPrevType(tyIterable, prev, c) + if n.len == 2: + let base = semTypeNode(c, n[1], nil) + addSonSkipIntLit(result, base, c.idgen) + else: + localError(c.config, n.info, errXExpectsOneTypeParam % "iterable") + result = newOrPrevType(tyError, prev, c) + proc semOrdinal(c: PContext, n: PNode, prev: PType): PType = result = newOrPrevType(tyOrdinal, prev, c) if n.len == 2: @@ -1844,6 +1853,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = of mRange: result = semRange(c, n, prev) of mSet: result = semSet(c, n, prev) of mOrdinal: result = semOrdinal(c, n, prev) + of mIterableType: result = semIterableType(c, n, prev) of mSeq: result = semContainer(c, n, tySequence, "seq", prev) if optSeqDestructors in c.config.globalOptions: @@ -2067,6 +2077,9 @@ proc processMagicType(c: PContext, m: PSym) = of mOrdinal: setMagicIntegral(c.config, m, tyOrdinal, szUncomputedSize) rawAddSon(m.typ, newTypeS(tyNone, c)) + of mIterableType: + setMagicIntegral(c.config, m, tyIterable, 0) + rawAddSon(m.typ, newTypeS(tyNone, c)) of mPNimrodNode: incl m.typ.flags, tfTriggersCompileTime incl m.typ.flags, tfCheckedForDestructor diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 87f7c273b..767f34dda 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -1119,6 +1119,8 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, return if x >= isGeneric: isGeneric else: x return isNone + of tyIterable: + if f.kind != tyIterable: return isNone of tyNot: case f.kind of tyNot: @@ -1421,6 +1423,15 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, of tyAlias, tySink: result = typeRel(c, lastSon(f), a, flags) + of tyIterable: + if a.kind == tyIterable: + if f.len == 1: + result = typeRel(c, lastSon(f), lastSon(a), flags) + else: + # f.len = 3, for some reason + result = isGeneric + else: + result = isNone of tyGenericInst: var prev = PType(idTableGet(c.bindings, f)) let origF = f @@ -2270,14 +2281,18 @@ proc prepareOperand(c: PContext; formal: PType; a: PNode): PNode = # a.typ == nil is valid result = a elif a.typ.isNil: - # XXX This is unsound! 'formal' can differ from overloaded routine to - # overloaded routine! - let flags = {efDetermineType, efAllowStmt} - #if formal.kind == tyIter: {efDetermineType, efWantIterator} - #else: {efDetermineType, efAllowStmt} - #elif formal.kind == tyTyped: {efDetermineType, efWantStmt} - #else: {efDetermineType} - result = c.semOperand(c, a, flags) + if formal.kind == tyIterable: + let flags = {efDetermineType, efAllowStmt, efWantIterator, efWantIterable} + result = c.semOperand(c, a, flags) + else: + # XXX This is unsound! 'formal' can differ from overloaded routine to + # overloaded routine! + let flags = {efDetermineType, efAllowStmt} + #if formal.kind == tyIterable: {efDetermineType, efWantIterator} + #else: {efDetermineType, efAllowStmt} + #elif formal.kind == tyTyped: {efDetermineType, efWantStmt} + #else: {efDetermineType} + result = c.semOperand(c, a, flags) else: result = a considerGenSyms(c, result) diff --git a/compiler/typeallowed.nim b/compiler/typeallowed.nim index 9075d9e38..270e7c028 100644 --- a/compiler/typeallowed.nim +++ b/compiler/typeallowed.nim @@ -97,6 +97,9 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, result = nil of tyUntyped, tyTyped: if kind notin {skParam, skResult} or taNoUntyped in flags: result = t + of tyIterable: + if kind notin {skParam} or taNoUntyped in flags: result = t + # tyIterable is only for templates and macros. of tyStatic: if kind notin {skParam}: result = t of tyVoid: diff --git a/compiler/types.nim b/compiler/types.nim index 1dbbc3261..ce4812506 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -460,8 +460,8 @@ const "lent ", "varargs[$1]", "UncheckedArray[$1]", "Error Type", "BuiltInTypeClass", "UserTypeClass", "UserTypeClassInst", "CompositeTypeClass", "inferred", - "and", "or", "not", "any", "static", "TypeFromExpr", "out ", - "void"] + "and", "or", "not", "any", "static", "TypeFromExpr", "concept", # xxx bugfix + "void", "iterable"] const preferToResolveSymbols = {preferName, preferTypeName, preferModuleInfo, preferGenericArg, preferResolved, preferMixed} @@ -645,6 +645,11 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string = of tyDistinct: result = "distinct " & typeToString(t[0], if prefer == preferModuleInfo: preferModuleInfo else: preferTypeName) + of tyIterable: + # xxx factor this pattern + result = "iterable" + if t.len > 0: + result &= "[" & typeToString(t[0]) & ']' of tyTuple: # we iterate over t.sons here, because t.n may be nil if t.n != nil: @@ -1191,7 +1196,7 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool = result = sameTypeOrNilAux(a[0], b[0], c) and sameValue(a.n[0], b.n[0]) and sameValue(a.n[1], b.n[1]) - of tyGenericInst, tyAlias, tyInferred: + of tyGenericInst, tyAlias, tyInferred, tyIterable: cycleCheck() result = sameTypeAux(a.lastSon, b.lastSon, c) of tyNone: result = false diff --git a/compiler/vmdeps.nim b/compiler/vmdeps.nim index a9157bc03..6191f9459 100644 --- a/compiler/vmdeps.nim +++ b/compiler/vmdeps.nim @@ -287,6 +287,7 @@ proc mapTypeToAstX(cache: IdentCache; t: PType; info: TLineInfo; of tyAnd: result = mapTypeToBracket("and", mAnd, t, info) of tyOr: result = mapTypeToBracket("or", mOr, t, info) of tyNot: result = mapTypeToBracket("not", mNot, t, info) + of tyIterable: result = mapTypeToBracket("iterable", mIterableType, t, info) of tyAnything: result = atomicType("anything", mNone) of tyInferred: assert false of tyStatic, tyFromExpr: diff --git a/doc/manual.rst b/doc/manual.rst index 1325d1c5f..ea444a0ac 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -2645,6 +2645,28 @@ Varargs matching See `Varargs <#types-varargs>`_. +iterable +-------- + +A called `iterator` yielding type `T` can be passed to a template or macro via +a parameter typed as `untyped` (for unresolved expressions) or the type class +`iterable` or `iterable[T]` (after type checking and overload resolution). + +.. code-block:: nim + iterator iota(n: int): int = + for i in 0..`_ for passing iterators to templates and macros. + Converters ========== @@ -5591,24 +5616,6 @@ is used to invoke templates/macros: unknownIdentifier.declareVar -Another common example is this: - -.. code-block:: nim - :test: "nim c $1" - :status: 1 - - from std/sequtils import toSeq - - iterator something: string = - yield "Hello" - yield "World" - - var info = something().toSeq - -The problem here is that the compiler already decided that `something()` as -an iterator is not callable in this context before `toSeq` gets its -chance to convert it into a sequence. - It is also not possible to use fully qualified identifiers with module symbol in method call syntax. The order in which the dot operator binds to symbols prohibits this. diff --git a/lib/system.nim b/lib/system.nim index bd9ae2762..6e84aca66 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -123,6 +123,10 @@ proc defined*(x: untyped): bool {.magic: "Defined", noSideEffect, compileTime.} ## # Do here programmer friendly expensive sanity checks. ## # Put here the normal code +when defined(nimHasIterable): + type + iterable*[T] {.magic: IterableType.} ## Represents an expression that yields `T` + when defined(nimHashOrdinalFixed): type Ordinal*[T] {.magic: Ordinal.} ## Generic ordinal type. Includes integer, diff --git a/testament/lib/stdtest/testutils.nim b/testament/lib/stdtest/testutils.nim index 58d136696..abffff24c 100644 --- a/testament/lib/stdtest/testutils.nim +++ b/testament/lib/stdtest/testutils.nim @@ -79,3 +79,9 @@ template whenVMorJs*(bodyIf, bodyElse) = else: when defined(js): bodyIf else: bodyElse + +template accept*(a) = + doAssert compiles(a) + +template reject*(a) = + doAssert not compiles(a) diff --git a/tests/types/titerable.nim b/tests/types/titerable.nim new file mode 100644 index 000000000..b0713961d --- /dev/null +++ b/tests/types/titerable.nim @@ -0,0 +1,143 @@ +discard """ + targets: "c js" +""" + +from stdtest/testutils import accept, reject, whenVMorJs + +# toSeq-like templates + +template toSeq2(a: iterable): auto = + var ret: seq[typeof(a)] + for ai in a: ret.add ai + ret + +template toSeq3(a: iterable[string]): auto = + var ret: seq[typeof(a)] + for ai in a: ret.add ai + ret + +template toSeq4[T](a: iterable[T]): auto = + var ret: seq[typeof(a)] + for ai in a: ret.add ai + ret + +template toSeq5[T: SomeInteger](a: iterable[T]): auto = + var ret: seq[typeof(a)] + for ai in a: ret.add ai + ret + +template toSeq6(a: iterable[int | float]): auto = + var ret: seq[typeof(a)] + for ai in a: ret.add ai + ret + +template toSeq7(a: iterable[seq]): auto = + var ret: seq[typeof(a)] + for ai in a: ret.add ai + ret + +template fn7b(a: untyped) = discard +template fn7c(a: typed) = discard +template fn7d(a: auto) = discard +template fn7e[T](a: T) = discard + +template fn8a(a: iterable) = discard +template fn8b[T](a: iterable[T]) = discard +template fn8c(a: iterable[int]) = discard + +template bad1 = + template fn4(a: int, b: iterable[float, int]) = + discard + +template bad2 = + proc fn4(a: iterable) = discard + +template bad3 = + proc fn4(a: iterable[int]) = discard + +template good4 = + template fn1(a: iterable) = discard + template fn2(a: iterable[int]) = discard + +# iterators +iterator iota(n: int): auto = + for i in 0.. Date: Sun, 11 Apr 2021 17:37:32 +0200 Subject: [PATCH 0618/2472] IC: integrity checking (#17695) * IC: integrity checking: the plumbing code * progress * progress + bugfix (yes, the code already found a bug) * implemented integrity checking --- compiler/ic/bitabs.nim | 4 + compiler/ic/ic.nim | 3 +- compiler/ic/integrity.nim | 151 ++++++++++++++++++++++++++++++++++++++ compiler/main.nim | 4 +- compiler/semdata.nim | 3 + compiler/semstmts.nim | 2 +- koch.nim | 2 +- testament/categories.nim | 2 +- 8 files changed, 166 insertions(+), 5 deletions(-) create mode 100644 compiler/ic/integrity.nim diff --git a/compiler/ic/bitabs.nim b/compiler/ic/bitabs.nim index 1f75b7759..0bce30b5d 100644 --- a/compiler/ic/bitabs.nim +++ b/compiler/ic/bitabs.nim @@ -36,6 +36,10 @@ const template idToIdx(x: LitId): int = x.int - idStart +proc hasLitId*[T](t: BiTable[T]; x: LitId): bool = + let idx = idToIdx(x) + result = idx >= 0 and idx < t.vals.len + proc enlarge[T](t: var BiTable[T]) = var n: seq[LitId] newSeq(n, len(t.keys) * 2) diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim index 46bb63e07..6769c6ffe 100644 --- a/compiler/ic/ic.nim +++ b/compiler/ic/ic.nim @@ -31,7 +31,7 @@ type moduleFlags: TSymFlags includes: seq[(LitId, string)] # first entry is the module filename itself imports: seq[LitId] # the modules this module depends on - toReplay: PackedTree # pragmas and VM specific state to replay. + toReplay*: PackedTree # pragmas and VM specific state to replay. topLevel*: PackedTree # top level statements bodies*: PackedTree # other trees. Referenced from typ.n and sym.ast by their position. #producedGenerics*: Table[GenericKey, SymId] @@ -167,6 +167,7 @@ proc addExported*(c: var PackedEncoder; m: var PackedModule; s: PSym) = m.exports.add((nameId, s.itemId.item)) proc addConverter*(c: var PackedEncoder; m: var PackedModule; s: PSym) = + assert c.thisModule == s.itemId.module m.converters.add(s.itemId.item) proc addTrmacro*(c: var PackedEncoder; m: var PackedModule; s: PSym) = diff --git a/compiler/ic/integrity.nim b/compiler/ic/integrity.nim new file mode 100644 index 000000000..b4545de94 --- /dev/null +++ b/compiler/ic/integrity.nim @@ -0,0 +1,151 @@ +# +# +# The Nim Compiler +# (c) Copyright 2021 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Integrity checking for a set of .rod files. +## The set must cover a complete Nim project. + +import std / sets +import ".." / [ast, modulegraphs] +import packed_ast, bitabs, ic + +type + CheckedContext = object + g: ModuleGraph + thisModule: int32 + checkedSyms: HashSet[ItemId] + checkedTypes: HashSet[ItemId] + +proc checkType(c: var CheckedContext; typeId: PackedItemId) +proc checkForeignSym(c: var CheckedContext; symId: PackedItemId) +proc checkNode(c: var CheckedContext; tree: PackedTree; n: NodePos) + +proc checkTypeObj(c: var CheckedContext; typ: PackedType) = + for child in typ.types: + checkType(c, child) + if typ.n != emptyNodeId: + checkNode(c, c.g.packed[c.thisModule].fromDisk.bodies, NodePos typ.n) + if typ.sym != nilItemId: + checkForeignSym(c, typ.sym) + if typ.owner != nilItemId: + checkForeignSym(c, typ.owner) + checkType(c, typ.typeInst) + +proc checkType(c: var CheckedContext; typeId: PackedItemId) = + if typeId == nilItemId: return + let itemId = translateId(typeId, c.g.packed, c.thisModule, c.g.config) + if not c.checkedTypes.containsOrIncl(itemId): + let oldThisModule = c.thisModule + c.thisModule = itemId.module + checkTypeObj c, c.g.packed[itemId.module].fromDisk.sh.types[itemId.item] + c.thisModule = oldThisModule + +proc checkSym(c: var CheckedContext; s: PackedSym) = + if s.name != LitId(0): + assert c.g.packed[c.thisModule].fromDisk.sh.strings.hasLitId s.name + checkType c, s.typ + if s.ast != emptyNodeId: + checkNode(c, c.g.packed[c.thisModule].fromDisk.bodies, NodePos s.ast) + if s.owner != nilItemId: + checkForeignSym(c, s.owner) + +proc checkLocalSym(c: var CheckedContext; item: int32) = + let itemId = ItemId(module: c.thisModule, item: item) + if not c.checkedSyms.containsOrIncl(itemId): + checkSym c, c.g.packed[c.thisModule].fromDisk.sh.syms[item] + +proc checkForeignSym(c: var CheckedContext; symId: PackedItemId) = + let itemId = translateId(symId, c.g.packed, c.thisModule, c.g.config) + if not c.checkedSyms.containsOrIncl(itemId): + let oldThisModule = c.thisModule + c.thisModule = itemId.module + checkSym c, c.g.packed[itemId.module].fromDisk.sh.syms[itemId.item] + c.thisModule = oldThisModule + +proc checkNode(c: var CheckedContext; tree: PackedTree; n: NodePos) = + if tree[n.int].typeId != nilItemId: + checkType(c, tree[n.int].typeId) + case n.kind + of nkEmpty, nkNilLit, nkType, nkNilRodNode: + discard + of nkIdent: + assert c.g.packed[c.thisModule].fromDisk.sh.strings.hasLitId n.litId + of nkSym: + checkLocalSym(c, tree.nodes[n.int].operand) + of directIntLit: + discard + of externIntLit: + assert c.g.packed[c.thisModule].fromDisk.sh.integers.hasLitId n.litId + of nkStrLit..nkTripleStrLit: + assert c.g.packed[c.thisModule].fromDisk.sh.strings.hasLitId n.litId + of nkFloatLit..nkFloat128Lit: + assert c.g.packed[c.thisModule].fromDisk.sh.floats.hasLitId n.litId + of nkModuleRef: + let (n1, n2) = sons2(tree, n) + assert n1.kind == nkInt32Lit + assert n2.kind == nkInt32Lit + checkForeignSym(c, PackedItemId(module: n1.litId, item: tree.nodes[n2.int].operand)) + else: + for n0 in sonsReadonly(tree, n): + checkNode(c, tree, n0) + +proc checkTree(c: var CheckedContext; t: PackedTree) = + for p in allNodes(t): checkNode(c, t, p) + +proc checkLocalSymIds(c: var CheckedContext; m: PackedModule; symIds: seq[int32]) = + for symId in symIds: + assert symId >= 0 and symId < m.sh.syms.len, $symId & " " & $m.sh.syms.len + +proc checkModule(c: var CheckedContext; m: PackedModule) = + # We check that: + # - Every symbol references existing types and symbols. + # - Every tree node references existing types and symbols. + for i in 0..high(m.sh.syms): + checkLocalSym c, int32(i) + + checkTree c, m.toReplay + checkTree c, m.topLevel + + for e in m.exports: + assert e[1] >= 0 and e[1] < m.sh.syms.len + assert e[0] == m.sh.syms[e[1]].name + + for e in m.compilerProcs: + assert e[1] >= 0 and e[1] < m.sh.syms.len + assert e[0] == m.sh.syms[e[1]].name + + checkLocalSymIds c, m, m.converters + checkLocalSymIds c, m, m.methods + checkLocalSymIds c, m, m.trmacros + checkLocalSymIds c, m, m.pureEnums + #[ + To do: Check all these fields: + + reexports*: seq[(LitId, PackedItemId)] + macroUsages*: seq[(PackedItemId, PackedLineInfo)] + + typeInstCache*: seq[(PackedItemId, PackedItemId)] + procInstCache*: seq[PackedInstantiation] + attachedOps*: seq[(TTypeAttachedOp, PackedItemId, PackedItemId)] + methodsPerType*: seq[(PackedItemId, int, PackedItemId)] + enumToStringProcs*: seq[(PackedItemId, PackedItemId)] + ]# + +proc checkIntegrity*(g: ModuleGraph) = + var c = CheckedContext(g: g) + for i in 0..high(g.packed): + # case statement here to enforce exhaustive checks. + case g.packed[i].status + of undefined: + discard "nothing to do" + of loading: + assert false, "cannot check integrity: Module still loading" + of stored, storing, outdated, loaded: + c.thisModule = int32 i + checkModule(c, g.packed[i].fromDisk) + diff --git a/compiler/main.nim b/compiler/main.nim index 26a19063f..3c5018546 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -21,7 +21,7 @@ import modules, modulegraphs, tables, lineinfos, pathutils, vmprofiler -import ic / cbackend +import ic / [cbackend, integrity] from ic / ic import rodViewer when not defined(leanCompiler): @@ -93,6 +93,8 @@ proc commandCompileToC(graph: ModuleGraph) = if conf.symbolFiles == disabledSf: cgenWriteModules(graph.backend, conf) else: + if isDefined(conf, "nimIcIntegrityChecks"): + checkIntegrity(graph) generateCode(graph) # graph.backend can be nil under IC when nothing changed at all: if graph.backend != nil: diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 51b2cdea1..daf1265e8 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -341,6 +341,9 @@ proc addConverter*(c: PContext, conv: LazySym) = assert conv.sym != nil if inclSym(c.converters, conv.sym): add(c.graph.ifaces[c.module.position].converters, conv) + +proc addConverterDef*(c: PContext, conv: LazySym) = + addConverter(c, conv) if c.config.symbolFiles != disabledSf: addConverter(c.encoder, c.packedRepr, conv.sym) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index bbfd49e2a..4c6a745f6 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -2093,7 +2093,7 @@ proc semConverterDef(c: PContext, n: PNode): PNode = var t = s.typ if t[0] == nil: localError(c.config, n.info, errXNeedsReturnType % "converter") if t.len != 2: localError(c.config, n.info, "a converter takes exactly one argument") - addConverter(c, LazySym(sym: s)) + addConverterDef(c, LazySym(sym: s)) proc semMacroDef(c: PContext, n: PNode): PNode = checkSonsLen(n, bodyPos + 1, c.config) diff --git a/koch.nim b/koch.nim index 3b00d3564..44d38589b 100644 --- a/koch.nim +++ b/koch.nim @@ -492,7 +492,7 @@ proc icTest(args: string) = for fragment in content.split("#!EDIT!#"): let file = inp.replace(".nim", "_temp.nim") writeFile(file, fragment) - var cmd = nimExe & " cpp --ic:on --listcmd " + var cmd = nimExe & " cpp --ic:on -d:nimIcIntegrityChecks --listcmd " if i == 0: cmd.add "-f " cmd.add quoteShell(file) diff --git a/testament/categories.nim b/testament/categories.nim index af6331dde..618046287 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -488,7 +488,7 @@ proc icTests(r: var TResults; testsDir: string, cat: Category, options: string) tooltests = ["compiler/nim.nim", "tools/nimgrep.nim"] writeOnly = " --incremental:writeonly " readOnly = " --incremental:readonly " - incrementalOn = " --incremental:on " + incrementalOn = " --incremental:on -d:nimIcIntegrityChecks " template test(x: untyped) = testSpecWithNimcache(r, makeRawTest(file, x & options, cat), nimcache) From e710b9cf33939fa20230ab3029ceb735e0a669ad Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 11 Apr 2021 20:14:44 +0200 Subject: [PATCH 0619/2472] macrocache.nim: removed trailing whitespace --- lib/core/macrocache.nim | 48 ++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/lib/core/macrocache.nim b/lib/core/macrocache.nim index e376ad87f..d4b103793 100644 --- a/lib/core/macrocache.nim +++ b/lib/core/macrocache.nim @@ -10,10 +10,10 @@ ## This module provides an API for macros to collect compile-time information ## across module boundaries. It should be used instead of global `{.compileTime.}` ## variables as those break incremental compilation. -## +## ## The main feature of this module is that if you create `CacheTable`s or ## any other `Cache` types with the same name in different modules, their -## content will be shared, meaning that you can fill a `CacheTable` in +## content will be shared, meaning that you can fill a `CacheTable` in ## one module, and iterate over its contents in another. runnableExamples: @@ -22,20 +22,20 @@ runnableExamples: const mcTable = CacheTable"myTable" const mcSeq = CacheSeq"mySeq" const mcCounter = CacheCounter"myCounter" - + static: # add new key "val" with the value `myval` let myval = newLit("hello ic") mcTable["val"] = myval assert mcTable["val"].kind == nnkStrLit - + # Can access the same cache from different static contexts # All the information is retained static: # get value from `mcTable` and add it to `mcSeq` mcSeq.add(mcTable["val"]) assert mcSeq.len == 1 - + static: assert mcSeq[0].strVal == "hello ic" @@ -49,7 +49,7 @@ type ## Compile-time sequence of `NimNode`s. CacheTable* = distinct string ## Compile-time table of key-value pairs. - ## + ## ## Keys are `string`s and values are `NimNode`s. CacheCounter* = distinct string ## Compile-time counter, uses `int` for storing the count. @@ -80,22 +80,22 @@ proc add*(s: CacheSeq; value: NimNode) {.magic: "NcsAdd".} = runnableExamples: import std/macros const mySeq = CacheSeq"addTest" - + static: mySeq.add(newLit(5)) mySeq.add(newLit("hello ic")) - + assert mySeq.len == 2 assert mySeq[1].strVal == "hello ic" -proc incl*(s: CacheSeq; value: NimNode) {.magic: "NcsIncl".} = +proc incl*(s: CacheSeq; value: NimNode) {.magic: "NcsIncl".} = ## Adds `value` to `s`. - ## + ## ## .. hint:: This doesn't do anything if `value` is already in `s`. runnableExamples: import std/macros const mySeq = CacheSeq"inclTest" - + static: mySeq.incl(newLit(5)) mySeq.incl(newLit(5)) @@ -113,7 +113,7 @@ proc len*(s: CacheSeq): int {.magic: "NcsLen".} = let val = newLit("helper") mySeq.add(val) assert mySeq.len == 1 - + mySeq.add(val) assert mySeq.len == 2 @@ -132,20 +132,20 @@ iterator items*(s: CacheSeq): NimNode = runnableExamples: import std/macros const myseq = CacheSeq"itemsTest" - + static: myseq.add(newLit(5)) myseq.add(newLit(42)) - + for val in myseq: # check that all values in `myseq` are int literals assert val.kind == nnkIntLit - + for i in 0 ..< len(s): yield s[i] proc `[]=`*(t: CacheTable; key: string, value: NimNode) {.magic: "NctPut".} = - ## Inserts a `(key, value)` pair into `t`. - ## + ## Inserts a `(key, value)` pair into `t`. + ## ## .. warning:: `key` has to be unique! Assigning `value` to a `key` that is already ## in the table will result in a compiler error. runnableExamples: @@ -155,11 +155,11 @@ proc `[]=`*(t: CacheTable; key: string, value: NimNode) {.magic: "NctPut".} = static: # assign newLit(5) to the key "value" mcTable["value"] = newLit(5) - - # check that we can get the value back - assert mcTable["value"].kind == nnkIntLit -proc len*(t: CacheTable): int {.magic: "NctLen".} = + # check that we can get the value back + assert mcTable["value"].kind == nnkIntLit + +proc len*(t: CacheTable): int {.magic: "NctLen".} = ## Returns the number of elements in `t`. runnableExamples: import std/macros @@ -177,7 +177,7 @@ proc `[]`*(t: CacheTable; key: string): NimNode {.magic: "NctGet".} = const mcTable = CacheTable"subTest" static: mcTable["toAdd"] = newStmtList() - + # get the NimNode back assert mcTable["toAdd"].kind == nnkStmtList @@ -189,8 +189,8 @@ iterator pairs*(t: CacheTable): (string, NimNode) = runnableExamples: import std/macros const mytabl = CacheTable"values" - - static: + + static: mytabl["intVal"] = newLit(5) mytabl["otherVal"] = newLit(6) for key, val in mytabl: From cae183915490846a0ec7dbcd52b29a74c7c59f05 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 11 Apr 2021 21:41:24 +0200 Subject: [PATCH 0620/2472] IC: added tcompiletime_counter test case (#17698) --- tests/ic/mcompiletime_counter.nim | 15 +++++++++++++++ tests/ic/tcompiletime_counter.nim | 24 ++++++++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 tests/ic/mcompiletime_counter.nim create mode 100644 tests/ic/tcompiletime_counter.nim diff --git a/tests/ic/mcompiletime_counter.nim b/tests/ic/mcompiletime_counter.nim new file mode 100644 index 000000000..6fc7b3f2a --- /dev/null +++ b/tests/ic/mcompiletime_counter.nim @@ -0,0 +1,15 @@ + +import std/macros +import std/macrocache + +const myCounter = CacheCounter"myCounter" + +proc getUniqueId*(): int {.compileTime.} = + inc myCounter + result = myCounter.value + +static: + myCounter.inc(3) + assert myCounter.value == 3 + + diff --git a/tests/ic/tcompiletime_counter.nim b/tests/ic/tcompiletime_counter.nim new file mode 100644 index 000000000..695a6ec27 --- /dev/null +++ b/tests/ic/tcompiletime_counter.nim @@ -0,0 +1,24 @@ +discard """ + output: '''id 4''' +""" + +import mcompiletime_counter + +const intId = getUniqueId() + +echo "id ", intId + +#!EDIT!# + +discard """ + output: '''id 4 5''' +""" + +import mcompiletime_counter + +const + intId = getUniqueId() + floatId = getUniqueId() + +echo "id ", intId, " ", floatId + From 0bc943ad54d707f93d47a6d52bf7d445c45388d8 Mon Sep 17 00:00:00 2001 From: shirleyquirk <31934565+shirleyquirk@users.noreply.github.com> Date: Mon, 12 Apr 2021 06:32:37 +0100 Subject: [PATCH 0621/2472] followup strformat PR. backslash escapes, tests, docs (#17700) * Allow use of colons inside fmt allowing colons inside fmt by replacing the format specifier delimiter lets arbitrary nim code be run within fmt expressions. Co-authored-by: flywind * formatting,documentation,backslash escapes Adding support for evaluating expressions by special-casing parentheses causes this regression: `&"""{ "(hello)" }"""` no longer parses. In addition, code such as &"""{(if open: '(' else: ')')}""" wouldn't work. To enable that, as well as the use of, e.g. Table constructors inside curlies, I've added backslash escapes. This also means that if/for/etc statements, unparenthesized, will work, if the colons are escaped, but i've left that under-documented. It's not exactly elegant having two types of escape, but I believe it's the least bad option. * changelog * added json strformat test * pulled my thumb out and wrote a parser Co-authored-by: Andreas Rumpf Co-authored-by: flywind --- changelogs/changelog_X_XX_X.md | 1 + lib/pure/strformat.nim | 33 ++++++++++++++++--- tests/stdlib/tstrformat.nim | 60 +++++++++++++++++++++++++++++++++- 3 files changed, 89 insertions(+), 5 deletions(-) diff --git a/changelogs/changelog_X_XX_X.md b/changelogs/changelog_X_XX_X.md index 77b421f33..d68ef4c20 100644 --- a/changelogs/changelog_X_XX_X.md +++ b/changelogs/changelog_X_XX_X.md @@ -13,6 +13,7 @@ The changes should go to changelog.md! - Changed `example.foo` to take additional `bar` parameter. +- Added support for evaluating parenthesised expressions in strformat ## Language changes diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim index c232c8a46..736b4d501 100644 --- a/lib/pure/strformat.nim +++ b/lib/pure/strformat.nim @@ -144,6 +144,19 @@ An expression like `&"{key} is {value:arg} {{z}}"` is transformed into: Parts of the string that are enclosed in the curly braces are interpreted as Nim code, to escape a `{` or `}`, double it. +Within a curly expression,however, '{','}', must be escaped with a backslash. + +To enable evaluating Nim expressions within curlies, inside parentheses +colons do not need to be escaped. +]## + +runnableExamples: + let x = "hello" + assert fmt"""{ "\{(" & x & ")\}" }""" == "{(hello)}" + assert fmt"""{{({ x })}}""" == "{(hello)}" + assert fmt"""{ $(\{x:1,"world":2\}) }""" == """[("hello", 1), ("world", 2)]""" + +##[ `&` delegates most of the work to an open overloaded set of `formatValue` procs. The required signature for a type `T` that supports formatting is usually `proc formatValue(result: var string; x: T; specifier: string)`. @@ -289,6 +302,7 @@ keep the hygiene of `myTemplate`, and we do not want `arg1` to be injected into the context where `myTemplate` is expanded, everything is wrapped in a `block`. + # Future directions A curly expression with commas in it like `{x, argA, argB}` could be @@ -588,10 +602,21 @@ proc strformatImpl(pattern: NimNode; openChar, closeChar: char): NimNode = var subexpr = "" var inParens = 0 - while i < f.len and f[i] != closeChar and (f[i] != ':' or inParens!=0): + var inSingleQuotes = false + var inDoubleQuotes = false + template notEscaped:bool = f[i-1]!='\\' + while i < f.len and f[i] != closeChar and (f[i] != ':' or inParens != 0): case f[i] - of '(': inc inParens - of ')': dec inParens + of '\\': + if i < f.len-1 and f[i+1] in {openChar,closeChar,':'}: inc i + of '\'': + if not inDoubleQuotes and notEscaped: inSingleQuotes = not inSingleQuotes + of '\"': + if notEscaped: inDoubleQuotes = not inDoubleQuotes + of '(': + if not (inSingleQuotes or inDoubleQuotes): inc inParens + of ')': + if not (inSingleQuotes or inDoubleQuotes): dec inParens of '=': let start = i inc i @@ -604,7 +629,7 @@ proc strformatImpl(pattern: NimNode; openChar, closeChar: char): NimNode = else: discard subexpr.add f[i] inc i - + var x: NimNode try: x = parseExpr(subexpr) diff --git a/tests/stdlib/tstrformat.nim b/tests/stdlib/tstrformat.nim index 86100e421..1863d3138 100644 --- a/tests/stdlib/tstrformat.nim +++ b/tests/stdlib/tstrformat.nim @@ -1,7 +1,7 @@ # xxx: test js target import genericstrformat -import std/[strformat, strutils, times] +import std/[strformat, strutils, times, tables, json] proc main() = block: # issue #7632 @@ -284,6 +284,20 @@ proc main() = doAssert fmt"{123.456=:>13e}" == "123.456= 1.234560e+02" doAssert fmt"{123.456=:13e}" == "123.456= 1.234560e+02" + let x = 3.14 + doAssert fmt"{(if x!=0: 1.0/x else: 0):.5}" == "0.31847" + doAssert fmt"""{(block: + var res: string + for i in 1..15: + res.add (if i mod 15 == 0: "FizzBuzz" + elif i mod 5 == 0: "Buzz" + elif i mod 3 == 0: "Fizz" + else: $i) & " " + res)}""" == "1 2 Fizz 4 Buzz Fizz 7 8 Fizz Buzz 11 Fizz 13 14 FizzBuzz " + + doAssert fmt"""{ "\{(" & msg & ")\}" }""" == "{(hello)}" + doAssert fmt"""{{({ msg })}}""" == "{(hello)}" + doAssert fmt"""{ $(\{msg:1,"world":2\}) }""" == """[("hello", 1), ("world", 2)]""" block: # tests for debug format string var name = "hello" let age = 21 @@ -496,6 +510,50 @@ proc main() = block: # test low(int64) doAssert &"{low(int64):-}" == "-9223372036854775808" + block: #expressions plus formatting + doAssert fmt"{if true\: 123.456 else\: 0=:>9.3f}" == "if true: 123.456 else: 0= 123.456" + doAssert fmt"{(if true: 123.456 else: 0)=}" == "(if true: 123.456 else: 0)=123.456" + doAssert fmt"{if true\: 123.456 else\: 0=:9.3f}" == "if true: 123.456 else: 0= 123.456" + doAssert fmt"{(if true: 123.456 else: 0)=:9.4f}" == "(if true: 123.456 else: 0)= 123.4560" + doAssert fmt"{(if true: 123.456 else: 0)=:>9.0f}" == "(if true: 123.456 else: 0)= 123." + doAssert fmt"{if true\: 123.456 else\: 0=:<9.4f}" == "if true: 123.456 else: 0=123.4560 " + doAssert fmt"""{(case true + of false: 0.0 + of true: 123.456)=:e}""" == """(case true + of false: 0.0 + of true: 123.456)=1.234560e+02""" + + doAssert fmt"""{block\: + var res = 0.000123456 + for _ in 0..5\: + res *= 10 + res=:>13e}""" == """block: + var res = 0.000123456 + for _ in 0..5: + res *= 10 + res= 1.234560e+02""" + #side effects + var x = 5 + doAssert fmt"{(x=7;123.456)=:13e}" == "(x=7;123.456)= 1.234560e+02" + doAssert x==7 + block: #curly bracket expressions and tuples + proc formatValue(result: var string; value:Table|bool|JsonNode; specifier:string) = result.add $value + + doAssert fmt"""{\{"a"\:1,"b"\:2\}.toTable() = }""" == """{"a":1,"b":2}.toTable() = {"a": 1, "b": 2}""" + doAssert fmt"""{(\{3: (1,"hi",0.9),4: (4,"lo",1.1)\}).toTable()}""" == """{3: (1, "hi", 0.9), 4: (4, "lo", 1.1)}""" + doAssert fmt"""{ (%* \{"name": "Isaac", "books": ["Robot Dreams"]\}) }""" == """{"name":"Isaac","books":["Robot Dreams"]}""" + doAssert """%( \%\* {"name": "Isaac"})*""".fmt('%','*') == """{"name":"Isaac"}""" + block: #parens in quotes that fool my syntax highlighter + doAssert fmt"{(if true: ')' else: '(')}" == ")" + doAssert fmt"{(if true: ']' else: ')')}" == "]" + doAssert fmt"""{(if true: "\")\"" else: "\"(")}""" == """")"""" + doAssert &"""{(if true: "\")" else: "")}""" == "\")" + doAssert &"{(if true: \"\\\")\" else: \"\")}" == "\")" + doAssert fmt"""{(if true: "')" else: "")}""" == "')" + doAssert fmt"""{(if true: "'" & "'" & ')' else: "")}""" == "'')" + doAssert &"""{(if true: "'" & "'" & ')' else: "")}""" == "'')" + doAssert &"{(if true: \"\'\" & \"'\" & ')' else: \"\")}" == "'')" + doAssert fmt"""{(if true: "'" & ')' else: "")}""" == "')" # xxx static: main() main() From 8b5769133cf30dae2a6ed15aa3ba9d2f2be713ec Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 12 Apr 2021 14:58:46 +0800 Subject: [PATCH 0622/2472] remove unnecessary assignment (#17702) --- lib/pure/coro.nim | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/pure/coro.nim b/lib/pure/coro.nim index ccd9fba4d..e3ed6aadd 100644 --- a/lib/pure/coro.nim +++ b/lib/pure/coro.nim @@ -274,7 +274,6 @@ proc start*(c: proc(), stacksize: int = defaultStackSize): CoroutineRef {.discar coro.execContext = CreateFiberEx(stacksize, stacksize, FIBER_FLAG_FLOAT_SWITCH, (proc(p: pointer) {.stdcall.} = runCurrentTask()), nil) - coro.stack.size = stacksize else: coro = cast[CoroutinePtr](alloc0(sizeof(Coroutine) + stacksize)) coro.stack.top = cast[pointer](cast[ByteAddress](coro) + sizeof(Coroutine)) From 4f5709e32648435ff253b056ac7066240c39d2b7 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 13 Apr 2021 07:23:29 +0200 Subject: [PATCH 0623/2472] IC: serialization bugfix (#17701) --- compiler/ic/ic.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim index 6769c6ffe..32bacba70 100644 --- a/compiler/ic/ic.nim +++ b/compiler/ic/ic.nim @@ -260,9 +260,9 @@ proc storeTypeLater(t: PType; c: var PackedEncoder; m: var PackedModule): Packed assert t.uniqueId.module >= 0 assert t.uniqueId.item > 0 return PackedItemId(module: toLitId(t.uniqueId.module.FileIndex, c, m), item: t.uniqueId.item) - assert t.itemId.module >= 0 + assert t.uniqueId.module >= 0 assert t.uniqueId.item > 0 - result = PackedItemId(module: toLitId(t.itemId.module.FileIndex, c, m), item: t.uniqueId.item) + result = PackedItemId(module: toLitId(t.uniqueId.module.FileIndex, c, m), item: t.uniqueId.item) addMissing(c, t) proc storeSymLater(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId = @@ -318,9 +318,9 @@ proc storeType(t: PType; c: var PackedEncoder; m: var PackedModule): PackedItemI # fill the reserved slot, nothing else: m.sh.types[t.uniqueId.item] = p - assert t.itemId.module >= 0 + assert t.uniqueId.module >= 0 assert t.uniqueId.item > 0 - result = PackedItemId(module: toLitId(t.itemId.module.FileIndex, c, m), item: t.uniqueId.item) + result = PackedItemId(module: toLitId(t.uniqueId.module.FileIndex, c, m), item: t.uniqueId.item) proc toPackedLib(l: PLib; c: var PackedEncoder; m: var PackedModule): PackedLib = ## the plib hangs off the psym via the .annex field From e92c78a9ac0ce0f7a9b2a0fada0c2c2557c40ae2 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 13 Apr 2021 00:33:28 -0700 Subject: [PATCH 0624/2472] add number literal jsbigints.big (#17707) --- lib/std/jsbigints.nim | 20 +++++++++++++------- tests/stdlib/tjsbigints.nim | 6 +++--- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/lib/std/jsbigints.nim b/lib/std/jsbigints.nim index 96ba90649..27df8fdad 100644 --- a/lib/std/jsbigints.nim +++ b/lib/std/jsbigints.nim @@ -13,16 +13,22 @@ func big*(integer: SomeInteger): JsBigInt {.importjs: "BigInt(#)".} = doAssert 0b1111100111.big == 0o1747.big and 0o1747.big == 999.big when nimvm: doAssert false, "JsBigInt can not be used at compile-time nor static context" else: discard -func big*(integer: cstring): JsBigInt {.importjs: "BigInt(#)".} = +func `'big`*(num: cstring): JsBigInt {.importjs: "BigInt(#)".} = ## Constructor for `JsBigInt`. runnableExamples: - doAssert big"-1" == big"1" - big"2" + doAssert -1'big == 1'big - 2'big # supports decimal, binary, octal, hex: - doAssert big"12" == 12.big - doAssert big"0b101" == 0b101.big - doAssert big"0o701" == 0o701.big - doAssert big"0xdeadbeaf" == 0xdeadbeaf.big - doAssert big"0xffffffffffffffff" == (1.big shl 64.big) - 1.big + doAssert -12'big == big"-12" + doAssert 12'big == 12.big + doAssert 0b101'big == 0b101.big + doAssert 0o701'big == 0o701.big + doAssert 0xdeadbeaf'big == 0xdeadbeaf.big + doAssert 0xffffffffffffffff'big == (1'big shl 64'big) - 1'big + doAssert not compiles(static(12'big)) + when nimvm: doAssert false, "JsBigInt can not be used at compile-time nor static context" else: discard + +func big*(integer: cstring): JsBigInt {.importjs: "BigInt(#)".} = + ## Alias for `'big` when nimvm: doAssert false, "JsBigInt can not be used at compile-time nor static context" else: discard func toCstring*(this: JsBigInt; radix: 2..36): cstring {.importjs: "#.toString(#)".} = diff --git a/tests/stdlib/tjsbigints.nim b/tests/stdlib/tjsbigints.nim index 34c5ddfbf..fcf699c67 100644 --- a/tests/stdlib/tjsbigints.nim +++ b/tests/stdlib/tjsbigints.nim @@ -26,7 +26,7 @@ doAssert big1.toCstring(10) == "2147483647".cstring doAssert big2 ** big3 == big(443556) var huge = big"999999999999999999999999999999999999999999999999999999999999999999999999999999999999999" huge.inc -huge = huge + big"-999999999999999999999999999999999999999999999999999999999999999999999999999999999999999" +huge = huge + -999999999999999999999999999999999999999999999999999999999999999999999999999999999999999'big doAssert huge == big"1" var list: seq[JsBigInt] for i in big"0" .. big"5": @@ -40,7 +40,7 @@ for i in big"0" ..< big"5": doAssert list == @[big"0", big"1", big"2", big"3", big"4"] block: - let b = big"2" - doAssert -b ** big"3" == big"-8" + let b = 2'big + doAssert -b ** 3'big == -8'big doAssert -b ** big"2" == big"4" # not -4 because of precedence doAssert -big"3" == big"-3" From 8e6b87a917a5d09a70d1e29f0b06915c29fcac6b Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 13 Apr 2021 10:43:16 +0200 Subject: [PATCH 0625/2472] IC: code cleanup (#17708) --- compiler/ic/ic.nim | 57 ++++++++++------------------------------------ 1 file changed, 12 insertions(+), 45 deletions(-) diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim index 32bacba70..e882b7a85 100644 --- a/compiler/ic/ic.nim +++ b/compiler/ic/ic.nim @@ -254,41 +254,31 @@ proc storeTypeLater(t: PType; c: var PackedEncoder; m: var PackedModule): Packed # we only write one tree into m.bodies after the other. if t.isNil: return nilItemId - if t.uniqueId.module != c.thisModule: - # XXX Assert here that it already was serialized in the foreign module! - # it is a foreign type: - assert t.uniqueId.module >= 0 - assert t.uniqueId.item > 0 - return PackedItemId(module: toLitId(t.uniqueId.module.FileIndex, c, m), item: t.uniqueId.item) assert t.uniqueId.module >= 0 assert t.uniqueId.item > 0 result = PackedItemId(module: toLitId(t.uniqueId.module.FileIndex, c, m), item: t.uniqueId.item) - addMissing(c, t) + if t.uniqueId.module == c.thisModule: + # the type belongs to this module, so serialize it here, eventually. + addMissing(c, t) proc storeSymLater(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId = if s.isNil: return nilItemId assert s.itemId.module >= 0 - if s.itemId.module != c.thisModule: - # XXX Assert here that it already was serialized in the foreign module! - # it is a foreign symbol: - assert s.itemId.module >= 0 - return PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), item: s.itemId.item) assert s.itemId.module >= 0 result = PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), item: s.itemId.item) - addMissing(c, s) + if s.itemId.module == c.thisModule: + # the sym belongs to this module, so serialize it here, eventually. + addMissing(c, s) proc storeType(t: PType; c: var PackedEncoder; m: var PackedModule): PackedItemId = ## serialize a ptype if t.isNil: return nilItemId - if t.uniqueId.module != c.thisModule: - # XXX Assert here that it already was serialized in the foreign module! - # it is a foreign type: - assert t.uniqueId.module >= 0 - assert t.uniqueId.item > 0 - return PackedItemId(module: toLitId(t.uniqueId.module.FileIndex, c, m), item: t.uniqueId.item) + assert t.uniqueId.module >= 0 + assert t.uniqueId.item > 0 + result = PackedItemId(module: toLitId(t.uniqueId.module.FileIndex, c, m), item: t.uniqueId.item) - if not c.typeMarker.containsOrIncl(t.uniqueId.item): + if t.uniqueId.module == c.thisModule and not c.typeMarker.containsOrIncl(t.uniqueId.item): if t.uniqueId.item >= m.sh.types.len: setLen m.sh.types, t.uniqueId.item+1 @@ -296,20 +286,9 @@ proc storeType(t: PType; c: var PackedEncoder; m: var PackedModule): PackedItemI size: t.size, align: t.align, nonUniqueId: t.itemId.item, paddingAtEnd: t.paddingAtEnd, lockLevel: t.lockLevel) storeNode(p, t, n) - - when false: - for op, s in pairs t.attachedOps: - c.addMissing s - p.attachedOps[op] = s.safeItemId(c, m) - p.typeInst = t.typeInst.storeType(c, m) for kid in items t.sons: p.types.add kid.storeType(c, m) - - when false: - for i, s in items t.methods: - c.addMissing s - p.methods.add (i, s.safeItemId(c, m)) c.addMissing t.sym p.sym = t.sym.safeItemId(c, m) c.addMissing t.owner @@ -318,10 +297,6 @@ proc storeType(t: PType; c: var PackedEncoder; m: var PackedModule): PackedItemI # fill the reserved slot, nothing else: m.sh.types[t.uniqueId.item] = p - assert t.uniqueId.module >= 0 - assert t.uniqueId.item > 0 - result = PackedItemId(module: toLitId(t.uniqueId.module.FileIndex, c, m), item: t.uniqueId.item) - proc toPackedLib(l: PLib; c: var PackedEncoder; m: var PackedModule): PackedLib = ## the plib hangs off the psym via the .annex field if l.isNil: return @@ -336,14 +311,9 @@ proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId if s.isNil: return nilItemId assert s.itemId.module >= 0 + result = PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), item: s.itemId.item) - if s.itemId.module != c.thisModule: - # XXX Assert here that it already was serialized in the foreign module! - # it is a foreign symbol: - assert s.itemId.module >= 0 - return PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), item: s.itemId.item) - - if not c.symMarker.containsOrIncl(s.itemId.item): + if s.itemId.module == c.thisModule and not c.symMarker.containsOrIncl(s.itemId.item): if s.itemId.item >= m.sh.syms.len: setLen m.sh.syms, s.itemId.item+1 @@ -375,9 +345,6 @@ proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId # fill the reserved slot, nothing else: m.sh.syms[s.itemId.item] = p - assert s.itemId.module >= 0 - result = PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), item: s.itemId.item) - proc addModuleRef(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule) = ## add a remote symbol reference to the tree let info = n.info.toPackedInfo(c, m) From fe77c3c72fe1fde4f41286158ba612c02083dfe2 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 13 Apr 2021 14:16:19 -0700 Subject: [PATCH 0626/2472] megatest: allow duplicate file names, show compilation progress (#17713) * megatest: allow duplicate file names * megatest: show compilation progress --- testament/categories.nim | 17 +++++++++++------ tests/misc/tjoinable.nim | 3 +++ tests/testament/tjoinable.nim | 1 + 3 files changed, 15 insertions(+), 6 deletions(-) create mode 100644 tests/misc/tjoinable.nim diff --git a/testament/categories.nim b/testament/categories.nim index 618046287..ee5c46519 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -571,6 +571,11 @@ proc quoted(a: string): string = proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) = ## returns a list of tests that have problems + #[ + xxx create a reusable megatest API after abstracting out testament specific code, + refs https://github.com/timotheecour/Nim/issues/655 + and https://github.com/nim-lang/gtk2/pull/28; it's useful in other contexts. + ]# var specs: seq[TSpec] = @[] for kind, dir in walkDir(testsDir): assert testsDir.startsWith(testsDir) @@ -595,16 +600,16 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) = var megatest: string # xxx (minor) put outputExceptedFile, outputGottenFile, megatestFile under here or `buildDir` var outDir = nimcacheDir(testsDir / "megatest", "", targetC) - const marker = "megatest:processing: " - + template toMarker(file, i): string = + "megatest:processing: [$1] $2" % [$i, file] for i, runSpec in specs: let file = runSpec.file - let file2 = outDir / ("megatest_$1.nim" % $i) + let file2 = outDir / ("megatest_a_$1.nim" % $i) # `include` didn't work with `trecmod2.nim`, so using `import` - let code = "echo \"$1\", $2\n" % [marker, quoted(file)] + let code = "echo $1\nstatic: echo \"CT:\", $1\n" % [toMarker(file, i).quoted] createDir(file2.parentDir) writeFile(file2, code) - megatest.add "import $1\nimport $2\n" % [quoted(file2), quoted(file)] + megatest.add "import $1\nimport $2 as megatest_b_$3\n" % [file2.quoted, file.quoted, $i] let megatestFile = testsDir / "megatest.nim" # so it uses testsDir / "config.nims" writeFile(megatestFile, megatest) @@ -626,7 +631,7 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) = writeFile(outputGottenFile, buf) var outputExpected = "" for i, runSpec in specs: - outputExpected.add marker & runSpec.file & "\n" + outputExpected.add toMarker(runSpec.file, i) & "\n" if runSpec.output.len > 0: outputExpected.add runSpec.output if not runSpec.output.endsWith "\n": diff --git a/tests/misc/tjoinable.nim b/tests/misc/tjoinable.nim new file mode 100644 index 000000000..f23fca0d4 --- /dev/null +++ b/tests/misc/tjoinable.nim @@ -0,0 +1,3 @@ +# checks that megatest allows duplicate names, see also `tests/testament/tjoinable.nim` +doAssert defined(testing) +doAssert defined(nimMegatest) diff --git a/tests/testament/tjoinable.nim b/tests/testament/tjoinable.nim index 7a0ad7985..c651780de 100644 --- a/tests/testament/tjoinable.nim +++ b/tests/testament/tjoinable.nim @@ -3,6 +3,7 @@ discard """ """ # checks that this is joinable +# checks that megatest allows duplicate names, see also `tests/misc/tjoinable.nim` doAssert defined(testing) doAssert defined(nimMegatest) echo "ok" # intentional to make sure this doesn't prevent `isJoinableSpec` From 840e13deb79db7b072595658e6aa8328d55d7889 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 13 Apr 2021 23:50:15 -0700 Subject: [PATCH 0627/2472] type with same name in different scope now works (#17710) * type with same name in different scope now works * fix tests/enum/tenum.nim which was wrong because it was affected by this bug --- compiler/semstmts.nim | 7 ++- compiler/sighashes.nim | 19 ++++-- tests/ccgbugs/tsamename3.nim | 111 +++++++++++++++++++++++++++++++++++ tests/enum/tenum.nim | 5 +- 4 files changed, 133 insertions(+), 9 deletions(-) create mode 100644 tests/ccgbugs/tsamename3.nim diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 4c6a745f6..2c257bd3d 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -1354,8 +1354,11 @@ proc typeSectionFinalPass(c: PContext, n: PNode) = checkConstructedType(c.config, s.info, s.typ) if s.typ.kind in {tyObject, tyTuple} and not s.typ.n.isNil: checkForMetaFields(c, s.typ.n) - # fix bug #5170: ensure locally scoped object types get a unique name: - if s.typ.kind == tyObject and not isTopLevel(c): incl(s.flags, sfGenSym) + + # fix bug #5170, bug #17162, bug #15526: ensure locally scoped types get a unique name: + if s.typ.kind in {tyEnum, tyRef, tyObject} and not isTopLevel(c): + incl(s.flags, sfGenSym) + #instAllTypeBoundOp(c, n.info) diff --git a/compiler/sighashes.nim b/compiler/sighashes.nim index 156bc66d7..f6df422e6 100644 --- a/compiler/sighashes.nim +++ b/compiler/sighashes.nim @@ -151,15 +151,24 @@ proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]) = c.hashTypeSym(t.sym) else: c.hashSym(t.sym) - if {sfAnon, sfGenSym} * t.sym.flags != {}: + + var symWithFlags: PSym + template hasFlag(sym): bool = + let ret = {sfAnon, sfGenSym} * sym.flags != {} + if ret: symWithFlags = sym + ret + if hasFlag(t.sym) or (t.kind == tyObject and t.owner.kind == skType and t.owner.typ.kind == tyRef and hasFlag(t.owner)): + # for `PFoo:ObjectType`, arising from `type PFoo = ref object` # Generated object names can be identical, so we need to # disambiguate furthermore by hashing the field types and names. if t.n.len > 0: - let oldFlags = t.sym.flags - # Mild hack to prevent endless recursion. - t.sym.flags.excl {sfAnon, sfGenSym} + let oldFlags = symWithFlags.flags + # Hack to prevent endless recursion + # xxx intead, use a hash table to indicate we've already visited a type, which + # would also be more efficient. + symWithFlags.flags.excl {sfAnon, sfGenSym} hashTree(c, t.n, flags + {CoHashTypeInsideNode}) - t.sym.flags = oldFlags + symWithFlags.flags = oldFlags else: # The object has no fields: we _must_ add something here in order to # make the hash different from the one we produce by hashing only the diff --git a/tests/ccgbugs/tsamename3.nim b/tests/ccgbugs/tsamename3.nim new file mode 100644 index 000000000..a69391e5c --- /dev/null +++ b/tests/ccgbugs/tsamename3.nim @@ -0,0 +1,111 @@ +block: # bug #15526 + block: + type Foo = ref object + x1: int + let f1 = Foo(x1: 1) + block: + type Foo = ref object + x2: int + let f2 = Foo(x2: 2) + +block: # ditto + template fn() = + block: + type Foo = ref object + x1: int + let f1 = Foo(x1: 1) + doAssert f1.x1 == 1 + block: + type Foo = ref object + x2: int + let f2 = Foo(x2: 2) + doAssert f2.x2 == 2 + static: fn() + fn() + +block: # bug #17162 + template fn = + var ret: string + block: + type A = enum a0, a1, a2 + for ai in A: + ret.add $ai + block: + type A = enum b0, b1, b2, b3 + for ai in A: + ret.add $ai + doAssert ret == "a0a1a2b0b1b2b3" + + static: fn() # ok + fn() # was bug + +block: # ditto + proc fn = + var ret: string + block: + type A = enum a0, a1, a2 + for ai in A: + ret.add $ai + block: + type A = enum b0, b1, b2, b3 + for ai in A: + ret.add $ai + doAssert ret == "a0a1a2b0b1b2b3" + + static: fn() # ok + fn() # was bug + +block: # bug #5170 + block: + type Foo = object + x1: int + let f1 = Foo(x1: 1) + block: + type Foo = object + x2: int + let f2 = Foo(x2: 2) + +block: # ditto + block: + type Foo = object + bar: bool + var f1: Foo + + block: + type Foo = object + baz: int + var f2: Foo + doAssert f2.baz == 0 + + block: + template fn() = + block: + type Foo = object + x1: int + let f1 = Foo(x1: 1) + doAssert f1.x1 == 1 + block: + type Foo = object + x2: int + let f2 = Foo(x2: 2) + doAssert f2.x2 == 2 + static: fn() + fn() + +when true: # ditto, refs https://github.com/nim-lang/Nim/issues/5170#issuecomment-582712132 + type Foo1 = object # at top level + bar: bool + var f1: Foo1 + + block: + type Foo1 = object + baz: int + var f2: Foo1 + doAssert f2.baz == 0 + +block: # make sure `hashType` doesn't recurse infinitely + type + PFoo = ref object + a, b: PFoo + c: int + var a: PFoo diff --git a/tests/enum/tenum.nim b/tests/enum/tenum.nim index aa338ee2c..97fd4c68c 100644 --- a/tests/enum/tenum.nim +++ b/tests/enum/tenum.nim @@ -6,7 +6,7 @@ ABCDC foo first0second32third64 my value A1my value Bconc2valueCabc4abc -my value A0my value Bconc1valueCabc3valueC +my value A0my value Bconc1valueCabc3abc ''' """ @@ -124,7 +124,8 @@ block tnamedfields: # trick the optimizer with a variable: var x = valueD echo valueA, ord(valueA), valueB, ord(valueB), valueC, valueD, ord(valueD), x - + doAssert $x == $valueD, $x + doAssert $x == "abc", $x block toptions: From 25b4a0ab0a03f35c58bc0dbfff51a344e108bfe3 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 14 Apr 2021 00:44:07 -0700 Subject: [PATCH 0628/2472] testament --verbose: prints commands; gitignore ic artifacts (#17716) --- .gitignore | 7 +++++-- testament/testament.nim | 14 +++++++++++--- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index e71fdf352..a22a4cd03 100644 --- a/.gitignore +++ b/.gitignore @@ -72,14 +72,17 @@ testament.db .*/ ~* -# testament cruft; TODO: generate these in a gitignore'd dir in the first place. +# testament cruft; TODO: generate these in a gitignore'd dir (./build) in the first place. testresults/ test.txt /test.ini tweeter.db tweeter_test.db -megatest.nim + +/tests/megatest.nim +/tests/ic/*_temp.nim + /outputExpected.txt /outputGotten.txt diff --git a/testament/testament.nim b/testament/testament.nim index 462c0a57c..ee7f67484 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -21,6 +21,11 @@ from lib/stdtest/specialpaths import splitTestFile var useColors = true var backendLogging = true var simulate = false +var optVerbose = false + +proc verboseCmd(cmd: string) = + if optVerbose: + echo "executing: ", cmd const failString* = "FAIL: " # ensures all failures can be searched with 1 keyword in CI logs @@ -38,7 +43,8 @@ Command: Arguments: arguments are passed to the compiler Options: - --print also print results to the console + --print print results to the console + --verbose print commands (compiling and running tests) --simulate see what tests would be run but don't run them (for debugging) --failing only show failing/ignored tests --targets:"c cpp js objc" run tests for specified targets (default: all) @@ -113,6 +119,7 @@ proc execCmdEx2(command: string, args: openArray[string]; workingDir, input: str for arg in args: result.cmdLine.add ' ' result.cmdLine.add quoteShell(arg) + verboseCmd(result.cmdLine) var p = startProcess(command, workingDir = workingDir, args = args, options = {poStdErrToStdOut, poUsePath}) var outp = outputStream(p) @@ -155,6 +162,7 @@ proc callCompiler(cmdTemplate, filename, options, nimcache: string, let c = prepareTestArgs(cmdTemplate, filename, options, nimcache, target, extraOptions) result.cmd = quoteShellCommand(c) + verboseCmd(c.quoteShellCommand) var p = startProcess(command = c[0], args = c[1 .. ^1], options = {poStdErrToStdOut, poUsePath}) let outp = p.outputStream @@ -491,7 +499,6 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, valgrindOptions.add "--leak-check=yes" args = valgrindOptions & exeCmd & args exeCmd = "valgrind" - # xxx honor `testament --verbose` here var (_, buf, exitCode) = execCmdEx2(exeCmd, args, input = expected.input) # Treat all failure codes from nodejs as 1. Older versions of nodejs used # to return other codes, but for us it is sufficient to know that it's not 0. @@ -663,7 +670,8 @@ proc main() = p.next() while p.kind in {cmdLongOption, cmdShortOption}: case p.key.normalize - of "print", "verbose": optPrintResults = true + of "print": optPrintResults = true + of "verbose": optVerbose = true of "failing": optFailing = true of "pedantic": discard # deadcode refs https://github.com/nim-lang/Nim/issues/16731 of "targets": From 3b47a689cfa56d9b54cd1b92dc1b57d3e4094937 Mon Sep 17 00:00:00 2001 From: zah Date: Wed, 14 Apr 2021 13:10:01 +0300 Subject: [PATCH 0629/2472] Remove the use of usrToCell in gcMark [backport:1.2] (#17709) * Remove the use of usrToCell in gcMark [backport:1.2] Recently, we've discovered a GC crash resulting from inlining of the memory allocation procs that allowed the compiler to avoid maintaining any references to the "user pointer" on the stack. Instead, a "cell pointer" appeared there and all field accesses were performed with adjusted offsets. This interfered with the ability of the GC to mark the correct cell in the conservative stack scans which lead to premature collection of objects. More details here: https://github.com/status-im/Nim/commit/af69b3ceae16281efd45cbee4ce1bedd14282304 This commit closes another theoretical loophole that may lead to the same problem. If a short proc is accessing both the object and its reference count in a short sequence of instructions, the compiler may be enticed to reduce the number of registers being used by storing only a single pointer to the object and using offsets when reading and writing fields. A perfectly good strategy would be to store only the cell pointer, so the reference count updates can be performed without applying offsets. Accessing the fields of the object requires offsets anyway, but these can be adjusted at compile-time without any loss. Following this strategy will lead to the same problem of marking a wrong cell during the conservative stack scan, leading to premature collection. The problem is avoided by not using `usrToCell` in `gcMark`. Since the cell discovery logic can already handle interior pointers, the user pointers don't need to be adjusted for the GC to function correctly. --- lib/system/gc.nim | 6 +++--- lib/system/gc_ms.nim | 5 ++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/system/gc.nim b/lib/system/gc.nim index 039a627ba..e50e80f11 100644 --- a/lib/system/gc.nim +++ b/lib/system/gc.nim @@ -710,16 +710,16 @@ proc collectCycles(gch: var GcHeap) {.raises: [].} = proc gcMark(gch: var GcHeap, p: pointer) {.inline.} = # the addresses are not as cells on the stack, so turn them to cells: sysAssert(allocInv(gch.region), "gcMark begin") - var cell = usrToCell(p) - var c = cast[ByteAddress](cell) + var c = cast[ByteAddress](p) if c >% PageSize: # fast check: does it look like a cell? - var objStart = cast[PCell](interiorAllocatedPtr(gch.region, cell)) + var objStart = cast[PCell](interiorAllocatedPtr(gch.region, p)) if objStart != nil: # mark the cell: incRef(objStart) add(gch.decStack, objStart) when false: + let cell = usrToCell(p) if isAllocatedPtr(gch.region, cell): sysAssert false, "allocated pointer but not interior?" # mark the cell: diff --git a/lib/system/gc_ms.nim b/lib/system/gc_ms.nim index b4221f7fd..c20e93699 100644 --- a/lib/system/gc_ms.nim +++ b/lib/system/gc_ms.nim @@ -455,11 +455,10 @@ proc markGlobals(gch: var GcHeap) = proc gcMark(gch: var GcHeap, p: pointer) {.inline.} = # the addresses are not as cells on the stack, so turn them to cells: - var cell = usrToCell(p) - var c = cast[ByteAddress](cell) + var c = cast[ByteAddress](p) if c >% PageSize: # fast check: does it look like a cell? - var objStart = cast[PCell](interiorAllocatedPtr(gch.region, cell)) + var objStart = cast[PCell](interiorAllocatedPtr(gch.region, p)) if objStart != nil: mark(gch, objStart) From 67e28c07f9cdb22a423b0ab08d4868252acc8ea4 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 14 Apr 2021 16:44:37 +0200 Subject: [PATCH 0630/2472] IC: first steps towards 'nim check --def --ic:on' (#17714) * IC: first steps towards 'nim check --def --ic:on' * IC navigator: deduplicate output lines * IC navigator: progress * IC navigator: use a different nimcache entry * IC navigator: special logic for templates/macros * IC navigator: proper error messages * IC navigator: prepare for testing code; document only what currently works somewhat --- compiler/commands.nim | 3 + compiler/ic/ic.nim | 28 +++++-- compiler/ic/navigator.nim | 147 +++++++++++++++++++++++++++++++++++++ compiler/ic/packed_ast.nim | 4 +- compiler/main.nim | 15 +++- compiler/msgs.nim | 4 +- compiler/options.nim | 7 +- compiler/sem.nim | 1 + compiler/semdata.nim | 10 +++ compiler/semexprs.nim | 1 + doc/advopt.txt | 3 + 11 files changed, 211 insertions(+), 12 deletions(-) create mode 100644 compiler/ic/navigator.nim diff --git a/compiler/commands.nim b/compiler/commands.nim index f36a4f515..e918979fb 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -859,6 +859,9 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; of "usages": expectNoArg(conf, switch, arg, pass, info) conf.ideCmd = ideUse + of "defusages": + expectNoArg(conf, switch, arg, pass, info) + conf.ideCmd = ideDus of "stdout": processOnOffSwitchG(conf, {optStdout}, arg, pass, info) of "listfullpaths": diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim index e882b7a85..4613b9cb8 100644 --- a/compiler/ic/ic.nim +++ b/compiler/ic/ic.nim @@ -89,14 +89,27 @@ proc rememberConfig(c: var PackedEncoder; m: var PackedModule; config: ConfigRef #primConfigFields rem m.cfg = pc +const + debugConfigDiff = defined(debugConfigDiff) + +when debugConfigDiff: + import std / [hashes, tables, intsets, sha1, strutils, sets] + proc configIdentical(m: PackedModule; config: ConfigRef): bool = result = m.definedSymbols == definedSymbolsAsString(config) - #if not result: - # echo "A ", m.definedSymbols, " ", definedSymbolsAsString(config) + when debugConfigDiff: + if not result: + var wordsA = m.definedSymbols.split(Whitespace).toHashSet() + var wordsB = definedSymbolsAsString(config).split(Whitespace).toHashSet() + for c in wordsA - wordsB: + echo "in A but not in B ", c + for c in wordsB - wordsA: + echo "in B but not in A ", c template eq(x) = result = result and m.cfg.x == config.x - #if not result: - # echo "B ", m.cfg.x, " ", config.x + when debugConfigDiff: + if m.cfg.x != config.x: + echo "B ", m.cfg.x, " ", config.x primConfigFields eq proc rememberStartupConfig*(dest: var PackedConfig, config: ConfigRef) = @@ -124,7 +137,7 @@ proc toLitId(x: FileIndex; c: var PackedEncoder; m: var PackedModule): LitId = c.filenames[x] = result c.lastFile = x c.lastLit = result - assert result != LitId(0) + assert result != LitId(0) proc toFileIndex*(x: LitId; m: PackedModule; config: ConfigRef): FileIndex = result = msgs.fileInfoIdx(config, AbsoluteFile m.sh.strings[x]) @@ -144,6 +157,8 @@ proc initEncoder*(c: var PackedEncoder; m: var PackedModule; moduleSym: PSym; co m.bodies = newTreeFrom(m.topLevel) m.toReplay = newTreeFrom(m.topLevel) + c.lastFile = FileIndex(-10) + let thisNimFile = FileIndex c.thisModule var h = msgs.getHash(config, thisNimFile) if h.len == 0: @@ -464,6 +479,9 @@ proc storeInstantiation*(c: var PackedEncoder; m: var PackedModule; s: PSym; i: concreteTypes: t) toPackedGeneratedProcDef(i.sym, c, m) +proc storeExpansion*(c: var PackedEncoder; m: var PackedModule; info: TLineInfo; s: PSym) = + toPackedNode(newSymNode(s, info), m.bodies, c, m) + proc loadError(err: RodFileError; filename: AbsoluteFile; config: ConfigRef;) = case err of cannotOpen: diff --git a/compiler/ic/navigator.nim b/compiler/ic/navigator.nim new file mode 100644 index 000000000..ad2b29f42 --- /dev/null +++ b/compiler/ic/navigator.nim @@ -0,0 +1,147 @@ +# +# +# The Nim Compiler +# (c) Copyright 2021 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Supports the "nim check --ic:on --def --track:FILE,LINE,COL" +## IDE-like features. It uses the set of .rod files to accomplish +## its task. The set must cover a complete Nim project. + +import std / sets + +from os import nil +from std/private/miscdollars import toLocation + +import ".." / [ast, modulegraphs, msgs, options] +import packed_ast, bitabs, ic + +type + NavContext = object + g: ModuleGraph + thisModule: int32 + trackPos: PackedLineInfo + alreadyEmitted: HashSet[string] + outputSep: char # for easier testing, use short filenames and spaces instead of tabs. + +proc isTracked(current, trackPos: PackedLineInfo, tokenLen: int): bool = + if current.file == trackPos.file and current.line == trackPos.line: + let col = trackPos.col + if col >= current.col and col < current.col+tokenLen: + return true + +proc searchLocalSym(c: var NavContext; s: PackedSym; info: PackedLineInfo): bool = + result = s.name != LitId(0) and + isTracked(info, c.trackPos, c.g.packed[c.thisModule].fromDisk.sh.strings[s.name].len) + +proc searchForeignSym(c: var NavContext; s: ItemId; info: PackedLineInfo): bool = + let name = c.g.packed[s.module].fromDisk.sh.syms[s.item].name + result = name != LitId(0) and + isTracked(info, c.trackPos, c.g.packed[s.module].fromDisk.sh.strings[name].len) + +const + EmptyItemId = ItemId(module: -1'i32, item: -1'i32) + +proc search(c: var NavContext; tree: PackedTree): ItemId = + # We use the linear representation here directly: + for i in 0..high(tree.nodes): + case tree.nodes[i].kind + of nkSym: + let item = tree.nodes[i].operand + if searchLocalSym(c, c.g.packed[c.thisModule].fromDisk.sh.syms[item], tree.nodes[i].info): + return ItemId(module: c.thisModule, item: item) + of nkModuleRef: + if tree.nodes[i].info.line == c.trackPos.line and tree.nodes[i].info.file == c.trackPos.file: + let (n1, n2) = sons2(tree, NodePos i) + assert n1.kind == nkInt32Lit + assert n2.kind == nkInt32Lit + let pId = PackedItemId(module: n1.litId, item: tree.nodes[n2.int].operand) + let itemId = translateId(pId, c.g.packed, c.thisModule, c.g.config) + if searchForeignSym(c, itemId, tree.nodes[i].info): + return itemId + else: discard + return EmptyItemId + +proc isDecl(tree: PackedTree; n: NodePos): bool = + # XXX This is not correct yet. + const declarativeNodes = procDefs + {nkMacroDef, nkTemplateDef, + nkLetSection, nkVarSection, nkUsingStmt, nkConstSection, nkTypeSection, + nkIdentDefs, nkEnumTy, nkVarTuple} + result = n.int >= 0 and tree[n.int].kind in declarativeNodes + +proc usage(c: var NavContext; info: PackedLineInfo; isDecl: bool) = + var m = "" + var file = c.g.packed[c.thisModule].fromDisk.sh.strings[info.file] + if c.outputSep == ' ': + file = os.extractFilename file + toLocation(m, file, info.line.int, info.col.int + ColOffset) + if not c.alreadyEmitted.containsOrIncl(m): + echo (if isDecl: "def" else: "usage"), c.outputSep, m + +proc list(c: var NavContext; tree: PackedTree; sym: ItemId) = + for i in 0..high(tree.nodes): + case tree.nodes[i].kind + of nkSym: + let item = tree.nodes[i].operand + if sym.item == item and sym.module == c.thisModule: + usage(c, tree.nodes[i].info, isDecl(tree, parent(NodePos i))) + of nkModuleRef: + let (n1, n2) = sons2(tree, NodePos i) + assert n1.kind == nkInt32Lit + assert n2.kind == nkInt32Lit + let pId = PackedItemId(module: n1.litId, item: tree.nodes[n2.int].operand) + let itemId = translateId(pId, c.g.packed, c.thisModule, c.g.config) + if itemId.item == sym.item and sym.module == itemId.module: + usage(c, tree.nodes[i].info, isDecl(tree, parent(NodePos i))) + else: discard + +proc nav(g: ModuleGraph) = + # translate the track position to a packed position: + let unpacked = g.config.m.trackPos + let mid = unpacked.fileIndex + let fileId = g.packed[int32 mid].fromDisk.sh.strings.getKeyId(toFullPath(g.config, mid)) + + if fileId == LitId(0): + internalError(g.config, unpacked, "cannot find a valid file ID") + return + + var c = NavContext( + g: g, + thisModule: int32 mid, + trackPos: PackedLineInfo(line: unpacked.line, col: unpacked.col, file: fileId), + outputSep: if isDefined(g.config, "nimIcNavigatorTests"): ' ' else: '\t' + ) + var symId = search(c, g.packed[int32 mid].fromDisk.topLevel) + if symId == EmptyItemId: + symId = search(c, g.packed[int32 mid].fromDisk.bodies) + + if symId == EmptyItemId: + localError(g.config, unpacked, "no symbol at this position") + return + + for i in 0..high(g.packed): + # case statement here to enforce exhaustive checks. + case g.packed[i].status + of undefined: + discard "nothing to do" + of loading: + assert false, "cannot check integrity: Module still loading" + of stored, storing, outdated, loaded: + c.thisModule = int32 i + list(c, g.packed[i].fromDisk.topLevel, symId) + list(c, g.packed[i].fromDisk.bodies, symId) + +proc navDefinition*(g: ModuleGraph) = nav(g) +proc navUsages*(g: ModuleGraph) = nav(g) +proc navDefusages*(g: ModuleGraph) = nav(g) + +proc writeRodFiles*(g: ModuleGraph) = + for i in 0..high(g.packed): + case g.packed[i].status + of undefined, loading, stored, loaded: + discard "nothing to do" + of storing, outdated: + closeRodFile(g, g.packed[i].module) diff --git a/compiler/ic/packed_ast.nim b/compiler/ic/packed_ast.nim index 1075664cb..5cc3b1476 100644 --- a/compiler/ic/packed_ast.nim +++ b/compiler/ic/packed_ast.nim @@ -258,9 +258,9 @@ iterator sonsWithoutLast2*(tree: PackedTree; n: NodePos): NodePos = proc parentImpl(tree: PackedTree; n: NodePos): NodePos = # finding the parent of a node is rather easy: var pos = n.int - 1 - while pos >= 0 and isAtom(tree, pos) or (pos + tree.nodes[pos].operand - 1 < n.int): + while pos >= 0 and (isAtom(tree, pos) or (pos + tree.nodes[pos].operand - 1 < n.int)): dec pos - assert pos >= 0, "node has no parent" + #assert pos >= 0, "node has no parent" result = NodePos(pos) template parent*(n: NodePos): NodePos = parentImpl(tree, n) diff --git a/compiler/main.nim b/compiler/main.nim index 3c5018546..475ba7773 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -21,7 +21,7 @@ import modules, modulegraphs, tables, lineinfos, pathutils, vmprofiler -import ic / [cbackend, integrity] +import ic / [cbackend, integrity, navigator] from ic / ic import rodViewer when not defined(leanCompiler): @@ -54,11 +54,20 @@ proc commandGenDepend(graph: ModuleGraph) = ' ' & changeFileExt(project, "dot").string) proc commandCheck(graph: ModuleGraph) = - graph.config.setErrorMaxHighMaybe - defineSymbol(graph.config.symbols, "nimcheck") + let conf = graph.config + conf.setErrorMaxHighMaybe + defineSymbol(conf.symbols, "nimcheck") semanticPasses(graph) # use an empty backend for semantic checking only compileProject(graph) + if conf.symbolFiles != disabledSf: + case conf.ideCmd + of ideDef: navDefinition(graph) + of ideUse: navUsages(graph) + of ideDus: navDefusages(graph) + else: discard + writeRodFiles(graph) + when not defined(leanCompiler): proc commandDoc2(graph: ModuleGraph; json: bool) = handleDocOutputOptions graph.config diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 24855ae18..452b1cd89 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -418,7 +418,9 @@ proc handleError(conf: ConfigRef; msg: TMsgKind, eh: TErrorHandling, s: string) inc(conf.errorCounter) conf.exitcode = 1'i8 if conf.errorCounter >= conf.errorMax: - quit(conf, msg) + # only really quit when we're not in the new 'nim check --def' mode: + if conf.ideCmd == ideNone: + quit(conf, msg) elif eh == doAbort and conf.cmd != cmdIdeTools: quit(conf, msg) elif eh == doRaise: diff --git a/compiler/options.nim b/compiler/options.nim index 7408ce675..9eeffb4fb 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -685,6 +685,11 @@ proc getOsCacheDir(): string = result = getHomeDir() / genSubDir.string proc getNimcacheDir*(conf: ConfigRef): AbsoluteDir = + proc nimcacheSuffix(conf: ConfigRef): string = + if conf.cmd == cmdCheck: "_check" + elif isDefined(conf, "release") or isDefined(conf, "danger"): "_r" + else: "_d" + # XXX projectName should always be without a file extension! result = if not conf.nimcacheDir.isEmpty: conf.nimcacheDir @@ -692,7 +697,7 @@ proc getNimcacheDir*(conf: ConfigRef): AbsoluteDir = conf.projectPath / genSubDir else: AbsoluteDir(getOsCacheDir() / splitFile(conf.projectName).name & - (if isDefined(conf, "release") or isDefined(conf, "danger"): "_r" else: "_d")) + nimcacheSuffix(conf)) proc pathSubs*(conf: ConfigRef; p, config: string): string = let home = removeTrailingDirSep(os.getHomeDir()) diff --git a/compiler/sem.nim b/compiler/sem.nim index 6b3f0c80d..ccc634b4d 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -452,6 +452,7 @@ const proc semMacroExpr(c: PContext, n, nOrig: PNode, sym: PSym, flags: TExprFlags = {}): PNode = + rememberExpansion(c, nOrig.info, sym) pushInfoContext(c.config, nOrig.info, sym.detailedInfo) let info = getCallLineInfo(n) diff --git a/compiler/semdata.nim b/compiler/semdata.nim index daf1265e8..6b0dcb0e2 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -570,3 +570,13 @@ proc sealRodFile*(c: PContext) = if m == c.module: addPragmaComputation(c, n) c.idgen.sealed = true # no further additions are allowed + +proc rememberExpansion*(c: PContext; info: TLineInfo; expandedSym: PSym) = + ## Templates and macros are very special in Nim; these have + ## inlining semantics so after semantic checking they leave no trace + ## in the sem'checked AST. This is very bad for IDE-like tooling + ## ("find all usages of this template" would not work). We need special + ## logic to remember macro/template expansions. This is done here and + ## delegated to the "rod" file mechanism. + if c.config.symbolFiles != disabledSf: + storeExpansion(c.encoder, c.packedRepr, info, expandedSym) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 256c008c9..cbd4318f7 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -27,6 +27,7 @@ const proc semTemplateExpr(c: PContext, n: PNode, s: PSym, flags: TExprFlags = {}): PNode = + rememberExpansion(c, n.info, s) let info = getCallLineInfo(n) markUsed(c, info, s) onUse(info, s) diff --git a/doc/advopt.txt b/doc/advopt.txt index 4842d090e..c7b63d50f 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -15,6 +15,7 @@ Advanced commands: //dump dump all defined conditionals and search paths see also: --dump.format:json (useful with: `| jq`) //check checks the project for syntax and semantic + (can be combined with --track and --defusages) Runtime checks (see -x): --objChecks:on|off turn obj conversion checks on|off @@ -27,6 +28,8 @@ Runtime checks (see -x): --infChecks:on|off turn Inf checks on|off Advanced options: + --track:FILE,LINE,COL set the tracking position for (--defusages) + --defusages find the definition and all usages of a symbol -o:FILE, --out:FILE set the output filename --outdir:DIR set the path where the output file will be written --usenimcache will use `outdir=$$nimcache`, whichever it resolves From 58f44c8b4e6486789a08555dccee6492d32575d4 Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 14 Apr 2021 22:45:22 +0800 Subject: [PATCH 0631/2472] Update channels.nim (#17717) --- lib/std/channels.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/std/channels.nim b/lib/std/channels.nim index 3bbc8a6b6..49def0913 100644 --- a/lib/std/channels.nim +++ b/lib/std/channels.nim @@ -307,8 +307,8 @@ proc sendUnbufferedMpmc(chan: ChannelRaw, data: sink pointer, size: int, nonBloc chan.head = 1 - release(chan.headLock) signal(chan.notEmptyCond) + release(chan.headLock) result = true proc sendMpmc(chan: ChannelRaw, data: sink pointer, size: int, nonBlocking: bool): bool = @@ -343,8 +343,8 @@ proc sendMpmc(chan: ChannelRaw, data: sink pointer, size: int, nonBlocking: bool if chan.tail == 2 * chan.size: chan.tail = 0 - release(chan.tailLock) signal(chan.notEmptyCond) + release(chan.tailLock) result = true proc recvUnbufferedMpmc(chan: ChannelRaw, data: pointer, size: int, nonBlocking: bool): bool = @@ -368,8 +368,8 @@ proc recvUnbufferedMpmc(chan: ChannelRaw, data: pointer, size: int, nonBlocking: chan.head = 0 - release(chan.headLock) signal(chan.notFullCond) + release(chan.headLock) result = true proc recvMpmc(chan: ChannelRaw, data: pointer, size: int, nonBlocking: bool): bool = @@ -404,8 +404,8 @@ proc recvMpmc(chan: ChannelRaw, data: pointer, size: int, nonBlocking: bool): bo if chan.head == 2 * chan.size: chan.head = 0 - release(chan.headLock) signal(chan.notFullCond) + release(chan.headLock) result = true proc channelCloseMpmc(chan: ChannelRaw): bool = From 44657b78c40475e60187b48157bae41ea5371252 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 15 Apr 2021 01:14:14 +0800 Subject: [PATCH 0632/2472] callback cannot be nil (#17718) `Task.callback` cannot be nil, we need to raise it at debug and release mode Situations: - if users create a Task object without using `toTask` and invoke the Task - if users already move the Task and invoke the Task --- lib/std/tasks.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/std/tasks.nim b/lib/std/tasks.nim index 6b7f86ce4..aed36f985 100644 --- a/lib/std/tasks.nim +++ b/lib/std/tasks.nim @@ -75,6 +75,7 @@ proc `=destroy`*(t: var Task) {.inline.} = proc invoke*(task: Task) {.inline.} = ## Invokes the `task`. + assert task.callback != nil task.callback(task.args) template checkIsolate(scratchAssignList: seq[NimNode], procParam, scratchDotExpr: NimNode) = From 56c37759d6183aa32d474e669de618ee9c4f7633 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20D=C3=B6ring?= Date: Wed, 14 Apr 2021 20:42:09 +0200 Subject: [PATCH 0633/2472] getCustomPragma is split up in more usable chunks (#11526) * getCustomPragma is split up in more usable chunks * changelog entry * fix for style checks * shitty typedesc special casing * Add since annotation and remove typedesc comments * Fix typo * Revert since annotation because it breaks bootstrapping * Export getCustomPragmaNode conditionally * Reduce code duplication * Update since * Update lib/core/macros.nim * Apply suggestions from code review Co-authored-by: Clyybber Co-authored-by: Andreas Rumpf --- changelog.md | 1 + lib/core/macros.nim | 254 ++++++++++++++++++++----------- tests/pragmas/tcustom_pragma.nim | 30 +++- 3 files changed, 191 insertions(+), 94 deletions(-) diff --git a/changelog.md b/changelog.md index 2f1d2d816..ec1690e06 100644 --- a/changelog.md +++ b/changelog.md @@ -32,6 +32,7 @@ implementations. Old behavior can be obtained with `-d:nimLegacyParseQueryStrict`. `cgi.decodeData` which uses the same underlying code is also updated the same way. +- Custom pragma values have now an API for use in macros. - In `std/os`, `getHomeDir`, `expandTilde`, `getTempDir`, `getConfigDir` now do not include trailing `DirSep`, unless `-d:nimLegacyHomeDir` is specified (for a transition period). diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 49c9a999c..ebe5a7187 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -1474,80 +1474,153 @@ macro expandMacros*(body: typed): untyped = echo body.toStrLit result = body -proc customPragmaNode(n: NimNode): NimNode = - expectKind(n, {nnkSym, nnkDotExpr, nnkBracketExpr, nnkTypeOfExpr, nnkCheckedFieldExpr}) - let - typ = n.getTypeInst() +proc findPragmaExprForFieldSym(arg, fieldSym: NimNode): NimNode = + case arg.kind + of nnkRecList, nnkRecCase: + for it in arg.children: + result = findPragmaExprForFieldSym(it, fieldSym) + if result != nil: + return + of nnkOfBranch: + return findPragmaExprForFieldSym(arg[1], fieldSym) + of nnkElse: + return findPragmaExprForFieldSym(arg[0], fieldSym) + of nnkIdentDefs: + for i in 0 ..< arg.len-2: + let child = arg[i] + result = findPragmaExprForFieldSym(child, fieldSym) + if result != nil: + return + of nnkAccQuoted, nnkIdent, nnkSym, nnkPostfix: + return nil + of nnkPragmaExpr: + var ident = arg[0] + if ident.kind == nnkPostfix: ident = ident[1] + if ident.kind == nnkAccQuoted: ident = ident[0] + if eqIdent(ident, fieldSym): + return arg[1] + else: + error("illegal arg: ", arg) - if typ.kind == nnkBracketExpr and typ.len > 1 and typ[1].kind == nnkProcTy: - return typ[1][1] - elif typ.typeKind == ntyTypeDesc: - let impl = typ[1].getImpl() - if impl[0].kind == nnkPragmaExpr: - return impl[0][1] +proc getPragmaByName(pragmaExpr: NimNode, name: string): NimNode = + if pragmaExpr.kind == nnkPragma: + for it in pragmaExpr: + if it.kind in nnkPragmaCallKinds: + if eqIdent(it[0], name): + return it + elif it.kind == nnkSym: + if eqIdent(it, name): + return it + +proc getCustomPragmaNodeFromProcSym(sym: NimNode, name: string): NimNode = + sym.expectKind nnkSym + if sym.symKind != nskProc: error("expected proc sym", sym) + + let impl = sym.getImpl + expectKind(impl, nnkProcDef) + result = getPragmaByName(impl[4], name) + +proc getCustomPragmaNodeFromObjFieldSym(sym: NimNode, name: string): NimNode = + sym.expectKind nnkSym + if sym.symKind != nskField: error("expected field sym", sym) + + # note this is not ``getTypeImpl``, because the result of + # ``getTypeImpl`` is cleaned up of any pragma expressions. + let impl = sym.owner.getImpl + impl.expectKind nnkTypeDef + + let objectTy = if impl[2].kind == nnkRefTy: impl[2][0] + else: impl[2] + + # only works on object types + objectTy.expectKind nnkObjectTy + + let recList = objectTy[2] + recList.expectKind nnkRecList + result = getPragmaByName(findPragmaExprForFieldSym(recList, sym), name) + +proc getCustomPragmaNodeFromTypeSym(sym: NimNode, name: string): NimNode = + sym.expectKind nnkSym + if sym.symKind != nskType: error("expected type sym", sym) + let impl = sym.getImpl + if impl.len > 0: + impl.expectKind nnkTypeDef + let pragmaExpr = impl[0] + if pragmaExpr.kind == nnkPragmaExpr: + result = getPragmaByName(pragmaExpr[1], name) + +proc getCustomPragmaNodeFromVarLetSym(sym: NimNode, name: string): NimNode = + sym.expectKind nnkSym + if sym.symKind notin {nskVar, nskLet}: error("expected var/let sym", sym) + let impl = sym.getImpl + impl.expectKind nnkIdentDefs + impl.expectLen 3 + let pragmaExpr = impl[0] + if pragmaExpr.kind == nnkPragmaExpr: + result = getPragmaByName(pragmaExpr[1], name) + +proc getCustomPragmaNode(sym: NimNode, name: string): NimNode = + sym.expectKind nnkSym + case sym.symKind + of nskField: + result = getCustomPragmaNodeFromObjFieldSym(sym, name) + of nskProc: + result = getCustomPragmaNodeFromProcSym(sym, name) + of nskType: + result = getCustomPragmaNodeFromTypeSym(sym, name) + of nskParam: + # When a typedesc parameter is passed to the macro, it will be of nskParam. + let typeInst = getTypeInst(sym) + if typeInst.kind == nnkBracketExpr and eqIdent(typeInst[0], "typeDesc"): + result = getCustomPragmaNodeFromTypeSym(typeInst[1], name) else: - return impl[0] # handle types which don't have macro at all + error("illegal sym kind for argument: " & $sym.symKind, sym) + of nskVar, nskLet: + # I think it is a bad idea to fall back to the typeSym. The API + # explicity requests a var/let symbol, not a type symbol. + result = + getCustomPragmaNodeFromVarLetSym(sym, name) or + getCustomPragmaNodeFromTypeSym(sym.getTypeInst, name) + else: + error("illegal sym kind for argument: " & $sym.symKind, sym) - if n.kind == nnkSym: # either an variable or a proc - let impl = n.getImpl() - if impl.kind in RoutineNodes: - return impl.pragma - elif impl.kind == nnkIdentDefs and impl[0].kind == nnkPragmaExpr: - return impl[0][1] +since (1, 5): + export getCustomPragmaNode + +proc hasCustomPragma*(n: NimNode, name: string): bool = + n.expectKind nnkSym + let pragmaNode = getCustomPragmaNode(n, name) + result = pragmaNode != nil + +proc getCustomPragmaNodeSmart(n: NimNode, name: string): NimNode = + case n.kind + of nnkDotExpr: + result = getCustomPragmaNode(n[1], name) + of nnkCheckedFieldExpr: + expectKind n[0], nnkDotExpr + result = getCustomPragmaNode(n[0][1], name) + of nnkSym: + result = getCustomPragmaNode(n, name) + of nnkTypeOfExpr: + var typeSym = n.getTypeInst + while typeSym.kind == nnkBracketExpr and typeSym[0].eqIdent "typeDesc": + typeSym = typeSym[1] + case typeSym.kind: + of nnkSym: + result = getCustomPragmaNode(typeSym, name) + of nnkProcTy: + # It is a bad idea to support this. The annotation can't be part + # of a symbol. + let pragmaExpr = typeSym[1] + result = getPragmaByName(pragmaExpr, name) else: - let timpl = typ.getImpl() - if timpl.len>0 and timpl[0].len>1: - return timpl[0][1] - else: - return timpl + typeSym.expectKind nnkSym + of nnkBracketExpr: + result = nil #false + else: + n.expectKind({nnkDotExpr, nnkCheckedFieldExpr, nnkSym, nnkTypeOfExpr}) - if n.kind in {nnkDotExpr, nnkCheckedFieldExpr}: - let name = $(if n.kind == nnkCheckedFieldExpr: n[0][1] else: n[1]) - let typInst = getTypeInst(if n.kind == nnkCheckedFieldExpr or n[0].kind == nnkHiddenDeref: n[0][0] else: n[0]) - var typDef = getImpl(if typInst.kind == nnkVarTy: typInst[0] else: typInst) - while typDef != nil: - typDef.expectKind(nnkTypeDef) - let typ = typDef[2] - typ.expectKind({nnkRefTy, nnkPtrTy, nnkObjectTy}) - let isRef = typ.kind in {nnkRefTy, nnkPtrTy} - if isRef and typ[0].kind in {nnkSym, nnkBracketExpr}: # defines ref type for another object(e.g. X = ref X) - typDef = getImpl(typ[0]) - else: # object definition, maybe an object directly defined as a ref type - let - obj = (if isRef: typ[0] else: typ) - var identDefsStack = newSeq[NimNode](obj[2].len) - for i in 0.. 0: - var identDefs = identDefsStack.pop() - if identDefs.kind == nnkRecCase: - identDefsStack.add(identDefs[0]) - for i in 1.. 0 and p[0].kind == nnkSym and p[0] == cp): - return newLit(true) - return newLit(false) + result = newLit(getCustomPragmaNodeSmart(n, $cp) != nil) macro getCustomPragmaVal*(n: typed, cp: typed{nkSym}): untyped = ## Expands to value of custom pragma `cp` of expression `n` which is expected @@ -1586,22 +1654,26 @@ macro getCustomPragmaVal*(n: typed, cp: typed{nkSym}): untyped = ## assert(o.myField.getCustomPragmaVal(serializationKey) == "mf") ## assert(o.getCustomPragmaVal(serializationKey) == "mo") ## assert(MyObj.getCustomPragmaVal(serializationKey) == "mo") - result = nil - let pragmaNode = customPragmaNode(n) - for p in pragmaNode: - if p.kind in nnkPragmaCallKinds and p.len > 0 and p[0].kind == nnkSym and p[0] == cp: - if p.len == 2: - result = p[1] - else: - let def = p[0].getImpl[3] - result = newTree(nnkPar) - for i in 1 ..< def.len: - let key = def[i][0] - let val = p[i] - result.add newTree(nnkExprColonExpr, key, val) - break - if result.kind == nnkEmpty: - error(n.repr & " doesn't have a pragma named " & cp.repr()) # returning an empty node results in most cases in a cryptic error, + n.expectKind({nnkDotExpr, nnkCheckedFieldExpr, nnkSym, nnkTypeOfExpr}) + let pragmaNode = getCustomPragmaNodeSmart(n, $cp) + + case pragmaNode.kind + of nnkPragmaCallKinds: + assert pragmaNode[0] == cp + if pragmaNode.len == 2: + result = pragmaNode[1] + else: + # create a named tuple expression for pragmas with multiple arguments + let def = pragmaNode[0].getImpl[3] + result = newTree(nnkPar) + for i in 1 ..< def.len: + let key = def[i][0] + let val = pragmaNode[i] + result.add nnkExprColonExpr.newTree(key, val) + of nnkSym: + error("The named pragma " & cp.repr & " in " & n.repr & " has no arguments and therefore no value.") + else: + error(n.repr & " doesn't have a pragma named " & cp.repr, n) macro unpackVarargs*(callee: untyped; args: varargs[untyped]): untyped = result = newCall(callee) diff --git a/tests/pragmas/tcustom_pragma.nim b/tests/pragmas/tcustom_pragma.nim index e9dac753d..5342b78e6 100644 --- a/tests/pragmas/tcustom_pragma.nim +++ b/tests/pragmas/tcustom_pragma.nim @@ -156,13 +156,20 @@ block: proc generic_proc[T]() = doAssert Annotated.hasCustomPragma(simpleAttr) - #-------------------------------------------------------------------------- # Pragma on proc type -let a: proc(x: int) {.defaultValue(5).} = nil +type + MyAnnotatedProcType {.defaultValue(4).} = proc(x: int) + +let a {.defaultValue(4).}: proc(x: int) = nil +var b: MyAnnotatedProcType = nil +var c: proc(x: int): void {.defaultValue(5).} = nil static: - doAssert hasCustomPragma(a.type, defaultValue) + doAssert hasCustomPragma(a, defaultValue) + doAssert hasCustomPragma(MyAnnotatedProcType, defaultValue) + doAssert hasCustomPragma(b, defaultValue) + doAssert hasCustomPragma(typeof(c), defaultValue) # bug #8371 template thingy {.pragma.} @@ -378,3 +385,20 @@ block: b {.world.}: int discard Hello(a: 1.0, b: 12) + +# issue #11511 +block: + template myAttr {.pragma.} + + type TObj = object + a {.myAttr.}: int + + macro hasMyAttr(t: typedesc): untyped = + let objTy = t.getType[1].getType + let recList = objTy[2] + let sym = recList[0] + assert sym.kind == nnkSym and sym.eqIdent("a") + let hasAttr = sym.hasCustomPragma("myAttr") + newLit(hasAttr) + + doAssert hasMyAttr(TObj) From d6242d7fe19849d6cc7079f0b006065fb2a8a19e Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 14 Apr 2021 12:22:22 -0700 Subject: [PATCH 0634/2472] simplify asyncfutures, asyncmacro (#17633) --- lib/pure/asyncfutures.nim | 20 ++++---- lib/pure/asyncmacro.nim | 83 +++++++++----------------------- tests/async/tasyncintemplate.nim | 28 ++++++++--- tests/errmsgs/tgcsafety.nim | 3 +- tests/pragmas/tcustom_pragma.nim | 20 +++----- 5 files changed, 62 insertions(+), 92 deletions(-) diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index 3c36c3453..fd62b2703 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -189,24 +189,22 @@ proc add(callbacks: var CallbackList, function: CallbackFunc) = last = last.next last.next = newCallback -proc complete*[T](future: Future[T], val: T) = - ## Completes `future` with value `val`. +proc completeImpl[T, U](future: Future[T], val: U, isVoid: static bool) = #assert(not future.finished, "Future already finished, cannot finish twice.") checkFinished(future) assert(future.error == nil) - future.value = val + when not isVoid: + future.value = val future.finished = true future.callbacks.call() when isFutureLoggingEnabled: logFutureFinish(future) -proc complete*(future: Future[void]) = - ## Completes a void `future`. - #assert(not future.finished, "Future already finished, cannot finish twice.") - checkFinished(future) - assert(future.error == nil) - future.finished = true - future.callbacks.call() - when isFutureLoggingEnabled: logFutureFinish(future) +proc complete*[T](future: Future[T], val: T) = + ## Completes `future` with value `val`. + completeImpl(future, val, false) + +proc complete*(future: Future[void], val = Future[void].default) = + completeImpl(future, (), true) proc complete*[T](future: FutureVar[T]) = ## Completes a `FutureVar`. diff --git a/lib/pure/asyncmacro.nim b/lib/pure/asyncmacro.nim index aaa5d9acd..ce538913f 100644 --- a/lib/pure/asyncmacro.nim +++ b/lib/pure/asyncmacro.nim @@ -11,7 +11,6 @@ import macros, strutils, asyncfutures - # TODO: Ref https://github.com/nim-lang/Nim/issues/5617 # TODO: Add more line infos proc newCallWithLineInfo(fromNode: NimNode; theProc: NimNode, args: varargs[NimNode]): NimNode = @@ -65,10 +64,7 @@ proc createFutureVarCompletions(futureVarIdents: seq[NimNode], fromNode: NimNode ) ) -proc processBody(node, retFutureSym: NimNode, - subTypeIsVoid: bool, - futureVarIdents: seq[NimNode]): NimNode = - #echo(node.treeRepr) +proc processBody(node, retFutureSym: NimNode, futureVarIdents: seq[NimNode]): NimNode = result = node case node.kind of nnkReturnStmt: @@ -78,14 +74,9 @@ proc processBody(node, retFutureSym: NimNode, result.add createFutureVarCompletions(futureVarIdents, node) if node[0].kind == nnkEmpty: - if not subTypeIsVoid: - result.add newCall(newIdentNode("complete"), retFutureSym, - newIdentNode("result")) - else: - result.add newCall(newIdentNode("complete"), retFutureSym) + result.add newCall(newIdentNode("complete"), retFutureSym, newIdentNode("result")) else: - let x = node[0].processBody(retFutureSym, subTypeIsVoid, - futureVarIdents) + let x = node[0].processBody(retFutureSym, futureVarIdents) if x.kind == nnkYieldStmt: result.add x else: result.add newCall(newIdentNode("complete"), retFutureSym, x) @@ -98,8 +89,7 @@ proc processBody(node, retFutureSym: NimNode, else: discard for i in 0 ..< result.len: - result[i] = processBody(result[i], retFutureSym, subTypeIsVoid, - futureVarIdents) + result[i] = processBody(result[i], retFutureSym, futureVarIdents) # echo result.repr @@ -146,7 +136,7 @@ proc asyncSingleProc(prc: NimNode): NimNode = if prc.kind == nnkProcTy: result = prc if prc[0][0].kind == nnkEmpty: - result[0][0] = parseExpr("Future[void]") + result[0][0] = quote do: Future[void] return result if prc.kind notin {nnkProcDef, nnkLambda, nnkMethodDef, nnkDo}: @@ -180,12 +170,7 @@ proc asyncSingleProc(prc: NimNode): NimNode = else: verifyReturnType(repr(returnType), returnType) - let subtypeIsVoid = returnType.kind == nnkEmpty or - (baseType.kind in {nnkIdent, nnkSym} and - baseType.eqIdent("void")) - let futureVarIdents = getFutureVarIdents(prc.params) - var outerProcBody = newNimNode(nnkStmtList, prc.body) # Extract the documentation comment from the original procedure declaration. @@ -213,42 +198,30 @@ proc asyncSingleProc(prc: NimNode): NimNode = # -> # -> complete(retFuture, result) var iteratorNameSym = genSym(nskIterator, $prcName & "Iter") - var procBody = prc.body.processBody(retFutureSym, subtypeIsVoid, - futureVarIdents) + var procBody = prc.body.processBody(retFutureSym, futureVarIdents) # don't do anything with forward bodies (empty) if procBody.kind != nnkEmpty: # fix #13899, defer should not escape its original scope procBody = newStmtList(newTree(nnkBlockStmt, newEmptyNode(), procBody)) - procBody.add(createFutureVarCompletions(futureVarIdents, nil)) + let resultIdent = ident"result" + procBody.insert(0): quote do: + {.push warning[resultshadowed]: off.} + when `subRetType` isnot void: + var `resultIdent`: `subRetType` + else: + var `resultIdent`: Future[void] + {.pop.} + procBody.add quote do: + complete(`retFutureSym`, `resultIdent`) - if not subtypeIsVoid: - procBody.insert(0, newNimNode(nnkPragma).add(newIdentNode("push"), - newNimNode(nnkExprColonExpr).add(newNimNode(nnkBracketExpr).add( - newIdentNode("warning"), newIdentNode("resultshadowed")), - newIdentNode("off")))) # -> {.push warning[resultshadowed]: off.} - - procBody.insert(1, newNimNode(nnkVarSection, prc.body).add( - newIdentDefs(newIdentNode("result"), baseType))) # -> var result: T - - procBody.insert(2, newNimNode(nnkPragma).add( - newIdentNode("pop"))) # -> {.pop.}) - - procBody.add( - newCall(newIdentNode("complete"), - retFutureSym, newIdentNode("result"))) # -> complete(retFuture, result) - else: - # -> complete(retFuture) - procBody.add(newCall(newIdentNode("complete"), retFutureSym)) - - var closureIterator = newProc(iteratorNameSym, [parseExpr("owned(FutureBase)")], + var closureIterator = newProc(iteratorNameSym, [quote do: owned(FutureBase)], procBody, nnkIteratorDef) closureIterator.pragma = newNimNode(nnkPragma, lineInfoFrom = prc.body) closureIterator.addPragma(newIdentNode("closure")) # If proc has an explicit gcsafe pragma, we add it to iterator as well. - if prc.pragma.findChild(it.kind in {nnkSym, nnkIdent} and $it == - "gcsafe") != nil: + if prc.pragma.findChild(it.kind in {nnkSym, nnkIdent} and $it == "gcsafe") != nil: closureIterator.addPragma(newIdentNode("gcsafe")) outerProcBody.add(closureIterator) @@ -266,12 +239,10 @@ proc asyncSingleProc(prc: NimNode): NimNode = outerProcBody.add newNimNode(nnkReturnStmt, prc.body[^1]).add(retFutureSym) result = prc - - if subtypeIsVoid: - # Add discardable pragma. - if returnType.kind == nnkEmpty: - # Add Future[void] - result.params[0] = parseExpr("owned(Future[void])") + # Add discardable pragma. + if returnType.kind == nnkEmpty: + # xxx consider removing `owned`? it's inconsistent with non-void case + result.params[0] = quote do: owned(Future[void]) # based on the yglukhov's patch to chronos: https://github.com/status-im/nim-chronos/pull/47 if procBody.kind != nnkEmpty: @@ -279,10 +250,6 @@ proc asyncSingleProc(prc: NimNode): NimNode = `outerProcBody` result.body = body2 - #echo(treeRepr(result)) - #if prcName == "recvLineInto": - # echo(toStrLit(result)) - macro async*(prc: untyped): untyped = ## Macro which processes async procedures into the appropriate ## iterators and yield statements. @@ -352,9 +319,3 @@ macro multisync*(prc: untyped): untyped = result = newStmtList() result.add(asyncSingleProc(asyncPrc)) result.add(sync) - # echo result.repr - -# overload for await as a fallback handler, based on the yglukhov's patch to chronos: https://github.com/status-im/nim-chronos/pull/47 -# template await*(f: typed): untyped = - # static: - # error "await only available within {.async.}" diff --git a/tests/async/tasyncintemplate.nim b/tests/async/tasyncintemplate.nim index 0f810b5be..4bddb1d18 100644 --- a/tests/async/tasyncintemplate.nim +++ b/tests/async/tasyncintemplate.nim @@ -1,6 +1,8 @@ discard """ output: ''' 42 +43 +43 1 2 3 @@ -8,16 +10,30 @@ discard """ ''' """ +# xxx move to tests/async/tasyncintemplate.nim import asyncdispatch -# bug #16159 -template foo() = - proc temp(): Future[int] {.async.} = return 42 - proc tempVoid(): Future[void] {.async.} = echo await temp() +block: # bug #16159 + template foo() = + proc temp(): Future[int] {.async.} = return 42 + proc tempVoid(): Future[void] {.async.} = echo await temp() + foo() + waitFor tempVoid() -foo() -waitFor tempVoid() +block: # aliasing `void` + template foo() = + type Foo = void + proc temp(): Future[int] {.async.} = return 43 + proc tempVoid(): Future[Foo] {.async.} = echo await temp() + proc tempVoid2() {.async.} = echo await temp() + foo() + waitFor tempVoid() + waitFor tempVoid2() +block: # sanity check + template foo() = + proc bad(): int {.async.} = discard + doAssert not compiles(bad()) block: # bug #16786 block: diff --git a/tests/errmsgs/tgcsafety.nim b/tests/errmsgs/tgcsafety.nim index 09ef92e75..563ead2e0 100644 --- a/tests/errmsgs/tgcsafety.nim +++ b/tests/errmsgs/tgcsafety.nim @@ -6,7 +6,8 @@ tgcsafety.nim(30, 18) Error: type mismatch: got .} diff --git a/tests/pragmas/tcustom_pragma.nim b/tests/pragmas/tcustom_pragma.nim index 5342b78e6..55761d107 100644 --- a/tests/pragmas/tcustom_pragma.nim +++ b/tests/pragmas/tcustom_pragma.nim @@ -262,6 +262,10 @@ block: doAssert input.treeRepr & "\n" == expectedRepr return input + macro expectedAstRepr(expectedRepr: static[string], input: untyped): untyped = + doAssert input.repr == expectedRepr + return input + const procTypeAst = """ ProcTy FormalParams @@ -280,20 +284,10 @@ ProcTy static: doAssert Foo is proc(x: int): Future[void] const asyncProcTypeAst = """ -ProcTy - FormalParams - BracketExpr - Ident "Future" - Ident "void" - IdentDefs - Ident "s" - Ident "string" - Empty - Pragma -""" - +proc (s: string): Future[void] {..}""" + # using expectedAst would show `OpenSymChoice` for Future[void], which is fragile. type - Bar = proc (s: string) {.async, expectedAst(asyncProcTypeAst).} + Bar = proc (s: string) {.async, expectedAstRepr(asyncProcTypeAst).} static: doAssert Bar is proc(x: string): Future[void] From eff9d75379d6f9f4a52c2040425dad1f408ede37 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 14 Apr 2021 22:35:18 +0200 Subject: [PATCH 0635/2472] IC: rodfiles: no floats (#17720) --- compiler/ic/ic.nim | 19 ++++++++----------- compiler/ic/integrity.nim | 6 ++---- compiler/ic/packed_ast.nim | 11 +++++++---- compiler/ic/rodfiles.nim | 3 +-- 4 files changed, 18 insertions(+), 21 deletions(-) diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim index 4613b9cb8..4f63ce03a 100644 --- a/compiler/ic/ic.nim +++ b/compiler/ic/ic.nim @@ -224,7 +224,7 @@ proc toLitId(x: string; m: var PackedModule): LitId = proc toLitId(x: BiggestInt; m: var PackedModule): LitId = ## store an integer as a literal - result = getOrIncl(m.sh.integers, x) + result = getOrIncl(m.sh.numbers, x) proc toPackedInfo(x: TLineInfo; c: var PackedEncoder; m: var PackedModule): PackedLineInfo = PackedLineInfo(line: x.line, col: x.col, file: toLitId(x.fileIndex, c, m)) @@ -400,7 +400,7 @@ proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var Pa typeId: storeTypeLater(n.typ, c, m), info: info) of externIntLit: ir.nodes.add PackedNode(kind: n.kind, flags: n.flags, - operand: int32 getOrIncl(m.sh.integers, n.intVal), + operand: int32 getOrIncl(m.sh.numbers, n.intVal), typeId: storeTypeLater(n.typ, c, m), info: info) of nkStrLit..nkTripleStrLit: ir.nodes.add PackedNode(kind: n.kind, flags: n.flags, @@ -408,7 +408,7 @@ proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var Pa typeId: storeTypeLater(n.typ, c, m), info: info) of nkFloatLit..nkFloat128Lit: ir.nodes.add PackedNode(kind: n.kind, flags: n.flags, - operand: int32 getOrIncl(m.sh.floats, n.floatVal), + operand: int32 getOrIncl(m.sh.numbers, cast[BiggestInt](n.floatVal)), typeId: storeTypeLater(n.typ, c, m), info: info) else: let patchPos = ir.prepare(n.kind, n.flags, @@ -521,8 +521,7 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef loadSeqSection depsSection, m.imports - loadTabSection integersSection, m.sh.integers - loadTabSection floatsSection, m.sh.floats + loadTabSection numbersSection, m.sh.numbers loadSeqSection exportsSection, m.exports @@ -587,8 +586,7 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac storeSeqSection depsSection, m.imports - storeTabSection integersSection, m.sh.integers - storeTabSection floatsSection, m.sh.floats + storeTabSection numbersSection, m.sh.numbers storeSeqSection exportsSection, m.exports @@ -700,11 +698,11 @@ proc loadNodes*(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; of directIntLit: result.intVal = tree.nodes[n.int].operand of externIntLit: - result.intVal = g[thisModule].fromDisk.sh.integers[n.litId] + result.intVal = g[thisModule].fromDisk.sh.numbers[n.litId] of nkStrLit..nkTripleStrLit: result.strVal = g[thisModule].fromDisk.sh.strings[n.litId] of nkFloatLit..nkFloat128Lit: - result.floatVal = g[thisModule].fromDisk.sh.floats[n.litId] + result.floatVal = cast[BiggestFloat](g[thisModule].fromDisk.sh.numbers[n.litId]) of nkModuleRef: let (n1, n2) = sons2(tree, n) assert n1.kind == nkInt32Lit @@ -1151,5 +1149,4 @@ proc rodViewer*(rodfile: AbsoluteFile; config: ConfigRef, cache: IdentCache) = echo "symbols: ", m.sh.syms.len, " types: ", m.sh.types.len, " top level nodes: ", m.topLevel.nodes.len, " other nodes: ", m.bodies.nodes.len, - " strings: ", m.sh.strings.len, " integers: ", m.sh.integers.len, - " floats: ", m.sh.floats.len + " strings: ", m.sh.strings.len, " numbers: ", m.sh.numbers.len diff --git a/compiler/ic/integrity.nim b/compiler/ic/integrity.nim index b4545de94..a28f2066a 100644 --- a/compiler/ic/integrity.nim +++ b/compiler/ic/integrity.nim @@ -79,12 +79,10 @@ proc checkNode(c: var CheckedContext; tree: PackedTree; n: NodePos) = checkLocalSym(c, tree.nodes[n.int].operand) of directIntLit: discard - of externIntLit: - assert c.g.packed[c.thisModule].fromDisk.sh.integers.hasLitId n.litId + of externIntLit, nkFloatLit..nkFloat128Lit: + assert c.g.packed[c.thisModule].fromDisk.sh.numbers.hasLitId n.litId of nkStrLit..nkTripleStrLit: assert c.g.packed[c.thisModule].fromDisk.sh.strings.hasLitId n.litId - of nkFloatLit..nkFloat128Lit: - assert c.g.packed[c.thisModule].fromDisk.sh.floats.hasLitId n.litId of nkModuleRef: let (n1, n2) = sons2(tree, n) assert n1.kind == nkInt32Lit diff --git a/compiler/ic/packed_ast.nim b/compiler/ic/packed_ast.nim index 5cc3b1476..0649b56ce 100644 --- a/compiler/ic/packed_ast.nim +++ b/compiler/ic/packed_ast.nim @@ -106,8 +106,8 @@ type syms*: seq[PackedSym] types*: seq[PackedType] strings*: BiTable[string] # we could share these between modules. - integers*: BiTable[BiggestInt] - floats*: BiTable[BiggestFloat] + numbers*: BiTable[BiggestInt] # we also store floats in here so + # that we can assure that every bit is kept #config*: ConfigRef PackedInstantiation* = object @@ -394,10 +394,13 @@ proc toString*(tree: PackedTree; n: NodePos; sh: Shared; nesting: int; result.addInt tree.nodes[pos].operand of externSIntLit: result.add " " - result.addInt sh.integers[LitId tree.nodes[pos].operand] + result.addInt sh.numbers[LitId tree.nodes[pos].operand] of externUIntLit: result.add " " - result.add $cast[uint64](sh.integers[LitId tree.nodes[pos].operand]) + result.add $cast[uint64](sh.numbers[LitId tree.nodes[pos].operand]) + of nkFloatLit..nkFloat128Lit: + result.add " " + result.add $cast[BiggestFloat](sh.numbers[LitId tree.nodes[pos].operand]) else: result.add "(\n" for i in 1..(nesting+1)*2: result.add ' ' diff --git a/compiler/ic/rodfiles.nim b/compiler/ic/rodfiles.nim index 39a5b6b73..fbe1cad33 100644 --- a/compiler/ic/rodfiles.nim +++ b/compiler/ic/rodfiles.nim @@ -16,8 +16,7 @@ type stringsSection checkSumsSection depsSection - integersSection - floatsSection + numbersSection exportsSection reexportsSection compilerProcsSection From ae9231cfebf800886c4febcf0fc7ccc380066108 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 14 Apr 2021 13:45:25 -0700 Subject: [PATCH 0636/2472] fix CI tests/errmsgs/tgcsafety.nim (#17721) --- tests/errmsgs/tgcsafety.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/errmsgs/tgcsafety.nim b/tests/errmsgs/tgcsafety.nim index 563ead2e0..6021d19e1 100644 --- a/tests/errmsgs/tgcsafety.nim +++ b/tests/errmsgs/tgcsafety.nim @@ -2,7 +2,7 @@ discard """ cmd: "nim check $file" errormsg: "type mismatch: got .}>" nimout: ''' -tgcsafety.nim(30, 18) Error: type mismatch: got .}> +tgcsafety.nim(31, 18) Error: type mismatch: got .}> but expected one of: proc serve(server: AsyncHttpServer; port: Port; callback: proc (request: Request): Future[void] {.closure, gcsafe.}; From f8dce493d36c10bfdfb3bd4ac87eae7b96b97f1a Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Thu, 15 Apr 2021 09:12:44 +0300 Subject: [PATCH 0637/2472] rst indentation fixes (ref #17340) (#17715) --- lib/packages/docutils/rst.nim | 95 +++++++---- lib/packages/docutils/rstast.nim | 18 +- nimdoc/rst2html/expected/rst_examples.html | 4 +- tests/stdlib/trst.nim | 185 ++++++++++++++++++++- tests/stdlib/trstgen.nim | 13 +- 5 files changed, 272 insertions(+), 43 deletions(-) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index c2385d517..7c4b92f88 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -1532,6 +1532,55 @@ proc parseUntilNewline(p: var RstParser, father: PRstNode) = of tkEof, tkIndent: break proc parseSection(p: var RstParser, result: PRstNode) {.gcsafe.} + +proc tokenAfterNewline(p: RstParser, start: int): int = + result = start + while true: + case p.tok[result].kind + of tkEof: + break + of tkIndent: + inc result + break + else: inc result + +proc tokenAfterNewline(p: RstParser): int {.inline.} = + result = tokenAfterNewline(p, p.idx) + +proc getWrappableIndent(p: RstParser): int = + ## Gets baseline indentation for bodies of field lists and directives. + ## Handles situations like this (with possible de-indent in [case.3]):: + ## + ## :field: definition [case.1] + ## + ## currInd currentTok(p).col + ## | | + ## v v + ## + ## .. Note:: defItem: [case.2] + ## definition + ## + ## ^ + ## | + ## nextIndent + ## + ## .. Note:: - point1 [case.3] + ## - point 2 + ## + ## ^ + ## | + ## nextIndent + if currentTok(p).kind == tkIndent: + result = currentTok(p).ival + else: + var nextIndent = p.tok[tokenAfterNewline(p)-1].ival + if nextIndent <= currInd(p): # parse only this line [case.1] + result = currentTok(p).col + elif nextIndent >= currentTok(p).col: # may be a definition list [case.2] + result = currentTok(p).col + else: + result = nextIndent # [case.3] + proc parseField(p: var RstParser): PRstNode = ## Returns a parsed rnField node. ## @@ -1541,13 +1590,12 @@ proc parseField(p: var RstParser): PRstNode = var fieldname = newRstNode(rnFieldName) parseUntil(p, fieldname, ":", false) var fieldbody = newRstNode(rnFieldBody) - if currentTok(p).kind != tkIndent: parseLine(p, fieldbody) - if currentTok(p).kind == tkIndent: - var indent = currentTok(p).ival - if indent > col: - pushInd(p, indent) - parseSection(p, fieldbody) - popInd(p) + if currentTok(p).kind == tkWhite: inc p.idx + let indent = getWrappableIndent(p) + if indent > col: + pushInd(p, indent) + parseSection(p, fieldbody) + popInd(p) result.add(fieldname) result.add(fieldbody) @@ -1652,20 +1700,6 @@ proc countTitles(p: var RstParser, n: PRstNode) = if p.s.hTitleCnt >= 2: break -proc tokenAfterNewline(p: RstParser, start: int): int = - result = start - while true: - case p.tok[result].kind - of tkEof: - break - of tkIndent: - inc result - break - else: inc result - -proc tokenAfterNewline(p: RstParser): int {.inline.} = - result = tokenAfterNewline(p, p.idx) - proc isAdornmentHeadline(p: RstParser, adornmentIdx: int): bool = ## check that underline/overline length is enough for the heading. ## No support for Unicode. @@ -1752,7 +1786,7 @@ proc whichSection(p: RstParser): RstNodeKind = return rnCodeBlock elif currentTok(p).symbol == "::": return rnLiteralBlock - elif currentTok(p).symbol == ".." and predNL(p) and + elif currentTok(p).symbol == ".." and nextTok(p).kind in {tkWhite, tkIndent}: return rnDirective case currentTok(p).kind @@ -1780,10 +1814,9 @@ proc whichSection(p: RstParser): RstNodeKind = elif match(p, tokenAfterNewline(p), "aI") and isAdornmentHeadline(p, tokenAfterNewline(p)): result = rnHeadline - elif predNL(p) and - currentTok(p).symbol in ["+", "*", "-"] and nextTok(p).kind == tkWhite: + elif currentTok(p).symbol in ["+", "*", "-"] and nextTok(p).kind == tkWhite: result = rnBulletList - elif match(p, p.idx, ":w:E") and predNL(p): + elif match(p, p.idx, ":w:E"): # (currentTok(p).symbol == ":") result = rnFieldList elif match(p, p.idx, "(e) ") or match(p, p.idx, "e) ") or @@ -2350,9 +2383,11 @@ proc parseDirective(p: var RstParser, k: RstNodeKind, flags: DirFlags): PRstNode parseLine(p, args) result.add(args) if hasOptions in flags: - if currentTok(p).kind == tkIndent and currentTok(p).ival >= 3 and + if currentTok(p).kind == tkIndent and currentTok(p).ival > currInd(p) and nextTok(p).symbol == ":": + pushInd(p, currentTok(p).ival) options = parseFields(p) + popInd(p) result.add(options) proc indFollows(p: RstParser): bool = @@ -2363,11 +2398,9 @@ proc parseBlockContent(p: var RstParser, father: var PRstNode, ## parse the final content part of explicit markup blocks (directives, ## footnotes, etc). Returns true if succeeded. if currentTok(p).kind != tkIndent or indFollows(p): - var nextIndent = p.tok[tokenAfterNewline(p)-1].ival - if nextIndent <= currInd(p): # parse only this line - nextIndent = currentTok(p).col - pushInd(p, nextIndent) - var content = contentParser(p) + let blockIndent = getWrappableIndent(p) + pushInd(p, blockIndent) + let content = contentParser(p) popInd(p) father.add content result = true diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim index 394cc2698..00f3f2b35 100644 --- a/lib/packages/docutils/rstast.nim +++ b/lib/packages/docutils/rstast.nim @@ -350,7 +350,7 @@ proc renderRstToJson*(node: PRstNode): string = proc renderRstToStr*(node: PRstNode, indent=0): string = ## Writes the parsed RST `node` into a compact string ## representation in the format (one line per every sub-node): - ## ``indent - kind - text - level - order - anchor (if non-zero)`` + ## ``indent - kind - [text|level|order|adType] - anchor (if non-zero)`` ## (suitable for debugging of RST parsing). if node == nil: result.add " ".repeat(indent) & "[nil]\n" @@ -358,21 +358,23 @@ proc renderRstToStr*(node: PRstNode, indent=0): string = result.add " ".repeat(indent) & $node.kind case node.kind of rnLeaf, rnSmiley: - result.add (if node.text == "": "" else: "\t'" & node.text & "'") + result.add (if node.text == "": "" else: " '" & node.text & "'") of rnEnumList: - result.add "\tlabelFmt=" & node.labelFmt + result.add " labelFmt=" & node.labelFmt of rnLineBlockItem: var txt: string - if node.lineIndent == "\n": txt = "\t(blank line)" - else: txt = "\tlineIndent=" & $node.lineIndent.len + if node.lineIndent == "\n": txt = " (blank line)" + else: txt = " lineIndent=" & $node.lineIndent.len result.add txt + of rnAdmonition: + result.add " adType=" & node.adType of rnHeadline, rnOverline, rnMarkdownHeadline: - result.add "\tlevel=" & $node.level + result.add " level=" & $node.level of rnFootnote, rnCitation, rnFootnoteRef, rnOptionListItem: - result.add (if node.order == 0: "" else: "\torder=" & $node.order) + result.add (if node.order == 0: "" else: " order=" & $node.order) else: discard - result.add (if node.anchor == "": "" else: "\tanchor='" & node.anchor & "'") + result.add (if node.anchor == "": "" else: " anchor='" & node.anchor & "'") result.add "\n" for son in node.sons: result.add renderRstToStr(son, indent=indent+2) diff --git a/nimdoc/rst2html/expected/rst_examples.html b/nimdoc/rst2html/expected/rst_examples.html index 32abc0f80..5c434193e 100644 --- a/nimdoc/rst2html/expected/rst_examples.html +++ b/nimdoc/rst2html/expected/rst_examples.html @@ -119,8 +119,8 @@ window.addEventListener('DOMContentLoaded', main);
    -

    - +

    Authors: Andreas Rumpf, Zahary Karadjov
    Version: |nimversion|
    +
    Authors:Andreas Rumpf, Zahary Karadjov
    Version:|nimversion|

    "Complexity" seems to be a lot like "energy": you can transfer it from the end-user to one/some of the other players, but the total amount seems to remain pretty much constant for a given task. -- Ran

    About this document

    Note: This document is a draft! Several of Nim's features may need more precise wording. This manual is constantly evolving into a proper specification.

    diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index 0645e4150..2398b92a8 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -1,6 +1,8 @@ discard """ output: ''' +[Suite] RST indentation + [Suite] RST include directive ''' """ @@ -9,9 +11,190 @@ discard """ import ../../lib/packages/docutils/rstgen import ../../lib/packages/docutils/rst -import unittest +import ../../lib/packages/docutils/rstast +import unittest, strutils +import std/private/miscdollars import os +proc toAst(input: string, + rstOptions: RstParseOptions = {roSupportMarkdown, roNimFile}, + error: ref string = nil, + warnings: ref seq[string] = nil): string = + ## If `error` is nil then no errors should be generated. + ## The same goes for `warnings`. + proc testMsgHandler(filename: string, line, col: int, msgkind: MsgKind, + arg: string) = + let mc = msgkind.whichMsgClass + let a = $msgkind % arg + var message: string + toLocation(message, filename, line, col + ColRstOffset) + message.add " $1: $2" % [$mc, a] + if mc == mcError: + doAssert error != nil, "unexpected RST error '" & message & "'" + error[] = message + # we check only first error because subsequent ones may be meaningless + raise newException(EParseError, message) + else: + doAssert warnings != nil, "unexpected RST warning '" & message & "'" + warnings[].add message + try: + const filen = "input" + + proc myFindFile(filename: string): string = + # we don't find any files in online mode: + result = "" + + var dummyHasToc = false + var rst = rstParse(input, filen, line=LineRstInit, column=ColRstInit, + dummyHasToc, rstOptions, myFindFile, testMsgHandler) + result = renderRstToStr(rst) + except EParseError: + discard + +suite "RST indentation": + test "nested bullet lists": + let input = dedent """ + * - bullet1 + - bullet2 + * - bullet3 + - bullet4 + """ + let output = input.toAst + check(output == dedent""" + rnBulletList + rnBulletItem + rnBulletList + rnBulletItem + rnInner + rnLeaf 'bullet1' + rnBulletItem + rnInner + rnLeaf 'bullet2' + rnBulletItem + rnBulletList + rnBulletItem + rnInner + rnLeaf 'bullet3' + rnBulletItem + rnInner + rnLeaf 'bullet4' + """) + + test "nested markup blocks": + let input = dedent""" + #) .. Hint:: .. Error:: none + #) .. Warning:: term0 + Definition0 + #) some + paragraph1 + #) term1 + Definition1 + term2 + Definition2 + """ + check(input.toAst == dedent""" + rnEnumList labelFmt=1) + rnEnumItem + rnAdmonition adType=hint + [nil] + [nil] + rnAdmonition adType=error + [nil] + [nil] + rnLeaf 'none' + rnEnumItem + rnAdmonition adType=warning + [nil] + [nil] + rnDefList + rnDefItem + rnDefName + rnLeaf 'term0' + rnDefBody + rnInner + rnLeaf 'Definition0' + rnEnumItem + rnInner + rnLeaf 'some' + rnLeaf ' ' + rnLeaf 'paragraph1' + rnEnumItem + rnDefList + rnDefItem + rnDefName + rnLeaf 'term1' + rnDefBody + rnInner + rnLeaf 'Definition1' + rnDefItem + rnDefName + rnLeaf 'term2' + rnDefBody + rnInner + rnLeaf 'Definition2' + """) + + test "code-block parsing": + let input1 = dedent""" + .. code-block:: nim + :test: "nim c $1" + + template additive(typ: typedesc) = + discard + """ + let input2 = dedent""" + .. code-block:: nim + :test: "nim c $1" + + template additive(typ: typedesc) = + discard + """ + let input3 = dedent""" + .. code-block:: nim + :test: "nim c $1" + template additive(typ: typedesc) = + discard + """ + let inputWrong = dedent""" + .. code-block:: nim + :test: "nim c $1" + + template additive(typ: typedesc) = + discard + """ + let ast = dedent""" + rnCodeBlock + rnDirArg + rnLeaf 'nim' + rnFieldList + rnField + rnFieldName + rnLeaf 'test' + rnFieldBody + rnInner + rnLeaf '"' + rnLeaf 'nim' + rnLeaf ' ' + rnLeaf 'c' + rnLeaf ' ' + rnLeaf '$' + rnLeaf '1' + rnLeaf '"' + rnField + rnFieldName + rnLeaf 'default-language' + rnFieldBody + rnLeaf 'Nim' + rnLiteralBlock + rnLeaf 'template additive(typ: typedesc) = + discard' + """ + check input1.toAst == ast + check input2.toAst == ast + check input3.toAst == ast + # "template..." should be parsed as a definition list attached to ":test:": + check inputWrong.toAst != ast + suite "RST include directive": test "Include whole": "other.rst".writeFile("**test1**") diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index ed5d72226..29747f4e8 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -1118,6 +1118,17 @@ Test1 """ check "
    55\n56\n
    " in input.toHtml + test "Nim code-block indentation": + let input = dedent """ + .. code-block:: nim + :number-lines: 55 + + x + """ + let output = input.toHtml + check "
    55\n
    " in output + check "x" in output + test "RST admonitions": # check that all admonitions are implemented let input0 = dedent """ @@ -1466,7 +1477,7 @@ Test1 """""" & """""" & """""" & - """""" & "\n
    field: text
    ") + """text""" & "\n") test "Field list: body after newline": let output = dedent """ From 51a40c1f38a8a03f745f1982cf74091a1c6f7315 Mon Sep 17 00:00:00 2001 From: Clyybber Date: Thu, 15 Apr 2021 13:55:50 +0200 Subject: [PATCH 0638/2472] Fix getCustomPragmaVal for some multi arg pragmas (#17723) * Fix getCustomPragmaVal for some multi arg pragmas * Bootstrap fix --- lib/core/macros.nim | 86 +++++++++++++++++--------------- tests/pragmas/tcustom_pragma.nim | 9 ++++ 2 files changed, 54 insertions(+), 41 deletions(-) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index ebe5a7187..fc68781da 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -1474,23 +1474,20 @@ macro expandMacros*(body: typed): untyped = echo body.toStrLit result = body -proc findPragmaExprForFieldSym(arg, fieldSym: NimNode): NimNode = +proc findPragmaNodeInRecList(arg, fieldSym: NimNode): NimNode = case arg.kind of nnkRecList, nnkRecCase: for it in arg.children: - result = findPragmaExprForFieldSym(it, fieldSym) - if result != nil: - return + result = findPragmaNodeInRecList(it, fieldSym) + if result != nil: return of nnkOfBranch: - return findPragmaExprForFieldSym(arg[1], fieldSym) + return findPragmaNodeInRecList(arg[1], fieldSym) of nnkElse: - return findPragmaExprForFieldSym(arg[0], fieldSym) + return findPragmaNodeInRecList(arg[0], fieldSym) of nnkIdentDefs: - for i in 0 ..< arg.len-2: - let child = arg[i] - result = findPragmaExprForFieldSym(child, fieldSym) - if result != nil: - return + for i in 0.. Date: Thu, 15 Apr 2021 15:45:38 +0200 Subject: [PATCH 0639/2472] Fix array's high & low return type for empty arrays (#17705) * fix array.high/low return type * Add test for empty array low return type Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- compiler/semexprs.nim | 2 ++ lib/system.nim | 8 ++++++++ tests/array/tarray.nim | 7 +++++++ 3 files changed, 17 insertions(+) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index cbd4318f7..13b53ef32 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -379,6 +379,8 @@ proc semLowHigh(c: PContext, n: PNode, m: TMagic): PNode = n.typ = getSysType(c.graph, n.info, tyInt) of tyArray: n.typ = typ[0] # indextype + if n.typ.kind == tyRange and emptyRange(n.typ.n[0], n.typ.n[1]): #Invalid range + n.typ = getSysType(c.graph, n.info, tyInt) of tyInt..tyInt64, tyChar, tyBool, tyEnum, tyUInt..tyUInt64, tyFloat..tyFloat64: n.typ = n[1].typ.skipTypes({tyTypeDesc}) of tyGenericParam: diff --git a/lib/system.nim b/lib/system.nim index 6e84aca66..38f6d64d3 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -348,6 +348,8 @@ proc high*[T](x: openArray[T]): int {.magic: "High", noSideEffect.} proc high*[I, T](x: array[I, T]): I {.magic: "High", noSideEffect.} ## Returns the highest possible index of an array `x`. ## + ## For empty arrays, the return type is `int`. + ## ## See also: ## * `low(array) <#low,array[I,T]>`_ ## @@ -360,6 +362,8 @@ proc high*[I, T](x: array[I, T]): I {.magic: "High", noSideEffect.} proc high*[I, T](x: typedesc[array[I, T]]): I {.magic: "High", noSideEffect.} ## Returns the highest possible index of an array type. ## + ## For empty arrays, the return type is `int`. + ## ## See also: ## * `low(typedesc[array]) <#low,typedesc[array[I,T]]>`_ ## @@ -420,6 +424,8 @@ proc low*[T](x: openArray[T]): int {.magic: "Low", noSideEffect.} proc low*[I, T](x: array[I, T]): I {.magic: "Low", noSideEffect.} ## Returns the lowest possible index of an array `x`. ## + ## For empty arrays, the return type is `int`. + ## ## See also: ## * `high(array) <#high,array[I,T]>`_ ## @@ -432,6 +438,8 @@ proc low*[I, T](x: array[I, T]): I {.magic: "Low", noSideEffect.} proc low*[I, T](x: typedesc[array[I, T]]): I {.magic: "Low", noSideEffect.} ## Returns the lowest possible index of an array type. ## + ## For empty arrays, the return type is `int`. + ## ## See also: ## * `high(typedesc[array]) <#high,typedesc[array[I,T]]>`_ ## diff --git a/tests/array/tarray.nim b/tests/array/tarray.nim index 81a43f203..2765ad06d 100644 --- a/tests/array/tarray.nim +++ b/tests/array/tarray.nim @@ -587,3 +587,10 @@ block t12466: a[0'u16 + i] = i for i in 0'u16 ..< 8'u16: a[0'u16 + i] = i + +block t17705: + # https://github.com/nim-lang/Nim/pull/17705 + var a = array[0, int].low + a = int(a) + var b = array[0, int].high + b = int(b) From 3f56b9944e5e81338630060bdc89454853340792 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 16 Apr 2021 01:25:03 +0800 Subject: [PATCH 0640/2472] follow up #17536 (#17727) --- tests/sets/t5792.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/sets/t5792.nim b/tests/sets/t5792.nim index efc38b1bc..297a1fc15 100644 --- a/tests/sets/t5792.nim +++ b/tests/sets/t5792.nim @@ -14,4 +14,4 @@ type of {b, c} - {a}: y: int -discard U(k: b, y: 1) +doAssert U(k: b, y: 1).y == 1 From 5c12c711f7076d0b095d6ca392de4e11834ceb0f Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 16 Apr 2021 01:25:35 +0800 Subject: [PATCH 0641/2472] follow up #17518 (#17726) --- tests/vm/tvmmisc.nim | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/vm/tvmmisc.nim b/tests/vm/tvmmisc.nim index f542fa560..f0d2c2dbf 100644 --- a/tests/vm/tvmmisc.nim +++ b/tests/vm/tvmmisc.nim @@ -279,6 +279,14 @@ block: # bug #10815 const a = P() doAssert $a == "" + +when defined osx: # xxx bug https://github.com/nim-lang/Nim/issues/10815#issuecomment-476380734 + block: + type CharSet {.union.} = object + cs: set[char] + vs: array[4, uint64] + const a = Charset(cs: {'a'..'z'}) + doAssert a.repr.len > 0 import tables From 95e8ddabb2bcc37e5f1bdd62bcc891b96f80083a Mon Sep 17 00:00:00 2001 From: shirleyquirk <31934565+shirleyquirk@users.noreply.github.com> Date: Thu, 15 Apr 2021 21:45:51 +0100 Subject: [PATCH 0642/2472] followup #17700 put changelog in wrong file (#17729) * Allow use of colons inside fmt allowing colons inside fmt by replacing the format specifier delimiter lets arbitrary nim code be run within fmt expressions. * oops * Update strformat.nim * one space. * Update lib/pure/strformat.nim Co-authored-by: Andreas Rumpf * Update lib/pure/strformat.nim Co-authored-by: Andreas Rumpf * changed parser to ignore ':' within parens * Update strformat.nim * Update lib/pure/strformat.nim Co-authored-by: flywind * formatting,documentation,backslash escapes Adding support for evaluating expressions by special-casing parentheses causes this regression: `&"""{ "(hello)" }"""` no longer parses. In addition, code such as &"""{(if open: '(' else: ')')}""" wouldn't work. To enable that, as well as the use of, e.g. Table constructors inside curlies, I've added backslash escapes. This also means that if/for/etc statements, unparenthesized, will work, if the colons are escaped, but i've left that under-documented. It's not exactly elegant having two types of escape, but I believe it's the least bad option. * not sure about how this works * changelog * changelog * added json strformat test * pulled my thumb out and wrote a parser * more escapes and string tests * ok, now i'm sure it's broken but cant get a failing test * found it * ok, that'll do * did i delete this? * spcng * spacing * messed up where changelogs are supposed to go * ocd Co-authored-by: Andreas Rumpf Co-authored-by: flywind --- changelog.md | 1 + changelogs/changelog_X_XX_X.md | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/changelog.md b/changelog.md index ec1690e06..aeaeb598d 100644 --- a/changelog.md +++ b/changelog.md @@ -38,6 +38,7 @@ unless `-d:nimLegacyHomeDir` is specified (for a transition period). ## Standard library additions and changes +- Added support for parenthesized expressions in `strformat` - Added `sections` iterator in `parsecfg`. diff --git a/changelogs/changelog_X_XX_X.md b/changelogs/changelog_X_XX_X.md index d68ef4c20..524599b11 100644 --- a/changelogs/changelog_X_XX_X.md +++ b/changelogs/changelog_X_XX_X.md @@ -13,8 +13,6 @@ The changes should go to changelog.md! - Changed `example.foo` to take additional `bar` parameter. -- Added support for evaluating parenthesised expressions in strformat - ## Language changes From 12783dbcf0075492a3d090e4dc3ead3628c18a3a Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 16 Apr 2021 14:50:44 +0800 Subject: [PATCH 0643/2472] make the copy operation of Thread an error (#17734) --- lib/system/threads.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/system/threads.nim b/lib/system/threads.nim index def35c238..7fbfe54ab 100644 --- a/lib/system/threads.nim +++ b/lib/system/threads.nim @@ -90,6 +90,8 @@ type dataFn: proc (m: TArg) {.nimcall, gcsafe.} data: TArg +proc `=copy`*[TArg](x: var Thread[TArg], y: Thread[TArg]) {.error.} + var threadDestructionHandlers {.rtlThreadVar.}: seq[proc () {.closure, gcsafe, raises: [].}] From 8161b02897a75c4b30593dbcc189cbd49d3832ea Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 16 Apr 2021 00:16:39 -0700 Subject: [PATCH 0644/2472] `import foo {.all.}` reboot (#17706) --- changelog.md | 9 ++++ compiler/ast.nim | 8 +-- compiler/ic/ic.nim | 58 ++++++++++++++------ compiler/ic/rodfiles.nim | 1 + compiler/importer.nim | 67 +++++++++++++++++------ compiler/lookups.nim | 12 +++-- compiler/magicsys.nim | 1 + compiler/modulegraphs.nim | 64 ++++++++++++++-------- compiler/modules.nim | 2 +- compiler/nimeval.nim | 2 +- compiler/options.nim | 2 +- compiler/semdata.nim | 8 ++- compiler/semmagic.nim | 5 ++ compiler/semstmts.nim | 1 + compiler/semtypes.nim | 7 +-- compiler/suggest.nim | 13 +++-- compiler/wordrecg.nim | 2 +- lib/std/importutils.nim | 34 ++++++++++++ testament/categories.nim | 1 + tests/ic/mimports.nim | 9 ++++ tests/ic/mimportsb.nim | 4 ++ tests/ic/timports.nim | 29 ++++++++++ tests/importalls/m1.nim | 70 ++++++++++++++++++++++++ tests/importalls/m2.nim | 3 ++ tests/importalls/m3.nim | 5 ++ tests/importalls/m4.nim | 10 ++++ tests/importalls/mt0.nim | 9 ++++ tests/importalls/mt1.nim | 23 ++++++++ tests/importalls/mt2.nim | 104 ++++++++++++++++++++++++++++++++++++ tests/importalls/mt3.nim | 12 +++++ tests/importalls/mt4.nim | 4 ++ tests/importalls/mt4b.nim | 3 ++ tests/importalls/mt5.nim | 9 ++++ tests/importalls/mt6.nim | 13 +++++ tests/importalls/mt7.nim | 14 +++++ tests/importalls/mt8.nim | 23 ++++++++ tests/importalls/mt9.nim | 12 +++++ tests/importalls/tmain2.nim | 7 +++ 38 files changed, 584 insertions(+), 76 deletions(-) create mode 100644 lib/std/importutils.nim create mode 100644 tests/ic/mimports.nim create mode 100644 tests/ic/mimportsb.nim create mode 100644 tests/ic/timports.nim create mode 100644 tests/importalls/m1.nim create mode 100644 tests/importalls/m2.nim create mode 100644 tests/importalls/m3.nim create mode 100644 tests/importalls/m4.nim create mode 100644 tests/importalls/mt0.nim create mode 100644 tests/importalls/mt1.nim create mode 100644 tests/importalls/mt2.nim create mode 100644 tests/importalls/mt3.nim create mode 100644 tests/importalls/mt4.nim create mode 100644 tests/importalls/mt4b.nim create mode 100644 tests/importalls/mt5.nim create mode 100644 tests/importalls/mt6.nim create mode 100644 tests/importalls/mt7.nim create mode 100644 tests/importalls/mt8.nim create mode 100644 tests/importalls/mt9.nim create mode 100644 tests/importalls/tmain2.nim diff --git a/changelog.md b/changelog.md index aeaeb598d..6dcfab9c6 100644 --- a/changelog.md +++ b/changelog.md @@ -307,6 +307,15 @@ - Added `iterable[T]` type class to match called iterators, which enables writing: `template fn(a: iterable)` instead of `template fn(a: untyped)` +- A new import syntax `import foo {.all.}` now allows to import all symbols (public or private) + from `foo`. It works in combination with all pre-existing import features. + This reduces or eliminates the need for workarounds such as using `include` (which has known issues) + when you need a private symbol for testing or making some internal APIs public just because + another internal module needs those. + It also helps mitigate the lack of cyclic imports in some cases. + +- Added a new module `std/importutils`, and an API `privateAccess`, which allows access to private fields + for an object type in the current scope. ## Compiler changes diff --git a/compiler/ast.nim b/compiler/ast.nim index 155af6375..4c835871f 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -605,6 +605,8 @@ type const routineKinds* = {skProc, skFunc, skMethod, skIterator, skConverter, skMacro, skTemplate} + ExportableSymKinds* = {skVar, skLet, skConst, skType, skEnumField, skStub, skAlias} + routineKinds + tfUnion* = tfNoSideEffect tfGcSafe* = tfThread tfObjHasKids* = tfEnumHasHoles @@ -691,7 +693,7 @@ type mInstantiationInfo, mGetTypeInfo, mGetTypeInfoV2, mNimvm, mIntDefine, mStrDefine, mBoolDefine, mRunnableExamples, mException, mBuiltinType, mSymOwner, mUncheckedArray, mGetImplTransf, - mSymIsInstantiationOf, mNodeId + mSymIsInstantiationOf, mNodeId, mPrivateAccess # things that we can evaluate safely at compile time, even if not asked for it: @@ -841,6 +843,7 @@ type depthLevel*: int symbols*: TStrTable parent*: PScope + allowPrivateAccess*: seq[PSym] # # enable access to private fields PScope* = ref TScope @@ -1011,9 +1014,6 @@ const NilableTypes*: TTypeKinds = {tyPointer, tyCString, tyRef, tyPtr, tyProc, tyError} # TODO PtrLikeKinds*: TTypeKinds = {tyPointer, tyPtr} # for VM - ExportableSymKinds* = {skVar, skConst, skProc, skFunc, skMethod, skType, - skIterator, - skMacro, skTemplate, skConverter, skEnumField, skLet, skStub, skAlias} PersistentNodeFlags*: TNodeFlags = {nfBase2, nfBase8, nfBase16, nfDotSetter, nfDotField, nfIsRef, nfIsPtr, nfPreventCg, nfLL, diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim index 4f63ce03a..1cd03ecfa 100644 --- a/compiler/ic/ic.nim +++ b/compiler/ic/ic.nim @@ -36,6 +36,7 @@ type bodies*: PackedTree # other trees. Referenced from typ.n and sym.ast by their position. #producedGenerics*: Table[GenericKey, SymId] exports*: seq[(LitId, int32)] + hidden*: seq[(LitId, int32)] reexports*: seq[(LitId, PackedItemId)] compilerProcs*: seq[(LitId, int32)] converters*, methods*, trmacros*, pureEnums*: seq[int32] @@ -177,6 +178,10 @@ proc addIncludeFileDep*(c: var PackedEncoder; m: var PackedModule; f: FileIndex) proc addImportFileDep*(c: var PackedEncoder; m: var PackedModule; f: FileIndex) = m.imports.add toLitId(f, c, m) +proc addHidden*(c: var PackedEncoder; m: var PackedModule; s: PSym) = + let nameId = getOrIncl(m.sh.strings, s.name.s) + m.hidden.add((nameId, s.itemId.item)) + proc addExported*(c: var PackedEncoder; m: var PackedModule; s: PSym) = let nameId = getOrIncl(m.sh.strings, s.name.s) m.exports.add((nameId, s.itemId.item)) @@ -524,7 +529,7 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef loadTabSection numbersSection, m.sh.numbers loadSeqSection exportsSection, m.exports - + loadSeqSection hiddenSection, m.hidden loadSeqSection reexportsSection, m.reexports loadSeqSection compilerProcsSection, m.compilerProcs @@ -589,7 +594,7 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac storeTabSection numbersSection, m.sh.numbers storeSeqSection exportsSection, m.exports - + storeSeqSection hiddenSection, m.hidden storeSeqSection reexportsSection, m.reexports storeSeqSection compilerProcsSection, m.compilerProcs @@ -655,7 +660,9 @@ type syms: seq[PSym] # indexed by itemId types: seq[PType] module*: PSym # the one true module symbol. - iface: Table[PIdent, seq[PackedItemId]] # PackedItemId so that it works with reexported symbols too + iface, ifaceHidden: Table[PIdent, seq[PackedItemId]] + # PackedItemId so that it works with reexported symbols too + # ifaceHidden includes private symbols PackedModuleGraph* = seq[LoadedModule] # indexed by FileIndex @@ -882,12 +889,22 @@ proc newPackage(config: ConfigRef; cache: IdentCache; fileIdx: FileIndex): PSym proc setupLookupTables(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; fileIdx: FileIndex; m: var LoadedModule) = m.iface = initTable[PIdent, seq[PackedItemId]]() - for e in m.fromDisk.exports: + m.ifaceHidden = initTable[PIdent, seq[PackedItemId]]() + template impl(iface, e) = let nameLit = e[0] - m.iface.mgetOrPut(cache.getIdent(m.fromDisk.sh.strings[nameLit]), @[]).add(PackedItemId(module: LitId(0), item: e[1])) - for re in m.fromDisk.reexports: - let nameLit = re[0] - m.iface.mgetOrPut(cache.getIdent(m.fromDisk.sh.strings[nameLit]), @[]).add(re[1]) + let e2 = + when e[1] is PackedItemId: e[1] + else: PackedItemId(module: LitId(0), item: e[1]) + iface.mgetOrPut(cache.getIdent(m.fromDisk.sh.strings[nameLit]), @[]).add(e2) + + for e in m.fromDisk.exports: + m.iface.impl(e) + m.ifaceHidden.impl(e) + for e in m.fromDisk.reexports: + m.iface.impl(e) + m.ifaceHidden.impl(e) + for e in m.fromDisk.hidden: + m.ifaceHidden.impl(e) let filename = AbsoluteFile toFullPath(conf, fileIdx) # We cannot call ``newSym`` here, because we have to circumvent the ID @@ -1053,16 +1070,21 @@ type values: seq[PackedItemId] i, module: int +template interfSelect(a: LoadedModule, importHidden: bool): auto = + var ret = a.iface.addr + if importHidden: ret = a.ifaceHidden.addr + ret[] + proc initRodIter*(it: var RodIter; config: ConfigRef, cache: IdentCache; g: var PackedModuleGraph; module: FileIndex; - name: PIdent): PSym = + name: PIdent, importHidden: bool): PSym = it.decoder = PackedDecoder( lastModule: int32(-1), lastLit: LitId(0), lastFile: FileIndex(-1), config: config, cache: cache) - it.values = g[int module].iface.getOrDefault(name) + it.values = g[int module].interfSelect(importHidden).getOrDefault(name) it.i = 0 it.module = int(module) if it.i < it.values.len: @@ -1070,7 +1092,7 @@ proc initRodIter*(it: var RodIter; config: ConfigRef, cache: IdentCache; inc it.i proc initRodIterAllSyms*(it: var RodIter; config: ConfigRef, cache: IdentCache; - g: var PackedModuleGraph; module: FileIndex): PSym = + g: var PackedModuleGraph; module: FileIndex, importHidden: bool): PSym = it.decoder = PackedDecoder( lastModule: int32(-1), lastLit: LitId(0), @@ -1079,7 +1101,7 @@ proc initRodIterAllSyms*(it: var RodIter; config: ConfigRef, cache: IdentCache; cache: cache) it.values = @[] it.module = int(module) - for v in g[int module].iface.values: + for v in g[int module].interfSelect(importHidden).values: it.values.add v it.i = 0 if it.i < it.values.len: @@ -1093,9 +1115,9 @@ proc nextRodIter*(it: var RodIter; g: var PackedModuleGraph): PSym = iterator interfaceSymbols*(config: ConfigRef, cache: IdentCache; g: var PackedModuleGraph; module: FileIndex; - name: PIdent): PSym = + name: PIdent, importHidden: bool): PSym = setupDecoder() - let values = g[int module].iface.getOrDefault(name) + let values = g[int module].interfSelect(importHidden).getOrDefault(name) for pid in values: let s = loadSym(decoder, g, int(module), pid) assert s != nil @@ -1103,9 +1125,9 @@ iterator interfaceSymbols*(config: ConfigRef, cache: IdentCache; proc interfaceSymbol*(config: ConfigRef, cache: IdentCache; g: var PackedModuleGraph; module: FileIndex; - name: PIdent): PSym = + name: PIdent, importHidden: bool): PSym = setupDecoder() - let values = g[int module].iface.getOrDefault(name) + let values = g[int module].interfSelect(importHidden).getOrDefault(name) result = loadSym(decoder, g, int(module), values[0]) proc idgenFromLoadedModule*(m: LoadedModule): IdGenerator = @@ -1140,6 +1162,10 @@ proc rodViewer*(rodfile: AbsoluteFile; config: ConfigRef, cache: IdentCache) = echo " ", m.sh.strings[ex[0]] # reexports*: seq[(LitId, PackedItemId)] + echo "hidden: " & $m.hidden.len + for ex in m.hidden: + echo " ", m.sh.strings[ex[0]], " local ID: ", ex[1] + echo "all symbols" for i in 0..high(m.sh.syms): if m.sh.syms[i].name != LitId(0): diff --git a/compiler/ic/rodfiles.nim b/compiler/ic/rodfiles.nim index fbe1cad33..47362da0b 100644 --- a/compiler/ic/rodfiles.nim +++ b/compiler/ic/rodfiles.nim @@ -18,6 +18,7 @@ type depsSection numbersSection exportsSection + hiddenSection reexportsSection compilerProcsSection trmacrosSection diff --git a/compiler/importer.nim b/compiler/importer.nim index f8ee0d483..692cdb604 100644 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -12,7 +12,7 @@ import intsets, ast, astalgo, msgs, options, idents, lookups, semdata, modulepaths, sigmatch, lineinfos, sets, - modulegraphs + modulegraphs, wordrecg proc readExceptSet*(c: PContext, n: PNode): IntSet = assert n.kind in {nkImportExceptStmt, nkExportExceptStmt} @@ -107,7 +107,25 @@ proc rawImportSymbol(c: PContext, s, origin: PSym; importSet: var IntSet) = if s.owner != origin: c.exportIndirections.incl((origin.id, s.id)) +proc splitPragmas(c: PContext, n: PNode): (PNode, seq[TSpecialWord]) = + template bail = globalError(c.config, n.info, "invalid pragma") + if n.kind == nkPragmaExpr: + if n.len == 2 and n[1].kind == nkPragma: + result[0] = n[0] + for ni in n[1]: + if ni.kind == nkIdent: result[1].add whichKeyword(ni.ident) + else: bail() + else: bail() + else: + result[0] = n + if result[0].safeLen > 0: + (result[0][^1], result[1]) = splitPragmas(c, result[0][^1]) + proc importSymbol(c: PContext, n: PNode, fromMod: PSym; importSet: var IntSet) = + let (n, kws) = splitPragmas(c, n) + if kws.len > 0: + globalError(c.config, n.info, "unexpected pragma") + let ident = lookups.considerQuotedIdent(c, n) let s = someSym(c.graph, fromMod, ident) if s == nil: @@ -204,18 +222,43 @@ proc importForwarded(c: PContext, n: PNode, exceptSet: IntSet; fromMod: PSym; im for i in 0..n.safeLen-1: importForwarded(c, n[i], exceptSet, fromMod, importSet) -proc importModuleAs(c: PContext; n: PNode, realModule: PSym): PSym = +proc importModuleAs(c: PContext; n: PNode, realModule: PSym, importHidden: bool): PSym = result = realModule c.unusedImports.add((realModule, n.info)) + template createModuleAliasImpl(ident): untyped = + createModuleAlias(realModule, nextSymId c.idgen, ident, realModule.info, c.config.options) if n.kind != nkImportAs: discard elif n.len != 2 or n[1].kind != nkIdent: localError(c.config, n.info, "module alias must be an identifier") elif n[1].ident.id != realModule.name.id: # some misguided guy will write 'import abc.foo as foo' ... - result = createModuleAlias(realModule, nextSymId c.idgen, n[1].ident, realModule.info, - c.config.options) + result = createModuleAliasImpl(n[1].ident) + if importHidden: + if result == realModule: # avoids modifying `realModule`, see D20201209T194412. + result = createModuleAliasImpl(realModule.name) + result.options.incl optImportHidden -proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym = +proc transformImportAs(c: PContext; n: PNode): tuple[node: PNode, importHidden: bool] = + var ret: typeof(result) + proc processPragma(n2: PNode): PNode = + let (result2, kws) = splitPragmas(c, n2) + result = result2 + for ai in kws: + case ai + of wImportHidden: ret.importHidden = true + else: globalError(c.config, n.info, "invalid pragma, expected: " & ${wImportHidden}) + + if n.kind == nkInfix and considerQuotedIdent(c, n[0]).s == "as": + ret.node = newNodeI(nkImportAs, n.info) + ret.node.add n[1].processPragma + ret.node.add n[2] + else: + ret.node = n.processPragma + return ret + +proc myImportModule(c: PContext, n: var PNode, importStmtResult: PNode): PSym = + let transf = transformImportAs(c, n) + n = transf.node let f = checkModuleName(c.config, n) if f != InvalidFileIdx: addImportFileDep(c, f) @@ -232,7 +275,7 @@ proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym = c.recursiveDep = err discard pushOptionEntry(c) - result = importModuleAs(c, n, c.graph.importModuleCallback(c.graph, c.module, f)) + result = importModuleAs(c, n, c.graph.importModuleCallback(c.graph, c.module, f), transf.importHidden) popOptionEntry(c) #echo "set back to ", L @@ -252,16 +295,8 @@ proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym = importStmtResult.add newSymNode(result, n.info) #newStrNode(toFullPath(c.config, f), n.info) -proc transformImportAs(c: PContext; n: PNode): PNode = - if n.kind == nkInfix and considerQuotedIdent(c, n[0]).s == "as": - result = newNodeI(nkImportAs, n.info) - result.add n[1] - result.add n[2] - else: - result = n - proc impMod(c: PContext; it: PNode; importStmtResult: PNode) = - let it = transformImportAs(c, it) + var it = it let m = myImportModule(c, it, importStmtResult) if m != nil: # ``addDecl`` needs to be done before ``importAllSymbols``! @@ -296,7 +331,6 @@ proc evalImport*(c: PContext, n: PNode): PNode = proc evalFrom*(c: PContext, n: PNode): PNode = result = newNodeI(nkImportStmt, n.info) checkMinSonsLen(n, 2, c.config) - n[0] = transformImportAs(c, n[0]) var m = myImportModule(c, n[0], result) if m != nil: n[0] = newSymNode(m) @@ -311,7 +345,6 @@ proc evalFrom*(c: PContext, n: PNode): PNode = proc evalImportExcept*(c: PContext, n: PNode): PNode = result = newNodeI(nkImportStmt, n.info) checkMinSonsLen(n, 2, c.config) - n[0] = transformImportAs(c, n[0]) var m = myImportModule(c, n[0], result) if m != nil: n[0] = newSymNode(m) diff --git a/compiler/lookups.nim b/compiler/lookups.nim index 15a22c778..7ceadfb96 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -75,7 +75,7 @@ proc closeScope*(c: PContext) = ensureNoMissingOrUnusedSymbols(c, c.currentScope) rawCloseScope(c) -iterator allScopes(scope: PScope): PScope = +iterator allScopes*(scope: PScope): PScope = var current = scope while current != nil: yield current @@ -311,11 +311,17 @@ proc addDeclAt*(c: PContext; scope: PScope, sym: PSym) = if conflict != nil: wrongRedefinition(c, sym.info, sym.name.s, conflict.info) -proc addInterfaceDeclAux(c: PContext, sym: PSym) = - if sfExported in sym.flags: +from ic / ic import addHidden + +proc addInterfaceDeclAux*(c: PContext, sym: PSym, forceExport = false) = + if sfExported in sym.flags or forceExport: # add to interface: if c.module != nil: exportSym(c, sym) else: internalError(c.config, sym.info, "addInterfaceDeclAux") + elif sym.kind in ExportableSymKinds and c.module != nil and isTopLevelInsideDeclaration(c, sym): + strTableAdd(semtabAll(c.graph, c.module), sym) + if c.config.symbolFiles != disabledSf: + addHidden(c.encoder, c.packedRepr, sym) proc addInterfaceDeclAt*(c: PContext, scope: PScope, sym: PSym) = addDeclAt(c, scope, sym) diff --git a/compiler/magicsys.nim b/compiler/magicsys.nim index e91bdf272..38d2345c8 100644 --- a/compiler/magicsys.nim +++ b/compiler/magicsys.nim @@ -50,6 +50,7 @@ proc getSysType*(g: ModuleGraph; info: TLineInfo; kind: TTypeKind): PType = result = g.sysTypes[kind] if result == nil: case kind + of tyVoid: result = sysTypeFromName("void") of tyInt: result = sysTypeFromName("int") of tyInt8: result = sysTypeFromName("int8") of tyInt16: result = sysTypeFromName("int16") diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index 02c745018..9df66269b 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -29,6 +29,7 @@ type patterns*: seq[LazySym] pureEnums*: seq[LazySym] interf: TStrTable + interfHidden: TStrTable uniqueName*: Rope Operators* = object @@ -160,9 +161,25 @@ proc toBase64a(s: cstring, len: int): string = result.add cb64[a shr 2] result.add cb64[(a and 3) shl 4] -template semtab*(m: PSym; g: ModuleGraph): TStrTable = +template interfSelect(iface: Iface, importHidden: bool): TStrTable = + var ret = iface.interf.addr # without intermediate ptr, it creates a copy and compiler becomes 15x slower! + if importHidden: ret = iface.interfHidden.addr + ret[] + +template semtab(g: ModuleGraph, m: PSym): TStrTable = g.ifaces[m.position].interf +template semtabAll*(g: ModuleGraph, m: PSym): TStrTable = + g.ifaces[m.position].interfHidden + +proc initStrTables*(g: ModuleGraph, m: PSym) = + initStrTable(semtab(g, m)) + initStrTable(semtabAll(g, m)) + +proc strTableAdds*(g: ModuleGraph, m: PSym, s: PSym) = + strTableAdd(semtab(g, m), s) + strTableAdd(semtabAll(g, m), s) + proc isCachedModule(g: ModuleGraph; module: int): bool {.inline.} = result = module < g.packed.len and g.packed[module].status == loaded @@ -187,39 +204,43 @@ type modIndex: int ti: TIdentIter rodIt: RodIter + importHidden: bool proc initModuleIter*(mi: var ModuleIter; g: ModuleGraph; m: PSym; name: PIdent): PSym = assert m.kind == skModule mi.modIndex = m.position mi.fromRod = isCachedModule(g, mi.modIndex) + mi.importHidden = optImportHidden in m.options if mi.fromRod: - result = initRodIter(mi.rodIt, g.config, g.cache, g.packed, FileIndex mi.modIndex, name) + result = initRodIter(mi.rodIt, g.config, g.cache, g.packed, FileIndex mi.modIndex, name, mi.importHidden) else: - result = initIdentIter(mi.ti, g.ifaces[mi.modIndex].interf, name) + result = initIdentIter(mi.ti, g.ifaces[mi.modIndex].interfSelect(mi.importHidden), name) proc nextModuleIter*(mi: var ModuleIter; g: ModuleGraph): PSym = if mi.fromRod: result = nextRodIter(mi.rodIt, g.packed) else: - result = nextIdentIter(mi.ti, g.ifaces[mi.modIndex].interf) + result = nextIdentIter(mi.ti, g.ifaces[mi.modIndex].interfSelect(mi.importHidden)) iterator allSyms*(g: ModuleGraph; m: PSym): PSym = + let importHidden = optImportHidden in m.options if isCachedModule(g, m): var rodIt: RodIter - var r = initRodIterAllSyms(rodIt, g.config, g.cache, g.packed, FileIndex m.position) + var r = initRodIterAllSyms(rodIt, g.config, g.cache, g.packed, FileIndex m.position, importHidden) while r != nil: yield r r = nextRodIter(rodIt, g.packed) else: - for s in g.ifaces[m.position].interf.data: + for s in g.ifaces[m.position].interfSelect(importHidden).data: if s != nil: yield s proc someSym*(g: ModuleGraph; m: PSym; name: PIdent): PSym = + let importHidden = optImportHidden in m.options if isCachedModule(g, m): - result = interfaceSymbol(g.config, g.cache, g.packed, FileIndex(m.position), name) + result = interfaceSymbol(g.config, g.cache, g.packed, FileIndex(m.position), name, importHidden) else: - result = strTableGet(g.ifaces[m.position].interf, name) + result = strTableGet(g.ifaces[m.position].interfSelect(importHidden), name) proc systemModuleSym*(g: ModuleGraph; name: PIdent): PSym = result = someSym(g, g.systemModule, name) @@ -343,26 +364,23 @@ proc hash*(u: SigHash): Hash = proc hash*(x: FileIndex): Hash {.borrow.} +template getPContext(): untyped = + when c is PContext: c + else: c.c + when defined(nimfind): template onUse*(info: TLineInfo; s: PSym) = - when compiles(c.c.graph): - if c.c.graph.onUsage != nil: c.c.graph.onUsage(c.c.graph, s, info) - else: - if c.graph.onUsage != nil: c.graph.onUsage(c.graph, s, info) + let c = getPContext() + if c.graph.onUsage != nil: c.graph.onUsage(c.graph, s, info) template onDef*(info: TLineInfo; s: PSym) = - when compiles(c.c.graph): - if c.c.graph.onDefinition != nil: c.c.graph.onDefinition(c.c.graph, s, info) - else: - if c.graph.onDefinition != nil: c.graph.onDefinition(c.graph, s, info) + let c = getPContext() + if c.graph.onDefinition != nil: c.graph.onDefinition(c.graph, s, info) template onDefResolveForward*(info: TLineInfo; s: PSym) = - when compiles(c.c.graph): - if c.c.graph.onDefinitionResolveForward != nil: - c.c.graph.onDefinitionResolveForward(c.c.graph, s, info) - else: - if c.graph.onDefinitionResolveForward != nil: - c.graph.onDefinitionResolveForward(c.graph, s, info) + let c = getPContext() + if c.graph.onDefinitionResolveForward != nil: + c.graph.onDefinitionResolveForward(c.graph, s, info) else: template onUse*(info: TLineInfo; s: PSym) = discard @@ -392,7 +410,7 @@ proc registerModule*(g: ModuleGraph; m: PSym) = g.ifaces[m.position] = Iface(module: m, converters: @[], patterns: @[], uniqueName: rope(uniqueModuleName(g.config, FileIndex(m.position)))) - initStrTable(g.ifaces[m.position].interf) + initStrTables(g, m) proc registerModuleById*(g: ModuleGraph; m: FileIndex) = registerModule(g, g.packed[int m].module) diff --git a/compiler/modules.nim b/compiler/modules.nim index 7d7a2b6f7..27b93e8df 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -115,7 +115,7 @@ proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): P elif graph.isDirty(result): result.flags.excl sfDirty # reset module fields: - initStrTable(result.semtab(graph)) + initStrTables(graph, result) result.ast = nil processModuleAux() graph.markClientsDirty(fileIdx) diff --git a/compiler/nimeval.nim b/compiler/nimeval.nim index 9be595cce..577c7b586 100644 --- a/compiler/nimeval.nim +++ b/compiler/nimeval.nim @@ -67,7 +67,7 @@ proc evalScript*(i: Interpreter; scriptStream: PLLStream = nil) = ## This can also be used to *reload* the script. assert i != nil assert i.mainModule != nil, "no main module selected" - initStrTable(i.mainModule.semtab(i.graph)) + initStrTables(i.graph, i.mainModule) i.mainModule.ast = nil let s = if scriptStream != nil: scriptStream diff --git a/compiler/options.nim b/compiler/options.nim index 9eeffb4fb..9ba4d62e4 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -41,7 +41,7 @@ type # please make sure we have under 32 options optMemTracker, optSinkInference # 'sink T' inference optCursorInference - + optImportHidden TOptions* = set[TOption] TGlobalOption* = enum # **keep binary compatible** diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 6b0dcb0e2..9e7258a69 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -361,12 +361,12 @@ proc addPattern*(c: PContext, p: LazySym) = addTrmacro(c.encoder, c.packedRepr, p.sym) proc exportSym*(c: PContext; s: PSym) = - strTableAdd(c.module.semtab(c.graph), s) + strTableAdds(c.graph, c.module, s) if c.config.symbolFiles != disabledSf: addExported(c.encoder, c.packedRepr, s) proc reexportSym*(c: PContext; s: PSym) = - strTableAdd(c.module.semtab(c.graph), s) + strTableAdds(c.graph, c.module, s) if c.config.symbolFiles != disabledSf: addReexport(c.encoder, c.packedRepr, s) @@ -536,6 +536,10 @@ proc checkMinSonsLen*(n: PNode, length: int; conf: ConfigRef) = proc isTopLevel*(c: PContext): bool {.inline.} = result = c.currentScope.depthLevel <= 2 +proc isTopLevelInsideDeclaration*(c: PContext, sym: PSym): bool {.inline.} = + # for routeKinds the scope isn't closed yet: + c.currentScope.depthLevel <= 2 + ord(sym.kind in routineKinds) + proc pushCaseContext*(c: PContext, caseNode: PNode) = c.p.caseContext.add((caseNode, 0)) diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim index c80e689a5..d8be85273 100644 --- a/compiler/semmagic.nim +++ b/compiler/semmagic.nim @@ -577,5 +577,10 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode, if n[1].typ.skipTypes(abstractInst).kind in {tyUInt..tyUInt64}: n[0].sym.magic = mSubU result = n + of mPrivateAccess: + let sym = n[1].typ[0].sym + assert sym != nil + c.currentScope.allowPrivateAccess.add sym + result = newNodeIT(nkEmpty, n.info, getSysType(c.graph, n.info, tyVoid)) else: result = n diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 2c257bd3d..bedc5a7e5 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -1067,6 +1067,7 @@ proc typeDefLeftSidePass(c: PContext, typeSection: PNode, i: int) = elif typsym.kind == skType and sfForward in typsym.flags: s = typsym addInterfaceDecl(c, s) + # PRTEMP no onDef here? else: localError(c.config, name.info, typsym.name.s & " is not a type that can be forwarded") s = typsym diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 34a90a3f2..4763f0fd3 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -138,9 +138,10 @@ proc semEnum(c: PContext, n: PNode, prev: PType): PType = identToReplace[] = symNode if e.position == 0: hasNull = true if result.sym != nil and sfExported in result.sym.flags: - incl(e.flags, sfUsed) - incl(e.flags, sfExported) - if not isPure: exportSym(c, e) + incl(e.flags, {sfUsed, sfExported}) + if result.sym != nil and not isPure: + addInterfaceDeclAux(c, e, forceExport = sfExported in result.sym.flags) + result.n.add symNode styleCheckDef(c.config, e) onDef(e.info, e) diff --git a/compiler/suggest.nim b/compiler/suggest.nim index cad776015..7a2c0d964 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -253,10 +253,15 @@ proc filterSymNoOpr(s: PSym; prefix: PNode; res: var PrefixMatch): bool {.inline proc fieldVisible*(c: PContext, f: PSym): bool {.inline.} = let fmoduleId = getModule(f).id result = sfExported in f.flags or fmoduleId == c.module.id - for module in c.friendModules: - if fmoduleId == module.id: - result = true - break + + if not result: + for module in c.friendModules: + if fmoduleId == module.id: return true + if f.kind == skField: + let symObj = f.owner + for scope in allScopes(c.currentScope): + for sym in scope.allowPrivateAccess: + if symObj.id == sym.id: return true proc getQuality(s: PSym): range[0..100] = result = 100 diff --git a/compiler/wordrecg.nim b/compiler/wordrecg.nim index 6a68e2c70..827efcfa0 100644 --- a/compiler/wordrecg.nim +++ b/compiler/wordrecg.nim @@ -109,7 +109,7 @@ type wStdIn = "stdin", wStdOut = "stdout", wStdErr = "stderr", wInOut = "inout", wByCopy = "bycopy", wByRef = "byref", wOneWay = "oneway", - wBitsize = "bitsize" + wBitsize = "bitsize", wImportHidden = "all", TSpecialWords* = set[TSpecialWord] diff --git a/lib/std/importutils.nim b/lib/std/importutils.nim new file mode 100644 index 000000000..4efe0e140 --- /dev/null +++ b/lib/std/importutils.nim @@ -0,0 +1,34 @@ +##[ +Utilities related to import and symbol resolution. + +Experimental API, subject to change. +]## + +#[ +Possible future APIs: +* module symbols (https://github.com/nim-lang/Nim/pull/9560) +* whichModule (subsumes canImport / moduleExists) (https://github.com/timotheecour/Nim/issues/376) +* getCurrentPkgDir (https://github.com/nim-lang/Nim/pull/10530) +* import from a computed string + related APIs (https://github.com/nim-lang/Nim/pull/10527) +]# + +when defined(nimImportutilsExample): + type Foo = object + x1: int # private + proc initFoo*(): auto = Foo() + +proc privateAccess*(t: typedesc) {.magic: "PrivateAccess".} = + ## Enables access to private fields of `t` in current scope. + runnableExamples("-d:nimImportutilsExample"): + # here we're importing a module containing: + # type Foo = object + # x1: int # private + # proc initFoo*(): auto = Foo() + var a = initFoo() + block: + assert not compiles(a.x1) + privateAccess(a.type) + a.x1 = 1 # accessible in this scope + block: + assert a.x1 == 1 # still in scope + assert not compiles(a.x1) diff --git a/testament/categories.nim b/testament/categories.nim index ee5c46519..18508d70c 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -500,6 +500,7 @@ proc icTests(r: var TResults; testsDir: string, cat: Category, options: string) const tempExt = "_temp.nim" for it in walkDirRec(testsDir / "ic"): + # for it in ["tests/ic/timports.nim"]: # debugging: to try a specific test if isTestFile(it) and not it.endsWith(tempExt): let nimcache = nimcacheDir(it, options, getTestSpecTarget()) removeDir(nimcache) diff --git a/tests/ic/mimports.nim b/tests/ic/mimports.nim new file mode 100644 index 000000000..50773e001 --- /dev/null +++ b/tests/ic/mimports.nim @@ -0,0 +1,9 @@ +from mimportsb {.all.} import fnb1, hfnb3 + +proc fn1*(): int = 1 +proc fn2*(): int = 2 +proc hfn3(): int = 3 +proc hfn4(): int = 4 +proc hfn5(): int = 5 + +export mimportsb.fnb2, hfnb3 diff --git a/tests/ic/mimportsb.nim b/tests/ic/mimportsb.nim new file mode 100644 index 000000000..3cae925c5 --- /dev/null +++ b/tests/ic/mimportsb.nim @@ -0,0 +1,4 @@ +proc fnb1*(): int = 1 +proc fnb2*(): int = 2 +proc hfnb3(): int = 3 +proc hfnb4(): int = 4 diff --git a/tests/ic/timports.nim b/tests/ic/timports.nim new file mode 100644 index 000000000..518a689f5 --- /dev/null +++ b/tests/ic/timports.nim @@ -0,0 +1,29 @@ +import mimports +doAssert fn1() == 1 +doAssert not declared(hfn3) + +#!EDIT!# + +import mimports {.all.} +doAssert fn1() == 1 +doAssert declared(hfn3) +doAssert hfn3() == 3 +doAssert mimports.hfn4() == 4 + +# reexports +doAssert not declared(fnb1) +doAssert not declared(hfnb4) +doAssert fnb2() == 2 +doAssert hfnb3() == 3 + +#!EDIT!# + +from mimports {.all.} import hfn3 +doAssert not declared(fn1) +from mimports {.all.} as bar import fn1 +doAssert fn1() == 1 +doAssert hfn3() == 3 +doAssert not declared(hfn4) +doAssert declared(mimports.hfn4) +doAssert mimports.hfn4() == 4 +doAssert bar.hfn4() == 4 diff --git a/tests/importalls/m1.nim b/tests/importalls/m1.nim new file mode 100644 index 000000000..b6ccbf5da --- /dev/null +++ b/tests/importalls/m1.nim @@ -0,0 +1,70 @@ +import ./m2 +import ./m3 {.all.} as m3 +from ./m3 as m3Bis import nil + +doAssert m3h2 == 2 +export m3h2 + +export m3Bis.m3p1 + +const foo0* = 2 +const foo1 = bar1 + +const foo1Aux = 2 +export foo1Aux + +doAssert not declared(bar2) +doAssert not compiles(bar2) + +var foo2 = 2 +let foo3 = 2 + +type Foo4 = enum + kg1, kg2 + +type Foo4b {.pure.} = enum + foo4b1, foo4b2 + +type Foo5 = object + z1: string + z2: Foo4 + z3: int + z4*: int + +proc `z3`*(a: Foo5): auto = + a.z3 * 10 + +proc foo6(): auto = 2 +proc foo6b*(): auto = 2 +template foo7: untyped = 2 +macro foo8(): untyped = discard +template foo9(a: int) = discard + +block: + template foo10: untyped = 2 + type Foo11 = enum + kg1b, kg2b + proc foo12(): auto = 2 + +proc initFoo5*(z3: int): Foo5 = Foo5(z3: z3) + +func foo13(): auto = 2 +iterator foo14a(): int = discard +iterator foo14b*(): int = discard +iterator foo14c(): int {.closure.} = discard +iterator foo14d(): int {.inline.} = discard + +# fwd declare +proc foo15(): int +proc foo15(): int = 2 + +proc foo16*(): int +proc foo16(): int = 2 + +proc foo17*(): int +proc foo17*(): int = 2 + +# other +type A1 = distinct int +type A2 = distinct int +converter foo18(x: A1): A2 = discard diff --git a/tests/importalls/m2.nim b/tests/importalls/m2.nim new file mode 100644 index 000000000..186650f68 --- /dev/null +++ b/tests/importalls/m2.nim @@ -0,0 +1,3 @@ +const bar1* = 2 +const bar2 = 2 +const bar3 = 3 diff --git a/tests/importalls/m3.nim b/tests/importalls/m3.nim new file mode 100644 index 000000000..caeae2a01 --- /dev/null +++ b/tests/importalls/m3.nim @@ -0,0 +1,5 @@ +# xxx use below naming convention in other test files: p: public, h: hidden +const m3p1* = 2 +const m3h2 = 2 +const m3h3 = 3 +const m3h4 = 4 diff --git a/tests/importalls/m4.nim b/tests/importalls/m4.nim new file mode 100644 index 000000000..b682b766a --- /dev/null +++ b/tests/importalls/m4.nim @@ -0,0 +1,10 @@ +{.warning[UnusedImport]: off.} # xxx bug: this shouldn't be needed since we have `export m3` +import ./m3 {.all.} +import ./m3 as m3b +export m3b +export m3h3 +export m3.m3h4 + +import ./m2 {.all.} as m2b +export m2b except bar3 + diff --git a/tests/importalls/mt0.nim b/tests/importalls/mt0.nim new file mode 100644 index 000000000..154d30ef9 --- /dev/null +++ b/tests/importalls/mt0.nim @@ -0,0 +1,9 @@ +import ./m1 as m +doAssert compiles(foo0) +doAssert not compiles(foo1) +doAssert foo6b() == 2 +doAssert m3h2 == 2 + +var f = initFoo5(z3=3) +doAssert f.z3 == 30 +doAssert z3(f) == 30 diff --git a/tests/importalls/mt1.nim b/tests/importalls/mt1.nim new file mode 100644 index 000000000..87c4eff16 --- /dev/null +++ b/tests/importalls/mt1.nim @@ -0,0 +1,23 @@ +import ./m1 {.all.} as m +doAssert foo1 == 2 +doAssert m.foo1 == 2 + +doAssert m.m3h2 == 2 +doAssert m3h2 == 2 +doAssert m.foo1Aux == 2 +doAssert m.m3p1 == 2 + +## field access +import std/importutils +privateAccess(Foo5) +# var x = Foo5(z1: "foo", z2: m.kg1) +# doAssert x.z1 == "foo" + +var f0: Foo5 +f0.z3 = 3 +doAssert f0.z3 == 3 +var f = initFoo5(z3=3) +doAssert f.z3 == 3 +doAssert z3(f) == 30 +doAssert m.z3(f) == 30 +doAssert not compiles(mt1.`z3`(f)) # z3 is an imported symbol diff --git a/tests/importalls/mt2.nim b/tests/importalls/mt2.nim new file mode 100644 index 000000000..52edbb69e --- /dev/null +++ b/tests/importalls/mt2.nim @@ -0,0 +1,104 @@ +from ./m1 {.all.} as r1 import foo1 +from ./m1 {.all.} as r2 import foo7 + +block: # different symbol kinds + doAssert foo1 == 2 + doAssert r1.foo1 == 2 + doAssert r1.foo2 == 2 + doAssert compiles(foo1) + doAssert compiles(r1.foo2) + doAssert not compiles(foo2) + doAssert not compiles(m3h2) + doAssert r1.foo3 == 2 + + block: # enum + var a: r1.Foo4 + let a1 = r1.kg1 + doAssert a1 == r1.Foo4.kg1 + type A = r1.Foo4 + doAssert a1 == A.kg1 + doAssert not compiles(kg1) + doAssert compiles(A.kg1) + var witness = false + for ai in r1.Foo4: + doAssert ai == a + doAssert ai == a1 + witness = true + break + doAssert witness + + block: # {.pure.} enum + var a: r1.Foo4b + doAssert not compiles(r1.foo4b1) # because pure + doAssert not compiles(foo4b1) + let a1 = r1.Foo4b.foo4b1 + doAssert a1 == a + type A = r1.Foo4b + doAssert a1 == A.foo4b1 + var witness = false + for ai in A: + doAssert ai == a + doAssert ai == a1 + witness = true + break + doAssert witness + + block: # object + doAssert compiles(r1.Foo5) + var a: r1.Foo5 + doAssert compiles(a.z4) + doAssert not compiles(a.z3) + + block: # remaining symbol kinds + doAssert r1.foo6() == 2 + doAssert r1.foo6b() == 2 + doAssert foo7() == 2 + doAssert r2.foo6b() == 2 + + r1.foo8() + r1.foo9(1) + doAssert r1.foo13() == 2 + for a in r1.foo14a(): discard + for a in r1.foo14b(): discard + for a in r1.foo14c(): discard + for a in r1.foo14d(): discard + doAssert r1.foo15() == 2 + doAssert r1.foo16() == 2 + doAssert r1.foo17() == 2 + doAssert compiles(r1.foo18) + doAssert declared(r1.foo18) + + block: # declarations at block scope should not be visible + doAssert declared(foo7) + doAssert declared(r1.foo6) + doAssert not declared(foo10) + doAssert not declared(foo6) + doAssert not declared(r1.Foo11) + doAssert not declared(r1.kg1b) + doAssert not declared(r1.foo12) + doAssert not compiles(r1.foo12()) + +## field access +import std/importutils +privateAccess(r1.Foo5) +var x = r1.Foo5(z1: "foo", z2: r1.kg1) +doAssert x.z1 == "foo" + +var f0: r1.Foo5 +f0.z3 = 3 +doAssert f0.z3 == 3 +var f = r1.initFoo5(z3=3) +doAssert f.z3 == 3 +doAssert r1.z3(f) == 30 + +import ./m1 as r3 +doAssert not declared(foo2) +doAssert not declared(r3.foo2) + +from ./m1 {.all.} as r4 import nil +doAssert not declared(foo2) +doAssert declared(r4.foo2) + +from ./m1 {.all.} import nil +doAssert not declared(foo2) +doAssert declared(m1.foo2) diff --git a/tests/importalls/mt3.nim b/tests/importalls/mt3.nim new file mode 100644 index 000000000..2bd7fd79d --- /dev/null +++ b/tests/importalls/mt3.nim @@ -0,0 +1,12 @@ +import ./m1 {.all.} + # D20201209T194412:here keep this as is, without `as`, so that mt8.nim test keeps + # checking that the original module symbol for `m1` isn't modified and that + # only the alias in `createModuleAlias` is affected. +doAssert declared(m1.foo1) +doAssert foo1 == 2 + + +doAssert m1.foo1 == 2 + +doAssert not compiles(mt3.foo0) # foo0 is an imported symbol +doAssert not compiles(mt3.foo1) # ditto diff --git a/tests/importalls/mt4.nim b/tests/importalls/mt4.nim new file mode 100644 index 000000000..cd7d32aad --- /dev/null +++ b/tests/importalls/mt4.nim @@ -0,0 +1,4 @@ +import ./m1 {.all.} except foo1 +doAssert foo2 == 2 +doAssert declared(foo2) +doAssert not compiles(foo1) diff --git a/tests/importalls/mt4b.nim b/tests/importalls/mt4b.nim new file mode 100644 index 000000000..4578bc54c --- /dev/null +++ b/tests/importalls/mt4b.nim @@ -0,0 +1,3 @@ +from ./m1 {.all.} as m2 import nil +doAssert not compiles(foo1) +doAssert m2.foo1 == 2 diff --git a/tests/importalls/mt5.nim b/tests/importalls/mt5.nim new file mode 100644 index 000000000..5c5785af6 --- /dev/null +++ b/tests/importalls/mt5.nim @@ -0,0 +1,9 @@ +import ./m1 {.all.} as m2 except foo1 +doAssert foo2 == 2 +doAssert not compiles(foo1) +doAssert m2.foo1 == 2 +doAssert compiles(m2.foo1) + +from system {.all.} as s import ThisIsSystem +doAssert ThisIsSystem +doAssert s.ThisIsSystem diff --git a/tests/importalls/mt6.nim b/tests/importalls/mt6.nim new file mode 100644 index 000000000..c6d8b2193 --- /dev/null +++ b/tests/importalls/mt6.nim @@ -0,0 +1,13 @@ +import ./m1 {.all.} as m2 +doAssert compiles(foo1) +doAssert compiles(m2.foo1) +doAssert declared(foo1) +doAssert declared(m2.foo0) # public: works fine + +doAssert m2.foo1 == 2 +doAssert declared(m2.foo1) +doAssert not declared(m2.nonexistant) + +# also tests the quoted `""` import +import "."/"m1" {.all.} as m1b +doAssert compiles(m1b.foo1) diff --git a/tests/importalls/mt7.nim b/tests/importalls/mt7.nim new file mode 100644 index 000000000..45a664610 --- /dev/null +++ b/tests/importalls/mt7.nim @@ -0,0 +1,14 @@ +include ./m1 +doAssert compiles(foo1) +doAssert compiles(mt7.foo1) +doAssert declared(foo1) +doAssert declared(mt7.foo1) +doAssert declared(mt7.foo0) + +var f0: Foo5 +f0.z3 = 3 +doAssert f0.z3 == 3 +var f = initFoo5(z3=3) +doAssert f.z3 == 3 +doAssert mt7.z3(f) == 30 +doAssert z3(f) == 30 diff --git a/tests/importalls/mt8.nim b/tests/importalls/mt8.nim new file mode 100644 index 000000000..f484e7b01 --- /dev/null +++ b/tests/importalls/mt8.nim @@ -0,0 +1,23 @@ +#[ +test multiple imports +]# + +{.warning[UnusedImport]: off.} +import ./m1, m2 {.all.}, ./m3 {.all.} + # make sure this keeps using `import ./m1` without as. + +# m1 is regularly imported +doAssert declared(m1.foo0) +doAssert declared(foo0) + +doAssert not declared(m1.foo1) + # if we didn't call `createModuleAlias` even for `import f1 {.all.}`, + # this would fail, see D20201209T194412. + +# m2 +doAssert declared(m2.bar2) +doAssert declared(bar2) + +# m3 +doAssert declared(m3.m3h2) +doAssert declared(m3h2) diff --git a/tests/importalls/mt9.nim b/tests/importalls/mt9.nim new file mode 100644 index 000000000..42d48ecbd --- /dev/null +++ b/tests/importalls/mt9.nim @@ -0,0 +1,12 @@ +# tests re-export of a module with import {.all.} + +import ./m4 + +doAssert m3p1 == 2 +doAssert not declared(m3h2) +doAssert m3h3 == 3 +doAssert m3h4 == 4 + +doAssert bar1 == 2 +doAssert bar2 == 2 +doAssert not declared(bar3) diff --git a/tests/importalls/tmain2.nim b/tests/importalls/tmain2.nim new file mode 100644 index 000000000..596f0f135 --- /dev/null +++ b/tests/importalls/tmain2.nim @@ -0,0 +1,7 @@ +discard """ +joinable: false # for clarity, but not necessary +""" + +{.warning[UnusedImport]: off.} +# only import `mt*.nim` here; these depend on `m*.nim` +import "."/[mt0,mt1,mt2,mt3,mt4,mt4b,mt5,mt6,mt7,mt8,mt9] From 606288974f5a424d4c5063bc6ce9c1b5cdc0a7bc Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 16 Apr 2021 12:20:43 +0200 Subject: [PATCH 0645/2472] ic navigator tests (#17735) * IC navigator: first basic test --- .gitignore | 1 + compiler/ic/ic.nim | 3 ++- compiler/ic/navigator.nim | 2 +- testament/categories.nim | 19 ++++++++++++------- testament/testament.nim | 10 +++++----- tests/navigator/tnav1.nim | 33 +++++++++++++++++++++++++++++++++ 6 files changed, 54 insertions(+), 14 deletions(-) create mode 100644 tests/navigator/tnav1.nim diff --git a/.gitignore b/.gitignore index a22a4cd03..e9e742404 100644 --- a/.gitignore +++ b/.gitignore @@ -82,6 +82,7 @@ tweeter_test.db /tests/megatest.nim /tests/ic/*_temp.nim +/tests/navigator/*_temp.nim /outputExpected.txt diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim index 1cd03ecfa..1f2d502ae 100644 --- a/compiler/ic/ic.nim +++ b/compiler/ic/ic.nim @@ -494,7 +494,8 @@ proc loadError(err: RodFileError; filename: AbsoluteFile; config: ConfigRef;) = of includeFileChanged: rawMessage(config, warnFileChanged, filename.string) else: - echo "Error: ", $err, " loading file: ", filename.string + rawMessage(config, warnCannotOpenFile, filename.string & " reason: " & $err) + #echo "Error: ", $err, " loading file: ", filename.string proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef; ignoreConfig = false): RodFileError = diff --git a/compiler/ic/navigator.nim b/compiler/ic/navigator.nim index ad2b29f42..b09275220 100644 --- a/compiler/ic/navigator.nim +++ b/compiler/ic/navigator.nim @@ -79,7 +79,7 @@ proc usage(c: var NavContext; info: PackedLineInfo; isDecl: bool) = file = os.extractFilename file toLocation(m, file, info.line.int, info.col.int + ColOffset) if not c.alreadyEmitted.containsOrIncl(m): - echo (if isDecl: "def" else: "usage"), c.outputSep, m + msgWriteln c.g.config, (if isDecl: "def" else: "usage") & c.outputSep & m proc list(c: var NavContext; tree: PackedTree; sym: ItemId) = for i in 0..high(tree.nodes): diff --git a/testament/categories.nim b/testament/categories.nim index 18508d70c..7b57adb8a 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -483,24 +483,27 @@ proc testNimblePackages(r: var TResults; cat: Category; packageFilter: string) = # ---------------- IC tests --------------------------------------------- -proc icTests(r: var TResults; testsDir: string, cat: Category, options: string) = +proc icTests(r: var TResults; testsDir: string, cat: Category, options: string; + isNavigatorTest: bool) = const tooltests = ["compiler/nim.nim", "tools/nimgrep.nim"] writeOnly = " --incremental:writeonly " readOnly = " --incremental:readonly " incrementalOn = " --incremental:on -d:nimIcIntegrityChecks " + navTestConfig = " --ic:on --defusages -d:nimIcNavigatorTests --hint[Conf]:off --warnings:off " template test(x: untyped) = testSpecWithNimcache(r, makeRawTest(file, x & options, cat), nimcache) template editedTest(x: untyped) = var test = makeTest(file, x & options, cat) + if isNavigatorTest: + test.spec.action = actionCompile test.spec.targets = {getTestSpecTarget()} testSpecWithNimcache(r, test, nimcache) const tempExt = "_temp.nim" - for it in walkDirRec(testsDir / "ic"): - # for it in ["tests/ic/timports.nim"]: # debugging: to try a specific test + for it in walkDirRec(testsDir): if isTestFile(it) and not it.endsWith(tempExt): let nimcache = nimcacheDir(it, options, getTestSpecTarget()) removeDir(nimcache) @@ -510,10 +513,10 @@ proc icTests(r: var TResults; testsDir: string, cat: Category, options: string) let file = it.replace(".nim", tempExt) writeFile(file, fragment) let oldPassed = r.passed - editedTest incrementalOn + editedTest(if isNavigatorTest: navTestConfig else: incrementalOn) if r.passed != oldPassed+1: break - when false: + if not isNavigatorTest and false: for file in tooltests: let nimcache = nimcacheDir(file, options, getTestSpecTarget()) removeDir(nimcache) @@ -528,7 +531,7 @@ proc icTests(r: var TResults; testsDir: string, cat: Category, options: string) # ---------------------------------------------------------------------------- -const AdditionalCategories = ["debugger", "examples", "lib", "ic"] +const AdditionalCategories = ["debugger", "examples", "lib", "ic", "navigator"] const MegaTestCat = "megatest" proc `&.?`(a, b: string): string = @@ -695,7 +698,9 @@ proc processCategory(r: var TResults, cat: Category, of "niminaction": testNimInAction(r, cat, options) of "ic": - icTests(r, testsDir, cat, options) + icTests(r, testsDir / cat2, cat, options, isNavigatorTest=false) + of "navigator": + icTests(r, testsDir / cat2, cat, options, isNavigatorTest=true) of "untestable": # These require special treatment e.g. because they depend on a third party # dependency; see `trunner_special` which runs some of those. diff --git a/testament/testament.nim b/testament/testament.nim index ee7f67484..b9e8f3b95 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -157,8 +157,8 @@ proc prepareTestArgs(cmdTemplate, filename, options, nimcache: string, "options", options, "file", filename.quoteShell, "filedir", filename.getFileDir(), "nim", compilerPrefix]) -proc callCompiler(cmdTemplate, filename, options, nimcache: string, - target: TTarget, extraOptions = ""): TSpec = +proc callNimCompiler(cmdTemplate, filename, options, nimcache: string, + target: TTarget, extraOptions = ""): TSpec = let c = prepareTestArgs(cmdTemplate, filename, options, nimcache, target, extraOptions) result.cmd = quoteShellCommand(c) @@ -465,11 +465,11 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, test.startTime = epochTime() case expected.action of actionCompile: - var given = callCompiler(expected.getCmd, test.name, test.options, nimcache, target, + var given = callNimCompiler(expected.getCmd, test.name, test.options, nimcache, target, extraOptions = " --stdout --hint[Path]:off --hint[Processing]:off") compilerOutputTests(test, target, given, expected, r) of actionRun: - var given = callCompiler(expected.getCmd, test.name, test.options, + var given = callNimCompiler(expected.getCmd, test.name, test.options, nimcache, target, extraOptions) if given.err != reSuccess: r.addResult(test, target, "", "$ " & given.cmd & '\n' & given.nimout, given.err) @@ -523,7 +523,7 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, else: compilerOutputTests(test, target, given, expected, r) of actionReject: - var given = callCompiler(expected.getCmd, test.name, test.options, + var given = callNimCompiler(expected.getCmd, test.name, test.options, nimcache, target) cmpMsgs(r, expected, given, test, target) diff --git a/tests/navigator/tnav1.nim b/tests/navigator/tnav1.nim new file mode 100644 index 000000000..d7c6f63e2 --- /dev/null +++ b/tests/navigator/tnav1.nim @@ -0,0 +1,33 @@ +discard """ + cmd: "nim check $options --track:$file,12,7 $file" + nimout: '''def tnav1_temp.nim(11, 10) +usage tnav1_temp.nim(12, 8) + ''' +""" + + +import std / [times] + +proc foo(x: int) = + echo x + +foo(3) +echo "yes", 1 != 3 + +#!EDIT!# +discard """ + cmd: "nim check $options --track:$file,15,2 $file" + nimout: '''def tnav1_temp.nim(12, 6) +usage tnav1_temp.nim(15, 1) + ''' +""" + + +import std / [times] + +proc foo(x: int) = + echo x + +foo(3) +echo "yes", 1 != 3 + From 611b88763f8ec88889b14da31ff220cb47789846 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 16 Apr 2021 03:54:49 -0700 Subject: [PATCH 0646/2472] start using import {.all.} (#17736) --- lib/pure/uri.nim | 22 ---------------------- tests/stdlib/turi.nim | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/lib/pure/uri.nim b/lib/pure/uri.nim index 4ac04d02e..d2a9f3848 100644 --- a/lib/pure/uri.nim +++ b/lib/pure/uri.nim @@ -555,25 +555,3 @@ proc getDataUri*(data, mime: string, encoding = "utf-8"): string {.since: (1, 3) runnableExamples: static: doAssert getDataUri("Nim", "text/plain") == "data:text/plain;charset=utf-8;base64,Tmlt" assert encoding.len > 0 and mime.len > 0 # Must *not* be URL-Safe, see RFC-2397 result = "data:" & mime & ";charset=" & encoding & ";base64," & base64.encode(data) - -when isMainModule and defined(testing): - # needed (pending https://github.com/nim-lang/Nim/pull/11865) because - # `removeDotSegments` is private, the other tests are in `turi`. - block: # removeDotSegments - # `removeDotSegments` is exported for -d:testing only - doAssert removeDotSegments("/foo/bar/baz") == "/foo/bar/baz" - doAssert removeDotSegments("") == "" # empty test - doAssert removeDotSegments(".") == "." # trailing period - doAssert removeDotSegments("a1/a2/../a3/a4/a5/./a6/a7/././") == "a1/a3/a4/a5/a6/a7/" - doAssert removeDotSegments("https://a1/a2/../a3/a4/a5/./a6/a7/././") == "https://a1/a3/a4/a5/a6/a7/" - doAssert removeDotSegments("http://a1/a2") == "http://a1/a2" - doAssert removeDotSegments("http://www.ai.") == "http://www.ai." - when false: # xxx these cases are buggy - # this should work, refs https://webmasters.stackexchange.com/questions/73934/how-can-urls-have-a-dot-at-the-end-e-g-www-bla-de - doAssert removeDotSegments("http://www.ai./") == "http://www.ai./" # fails - echo removeDotSegments("http://www.ai./") # http://www.ai/ - echo removeDotSegments("a/b.../c") # b.c - echo removeDotSegments("a/b../c") # bc - echo removeDotSegments("a/.../c") # .c - echo removeDotSegments("a//../b") # a/b - echo removeDotSegments("a/b/c//") # a/b/c// diff --git a/tests/stdlib/turi.nim b/tests/stdlib/turi.nim index d1a30bbfc..a3b6afe2c 100644 --- a/tests/stdlib/turi.nim +++ b/tests/stdlib/turi.nim @@ -3,6 +3,7 @@ discard """ """ import std/uri +from std/uri {.all.} as uri2 import removeDotSegments from std/sequtils import toSeq template main() = @@ -14,6 +15,24 @@ template main() = doAssert decodeUrl(encodeUrl(test1, false), false) == test1 doAssert decodeUrl(encodeUrl(test1)) == test1 + block: # removeDotSegments + doAssert removeDotSegments("/foo/bar/baz") == "/foo/bar/baz" + doAssert removeDotSegments("") == "" # empty test + doAssert removeDotSegments(".") == "." # trailing period + doAssert removeDotSegments("a1/a2/../a3/a4/a5/./a6/a7/././") == "a1/a3/a4/a5/a6/a7/" + doAssert removeDotSegments("https://a1/a2/../a3/a4/a5/./a6/a7/././") == "https://a1/a3/a4/a5/a6/a7/" + doAssert removeDotSegments("http://a1/a2") == "http://a1/a2" + doAssert removeDotSegments("http://www.ai.") == "http://www.ai." + when false: # xxx these cases are buggy + # this should work, refs https://webmasters.stackexchange.com/questions/73934/how-can-urls-have-a-dot-at-the-end-e-g-www-bla-de + doAssert removeDotSegments("http://www.ai./") == "http://www.ai./" # fails + echo removeDotSegments("http://www.ai./") # http://www.ai/ + echo removeDotSegments("a/b.../c") # b.c + echo removeDotSegments("a/b../c") # bc + echo removeDotSegments("a/.../c") # .c + echo removeDotSegments("a//../b") # a/b + echo removeDotSegments("a/b/c//") # a/b/c// + block: # parseUri block: let org = "udp://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:8080" From d19e4310dc16cae2329c55dfa8feb94e0981dc0c Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 16 Apr 2021 05:21:26 -0700 Subject: [PATCH 0647/2472] std/hashes: hash(ref|ptr|pointer) + other improvements (#17731) --- lib/pure/hashes.nim | 86 +++++++++++++++++++---------- testament/lib/stdtest/testutils.nim | 4 ++ tests/collections/ttables.nim | 14 +++++ tests/stdlib/thashes.nim | 22 +++++++- tests/stdlib/tstrutils.nim | 6 +- 5 files changed, 96 insertions(+), 36 deletions(-) diff --git a/lib/pure/hashes.nim b/lib/pure/hashes.nim index 3339adba2..2ef0e2454 100644 --- a/lib/pure/hashes.nim +++ b/lib/pure/hashes.nim @@ -182,7 +182,7 @@ proc hashData*(data: pointer, size: int): Hash = var h: Hash = 0 when defined(js): var p: cstring - asm """`p` = `Data`;""" + asm """`p` = `Data`""" else: var p = cast[cstring](data) var i = 0 @@ -193,32 +193,6 @@ proc hashData*(data: pointer, size: int): Hash = dec(s) result = !$h -when defined(js): - var objectID = 0 - -proc hash*(x: pointer): Hash {.inline.} = - ## Efficient hashing of pointers. - when defined(js): - asm """ - if (typeof `x` == "object") { - if ("_NimID" in `x`) - `result` = `x`["_NimID"]; - else { - `result` = ++`objectID`; - `x`["_NimID"] = `result`; - } - } - """ - else: - result = cast[Hash](cast[uint](x) shr 3) # skip the alignment - -proc hash*[T: proc](x: T): Hash {.inline.} = - ## Efficient hashing of proc vars. Closures are supported too. - when T is "closure": - result = hash(rawProc(x)) !& hash(rawEnv(x)) - else: - result = hash(pointer(x)) - proc hashIdentity*[T: Ordinal|enum](x: T): Hash {.inline, since: (1, 3).} = ## The identity hash, i.e. `hashIdentity(x) = x`. cast[Hash](ord(x)) @@ -232,6 +206,59 @@ else: ## Efficient hashing of integers. hashWangYi1(uint64(ord(x))) +when defined(js): + var objectID = 0 + proc getObjectId(x: pointer): int = + asm """ + if (typeof `x` == "object") { + if ("_NimID" in `x`) + `result` = `x`["_NimID"]; + else { + `result` = ++`objectID`; + `x`["_NimID"] = `result`; + } + } + """ + +proc hash*(x: pointer): Hash {.inline.} = + ## Efficient `hash` overload. + when defined(js): + let y = getObjectId(x) + else: + let y = cast[int](x) + hash(y) # consistent with code expecting scrambled hashes depending on `nimIntHash1`. + +proc hash*[T](x: ref[T] | ptr[T]): Hash {.inline.} = + ## Efficient `hash` overload. + runnableExamples: + var a: array[10, uint8] + assert a[0].addr.hash != a[1].addr.hash + assert cast[pointer](a[0].addr).hash == a[0].addr.hash + runnableExamples: + type A = ref object + x: int + let a = A(x: 3) + let ha = a.hash + assert ha != A(x: 3).hash # A(x: 3) is a different ref object from `a`. + a.x = 4 + assert ha == a.hash # the hash only depends on the address + runnableExamples: + # you can overload `hash` if you want to customize semantics + type A[T] = ref object + x, y: T + proc hash(a: A): Hash = hash(a.x) + assert A[int](x: 3, y: 4).hash == A[int](x: 3, y: 5).hash + # xxx pending bug #17733, merge as `proc hash*(pointer | ref | ptr): Hash` + # or `proc hash*[T: ref | ptr](x: T): Hash` + hash(cast[pointer](x)) + +proc hash*[T: proc](x: T): Hash {.inline.} = + ## Efficient hashing of proc vars. Closures are supported too. + when T is "closure": + result = hash((rawProc(x), rawEnv(x))) + else: + result = hash(pointer(x)) + proc hash*(x: float): Hash {.inline.} = ## Efficient hashing of floats. let y = x + 0.0 # for denormalization @@ -484,10 +511,9 @@ proc hashIgnoreCase*(sBuf: string, sPos, ePos: int): Hash = h = h !& ord(c) result = !$h - proc hash*[T: tuple | object](x: T): Hash = - ## Efficient hashing of tuples and objects. - ## There must be a `hash` proc defined for each of the field types. + ## Efficient `hash` overload. + ## `hash` must be defined for each component of `x`. runnableExamples: type Obj = object x: int diff --git a/testament/lib/stdtest/testutils.nim b/testament/lib/stdtest/testutils.nim index abffff24c..35423de17 100644 --- a/testament/lib/stdtest/testutils.nim +++ b/testament/lib/stdtest/testutils.nim @@ -85,3 +85,7 @@ template accept*(a) = template reject*(a) = doAssert not compiles(a) + +template disableVm*(body) = + when nimvm: discard + else: body diff --git a/tests/collections/ttables.nim b/tests/collections/ttables.nim index 61197e9f0..c2864b75f 100644 --- a/tests/collections/ttables.nim +++ b/tests/collections/ttables.nim @@ -7,7 +7,11 @@ And we get here 3 ''' joinable: false +targets: "c cpp js" """ + +# xxx wrap in a template to test in VM, see https://github.com/timotheecour/Nim/issues/534#issuecomment-769565033 + import hashes, sequtils, tables, algorithm proc sortedPairs[T](t: T): auto = toSeq(t.pairs).sorted @@ -444,3 +448,13 @@ block emptyOrdered: var t2: OrderedTable[int, string] doAssert t1 == t2 +block: # Table[ref, int] + type A = ref object + x: int + var t: OrderedTable[A, int] + let a1 = A(x: 3) + let a2 = A(x: 3) + t[a1] = 10 + t[a2] = 11 + doAssert t[a1] == 10 + doAssert t[a2] == 11 diff --git a/tests/stdlib/thashes.nim b/tests/stdlib/thashes.nim index 044259f88..66857d3ca 100644 --- a/tests/stdlib/thashes.nim +++ b/tests/stdlib/thashes.nim @@ -3,7 +3,7 @@ discard """ """ import std/hashes - +from stdtest/testutils import disableVm, whenVMorJs when not defined(js) and not defined(cpp): block: @@ -177,5 +177,25 @@ proc main() = doAssert hash(Obj5(t: false, x: 1)) == hash(Obj5(t: true, y: 1)) doAssert hash(Obj5(t: false, x: 1)) != hash(Obj5(t: true, y: 2)) + block: # hash(ref|ptr|pointer) + var a: array[10, uint8] + # disableVm: + whenVMorJs: + # pending fix proposed in https://github.com/nim-lang/Nim/issues/15952#issuecomment-786312417 + discard + do: + assert a[0].addr.hash != a[1].addr.hash + assert cast[pointer](a[0].addr).hash == a[0].addr.hash + + block: # hash(ref) + type A = ref object + x: int + let a = A(x: 3) + disableVm: # xxx Error: VM does not support 'cast' from tyRef to tyPointer + let ha = a.hash + assert ha != A(x: 3).hash # A(x: 3) is a different ref object from `a`. + a.x = 4 + assert ha == a.hash # the hash only depends on the address + static: main() main() diff --git a/tests/stdlib/tstrutils.nim b/tests/stdlib/tstrutils.nim index a6248d1e3..771dddcaf 100644 --- a/tests/stdlib/tstrutils.nim +++ b/tests/stdlib/tstrutils.nim @@ -3,7 +3,7 @@ discard """ """ import std/strutils - +from stdtest/testutils import disableVm # xxx each instance of `disableVm` and `when not defined js:` should eventually be fixed template rejectParse(e) = @@ -12,10 +12,6 @@ template rejectParse(e) = raise newException(AssertionDefect, "This was supposed to fail: $#!" % astToStr(e)) except ValueError: discard -template disableVm(body) = - when nimvm: discard - else: body - template main() = block: # strip doAssert strip(" ha ") == "ha" From fdd4391534578d6a5a655eef99ef96e53ff2b4f1 Mon Sep 17 00:00:00 2001 From: shirleyquirk <31934565+shirleyquirk@users.noreply.github.com> Date: Fri, 16 Apr 2021 13:22:51 +0100 Subject: [PATCH 0648/2472] Fix buffer-overrun bug in net (#17728) [backport:1.0] --- changelog.md | 2 ++ lib/pure/net.nim | 7 +++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/changelog.md b/changelog.md index 6dcfab9c6..cfdcd3c62 100644 --- a/changelog.md +++ b/changelog.md @@ -40,6 +40,8 @@ ## Standard library additions and changes - Added support for parenthesized expressions in `strformat` +- Fixed buffer overflow bugs in `net` + - Added `sections` iterator in `parsecfg`. - Make custom op in macros.quote work for all statements. diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 9be9c6acb..343cdc9b1 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -690,12 +690,11 @@ when defineSsl: let ctx = SslContext(context: ssl.SSL_get_SSL_CTX) let hintString = if hint == nil: "" else: $hint let (identityString, pskString) = (ctx.clientGetPskFunc)(hintString) - if psk.len.cuint > max_psk_len: + if pskString.len.cuint > max_psk_len: return 0 if identityString.len.cuint >= max_identity_len: return 0 - - copyMem(identity, identityString.cstring, pskString.len + 1) # with the last zero byte + copyMem(identity, identityString.cstring, identityString.len + 1) # with the last zero byte copyMem(psk, pskString.cstring, pskString.len) return pskString.len.cuint @@ -716,7 +715,7 @@ when defineSsl: max_psk_len: cint): cuint {.cdecl.} = let ctx = SslContext(context: ssl.SSL_get_SSL_CTX) let pskString = (ctx.serverGetPskFunc)($identity) - if psk.len.cint > max_psk_len: + if pskString.len.cint > max_psk_len: return 0 copyMem(psk, pskString.cstring, pskString.len) From 1b65b9cc193791285f63fcf9d52f19da72fffd4f Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 16 Apr 2021 05:55:51 -0700 Subject: [PATCH 0649/2472] refs #17292 fix `repr`: `(discard)` now does't render as `discard` which gave illegal code (#17455) * refs #17292 fix `repr` with (discard) * add tests * add more tests --- compiler/renderer.nim | 8 ++++++- tests/stdlib/trepr.nim | 50 ++++++++++++++++++++++++++++++++---------- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/compiler/renderer.nim b/compiler/renderer.nim index b3b0adc01..4c0de9ed5 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -1461,7 +1461,13 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext, fromStmtList = false) = put(g, tkSpaces, Space) putWithSpace(g, tkEquals, "=") gsub(g, n, 1) - of nkStmtList, nkStmtListExpr, nkStmtListType: gstmts(g, n, emptyContext) + of nkStmtList, nkStmtListExpr, nkStmtListType: + if n.len == 1 and n[0].kind == nkDiscardStmt: + put(g, tkParLe, "(") + gsub(g, n[0]) + put(g, tkParRi, ")") + else: + gstmts(g, n, emptyContext) of nkIfStmt: putWithSpace(g, tkIf, "if") gif(g, n) diff --git a/tests/stdlib/trepr.nim b/tests/stdlib/trepr.nim index f72bbb34e..86f3dee30 100644 --- a/tests/stdlib/trepr.nim +++ b/tests/stdlib/trepr.nim @@ -5,9 +5,10 @@ discard """ # if excessive, could remove 'cpp' from targets -from strutils import endsWith, contains +from strutils import endsWith, contains, strip from std/macros import newLit -macro deb(a): string = newLit a.repr + +macro deb(a): string = newLit a.repr.strip template main() = doAssert repr({3,5}) == "{3, 5}" @@ -67,17 +68,16 @@ template main() = else: doAssert reprOpenarray(arr) == "[1, 2, 3]" - block: # bug #17292 + block: # bug #17292 repr with `do` template foo(a, b, c, d) = discard block: let a = deb: foo(1, 2, 3, 4) - doAssert a == "\nfoo(1, 2, 3, 4)" + doAssert a == "foo(1, 2, 3, 4)" block: let a = deb: foo(1, 2, 3): 4 doAssert a == """ - foo(1, 2, 3): 4""" @@ -86,7 +86,6 @@ foo(1, 2, 3): foo(1, 2): 3 do: 4 doAssert a == """ - foo(1, 2): 3 do: @@ -98,7 +97,6 @@ do: do: 3 do: 4 doAssert a == """ - foo(1): 3 do: @@ -118,7 +116,6 @@ do: 4 doAssert a == """ - foo(1): 3 do: @@ -135,7 +132,6 @@ do: do: 3 do: 4 doAssert a == """ - foo: 1 do: @@ -145,13 +141,44 @@ do: do: 4""" + block: # bug #17292 repr with `(discard)` (`discard` would result in illegal code) + let a = deb: + let f {.inject.} = () => (discard) + doAssert a == """ +let f {.inject.} = () => + (discard )""" + + let a2 = deb: + block: + discard + discard + + block: + when true: discard + + # let a = b => discard # illegal + discard b => (discard) # legal + + block: + return + doAssert a2 == """ +block: + discard +discard +block: + when true: + discard +discard b => + (discard ) +block: + return""" + block: # bug #17292 (bug 4) let a = deb: proc `=destroy`() = discard proc `'foo`(): int = discard proc `foo bar baz`(): int = discard let a2 = """ - proc `=destroy`() = discard @@ -159,8 +186,7 @@ proc `'foo`(): int = discard proc `foo bar baz`(): int = - discard -""" + discard""" doAssert a2 == a block: # setters: `foo=` From 957478ce264d0496f9a0c33de4af77ad0846b42d Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 16 Apr 2021 15:23:17 +0200 Subject: [PATCH 0650/2472] removed nimfind tool, succeeded by 'nim check --defusages' (#17737) --- koch.nim | 2 - tools/nimfind.nim | 234 --------------------------------------------- tools/nimfind.nims | 2 - 3 files changed, 238 deletions(-) delete mode 100644 tools/nimfind.nim delete mode 100644 tools/nimfind.nims diff --git a/koch.nim b/koch.nim index 44d38589b..3e41310d7 100644 --- a/koch.nim +++ b/koch.nim @@ -216,8 +216,6 @@ proc buildTools(args: string = "") = options = "-d:release " & args) when defined(windows): buildVccTool(args) bundleNimpretty(args) - nimCompileFold("Compile nimfind", "tools/nimfind.nim", - options = "-d:release " & args) nimCompileFold("Compile testament", "testament/testament.nim", options = "-d:release " & args) diff --git a/tools/nimfind.nim b/tools/nimfind.nim deleted file mode 100644 index 9c8247606..000000000 --- a/tools/nimfind.nim +++ /dev/null @@ -1,234 +0,0 @@ -# -# -# The Nim Compiler -# (c) Copyright 2018 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Nimfind is a tool that helps to give editors IDE like capabilities. - -when not defined(nimcore): - {.error: "nimcore MUST be defined for Nim's core tooling".} -when not defined(nimfind): - {.error: "nimfind MUST be defined for Nim's nimfind tool".} - -const Usage = """ -Nimfind - Tool to find declarations or usages for Nim symbols -Usage: - nimfind [options] file.nim:line:col - -Options: - --help, -h show this help - --rebuild rebuild the index - --project:file.nim use file.nim as the entry point - -In addition, all command line options of Nim that do not affect code generation -are supported. -""" - -import strutils, os, parseopt - -import "../compiler" / [options, commands, modules, sem, - passes, passaux, msgs, ast, - idents, modulegraphs, lineinfos, cmdlinehelper, - pathutils] - -import db_sqlite - -proc createDb(db: DbConn) = - db.exec(sql""" - create table if not exists filenames( - id integer primary key, - fullpath varchar(8000) not null - ); - """) - db.exec sql"create index if not exists FilenameIx on filenames(fullpath);" - - # every sym can have potentially 2 different definitions due to forward - # declarations. - db.exec(sql""" - create table if not exists syms( - id integer primary key, - nimid integer not null, - name varchar(256) not null, - defline integer not null, - defcol integer not null, - deffile integer not null, - deflineB integer not null default 0, - defcolB integer not null default 0, - deffileB integer not null default 0, - foreign key (deffile) references filenames(id), - foreign key (deffileB) references filenames(id) - ); - """) - - db.exec(sql""" - create table if not exists usages( - id integer primary key, - nimid integer not null, - line integer not null, - col integer not null, - colB integer not null, - file integer not null, - foreign key (file) references filenames(id), - foreign key (nimid) references syms(nimid) - ); - """) - db.exec sql"create index if not exists UsagesIx on usages(file, line);" - -proc toDbFileId*(db: DbConn; conf: ConfigRef; fileIdx: FileIndex): int = - if fileIdx == FileIndex(-1): return -1 - let fullpath = toFullPath(conf, fileIdx) - let row = db.getRow(sql"select id from filenames where fullpath = ?", fullpath) - let id = row[0] - if id.len == 0: - result = int db.insertID(sql"insert into filenames(fullpath) values (?)", - fullpath) - else: - result = parseInt(id) - -type - FinderRef = ref object of RootObj - db: DbConn - -proc writeDef(graph: ModuleGraph; s: PSym; info: TLineInfo) = - let f = FinderRef(graph.backend) - f.db.exec(sql"""insert into syms(nimid, name, defline, defcol, deffile) values (?, ?, ?, ?, ?)""", - s.id, s.name.s, info.line, info.col, - toDbFileId(f.db, graph.config, info.fileIndex)) - -proc writeDefResolveForward(graph: ModuleGraph; s: PSym; info: TLineInfo) = - let f = FinderRef(graph.backend) - f.db.exec(sql"""update syms set deflineB = ?, defcolB = ?, deffileB = ? - where nimid = ?""", info.line, info.col, - toDbFileId(f.db, graph.config, info.fileIndex), s.id) - -proc writeUsage(graph: ModuleGraph; s: PSym; info: TLineInfo) = - let f = FinderRef(graph.backend) - f.db.exec(sql"""insert into usages(nimid, line, col, colB, file) values (?, ?, ?, ?, ?)""", - s.id, info.line, info.col, info.col + s.name.s.len - 1, - toDbFileId(f.db, graph.config, info.fileIndex)) - -proc performSearch(conf: ConfigRef; dbfile: AbsoluteFile) = - var db = open(connection=string dbfile, user="nim", password="", - database="nim") - let pos = conf.m.trackPos - let fid = toDbFileId(db, conf, pos.fileIndex) - let known = toFullPath(conf, pos.fileIndex) - let nimids = db.getRow(sql"""select distinct nimid from usages where line = ? and file = ? and ? between col and colB""", - pos.line, fid, pos.col) - if nimids.len > 0: - var idSet = "" - for id in nimids: - if idSet.len > 0: idSet.add ", " - idSet.add id - var outputLater = "" - for r in db.rows(sql"""select line, col, filenames.fullpath from usages - inner join filenames on filenames.id = file - where nimid in (?)""", idSet): - let line = parseInt(r[0]) - let col = parseInt(r[1]) - let file = r[2] - if file == known and line == pos.line.int: - # output the line we already know last: - outputLater.add file & ":" & $line & ":" & $(col+1) & "\n" - else: - echo file, ":", line, ":", col+1 - if outputLater.len > 0: stdout.write outputLater - close(db) - -proc setupDb(g: ModuleGraph; dbfile: AbsoluteFile) = - var f = FinderRef() - removeFile(dbfile) - f.db = open(connection=string dbfile, user="nim", password="", - database="nim") - createDb(f.db) - f.db.exec(sql"pragma journal_mode=off") - # This MUST be turned off, otherwise it's way too slow even for testing purposes: - f.db.exec(sql"pragma SYNCHRONOUS=off") - f.db.exec(sql"pragma LOCKING_MODE=exclusive") - g.backend = f - -proc mainCommand(graph: ModuleGraph) = - let conf = graph.config - let dbfile = getNimcacheDir(conf) / RelativeFile"nimfind.db" - if not fileExists(dbfile) or optForceFullMake in conf.globalOptions: - clearPasses(graph) - registerPass graph, verbosePass - registerPass graph, semPass - conf.setCmd cmdIdeTools - wantMainModule(conf) - setupDb(graph, dbfile) - - graph.onDefinition = writeUsage # writeDef - graph.onDefinitionResolveForward = writeUsage # writeDefResolveForward - graph.onUsage = writeUsage - - if not fileExists(conf.projectFull): - quit "cannot find file: " & conf.projectFull.string - add(conf.searchPaths, conf.libpath) - conf.setErrorMaxHighMaybe - try: - compileProject(graph) - finally: - close(FinderRef(graph.backend).db) - performSearch(conf, dbfile) - -proc processCmdLine*(pass: TCmdLinePass, cmd: string; conf: ConfigRef) = - var p = parseopt.initOptParser(cmd) - while true: - parseopt.next(p) - case p.kind - of cmdEnd: break - of cmdLongOption, cmdShortOption: - case p.key.normalize - of "help", "h": - stdout.writeLine(Usage) - quit() - of "project": - conf.projectName = p.val - of "rebuild": - incl conf.globalOptions, optForceFullMake - else: processSwitch(pass, p, conf) - of cmdArgument: - let info = p.key.split(':') - if info.len == 3: - conf.projectName = findProjectNimFile(conf, info[0].splitFile.dir) - if conf.projectName.len == 0: conf.projectName = info[0] - try: - conf.m.trackPos = newLineInfo(conf, AbsoluteFile info[0], - parseInt(info[1]), parseInt(info[2])-1) - except ValueError: - quit "invalid command line" - else: - quit "invalid command line" - -proc handleCmdLine(cache: IdentCache; conf: ConfigRef) = - let self = NimProg( - suggestMode: true, - processCmdLine: processCmdLine - ) - self.initDefinesProg(conf, "nimfind") - - if paramCount() == 0: - stdout.writeLine(Usage) - return - - self.processCmdLineAndProjectPath(conf) - - # Find Nim's prefix dir. - let binaryPath = findExe("nim") - if binaryPath == "": - raise newException(IOError, - "Cannot find Nim standard library: Nim compiler not in PATH") - conf.prefixDir = AbsoluteDir binaryPath.splitPath().head.parentDir() - if not dirExists(conf.prefixDir / RelativeDir"lib"): - conf.prefixDir = AbsoluteDir"" - - var graph = newModuleGraph(cache, conf) - if self.loadConfigsAndProcessCmdLine(cache, conf, graph): - mainCommand(graph) - -handleCmdLine(newIdentCache(), newConfigRef()) diff --git a/tools/nimfind.nims b/tools/nimfind.nims deleted file mode 100644 index 3013c360d..000000000 --- a/tools/nimfind.nims +++ /dev/null @@ -1,2 +0,0 @@ ---define:nimfind ---define:nimcore From 201ac2b9c93a8c3677b4910a1986c91741aee0e3 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 16 Apr 2021 11:26:05 -0700 Subject: [PATCH 0651/2472] fix nim CI following #17455 (same postmortem as #17721) (#17740) --- tests/stdlib/trepr.nim | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/stdlib/trepr.nim b/tests/stdlib/trepr.nim index 86f3dee30..356764903 100644 --- a/tests/stdlib/trepr.nim +++ b/tests/stdlib/trepr.nim @@ -193,10 +193,8 @@ proc `foo bar baz`(): int = let a = deb: proc `foo=`() = discard doAssert a == """ - proc `foo=`() = - discard -""" + discard""" block: # bug #14850 block: @@ -221,7 +219,6 @@ proc `foo=`() = 4) doAssert a == """ - template bar(): untyped = foo1: discard @@ -244,8 +241,7 @@ template bar(): untyped = 4 a.add(foo7 do: echo "baz" - 4) -""" + 4)""" static: main() main() From 8e474fbb57c2f7b58a840b5b30230c2267633c8e Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 17 Apr 2021 01:12:12 +0200 Subject: [PATCH 0652/2472] IC: yet another embarrassing omission (#17743) * IC: yet another embarrassing omission * VM: fewer hacks that kept IC from working --- compiler/ast.nim | 2 +- compiler/ic/ic.nim | 3 ++- compiler/vmdef.nim | 3 +++ compiler/vmgen.nim | 14 ++++++-------- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 4c835871f..bf0c9669a 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -1054,7 +1054,7 @@ const defaultSize = -1 defaultAlignment = -1 - defaultOffset = -1 + defaultOffset* = -1 proc getPIdent*(a: PNode): PIdent {.inline.} = ## Returns underlying `PIdent` for `{nkSym, nkIdent}`, or `nil`. diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim index 1f2d502ae..da0af2edf 100644 --- a/compiler/ic/ic.nim +++ b/compiler/ic/ic.nim @@ -765,7 +765,8 @@ proc symHeaderFromPacked(c: var PackedDecoder; g: var PackedModuleGraph; kind: s.kind, magic: s.magic, flags: s.flags, info: translateLineInfo(c, g, si, s.info), options: s.options, - position: s.position, + position: if s.kind in {skForVar, skVar, skLet, skTemp}: 0 else: s.position, + offset: if s.kind in routineKinds: defaultOffset else: s.offset, name: getIdent(c.cache, g[si].fromDisk.sh.strings[s.name]) ) diff --git a/compiler/vmdef.nim b/compiler/vmdef.nim index e8b5cdda9..068ade4b0 100644 --- a/compiler/vmdef.nim +++ b/compiler/vmdef.nim @@ -10,6 +10,8 @@ ## This module contains the type definitions for the new evaluation engine. ## An instruction is 1-3 int32s in memory, it is a register based VM. +import std / tables + import ast, idents, options, modulegraphs, lineinfos type TInstrType* = uint64 @@ -266,6 +268,7 @@ type profiler*: Profiler templInstCounter*: ref int # gives every template instantiation a unique ID, needed here for getAst vmstateDiff*: seq[(PSym, PNode)] # we remember the "diff" to global state here (feature for IC) + procToCodePos*: Table[int, int] PStackFrame* = ref TStackFrame TStackFrame* = object diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 69d7ec4b3..951d47d2f 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -27,6 +27,8 @@ # solves the opcLdConst vs opcAsgnConst issue. Of course whether we need # this copy depends on the involved types. +import std / tables + import strutils, ast, types, msgs, renderer, vmdef, intsets, magicsys, options, lowerings, lineinfos, transf @@ -2248,8 +2250,8 @@ proc optimizeJumps(c: PCtx; start: int) = else: discard proc genProc(c: PCtx; s: PSym): int = - var x = s.ast[miscPos] - if x.kind == nkEmpty or x[0].kind == nkEmpty: + let pos = c.procToCodePos.getOrDefault(s.id) + if pos == 0: #if s.name.s == "outterMacro" or s.name.s == "innerProc": # echo "GENERATING CODE FOR ", s.name.s let last = c.code.len-1 @@ -2260,11 +2262,7 @@ proc genProc(c: PCtx; s: PSym): int = c.debug.setLen(last) #c.removeLastEof result = c.code.len+1 # skip the jump instruction - if x.kind == nkEmpty: - x = newTree(nkBracket, newIntNode(nkIntLit, result), x) - else: - x[0] = newIntNode(nkIntLit, result) - s.ast[miscPos] = x + c.procToCodePos[s.id] = result # thanks to the jmp we can add top level statements easily and also nest # procs easily: let body = transformBody(c.graph, c.idgen, s, cache = not isCompileTimeProc(s)) @@ -2297,4 +2295,4 @@ proc genProc(c: PCtx; s: PSym): int = c.prc = oldPrc else: c.prc.maxSlots = s.offset - result = x[0].intVal.int + result = pos From 7e94420847d2d18347410099e97195ddebe8b85f Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 17 Apr 2021 02:14:09 -0700 Subject: [PATCH 0653/2472] cString => cSourceString; tyCString => tyCstring so that error msgs show cstring, not cString (#17744) --- compiler/ast.nim | 6 +++--- compiler/ccgcalls.nim | 2 +- compiler/ccgexprs.nim | 14 +++++++------- compiler/ccgreset.nim | 2 +- compiler/ccgtypes.nim | 6 +++--- compiler/concepts.nim | 4 ++-- compiler/evalffi.nim | 8 ++++---- compiler/jsgen.nim | 20 ++++++++++---------- compiler/jstypes.nim | 2 +- compiler/liftdestructors.nim | 2 +- compiler/magicsys.nim | 2 +- compiler/nilcheck.nim | 2 +- compiler/semexprs.nim | 4 ++-- compiler/semfold.nim | 4 ++-- compiler/semmacrosanity.nim | 2 +- compiler/sempass2.nim | 2 +- compiler/semtypes.nim | 6 +++--- compiler/sigmatch.nim | 10 +++++----- compiler/sizealignoffsetimpl.nim | 2 +- compiler/sourcemap.nim | 10 +++++----- compiler/transf.nim | 4 ++-- compiler/typeallowed.nim | 2 +- compiler/types.nim | 6 +++--- compiler/vm.nim | 2 +- compiler/vmdeps.nim | 2 +- compiler/vmgen.nim | 12 ++++++------ compiler/vmmarshal.nim | 4 ++-- lib/core/typeinfo.nim | 8 ++++---- lib/system/assign.nim | 2 +- lib/system/hti.nim | 2 +- lib/system/repr.nim | 2 +- lib/system/reprjs.nim | 2 +- 32 files changed, 79 insertions(+), 79 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index bf0c9669a..02a2ae7a4 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -379,7 +379,7 @@ type tySequence, tyProc, tyPointer, tyOpenArray, - tyString, tyCString, tyForward, + tyString, tyCstring, tyForward, tyInt, tyInt8, tyInt16, tyInt32, tyInt64, # signed integers tyFloat, tyFloat32, tyFloat64, tyFloat128, tyUInt, tyUInt8, tyUInt16, tyUInt32, tyUInt64, @@ -1005,13 +1005,13 @@ const tyBool, tyChar, tyEnum, tyArray, tyObject, tySet, tyTuple, tyRange, tyPtr, tyRef, tyVar, tyLent, tySequence, tyProc, tyPointer, - tyOpenArray, tyString, tyCString, tyInt..tyInt64, tyFloat..tyFloat128, + tyOpenArray, tyString, tyCstring, tyInt..tyInt64, tyFloat..tyFloat128, tyUInt..tyUInt64} IntegralTypes* = {tyBool, tyChar, tyEnum, tyInt..tyInt64, tyFloat..tyFloat128, tyUInt..tyUInt64} # weird name because it contains tyFloat ConstantDataTypes*: TTypeKinds = {tyArray, tySet, tyTuple, tySequence} - NilableTypes*: TTypeKinds = {tyPointer, tyCString, tyRef, tyPtr, + NilableTypes*: TTypeKinds = {tyPointer, tyCstring, tyRef, tyPtr, tyProc, tyError} # TODO PtrLikeKinds*: TTypeKinds = {tyPointer, tyPtr} # for VM PersistentNodeFlags*: TNodeFlags = {nfBase2, nfBase8, nfBase16, diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index 2a93cb5af..616028ac6 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -178,7 +178,7 @@ proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType): (Rope, else: result = ("($3*)($1)+($2)" % [rdLoc(a), rdLoc(b), dest], lengthExpr) - of tyUncheckedArray, tyCString: + of tyUncheckedArray, tyCstring: result = ("($3*)($1)+($2)" % [rdLoc(a), rdLoc(b), dest], lengthExpr) of tyString, tySequence: diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 3872ed360..0e54adf49 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -399,7 +399,7 @@ proc genAssignment(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) = [rdLoc(dest), rdLoc(src), getSize(p.config, dest.t)]) else: linefmt(p, cpsStmts, "$1 = $2;$n", [rdLoc(dest), rdLoc(src)]) - of tyPtr, tyPointer, tyChar, tyBool, tyEnum, tyCString, + of tyPtr, tyPointer, tyChar, tyBool, tyEnum, tyCstring, tyInt..tyUInt64, tyRange, tyVar, tyLent, tyNil: linefmt(p, cpsStmts, "$1 = $2;$n", [rdLoc(dest), rdLoc(src)]) else: internalError(p.config, "genAssignment: " & $ty.kind) @@ -449,7 +449,7 @@ proc genDeepCopy(p: BProc; dest, src: TLoc) = [rdLoc(dest), rdLoc(src), getSize(p.config, dest.t)]) else: linefmt(p, cpsStmts, "$1 = $2;$n", [rdLoc(dest), rdLoc(src)]) - of tyPointer, tyChar, tyBool, tyEnum, tyCString, + of tyPointer, tyChar, tyBool, tyEnum, tyCstring, tyInt..tyUInt64, tyRange, tyVar, tyLent: linefmt(p, cpsStmts, "$1 = $2;$n", [rdLoc(dest), rdLoc(src)]) else: internalError(p.config, "genDeepCopy: " & $ty.kind) @@ -1016,7 +1016,7 @@ proc genBracketExpr(p: BProc; n: PNode; d: var TLoc) = of tyArray: genArrayElem(p, n, n[0], n[1], d) of tyOpenArray, tyVarargs: genOpenArrayElem(p, n, n[0], n[1], d) of tySequence, tyString: genSeqElem(p, n, n[0], n[1], d) - of tyCString: genCStringElem(p, n, n[0], n[1], d) + of tyCstring: genCStringElem(p, n, n[0], n[1], d) of tyTuple: genTupleElem(p, n, d) else: internalError(p.config, n.info, "expr(nkBracketExpr, " & $ty.kind & ')') discard getTypeDesc(p.module, n.typ) @@ -1666,7 +1666,7 @@ proc genRepr(p: BProc, e: PNode, d: var TLoc) = putIntoDest(p, d, e, ropecg(p.module, "#reprOpenArray($1, $2)", [rdLoc(b), genTypeInfoV1(p.module, elemType(t), e.info)]), a.storage) - of tyCString, tyArray, tyRef, tyPtr, tyPointer, tyNil, tySequence: + of tyCstring, tyArray, tyRef, tyPtr, tyPointer, tyNil, tySequence: putIntoDest(p, d, e, ropecg(p.module, "#reprAny($1, $2)", [ rdLoc(a), genTypeInfoV1(p.module, t, e.info)]), a.storage) @@ -1745,7 +1745,7 @@ proc genArrayLen(p: BProc, e: PNode, d: var TLoc, op: TMagic) = else: if op == mHigh: unaryExpr(p, e, d, "($1.Field1-1)") else: unaryExpr(p, e, d, "$1.Field1") - of tyCString: + of tyCstring: if op == mHigh: unaryExpr(p, e, d, "($1 ? (#nimCStrLen($1)-1) : -1)") else: unaryExpr(p, e, d, "($1 ? #nimCStrLen($1) : 0)") of tyString: @@ -2920,7 +2920,7 @@ proc getDefaultValue(p: BProc; typ: PType; info: TLineInfo): Rope = of tyBool: result = rope"NIM_FALSE" of tyEnum, tyChar, tyInt..tyInt64, tyUInt..tyUInt64: result = rope"0" of tyFloat..tyFloat128: result = rope"0.0" - of tyCString, tyVar, tyLent, tyPointer, tyPtr, tyUntyped, + of tyCstring, tyVar, tyLent, tyPointer, tyPtr, tyUntyped, tyTyped, tyTypeDesc, tyStatic, tyRef, tyNil: result = rope"NIM_NIL" of tyString, tySequence: @@ -3187,7 +3187,7 @@ proc genBracedInit(p: BProc, n: PNode; isConst: bool; optionalType: PType): Rope of tyObject: result = genConstObjConstr(p, n, isConst) - of tyString, tyCString: + of tyString, tyCstring: if optSeqDestructors in p.config.globalOptions and n.kind != nkNilLit and ty == tyString: result = genStringLiteralV2Const(p.module, n, isConst) else: diff --git a/compiler/ccgreset.nim b/compiler/ccgreset.nim index ef1505f57..fc7370d81 100644 --- a/compiler/ccgreset.nim +++ b/compiler/ccgreset.nim @@ -85,7 +85,7 @@ proc specializeResetT(p: BProc, accessor: Rope, typ: PType) = lineCg(p, cpsStmts, "$1 = NIM_NIL;$n", [accessor]) of tyChar, tyBool, tyEnum, tyInt..tyUInt64: lineCg(p, cpsStmts, "$1 = 0;$n", [accessor]) - of tyCString, tyPointer, tyPtr, tyVar, tyLent: + of tyCstring, tyPointer, tyPtr, tyVar, tyLent: lineCg(p, cpsStmts, "$1 = NIM_NIL;$n", [accessor]) else: discard diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 9d95e3f67..3d4619764 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -189,7 +189,7 @@ proc mapType(conf: ConfigRef; typ: PType; kind: TSymKind): TCTypeKind = of tySequence: result = ctNimSeq of tyProc: result = if typ.callConv != ccClosure: ctProc else: ctStruct of tyString: result = ctNimStr - of tyCString: result = ctCString + of tyCstring: result = ctCString of tyInt..tyUInt64: result = TCTypeKind(ord(typ.kind) - ord(tyInt) + ord(ctInt)) of tyStatic: @@ -292,7 +292,7 @@ proc getSimpleTypeDesc(m: BModule, typ: PType): Rope = else: discard cgsym(m, "NimStringDesc") result = typeNameOrLiteral(m, typ, "NimStringDesc*") - of tyCString: result = typeNameOrLiteral(m, typ, "NCSTRING") + of tyCstring: result = typeNameOrLiteral(m, typ, "NCSTRING") of tyBool: result = typeNameOrLiteral(m, typ, "NIM_BOOL") of tyChar: result = typeNameOrLiteral(m, typ, "NIM_CHAR") of tyNil: result = typeNameOrLiteral(m, typ, "void*") @@ -1452,7 +1452,7 @@ proc genTypeInfoV1(m: BModule, t: PType; info: TLineInfo): Rope = case t.kind of tyEmpty, tyVoid: result = rope"0" - of tyPointer, tyBool, tyChar, tyCString, tyString, tyInt..tyUInt64, tyVar, tyLent: + of tyPointer, tyBool, tyChar, tyCstring, tyString, tyInt..tyUInt64, tyVar, tyLent: genTypeInfoAuxBase(m, t, t, result, rope"0", info) of tyStatic: if t.n != nil: result = genTypeInfoV1(m, lastSon t, info) diff --git a/compiler/concepts.nim b/compiler/concepts.nim index 54579f73f..de994f1b8 100644 --- a/compiler/concepts.nim +++ b/compiler/concepts.nim @@ -139,7 +139,7 @@ proc matchType(c: PContext; f, a: PType; m: var MatchCon): bool = if result: when logBindings: echo "A adding ", f, " ", ak m.inferred.add((f, ak)) - elif m.magic == mArrGet and ak.kind in {tyArray, tyOpenArray, tySequence, tyVarargs, tyCString, tyString}: + elif m.magic == mArrGet and ak.kind in {tyArray, tyOpenArray, tySequence, tyVarargs, tyCstring, tyString}: when logBindings: echo "B adding ", f, " ", lastSon ak m.inferred.add((f, lastSon ak)) result = true @@ -165,7 +165,7 @@ proc matchType(c: PContext; f, a: PType; m: var MatchCon): bool = result = false of tyEnum, tyObject, tyDistinct: result = sameType(f, a) - of tyEmpty, tyString, tyCString, tyPointer, tyNil, tyUntyped, tyTyped, tyVoid: + of tyEmpty, tyString, tyCstring, tyPointer, tyNil, tyUntyped, tyTyped, tyVoid: result = a.skipTypes(ignorableForArgType).kind == f.kind of tyBool, tyChar, tyInt..tyUInt64: let ak = a.skipTypes(ignorableForArgType) diff --git a/compiler/evalffi.nim b/compiler/evalffi.nim index 0d471cf98..5004011ed 100644 --- a/compiler/evalffi.nim +++ b/compiler/evalffi.nim @@ -92,7 +92,7 @@ proc mapType(conf: ConfigRef, t: ast.PType): ptr libffi.Type = else: result = nil of tyFloat, tyFloat64: result = addr libffi.type_double of tyFloat32: result = addr libffi.type_float - of tyVar, tyLent, tyPointer, tyPtr, tyRef, tyCString, tySequence, tyString, tyUntyped, + of tyVar, tyLent, tyPointer, tyPtr, tyRef, tyCstring, tySequence, tyString, tyUntyped, tyTyped, tyTypeDesc, tyProc, tyArray, tyStatic, tyNil: result = addr libffi.type_pointer of tyDistinct, tyAlias, tySink: @@ -205,7 +205,7 @@ proc pack(conf: ConfigRef, v: PNode, typ: PType, res: pointer) = of tyFloat32: awr(float32, v.floatVal) of tyFloat64: awr(float64, v.floatVal) - of tyPointer, tyProc, tyCString, tyString: + of tyPointer, tyProc, tyCstring, tyString: if v.kind == nkNilLit: # nothing to do since the memory is 0 initialized anyway discard @@ -386,7 +386,7 @@ proc unpack(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode = result = unpackObject(conf, x, typ, n) of tyArray: result = unpackArray(conf, x, typ, n) - of tyCString, tyString: + of tyCstring, tyString: let p = rd(cstring, x) if p.isNil: setNil() @@ -402,7 +402,7 @@ proc unpack(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode = proc fficast*(conf: ConfigRef, x: PNode, destTyp: PType): PNode = if x.kind == nkPtrLit and x.typ.kind in {tyPtr, tyRef, tyVar, tyLent, tyPointer, - tyProc, tyCString, tyString, + tyProc, tyCstring, tyString, tySequence}: result = newNodeIT(x.kind, x.info, destTyp) result.intVal = x.intVal diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index ca716b1c0..350563d6b 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -210,7 +210,7 @@ proc mapType(typ: PType): TJSTypeKind = if t.n != nil: result = mapType(lastSon t) else: result = etyNone of tyProc: result = etyProc - of tyCString: result = etyString + of tyCstring: result = etyString of tyConcept, tyIterable: doAssert false proc mapType(p: PProc; typ: PType): TJSTypeKind = @@ -1047,14 +1047,14 @@ proc needsNoCopy(p: PProc; y: PNode): bool = return y.kind in nodeKindsNeedNoCopy or ((mapType(y.typ) != etyBaseIndex or (y.kind == nkSym and y.sym.kind == skParam)) and (skipTypes(y.typ, abstractInst).kind in - {tyRef, tyPtr, tyLent, tyVar, tyCString, tyProc, tyOwned} + IntegralTypes)) + {tyRef, tyPtr, tyLent, tyVar, tyCstring, tyProc, tyOwned} + IntegralTypes)) proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) = var a, b: TCompRes var xtyp = mapType(p, x.typ) # disable `[]=` for cstring - if x.kind == nkBracketExpr and x.len >= 2 and x[0].typ.skipTypes(abstractInst).kind == tyCString: + if x.kind == nkBracketExpr and x.len >= 2 and x[0].typ.skipTypes(abstractInst).kind == tyCstring: localError(p.config, x.info, "cstring doesn't support `[]=` operator") gen(p, x, a) @@ -1263,14 +1263,14 @@ proc genArrayAccess(p: PProc, n: PNode, r: var TCompRes) = var ty = skipTypes(n[0].typ, abstractVarRange) if ty.kind in {tyRef, tyPtr, tyLent, tyOwned}: ty = skipTypes(ty.lastSon, abstractVarRange) case ty.kind - of tyArray, tyOpenArray, tySequence, tyString, tyCString, tyVarargs: + of tyArray, tyOpenArray, tySequence, tyString, tyCstring, tyVarargs: genArrayAddr(p, n, r) of tyTuple: genFieldAddr(p, n, r) else: internalError(p.config, n.info, "expr(nkBracketExpr, " & $ty.kind & ')') r.typ = mapType(n.typ) if r.res == nil: internalError(p.config, n.info, "genArrayAccess") - if ty.kind == tyCString: + if ty.kind == tyCstring: r.res = "$1.charCodeAt($2)" % [r.address, r.res] elif r.typ == etyBaseIndex: if needsTemp(p, n[0]): @@ -1340,7 +1340,7 @@ proc genAddr(p: PProc, n: PNode, r: var TCompRes) = else: let kindOfIndexedExpr = skipTypes(n[0][0].typ, abstractVarRange).kind case kindOfIndexedExpr - of tyArray, tyOpenArray, tySequence, tyString, tyCString, tyVarargs: + of tyArray, tyOpenArray, tySequence, tyString, tyCstring, tyVarargs: genArrayAddr(p, n[0], r) of tyTuple: genFieldAddr(p, n[0], r) @@ -1744,7 +1744,7 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope = result = putToSeq("null", indirect) of tySequence, tyString: result = putToSeq("[]", indirect) - of tyCString, tyProc: + of tyCstring, tyProc: result = putToSeq("null", indirect) of tyStatic: if t.n != nil: @@ -2030,7 +2030,7 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) = gen(p, n[1], lhs) gen(p, n[2], rhs) - if skipTypes(n[1].typ, abstractVarRange).kind == tyCString: + if skipTypes(n[1].typ, abstractVarRange).kind == tyCstring: let (b, tmp) = maybeMakeTemp(p, n[2], rhs) r.res = "if (null != $1) { if (null == $2) $2 = $3; else $2 += $3; }" % [b, lhs.rdLoc, tmp] @@ -2087,7 +2087,7 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) = of mLengthStr, mLengthSeq, mLengthOpenArray, mLengthArray: var x: TCompRes gen(p, n[1], x) - if skipTypes(n[1].typ, abstractInst).kind == tyCString: + if skipTypes(n[1].typ, abstractInst).kind == tyCstring: let (a, tmp) = maybeMakeTemp(p, n[1], x) r.res = "(($1) == null ? 0 : ($2).length)" % [a, tmp] else: @@ -2096,7 +2096,7 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) = of mHigh: var x: TCompRes gen(p, n[1], x) - if skipTypes(n[1].typ, abstractInst).kind == tyCString: + if skipTypes(n[1].typ, abstractInst).kind == tyCstring: let (a, tmp) = maybeMakeTemp(p, n[1], x) r.res = "(($1) == null ? -1 : ($2).length - 1)" % [a, tmp] else: diff --git a/compiler/jstypes.nim b/compiler/jstypes.nim index 2073c252e..81f03dc12 100644 --- a/compiler/jstypes.nim +++ b/compiler/jstypes.nim @@ -128,7 +128,7 @@ proc genTypeInfo(p: PProc, typ: PType): Rope = case t.kind of tyDistinct: result = genTypeInfo(p, t[0]) - of tyPointer, tyProc, tyBool, tyChar, tyCString, tyString, tyInt..tyUInt64: + of tyPointer, tyProc, tyBool, tyChar, tyCstring, tyString, tyInt..tyUInt64: var s = "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" % [result, rope(ord(t.kind))] diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim index 436665827..ec0435f95 100644 --- a/compiler/liftdestructors.nim +++ b/compiler/liftdestructors.nim @@ -825,7 +825,7 @@ proc ownedClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode) = case t.kind of tyNone, tyEmpty, tyVoid: discard - of tyPointer, tySet, tyBool, tyChar, tyEnum, tyInt..tyUInt64, tyCString, + of tyPointer, tySet, tyBool, tyChar, tyEnum, tyInt..tyUInt64, tyCstring, tyPtr, tyUncheckedArray, tyVar, tyLent: defaultOp(c, t, body, x, y) of tyRef: diff --git a/compiler/magicsys.nim b/compiler/magicsys.nim index 38d2345c8..ab234a2a8 100644 --- a/compiler/magicsys.nim +++ b/compiler/magicsys.nim @@ -68,7 +68,7 @@ proc getSysType*(g: ModuleGraph; info: TLineInfo; kind: TTypeKind): PType = of tyBool: result = sysTypeFromName("bool") of tyChar: result = sysTypeFromName("char") of tyString: result = sysTypeFromName("string") - of tyCString: result = sysTypeFromName("cstring") + of tyCstring: result = sysTypeFromName("cstring") of tyPointer: result = sysTypeFromName("pointer") of tyNil: result = newSysType(g, tyNil, g.config.target.ptrSize) else: internalError(g.config, "request for typekind: " & $kind) diff --git a/compiler/nilcheck.nim b/compiler/nilcheck.nim index b779830d6..f3ec893f7 100644 --- a/compiler/nilcheck.nim +++ b/compiler/nilcheck.nim @@ -1283,7 +1283,7 @@ proc typeNilability(typ: PType): Nilability = # echo "typeNilability ", $typ.flags, " ", $typ.kind result = if tfNotNil in typ.flags: Safe - elif typ.kind in {tyRef, tyCString, tyPtr, tyPointer}: + elif typ.kind in {tyRef, tyCstring, tyPtr, tyPointer}: # # tyVar ? tyVarargs ? tySink ? tyLent ? # TODO spec? tests? diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 13b53ef32..1582cc0fd 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -375,7 +375,7 @@ proc semLowHigh(c: PContext, n: PNode, m: TMagic): PNode = n[1] = semExprWithType(c, n[1], {efDetermineType}) var typ = skipTypes(n[1].typ, abstractVarRange + {tyTypeDesc, tyUserTypeClassInst}) case typ.kind - of tySequence, tyString, tyCString, tyOpenArray, tyVarargs: + of tySequence, tyString, tyCstring, tyOpenArray, tyVarargs: n.typ = getSysType(c.graph, n.info, tyInt) of tyArray: n.typ = typ[0] # indextype @@ -1508,7 +1508,7 @@ proc semSubscript(c: PContext, n: PNode, flags: TExprFlags): PNode = arr = arr.base case arr.kind - of tyArray, tyOpenArray, tyVarargs, tySequence, tyString, tyCString, + of tyArray, tyOpenArray, tyVarargs, tySequence, tyString, tyCstring, tyUncheckedArray: if n.len != 2: return nil n[0] = makeDeref(n[0]) diff --git a/compiler/semfold.nim b/compiler/semfold.nim index 0d4f6a0bd..04ed73209 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -143,7 +143,7 @@ proc evalOp(m: TMagic, n, a, b, c: PNode; idgen: IdGenerator; g: ModuleGraph): P elif a.kind in {nkStrLit..nkTripleStrLit}: if a.typ.kind == tyString: result = newIntNodeT(toInt128(a.strVal.len), n, idgen, g) - elif a.typ.kind == tyCString: + elif a.typ.kind == tyCstring: result = newIntNodeT(toInt128(nimCStrLen(a.strVal)), n, idgen, g) else: result = newIntNodeT(toInt128(a.len), n, idgen, g) @@ -578,7 +578,7 @@ proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode result = newIntNodeT(firstOrd(g.config, n[1].typ), n, idgen, g) of mHigh: if skipTypes(n[1].typ, abstractVar+{tyUserTypeClassInst}).kind notin - {tySequence, tyString, tyCString, tyOpenArray, tyVarargs}: + {tySequence, tyString, tyCstring, tyOpenArray, tyVarargs}: if skipTypes(n[1].typ, abstractVarRange).kind in tyFloat..tyFloat64: result = newFloatNodeT(lastFloat(n[1].typ), n, g) else: diff --git a/compiler/semmacrosanity.nim b/compiler/semmacrosanity.nim index a8eb62067..aebee8998 100644 --- a/compiler/semmacrosanity.nim +++ b/compiler/semmacrosanity.nim @@ -92,7 +92,7 @@ proc annotateType*(n: PNode, t: PType; conf: ConfigRef) = else: globalError(conf, n.info, "integer literal must have some int type") of nkStrLit..nkTripleStrLit: - if x.kind in {tyString, tyCString}: + if x.kind in {tyString, tyCstring}: n.typ = t else: globalError(conf, n.info, "string literal must be of some string type") diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index 52998c214..e66c9596a 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -696,7 +696,7 @@ proc paramType(op: PType, i: int): PType = if op != nil and i < op.len: result = op[i] proc cstringCheck(tracked: PEffects; n: PNode) = - if n[0].typ.kind == tyCString and (let a = skipConv(n[1]); + if n[0].typ.kind == tyCstring and (let a = skipConv(n[1]); a.typ.kind == tyString and a.kind notin {nkStrLit..nkTripleStrLit}): message(tracked.config, n.info, warnUnsafeCode, renderTree(n)) diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 4763f0fd3..f7c42e638 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -95,7 +95,7 @@ proc semEnum(c: PContext, n: PNode, prev: PType): PType = of tyTuple: if v.len == 2: strVal = v[1] # second tuple part is the string value - if skipTypes(strVal.typ, abstractInst).kind in {tyString, tyCString}: + if skipTypes(strVal.typ, abstractInst).kind in {tyString, tyCstring}: if not isOrdinalType(v[0].typ, allowEnumWithHoles=true): localError(c.config, v[0].info, errOrdinalTypeExpected & "; given: " & typeToString(v[0].typ, preferDesc)) x = toInt64(getOrdValue(v[0])) # first tuple part is the ordinal @@ -104,7 +104,7 @@ proc semEnum(c: PContext, n: PNode, prev: PType): PType = localError(c.config, strVal.info, errStringLiteralExpected) else: localError(c.config, v.info, errWrongNumberOfVariables) - of tyString, tyCString: + of tyString, tyCstring: strVal = v x = counter else: @@ -2037,7 +2037,7 @@ proc processMagicType(c: PContext, m: PSym) = if optSeqDestructors in c.config.globalOptions: incl m.typ.flags, tfHasAsgn of mCstring: - setMagicIntegral(c.config, m, tyCString, c.config.target.ptrSize) + setMagicIntegral(c.config, m, tyCstring, c.config.target.ptrSize) rawAddSon(m.typ, getSysType(c.graph, m.info, tyChar)) of mPointer: setMagicIntegral(c.config, m, tyPointer, c.config.target.ptrSize) of mNil: setMagicType(c.config, m, tyNil, c.config.target.ptrSize) diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 767f34dda..2563c38e9 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -260,7 +260,7 @@ proc sumGeneric(t: PType): int = of tyGenericParam, tyUntyped, tyTyped: break of tyAlias, tySink: t = t.lastSon of tyBool, tyChar, tyEnum, tyObject, tyPointer, - tyString, tyCString, tyInt..tyInt64, tyFloat..tyFloat128, + tyString, tyCstring, tyInt..tyInt64, tyFloat..tyFloat128, tyUInt..tyUInt64, tyCompositeTypeClass: return isvar else: @@ -1383,7 +1383,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, # 'pointer' is NOT compatible to regionized pointers # so 'dealloc(regionPtr)' fails: if a.len == 1: result = isConvertible - of tyCString: result = isConvertible + of tyCstring: result = isConvertible else: discard of tyString: case a.kind @@ -1394,10 +1394,10 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, result = isEqual of tyNil: result = isNone else: discard - of tyCString: + of tyCstring: # conversion from string to cstring is automatic: case a.kind - of tyCString: + of tyCstring: if tfNotNil in f.flags and tfNotNil notin a.flags: result = isNilConversion else: @@ -2436,7 +2436,7 @@ proc matchesAux(c: PContext, n, nOrig: PNode, m: var TCandidate, marker: var Int n[a] = prepareOperand(c, n[a]) if skipTypes(n[a].typ, abstractVar-{tyTypeDesc}).kind==tyString: m.call.add implicitConv(nkHiddenStdConv, - getSysType(c.graph, n[a].info, tyCString), + getSysType(c.graph, n[a].info, tyCstring), copyTree(n[a]), m, c) else: m.call.add copyTree(n[a]) diff --git a/compiler/sizealignoffsetimpl.nim b/compiler/sizealignoffsetimpl.nim index b50777c9e..44ba32579 100644 --- a/compiler/sizealignoffsetimpl.nim +++ b/compiler/sizealignoffsetimpl.nim @@ -242,7 +242,7 @@ proc computeSizeAlign(conf: ConfigRef; typ: PType) = else: typ.size = conf.target.ptrSize typ.align = int16(conf.target.ptrSize) - of tyCString, tySequence, tyPtr, tyRef, tyVar, tyLent: + of tyCstring, tySequence, tyPtr, tyRef, tyVar, tyLent: let base = typ.lastSon if base == typ: # this is not the correct location to detect ``type A = ptr A`` diff --git a/compiler/sourcemap.nim b/compiler/sourcemap.nim index 2ec40227b..b87de75f3 100644 --- a/compiler/sourcemap.nim +++ b/compiler/sourcemap.nim @@ -8,13 +8,13 @@ type name*: string children*: seq[Child] - C = enum cSourceNode, cString + C = enum cSourceNode, cSourceString Child* = ref object case kind*: C: of cSourceNode: node*: SourceNode - of cString: + of cSourceString: s*: string SourceMap* = ref object @@ -44,7 +44,7 @@ type proc child*(s: string): Child = - Child(kind: cString, s: s) + Child(kind: cSourceString, s: s) proc child*(node: SourceNode): Child = @@ -72,7 +72,7 @@ proc text*(sourceNode: SourceNode, depth: int): string = let empty = " " result = &"{repeat(empty, depth)}SourceNode({sourceNode.source}:{sourceNode.line}:{sourceNode.column}):\n" for child in sourceNode.children: - if child.kind == cString: + if child.kind == cSourceString: result.add(&"{repeat(empty, depth + 1)}{child.s}\n") else: result.add(child.node.text(depth + 1)) @@ -308,7 +308,7 @@ proc addMapping*(map: SourceMapGenerator, mapping: Mapping) = proc walk*(node: SourceNode, fn: proc(line: string, original: SourceNode)) = for child in node.children: - if child.kind == cString and child.s.len > 0: + if child.kind == cSourceString and child.s.len > 0: fn(child.s, node) else: child.node.walk(fn) diff --git a/compiler/transf.nim b/compiler/transf.nim index 48c737f46..fb59887b0 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -497,14 +497,14 @@ proc transformConv(c: PTransf, n: PNode): PNode = result.typ = takeType(n.typ, n[1].typ, c.graph, c.idgen) #echo n.info, " came here and produced ", typeToString(result.typ), # " from ", typeToString(n.typ), " and ", typeToString(n[1].typ) - of tyCString: + of tyCstring: if source.kind == tyString: result = newTransNode(nkStringToCString, n, 1) result[0] = transform(c, n[1]) else: result = transformSons(c, n) of tyString: - if source.kind == tyCString: + if source.kind == tyCstring: result = newTransNode(nkCStringToString, n, 1) result[0] = transform(c, n[1]) else: diff --git a/compiler/typeallowed.nim b/compiler/typeallowed.nim index 270e7c028..2f0c039a4 100644 --- a/compiler/typeallowed.nim +++ b/compiler/typeallowed.nim @@ -116,7 +116,7 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, result = t of tyNil: if kind != skConst and kind != skParam: result = t - of tyString, tyBool, tyChar, tyEnum, tyInt..tyUInt64, tyCString, tyPointer: + of tyString, tyBool, tyChar, tyEnum, tyInt..tyUInt64, tyCstring, tyPointer: result = nil of tyOrdinal: if kind != skParam: result = t diff --git a/compiler/types.nim b/compiler/types.nim index ce4812506..e56fcffe4 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -503,7 +503,7 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string = result = typeToString(t[0]) elif prefer in {preferResolved, preferMixed}: case t.kind - of IntegralTypes + {tyFloat..tyFloat128} + {tyString, tyCString}: + of IntegralTypes + {tyFloat..tyFloat128} + {tyString, tyCstring}: result = typeToStr[t.kind] of tyGenericBody: result = typeToString(t.lastSon) @@ -762,7 +762,7 @@ proc firstOrd*(conf: ConfigRef; t: PType): Int128 = of tyOrdinal: if t.len > 0: result = firstOrd(conf, lastSon(t)) else: internalError(conf, "invalid kind for firstOrd(" & $t.kind & ')') - of tyUncheckedArray, tyCString: + of tyUncheckedArray, tyCstring: result = Zero else: internalError(conf, "invalid kind for firstOrd(" & $t.kind & ')') @@ -1129,7 +1129,7 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool = return true case a.kind - of tyEmpty, tyChar, tyBool, tyNil, tyPointer, tyString, tyCString, + of tyEmpty, tyChar, tyBool, tyNil, tyPointer, tyString, tyCstring, tyInt..tyUInt64, tyTyped, tyUntyped, tyVoid: result = sameFlags(a, b) of tyStatic, tyFromExpr: diff --git a/compiler/vm.nim b/compiler/vm.nim index e3ba2994f..5c993d77c 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -399,7 +399,7 @@ proc opConv(c: PCtx; dest: var TFullReg, src: TFullReg, desttyp, srctyp: PType): dest.node.strVal = $src.floatVal of tyString: dest.node.strVal = src.node.strVal - of tyCString: + of tyCstring: if src.node.kind == nkBracket: # Array of chars var strVal = "" diff --git a/compiler/vmdeps.nim b/compiler/vmdeps.nim index 6191f9459..b67e2e48a 100644 --- a/compiler/vmdeps.nim +++ b/compiler/vmdeps.nim @@ -257,7 +257,7 @@ proc mapTypeToAstX(cache: IdentCache; t: PType; info: TLineInfo; result.add t.n[1].copyTree of tyPointer: result = atomicType("pointer", mPointer) of tyString: result = atomicType("string", mString) - of tyCString: result = atomicType("cstring", mCstring) + of tyCstring: result = atomicType("cstring", mCstring) of tyInt: result = atomicType("int", mInt) of tyInt8: result = atomicType("int8", mInt8) of tyInt16: result = atomicType("int16", mInt16) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 951d47d2f..c60e81020 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -190,7 +190,7 @@ proc getSlotKind(t: PType): TSlotKind = case t.skipTypes(abstractRange-{tyTypeDesc}).kind of tyBool, tyChar, tyEnum, tyOrdinal, tyInt..tyInt64, tyUInt..tyUInt64: slotTempInt - of tyString, tyCString: + of tyString, tyCstring: slotTempStr of tyFloat..tyFloat128: slotTempFloat @@ -1053,7 +1053,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = of mLengthStr: case n[1].typ.kind of tyString: genUnaryABI(c, n, dest, opcLenStr) - of tyCString: genUnaryABI(c, n, dest, opcLenCstring) + of tyCstring: genUnaryABI(c, n, dest, opcLenCstring) else: doAssert false, $n[1].typ.kind of mIncl, mExcl: unused(c, n, dest) @@ -1199,7 +1199,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = let tmp = c.genx(n[1]) case n[1].typ.skipTypes(abstractVar-{tyTypeDesc}).kind: of tyString: c.gABI(n, opcLenStr, dest, tmp, 1) - of tyCString: c.gABI(n, opcLenCstring, dest, tmp, 1) + of tyCstring: c.gABI(n, opcLenCstring, dest, tmp, 1) else: c.gABI(n, opcLenSeq, dest, tmp, 1) c.freeTemp(tmp) of mEcho: @@ -1530,7 +1530,7 @@ proc genAsgn(c: PCtx; le, ri: PNode; requiresCopy: bool) = let idx = c.genIndex(le[1], le[0].typ) let tmp = c.genx(ri) if le[0].typ.skipTypes(abstractVarRange-{tyTypeDesc}).kind in { - tyString, tyCString}: + tyString, tyCstring}: c.preventFalseAlias(le, opcWrStrIdx, dest, idx, tmp) else: c.preventFalseAlias(le, opcWrArr, dest, idx, tmp) @@ -1775,7 +1775,7 @@ proc genCheckedObjAccess(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) = proc genArrAccess(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) = let arrayType = n[0].typ.skipTypes(abstractVarRange-{tyTypeDesc}).kind - if arrayType in {tyString, tyCString}: + if arrayType in {tyString, tyCstring}: let opc = if gfNodeAddr in flags: opcLdStrIdxAddr else: opcLdStrIdx genArrAccessOpcode(c, n, dest, opc, flags) elif arrayType == tyTypeDesc: @@ -1813,7 +1813,7 @@ proc getNullValue(typ: PType, info: TLineInfo; conf: ConfigRef): PNode = result = newNodeIT(nkUIntLit, info, t) of tyFloat..tyFloat128: result = newNodeIT(nkFloatLit, info, t) - of tyCString, tyString: + of tyCstring, tyString: result = newNodeIT(nkStrLit, info, t) result.strVal = "" of tyVar, tyLent, tyPointer, tyPtr, tyUntyped, diff --git a/compiler/vmmarshal.nim b/compiler/vmmarshal.nim index ffd8e16d7..d48d2408c 100644 --- a/compiler/vmmarshal.nim +++ b/compiler/vmmarshal.nim @@ -121,7 +121,7 @@ proc storeAny(s: var string; t: PType; a: PNode; stored: var IntSet; s.add(", ") storeAny(s, t.lastSon, a, stored, conf) s.add("]") - of tyString, tyCString: + of tyString, tyCstring: if a.kind == nkNilLit: s.add("null") else: s.add(escapeJson(a.strVal)) of tyInt..tyInt64, tyUInt..tyUInt64: s.add($a.intVal) @@ -255,7 +255,7 @@ proc loadAny(p: var JsonParser, t: PType, if p.kind == jsonArrayEnd: next(p) else: raiseParseErr(p, "']' end of ref-address pair expected") else: raiseParseErr(p, "int for pointer type expected") - of tyString, tyCString: + of tyString, tyCstring: case p.kind of jsonNull: result = newNode(nkNilLit) diff --git a/lib/core/typeinfo.nim b/lib/core/typeinfo.nim index 08d567351..d66dc1898 100644 --- a/lib/core/typeinfo.nim +++ b/lib/core/typeinfo.nim @@ -322,12 +322,12 @@ proc base*(x: Any): Any = proc isNil*(x: Any): bool = ## `isNil` for an `x` that represents a cstring, proc or ## some pointer type. - assert x.rawType.kind in {tyCString, tyRef, tyPtr, tyPointer, tyProc} + assert x.rawType.kind in {tyCstring, tyRef, tyPtr, tyPointer, tyProc} result = isNil(cast[ppointer](x.value)[]) const pointerLike = - when defined(gcDestructors): {tyCString, tyRef, tyPtr, tyPointer, tyProc} - else: {tyString, tyCString, tyRef, tyPtr, tyPointer, tySequence, tyProc} + when defined(gcDestructors): {tyCstring, tyRef, tyPtr, tyPointer, tyProc} + else: {tyString, tyCstring, tyRef, tyPtr, tyPointer, tySequence, tyProc} proc getPointer*(x: Any): pointer = ## Retrieves the pointer value out of `x`. `x` needs to be of kind @@ -669,7 +669,7 @@ proc setString*(x: Any, y: string) = proc getCString*(x: Any): cstring = ## Retrieves the `cstring` value out of `x`. `x` needs to represent a `cstring`. - assert x.rawType.kind == tyCString + assert x.rawType.kind == tyCstring result = cast[ptr cstring](x.value)[] proc assign*(x, y: Any) = diff --git a/lib/system/assign.nim b/lib/system/assign.nim index faf16fc90..20b854107 100644 --- a/lib/system/assign.nim +++ b/lib/system/assign.nim @@ -167,7 +167,7 @@ when false: of tyPointer: k = "range" of tyOpenArray: k = "openarray" of tyString: k = "string" - of tyCString: k = "cstring" + of tyCstring: k = "cstring" of tyInt: k = "int" of tyInt32: k = "int32" else: k = "other" diff --git a/lib/system/hti.nim b/lib/system/hti.nim index 3dbcd7615..9acaae88b 100644 --- a/lib/system/hti.nim +++ b/lib/system/hti.nim @@ -40,7 +40,7 @@ type tyPointer, tyOpenArray, tyString, - tyCString, + tyCstring, tyForward, tyInt, tyInt8, diff --git a/lib/system/repr.nim b/lib/system/repr.nim index 7424500eb..020c2281f 100644 --- a/lib/system/repr.nim +++ b/lib/system/repr.nim @@ -281,7 +281,7 @@ when not defined(useNimRtl): of tyString: let sp = cast[ptr string](p) reprStrAux(result, sp[].cstring, sp[].len) - of tyCString: + of tyCstring: let cs = cast[ptr cstring](p)[] if cs.isNil: add result, "nil" else: reprStrAux(result, cs, cs.len) diff --git a/lib/system/reprjs.nim b/lib/system/reprjs.nim index 36972024a..cdc3403f8 100644 --- a/lib/system/reprjs.nim +++ b/lib/system/reprjs.nim @@ -200,7 +200,7 @@ proc reprAux(result: var string, p: pointer, typ: PNimType, var fp: int {. emit: "`fp` = `p`;\n" .} add(result, reprStr(cast[string](p))) - of tyCString: + of tyCstring: var fp: cstring {. emit: "`fp` = `p`;\n" .} if fp.isNil: From 5c5f356347b15a82491c7f23e9bf2dd30a0c80f2 Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 17 Apr 2021 17:48:22 +0800 Subject: [PATCH 0654/2472] replace defer with try ... finally (#17753) --- lib/core/rlocks.nim | 10 +++++----- tests/stdlib/tuserlocks.nim | 11 +++++++++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/lib/core/rlocks.nim b/lib/core/rlocks.nim index 27a9c5e60..f9224adb8 100644 --- a/lib/core/rlocks.nim +++ b/lib/core/rlocks.nim @@ -47,9 +47,9 @@ proc release*(lock: var RLock) = template withRLock*(lock: var RLock, code: untyped): untyped = ## Acquires the given lock and then executes the code. - block: - acquire(lock) - defer: - release(lock) - {.locks: [lock].}: + acquire(lock) + {.locks: [lock].}: + try: code + finally: + release(lock) diff --git a/tests/stdlib/tuserlocks.nim b/tests/stdlib/tuserlocks.nim index f6eafa05d..9251fa9e2 100644 --- a/tests/stdlib/tuserlocks.nim +++ b/tests/stdlib/tuserlocks.nim @@ -1,8 +1,8 @@ discard """ - cmd: "nim $target --threads:on $options $file" + matrix: "--threads:on" """ -import rlocks +import std/rlocks var r: RLock r.initRLock() @@ -10,4 +10,11 @@ doAssert r.tryAcquire() doAssert r.tryAcquire() r.release() r.release() + +block: + var x = 12 + withRLock r: + inc x + doAssert x == 13 + r.deinitRLock() From 5c9c1988f6bb92e8cfdc639fc0f83e78cb5efbd2 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 17 Apr 2021 02:49:54 -0700 Subject: [PATCH 0655/2472] -d:nimDebug: calls doAssert false instead of quit (#17739) --- compiler/ccgexprs.nim | 4 ++-- compiler/docgen.nim | 2 +- compiler/extccomp.nim | 2 +- compiler/ic/ic.nim | 3 +-- compiler/main.nim | 6 +++--- compiler/msgs.nim | 3 ++- compiler/options.nim | 10 ++++++++-- compiler/sigmatch.nim | 2 +- compiler/syntaxes.nim | 2 +- 9 files changed, 20 insertions(+), 14 deletions(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 0e54adf49..a42c30aae 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -2387,13 +2387,13 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = localError(p.config, e.info, strutils.`%`(errXMustBeCompileTime, e[0].sym.name.s)) of mSpawn: when defined(leanCompiler): - quit "compiler built without support for the 'spawn' statement" + p.config.quitOrRaise "compiler built without support for the 'spawn' statement" else: let n = spawn.wrapProcForSpawn(p.module.g.graph, p.module.idgen, p.module.module, e, e.typ, nil, nil) expr(p, n, d) of mParallel: when defined(leanCompiler): - quit "compiler built without support for the 'parallel' statement" + p.config.quitOrRaise "compiler built without support for the 'parallel' statement" else: let n = semparallel.liftParallel(p.module.g.graph, p.module.idgen, p.module.module, e) expr(p, n, d) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 2f7415241..4d9f09ca6 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -498,7 +498,7 @@ proc runAllExamples(d: PDoc) = "docCmd", group.docCmd, ] if os.execShellCmd(cmd) != 0: - quit "[runnableExamples] failed: generated file: '$1' group: '$2' cmd: $3" % [outp.string, group[].prettyString, cmd] + d.conf.quitOrRaise "[runnableExamples] failed: generated file: '$1' group: '$2' cmd: $3" % [outp.string, group[].prettyString, cmd] else: # keep generated source file `outp` to allow inspection. rawMessage(d.conf, hintSuccess, ["runnableExamples: " & outp.string]) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index e0fd5e206..8d1432aa8 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -1132,7 +1132,7 @@ proc runJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile) = except: let e = getCurrentException() - quit "\ncaught exception:\n" & e.msg & "\nstacktrace:\n" & e.getStackTrace() & + conf.quitOrRaise "\ncaught exception:\n" & e.msg & "\nstacktrace:\n" & e.getStackTrace() & "error evaluating JSON file: " & jsonFile.string proc genMappingFiles(conf: ConfigRef; list: CfileList): Rope = diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim index da0af2edf..6f208a2a4 100644 --- a/compiler/ic/ic.nim +++ b/compiler/ic/ic.nim @@ -1149,8 +1149,7 @@ proc rodViewer*(rodfile: AbsoluteFile; config: ConfigRef, cache: IdentCache) = var m: PackedModule let err = loadRodFile(rodfile, m, config, ignoreConfig=true) if err != ok: - echo "Error: could not load: ", rodfile.string, " reason: ", err - quit 1 + config.quitOrRaise "Error: could not load: " & $rodfile.string & " reason: " & $err when true: echo "exports:" diff --git a/compiler/main.nim b/compiler/main.nim index 475ba7773..6eb830164 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -229,7 +229,7 @@ proc mainCommand*(graph: ModuleGraph) = template docLikeCmd(body) = when defined(leanCompiler): - quit "compiler wasn't built with documentation generator" + conf.quitOrRaise "compiler wasn't built with documentation generator" else: wantMainModule(conf) loadConfigs(DocConfig, cache, conf, graph.idgen) @@ -278,7 +278,7 @@ proc mainCommand*(graph: ModuleGraph) = conf.setNoteDefaults(warn, true) conf.setNoteDefaults(warnRedefinitionOfLabel, false) # similar to issue #13218 when defined(leanCompiler): - quit "compiler wasn't built with documentation generator" + conf.quitOrRaise "compiler wasn't built with documentation generator" else: loadConfigs(DocConfig, cache, conf, graph.idgen) commandRst2Html(cache, conf) @@ -288,7 +288,7 @@ proc mainCommand*(graph: ModuleGraph) = warnFieldXNotSupported, warnRstStyle]: conf.setNoteDefaults(warn, true) when defined(leanCompiler): - quit "compiler wasn't built with documentation generator" + conf.quitOrRaise "compiler wasn't built with documentation generator" else: loadConfigs(DocTexConfig, cache, conf, graph.idgen) commandRst2TeX(cache, conf) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 452b1cd89..c1e1f9e39 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -398,7 +398,8 @@ proc log*(s: string) = close(f) proc quit(conf: ConfigRef; msg: TMsgKind) {.gcsafe.} = - if defined(debug) or msg == errInternal or conf.hasHint(hintStackTrace): + if conf.isDefined("nimDebug"): quitOrRaise(conf, $msg) + elif defined(debug) or msg == errInternal or conf.hasHint(hintStackTrace): {.gcsafe.}: if stackTraceAvailable() and isNil(conf.writelnHook): writeStackTrace() diff --git a/compiler/options.nim b/compiler/options.nim index 9ba4d62e4..5475410a4 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -576,6 +576,13 @@ proc isDefined*(conf: ConfigRef; symbol: string): bool = osDragonfly, osMacosx} else: discard +template quitOrRaise*(conf: ConfigRef, msg = "") = + # xxx in future work, consider whether to also intercept `msgQuit` calls + if conf.isDefined("nimDebug"): + doAssert false, msg + else: + quit(msg) # quits with QuitFailure + proc importantComments*(conf: ConfigRef): bool {.inline.} = conf.cmd in cmdDocLike + {cmdIdeTools} proc usesWriteBarrier*(conf: ConfigRef): bool {.inline.} = conf.selectedGC >= gcRefc @@ -732,8 +739,7 @@ proc completeGeneratedFilePath*(conf: ConfigRef; f: AbsoluteFile, try: createDir(subdir.string) except OSError: - writeLine(stdout, "cannot create directory: " & subdir.string) - quit(1) + conf.quitOrRaise "cannot create directory: " & subdir.string result = subdir / RelativeFile f.string.splitPath.tail #echo "completeGeneratedFilePath(", f, ") = ", result diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 2563c38e9..6a36c1381 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -1011,7 +1011,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, when declared(deallocatedRefId): let corrupt = deallocatedRefId(cast[pointer](f)) if corrupt != 0: - quit "it's corrupt " & $corrupt + c.c.config.quitOrRaise "it's corrupt " & $corrupt if f.kind == tyUntyped: if aOrig != nil: put(c, f, aOrig) diff --git a/compiler/syntaxes.nim b/compiler/syntaxes.nim index e745e28ba..03a9702a3 100644 --- a/compiler/syntaxes.nim +++ b/compiler/syntaxes.nim @@ -50,7 +50,7 @@ proc parsePipe(filename: AbsoluteFile, inputStream: PLLStream; cache: IdentCache if i+1 < line.len and line[i] == '#' and line[i+1] == '?': when defined(nimpretty): # XXX this is a bit hacky, but oh well... - quit "can't nimpretty a source code filter" + config.quitOrRaise "can't nimpretty a source code filter: " & $filename else: inc(i, 2) while i < line.len and line[i] in Whitespace: inc(i) From 8e124da75d7b3cafe27c197a027d34f1aca012e3 Mon Sep 17 00:00:00 2001 From: Tanguy Cizain Date: Sat, 17 Apr 2021 15:34:00 +0200 Subject: [PATCH 0656/2472] fix range to unsigned conversion (#17754) --- compiler/sigmatch.nim | 2 +- tests/range/trange.nim | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 6a36c1381..b2003c0a8 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -409,7 +409,7 @@ proc handleRange(f, a: PType, min, max: TTypeKind): TTypeRelation = elif a.kind == tyRange and # Make sure the conversion happens between types w/ same signedness (f.kind in {tyInt..tyInt64} and a[0].kind in {tyInt..tyInt64} or - f.kind in {tyUInt8..tyUInt32} and a[0].kind in {tyUInt8..tyInt32}) and + f.kind in {tyUInt8..tyUInt32} and a[0].kind in {tyUInt8..tyUInt32}) and a.n[0].intVal >= firstOrd(nil, f) and a.n[1].intVal <= lastOrd(nil, f): # passing 'nil' to firstOrd/lastOrd here as type checking rules should # not depend on the target integer size configurations! diff --git a/tests/range/trange.nim b/tests/range/trange.nim index c864387f6..92ab5ea86 100644 --- a/tests/range/trange.nim +++ b/tests/range/trange.nim @@ -142,3 +142,15 @@ var a: array[4'u, string] for i in 0.. Date: Sat, 17 Apr 2021 16:27:10 +0100 Subject: [PATCH 0657/2472] make cuchar alias uint8 instead of char (#17738) --- changelog.md | 2 ++ lib/system.nim | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index cfdcd3c62..5efde5659 100644 --- a/changelog.md +++ b/changelog.md @@ -4,6 +4,8 @@ ## Changes affecting backward compatibility +- `cuchar` now aliases `uint8` instead of `char` + - `repr` now doesn't insert trailing newline; previous behavior was very inconsistent, see #16034. Use `-d:nimLegacyReprWithNewline` for previous behavior. diff --git a/lib/system.nim b/lib/system.nim index 38f6d64d3..b7a4b45cf 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1452,7 +1452,7 @@ type # these work for most platforms: ## This is the same as the type `long double` in *C*. ## This C type is not supported by Nim's code generator. - cuchar* {.importc: "unsigned char", nodecl.} = char + cuchar* {.importc: "unsigned char", nodecl.} = uint8 ## This is the same as the type `unsigned char` in *C*. cushort* {.importc: "unsigned short", nodecl.} = uint16 ## This is the same as the type `unsigned short` in *C*. From 7b6ab5109fe76afefbf5744b5f9f534e86d94629 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 18 Apr 2021 07:27:03 +0200 Subject: [PATCH 0658/2472] IC exposes typedesc implementation shenanigans (#17759) * IC exposes typedesc implementation shenanigans; so I change system.default's definition to what it should have been to begin with * Update lib/system.nim Co-authored-by: Timothee Cour --- lib/system.nim | 2 +- tests/ic/tstdlib_import_changed.nim | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 tests/ic/tstdlib_import_changed.nim diff --git a/lib/system.nim b/lib/system.nim index b7a4b45cf..532883853 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -980,7 +980,7 @@ proc `@`* [IDX, T](a: sink array[IDX, T]): seq[T] {.magic: "ArrToSeq", noSideEff ## echo @a # => @[1, 3, 5] ## echo @b # => @['f', 'o', 'o'] -proc default*(T: typedesc): T {.magic: "Default", noSideEffect.} = +proc default*[T](_: typedesc[T]): T {.magic: "Default", noSideEffect.} = ## returns the default value of the type `T`. runnableExamples: assert (int, float).default == (0, 0.0) diff --git a/tests/ic/tstdlib_import_changed.nim b/tests/ic/tstdlib_import_changed.nim new file mode 100644 index 000000000..da69a53b5 --- /dev/null +++ b/tests/ic/tstdlib_import_changed.nim @@ -0,0 +1,15 @@ +discard """ + output: '''yes''' +""" + +echo "yes" + +#!EDIT!# + +discard """ + output: '''yes2''' +""" + +import std / [monotimes] +#discard getMonoTime() +echo "yes2" From ca3fe63bab54779e6dc2df3c9a72b9c4280c0eaf Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 18 Apr 2021 21:30:30 +0800 Subject: [PATCH 0659/2472] remove jstin from important packages (#17765) Ref https://github.com/nim-lang/Nim/pull/17759 https://github.com/LemonBoy/jstin/blob/master/src/jstin.nim is already archived and cannot pass CI anymore. --- testament/important_packages.nim | 1 - 1 file changed, 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 620ad3a00..28d7dc367 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -77,7 +77,6 @@ pkg "illwill", "nimble examples" pkg "inim" pkg "itertools", "nim doc src/itertools.nim" pkg "iterutils" -pkg "jstin" pkg "karax", "nim c -r tests/tester.nim" pkg "kdtree", "nimble test", "https://github.com/jblindsay/kdtree" pkg "loopfusion" From 42c6eec4ef7752c4f48ace2899a44840df95da9c Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 18 Apr 2021 06:34:29 -0700 Subject: [PATCH 0660/2472] fix #17749 ignore SIGPIPE signals, fix nim CI #17748 (#17752) * fix #17749 SIGPIPE * fix for windows --- changelog.md | 20 ++++++++++++-------- koch.nim | 4 ++++ lib/system/excpt.nim | 16 +++++++++++++--- tests/stdlib/tosproc.nim | 23 +++++++++++++++++++++-- tests/system/tsigexitcode.nim | 11 +++++++---- 5 files changed, 57 insertions(+), 17 deletions(-) diff --git a/changelog.md b/changelog.md index 5efde5659..8fab425f8 100644 --- a/changelog.md +++ b/changelog.md @@ -39,6 +39,18 @@ - In `std/os`, `getHomeDir`, `expandTilde`, `getTempDir`, `getConfigDir` now do not include trailing `DirSep`, unless `-d:nimLegacyHomeDir` is specified (for a transition period). +- On POSIX systems, the default signal handlers used for Nim programs (it's + used for printing the stacktrace on fatal signals) will now re-raise the + signal for the OS default handlers to handle. + + This lets the OS perform its default actions, which might include core + dumping (on select signals) and notifying the parent process about the cause + of termination. + +- On POSIX systems, we now ignore `SIGPIPE` signals, use `-d:nimLegacySigpipeHandler` + for previous behavior. + + ## Standard library additions and changes - Added support for parenthesized expressions in `strformat` @@ -219,14 +231,6 @@ `ValueError` when the real command line is not available. `parseopt` was previously excluded from `prelude` for JS, as it could not be imported. -- On POSIX systems, the default signal handlers used for Nim programs (it's - used for printing the stacktrace on fatal signals) will now re-raise the - signal for the OS default handlers to handle. - - This lets the OS perform its default actions, which might include core - dumping (on select signals) and notifying the parent process about the cause - of termination. - - Added `system.prepareStrMutation` for better support of low level `moveMem`, `copyMem` operations for Orc's copy-on-write string implementation. diff --git a/koch.nim b/koch.nim index 3e41310d7..49540eb6a 100644 --- a/koch.nim +++ b/koch.nim @@ -542,6 +542,10 @@ proc runCI(cmd: string) = # boot without -d:nimHasLibFFI to make sure this still works kochExecFold("Boot in release mode", "boot -d:release -d:nimStrictMode") + when false: # debugging: when you need to run only 1 test in CI, use something like this: + execFold("debugging test", "nim r tests/stdlib/tosproc.nim") + doAssert false, "debugging only" + ## build nimble early on to enable remainder to depend on it if needed kochExecFold("Build Nimble", "nimble") diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index 06fa45097..da034e57e 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -602,6 +602,9 @@ when defined(cpp) and appType != "lib" and not gotoBasedExceptions and quit 1 when not defined(noSignalHandler) and not defined(useNimRtl): + type Sighandler = proc (a: cint) {.noconv, benign.} + # xxx factor with ansi_c.CSighandlerT, posix.Sighandler + proc signalHandler(sign: cint) {.exportc: "signalHandler", noconv.} = template processSignal(s, action: untyped) {.dirty.} = if s == SIGINT: action("SIGINT: Interrupted by Ctrl-C.\n") @@ -652,7 +655,11 @@ when not defined(noSignalHandler) and not defined(useNimRtl): else: quit(1) + var SIG_IGN {.importc: "SIG_IGN", header: "".}: Sighandler + proc registerSignalHandler() = + # xxx `signal` is deprecated and has many caveats, we should use `sigaction` instead, e.g. + # https://stackoverflow.com/questions/231912/what-is-the-difference-between-sigaction-and-signal c_signal(SIGINT, signalHandler) c_signal(SIGSEGV, signalHandler) c_signal(SIGABRT, signalHandler) @@ -661,14 +668,17 @@ when not defined(noSignalHandler) and not defined(useNimRtl): when declared(SIGBUS): c_signal(SIGBUS, signalHandler) when declared(SIGPIPE): - c_signal(SIGPIPE, signalHandler) + when defined(nimLegacySigpipeHandler): + c_signal(SIGPIPE, signalHandler) + else: + c_signal(SIGPIPE, SIG_IGN) registerSignalHandler() # call it in initialization section proc setControlCHook(hook: proc () {.noconv.}) = # ugly cast, but should work on all architectures: - type SignalHandler = proc (sign: cint) {.noconv, benign.} - c_signal(SIGINT, cast[SignalHandler](hook)) + when declared(Sighandler): + c_signal(SIGINT, cast[Sighandler](hook)) when not defined(noSignalHandler) and not defined(useNimRtl): proc unsetControlCHook() = diff --git a/tests/stdlib/tosproc.nim b/tests/stdlib/tosproc.nim index 96cff1468..67731322a 100644 --- a/tests/stdlib/tosproc.nim +++ b/tests/stdlib/tosproc.nim @@ -206,8 +206,8 @@ else: # main driver var line = newStringOfCap(120) while true: if outp.readLine(line): - result[0].string.add(line.string) - result[0].string.add("\n") + result[0].add(line) + result[0].add("\n") else: result[1] = peekExitCode(p) if result[1] != -1: break @@ -279,3 +279,22 @@ else: # main driver when defined(posix): doAssert execCmdEx("echo $FO", env = newStringTable({"FO": "B"})) == ("B\n", 0) doAssert execCmdEx("echo $PWD", workingDir = "/") == ("/\n", 0) + + block: # bug #17749 + let output = compileNimProg("-d:case_testfile4", "D20210417T011153") + var p = startProcess(output, dir) + let inp = p.inputStream + var count = 0 + when defined(windows): + # xxx we should make osproc.hsWriteData raise IOError on windows, consistent + # with posix; we could also (in addition) make IOError a subclass of OSError. + type SIGPIPEError = OSError + else: + type SIGPIPEError = IOError + doAssertRaises(SIGPIPEError): + for i in 0..<100000: + count.inc + inp.writeLine "ok" # was giving SIGPIPE and crashing + doAssert count >= 100 + doAssert waitForExit(p) == QuitFailure + close(p) # xxx isn't that missing in other places? diff --git a/tests/system/tsigexitcode.nim b/tests/system/tsigexitcode.nim index 6922cb8eb..249256b40 100644 --- a/tests/system/tsigexitcode.nim +++ b/tests/system/tsigexitcode.nim @@ -11,10 +11,13 @@ proc main() = discard posix.raise(signal) else: # synchronize this list with lib/system/except.nim:registerSignalHandler() - let fatalSigs = [SIGINT, SIGSEGV, SIGABRT, SIGFPE, SIGILL, SIGBUS, - SIGPIPE] - for s in fatalSigs: + let sigs = [SIGINT, SIGSEGV, SIGABRT, SIGFPE, SIGILL, SIGBUS, SIGPIPE] + for s in sigs: let (_, exitCode) = execCmdEx(quoteShellCommand [getAppFilename(), $s]) - doAssert exitCode == 128 + s, "mismatched exit code for signal " & $s + if s == SIGPIPE: + # SIGPIPE should be ignored + doAssert exitCode == 0, $(exitCode, s) + else: + doAssert exitCode == 128+s, $(exitCode, s) main() From c7b77829fe26d9053df2181d96ffc358409674a6 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 18 Apr 2021 23:38:07 +0200 Subject: [PATCH 0661/2472] IC: fixes a long standing bug about DCE set computations (#17763) --- compiler/ic/cbackend.nim | 58 +++++++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/compiler/ic/cbackend.nim b/compiler/ic/cbackend.nim index 2f3910b32..c7e5c7266 100644 --- a/compiler/ic/cbackend.nim +++ b/compiler/ic/cbackend.nim @@ -104,14 +104,35 @@ proc aliveSymsChanged(config: ConfigRef; position: int; alive: AliveSyms): bool let oldAsSet = toPackedSet[int32](oldData) let newAsSet = toPackedSet[int32](s) echo "set of live symbols changed ", asymFile.changeFileExt("rod"), " ", position, " ", f2.err - echo "in old but not in new ", oldAsSet.difference(newAsSet) - echo "in new but not in old ", newAsSet.difference(oldAsSet) - - if execShellCmd(getAppFilename() & " rod " & quoteShell(asymFile.changeFileExt("rod"))) != 0: - echo "command failed" + echo "in old but not in new ", oldAsSet.difference(newAsSet), " number of entries in old ", oldAsSet.len + echo "in new but not in old ", newAsSet.difference(oldAsSet), " number of entries in new ", newAsSet.len + #if execShellCmd(getAppFilename() & " rod " & quoteShell(asymFile.changeFileExt("rod"))) != 0: + # echo "command failed" result = true storeAliveSymsImpl(asymFile, s) +proc genPackedModule(g: ModuleGraph, i: int; alive: var AliveSyms) = + # case statement here to enforce exhaustive checks. + case g.packed[i].status + of undefined: + discard "nothing to do" + of loading, stored: + assert false + of storing, outdated: + storeAliveSyms(g.config, g.packed[i].module.position, alive) + generateCodeForModule(g, g.packed[i], alive) + closeRodFile(g, g.packed[i].module) + of loaded: + if g.packed[i].loadedButAliveSetChanged: + generateCodeForModule(g, g.packed[i], alive) + else: + addFileToLink(g.config, g.packed[i].module) + replayTypeInfo(g, g.packed[i], FileIndex(i)) + + if g.backend == nil: + g.backend = cgendata.newModuleList(g) + registerInitProcs(BModuleList(g.backend), g.packed[i].module, g.packed[i].fromDisk.backendFlags) + proc generateCode*(g: ModuleGraph) = ## The single entry point, generate C(++) code for the entire ## Nim program aka `ModuleGraph`. @@ -143,24 +164,11 @@ proc generateCode*(g: ModuleGraph) = setupBackendModule(g, g.packed[i]) # Second pass: Code generation. + let mainModuleIdx = g.config.projectMainIdx2.int + # We need to generate the main module last, because only then + # all init procs have been registered: for i in 0..high(g.packed): - # case statement here to enforce exhaustive checks. - case g.packed[i].status - of undefined: - discard "nothing to do" - of loading, stored: - assert false - of storing, outdated: - generateCodeForModule(g, g.packed[i], alive) - closeRodFile(g, g.packed[i].module) - storeAliveSyms(g.config, g.packed[i].module.position, alive) - of loaded: - if g.packed[i].loadedButAliveSetChanged: - generateCodeForModule(g, g.packed[i], alive) - else: - addFileToLink(g.config, g.packed[i].module) - replayTypeInfo(g, g.packed[i], FileIndex(i)) - - if g.backend == nil: - g.backend = cgendata.newModuleList(g) - registerInitProcs(BModuleList(g.backend), g.packed[i].module, g.packed[i].fromDisk.backendFlags) + if i != mainModuleIdx: + genPackedModule(g, i, alive) + if mainModuleIdx >= 0: + genPackedModule(g, mainModuleIdx, alive) From 6916faf83c8af609304bde9d2ea2ed62c514cbf1 Mon Sep 17 00:00:00 2001 From: Dankr4d Date: Sun, 18 Apr 2021 23:39:05 +0200 Subject: [PATCH 0662/2472] Fix #17755 (#17766) Signed-off-by: Dankr4d --- lib/wrappers/openssl.nim | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/wrappers/openssl.nim b/lib/wrappers/openssl.nim index 9c33386c5..1a8893957 100644 --- a/lib/wrappers/openssl.nim +++ b/lib/wrappers/openssl.nim @@ -52,14 +52,23 @@ when sslVersion != "": from posix import SocketHandle elif useWinVersion: - when not defined(nimOldDlls) and defined(cpu64): + when defined(openssl10) or defined(nimOldDlls): + when defined(cpu64): + const + DLLSSLName* = "(ssleay32|ssleay64).dll" + DLLUtilName* = "(libeay32|libeay64).dll" + else: + const + DLLSSLName* = "ssleay32.dll" + DLLUtilName* = "libeay32.dll" + elif defined(cpu64): const DLLSSLName* = "(libssl-1_1-x64|ssleay64|libssl64).dll" DLLUtilName* = "(libcrypto-1_1-x64|libeay64).dll" else: const DLLSSLName* = "(libssl-1_1|ssleay32|libssl32).dll" - DLLUtilName* = "(libcrypto-1_1|libeay32).dll" + DLLUtilName* = "(libcrypto-1_1|libeay32).dll" from winlean import SocketHandle else: From d6c8efa5d444f6849102dca192a199f12c8d55eb Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 18 Apr 2021 15:03:24 -0700 Subject: [PATCH 0663/2472] declaredlocs followup: handle `cannot instantiate` errors (#17745) --- compiler/semtypes.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index f7c42e638..8be210067 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -1463,9 +1463,9 @@ proc semGeneric(c: PContext, n: PNode, s: PSym, prev: PType): PType = matches(c, n, copyTree(n), m) if m.state != csMatch: - let err = "cannot instantiate " & typeToString(t) & "\n" & - "got: <" & describeArgs(c, n) & ">\n" & - "but expected: <" & describeArgs(c, t.n, 0) & ">" + var err = "cannot instantiate " + err.addTypeHeader(c.config, t) + err.add "\ngot: <$1>\nbut expected: <$2>" % [describeArgs(c, n), describeArgs(c, t.n, 0)] localError(c.config, n.info, errGenerated, err) return newOrPrevType(tyError, prev, c) From 0a10af5a2ce27638c8298e96866a779928122269 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 18 Apr 2021 15:15:58 -0700 Subject: [PATCH 0664/2472] privateAccess now works with ref | ptr (#17760) --- compiler/semmagic.nim | 6 +-- compiler/suggest.nim | 4 +- lib/std/importutils.nim | 2 +- testament/lib/stdtest/testutils.nim | 20 +++++++ tests/importalls/mt1.nim | 4 +- tests/stdlib/mimportutils.nim | 17 ++++++ tests/stdlib/timportutils.nim | 82 +++++++++++++++++++++++++++++ tests/stdlib/ttestutils.nim | 29 +++++++--- 8 files changed, 149 insertions(+), 15 deletions(-) create mode 100644 tests/stdlib/mimportutils.nim create mode 100644 tests/stdlib/timportutils.nim diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim index d8be85273..c6cf63767 100644 --- a/compiler/semmagic.nim +++ b/compiler/semmagic.nim @@ -578,9 +578,9 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode, n[0].sym.magic = mSubU result = n of mPrivateAccess: - let sym = n[1].typ[0].sym - assert sym != nil - c.currentScope.allowPrivateAccess.add sym + var t = n[1].typ[0] + if t.kind in {tyRef, tyPtr}: t = t[0] + c.currentScope.allowPrivateAccess.add t.sym result = newNodeIT(nkEmpty, n.info, getSysType(c.graph, n.info, tyVoid)) else: result = n diff --git a/compiler/suggest.nim b/compiler/suggest.nim index 7a2c0d964..f3add0849 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -258,7 +258,9 @@ proc fieldVisible*(c: PContext, f: PSym): bool {.inline.} = for module in c.friendModules: if fmoduleId == module.id: return true if f.kind == skField: - let symObj = f.owner + var symObj = f.owner + if symObj.typ.kind in {tyRef, tyPtr}: + symObj = symObj.typ[0].sym for scope in allScopes(c.currentScope): for sym in scope.allowPrivateAccess: if symObj.id == sym.id: return true diff --git a/lib/std/importutils.nim b/lib/std/importutils.nim index 4efe0e140..f7a3cbd03 100644 --- a/lib/std/importutils.nim +++ b/lib/std/importutils.nim @@ -17,7 +17,7 @@ when defined(nimImportutilsExample): x1: int # private proc initFoo*(): auto = Foo() -proc privateAccess*(t: typedesc) {.magic: "PrivateAccess".} = +proc privateAccess*(t: typedesc[object|ref|ptr]) {.magic: "PrivateAccess".} = ## Enables access to private fields of `t` in current scope. runnableExamples("-d:nimImportutilsExample"): # here we're importing a module containing: diff --git a/testament/lib/stdtest/testutils.nim b/testament/lib/stdtest/testutils.nim index 35423de17..44048adbd 100644 --- a/testament/lib/stdtest/testutils.nim +++ b/testament/lib/stdtest/testutils.nim @@ -1,5 +1,6 @@ import std/private/miscdollars from std/os import getEnv +import std/[macros, genasts] template flakyAssert*(cond: untyped, msg = "", notifySuccess = true) = ## API to deal with flaky or failing tests. This avoids disabling entire tests @@ -89,3 +90,22 @@ template reject*(a) = template disableVm*(body) = when nimvm: discard else: body + +template typeOrVoid[T](a: T): type = + # FACTOR with asyncjs.typeOrVoid + T + +macro assertAll*(body) = + ## works in VM, unlike `check`, `require` + runnableExamples: + assertAll: + 1+1 == 2 + var a = @[1, 2] # statements work + a.len == 2 + # remove this once these support VM, pending #10129 (closed but not yet fixed) + result = newStmtList() + for a in body: + result.add genAst(a) do: + # better than: `when not compiles(typeof(a)):` + when typeOrVoid(a) is void: a + else: doAssert a diff --git a/tests/importalls/mt1.nim b/tests/importalls/mt1.nim index 87c4eff16..b7983945c 100644 --- a/tests/importalls/mt1.nim +++ b/tests/importalls/mt1.nim @@ -10,8 +10,8 @@ doAssert m.m3p1 == 2 ## field access import std/importutils privateAccess(Foo5) -# var x = Foo5(z1: "foo", z2: m.kg1) -# doAssert x.z1 == "foo" +var x = Foo5(z1: "foo", z2: m.kg1) +doAssert x.z1 == "foo" var f0: Foo5 f0.z3 = 3 diff --git a/tests/stdlib/mimportutils.nim b/tests/stdlib/mimportutils.nim new file mode 100644 index 000000000..d2b185cd3 --- /dev/null +++ b/tests/stdlib/mimportutils.nim @@ -0,0 +1,17 @@ +type + A* = object + a0*: int + ha1: float + B = object + b0*: int + hb1: float + C* = ref object + c0: int + hc1: float + D* = ptr object + d0: int + hd1: float + PA* = ref A + PtA* = ptr A + +proc initB*(): B = B() diff --git a/tests/stdlib/timportutils.nim b/tests/stdlib/timportutils.nim new file mode 100644 index 000000000..37e2b7102 --- /dev/null +++ b/tests/stdlib/timportutils.nim @@ -0,0 +1,82 @@ +import std/importutils +import stdtest/testutils +import mimportutils + +template main = + block: # privateAccess + assertAll: + var a: A + var b = initB() # B is private + compiles(a.a0) + compiles(b.b0) + not compiles(a.ha1) + not compiles(b.hb1) + + block: + assertAll: + privateAccess A + compiles(a.ha1) + a.ha1 == 0.0 + not compiles(a.hb1) + privateAccess b.typeof + b.hb1 = 3 + type B2 = b.typeof + let b2 = B2(b0: 4, hb1: 5) + b.hb1 == 3 + b2 == B2(b0: 4, hb1: 5) + + assertAll: + not compiles(a.ha1) + not compiles(b.hb1) + + block: + assertAll: + not compiles(C(c0: 1, hc1: 2)) + privateAccess C + let c = C(c0: 1, hc1: 2) + c.hc1 == 2 + + block: + assertAll: + privateAccess PA + var pa = PA(a0: 1, ha1: 2) + pa.ha1 == 2 + pa.ha1 = 3 + pa.ha1 == 3 + + block: + assertAll: + var a = A(a0: 1) + var a2 = a.addr + not compiles(a2.ha1) + privateAccess PtA + a2.type is PtA + a2.ha1 = 2 + a2.ha1 == 2 + a.ha1 = 3 + a2.ha1 == 3 + + block: + disableVm: + assertAll: + var a = A.create() + defer: dealloc(a) + a is PtA + a.typeof is PtA + not compiles(a.ha1) + privateAccess a.typeof + a.ha1 = 2 + a.ha1 == 2 + a[].ha1 = 3 + a.ha1 == 3 + + block: + disableVm: + assertAll: + var a = A.create() + defer: dealloc(a) + privateAccess PtA + a.ha1 == 0 + +static: main() +main() diff --git a/tests/stdlib/ttestutils.nim b/tests/stdlib/ttestutils.nim index 7e39c9ae3..0d789cd76 100644 --- a/tests/stdlib/ttestutils.nim +++ b/tests/stdlib/ttestutils.nim @@ -1,11 +1,24 @@ import stdtest/testutils -block: # greedyOrderedSubsetLines - doAssert greedyOrderedSubsetLines("a1\na3", "a0\na1\na2\na3\na4") - doAssert not greedyOrderedSubsetLines("a3\na1", "a0\na1\na2\na3\na4") # out of order - doAssert not greedyOrderedSubsetLines("a1\na5", "a0\na1\na2\na3\na4") # a5 not in lhs +block: # assertAll + assertAll: + 1+1 == 2 + var a = 3 + a == 3 - doAssert not greedyOrderedSubsetLines("a1\na5", "a0\na1\na2\na3\na4\nprefix:a5") - doAssert not greedyOrderedSubsetLines("a1\na5", "a0\na1\na2\na3\na4\na5:suffix") - doAssert not greedyOrderedSubsetLines("a5", "a0\na1\na2\na3\na4\nprefix:a5") - doAssert not greedyOrderedSubsetLines("a5", "a0\na1\na2\na3\na4\na5:suffix") + doAssertRaises(AssertionDefect): + assertAll: + 1+1 == 2 + var a = 3 + a == 4 + +block: # greedyOrderedSubsetLines + assertAll: + greedyOrderedSubsetLines("a1\na3", "a0\na1\na2\na3\na4") + not greedyOrderedSubsetLines("a3\na1", "a0\na1\na2\na3\na4") # out of order + not greedyOrderedSubsetLines("a1\na5", "a0\na1\na2\na3\na4") # a5 not in lhs + + not greedyOrderedSubsetLines("a1\na5", "a0\na1\na2\na3\na4\nprefix:a5") + not greedyOrderedSubsetLines("a1\na5", "a0\na1\na2\na3\na4\na5:suffix") + not greedyOrderedSubsetLines("a5", "a0\na1\na2\na3\na4\nprefix:a5") + not greedyOrderedSubsetLines("a5", "a0\na1\na2\na3\na4\na5:suffix") From 5042cb956be1cfc1dfb106d3dbc9d897e10f72d8 Mon Sep 17 00:00:00 2001 From: Saem Ghani Date: Sun, 18 Apr 2021 22:40:08 -0700 Subject: [PATCH 0665/2472] [ci skip] document compiler/ic/rodfiles.nim (#17771) * [ci skip] document compiler/ic/rodfiles.nim Why? * understand how rodfile module works and a bit of the format * leave notes behind for others * rather than Araq guess what others need, he can fix what other glean * possible model for making the compiler more aproachable Bonus: * might have found a minor bug in `loadSection` * Update compiler/ic/rodfiles.nim Co-authored-by: Andreas Rumpf --- compiler/ic/rodfiles.nim | 72 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/compiler/ic/rodfiles.nim b/compiler/ic/rodfiles.nim index 47362da0b..86d8ba203 100644 --- a/compiler/ic/rodfiles.nim +++ b/compiler/ic/rodfiles.nim @@ -7,8 +7,65 @@ # distribution, for details about the copyright. # +## Low level binary format used by the compiler to store and load various AST +## and related data. +## +## NB: this is incredibly low level and if you're interested in how the +## compiler works and less a storage format, you're probably looking for +## the `ic` or `packed_ast` modules to understand the logical format. + from typetraits import supportsCopyMem +## Overview +## ======== +## `RodFile` represents a Rod File (versioned binary format), and the +## associated data for common interactions such as IO and error tracking +## (`RodFileError`). The file format broken up into sections (`RodSection`) +## and preceeded by a header (see: `cookie`). The precise layout, section +## ordering and data following the section are determined by the user. See +## `ic.loadRoadFile`. +## +## A basic but "wrong" example of the lifecycle: +## --------------------------------------------- +## 1. `create` or `open` - create a new one or open an existing +## 2. `storeHeader` - header info +## 3. `storePrim` or `storeSeq` - save your stuff +## 4. `close` - and we're done +## +## Now read the bits below to understand what's missing. +## +## Issues with the Example +## ``````````````````````` +## Missing Sections: +## This is a low level API, so headers and sections need to be stored and +## loaded by the user, see `storeHeader` & `loadHeader` and `storeSection` & +## `loadSection`, respectively. +## +## No Error Handling: +## The API is centered around IO and prone to error, each operation checks or +## sets the `RodFile.err` field. A user of this API needs to handle these +## appropriately. +## +## API Notes +## ========= +## +## Valid inputs for Rod files +## -------------------------- +## ASTs, hopes, dreams, and anything as long as it and any children it may have +## support `copyMem`. This means anything that is not a pointer and that does not contain a pointer. At a glance these are: +## * string +## * objects & tuples (fields are recursed) +## * sequences AKA `seq[T]` +## +## Note on error handling style +## ---------------------------- +## A flag based approach is used where operations no-op in case of a +## preexisting error and set the flag if they encounter one. +## +## Misc +## ---- +## * 'Prim' is short for 'primitive', as in a non-sequence type + type RodSection* = enum versionSection @@ -60,6 +117,8 @@ proc setError(f: var RodFile; err: RodFileError) {.inline.} = #raise newException(IOError, "IO error") proc storePrim*(f: var RodFile; s: string) = + ## Stores a string. + ## The len is prefixed to allow for later retreival. if f.err != ok: return if s.len >= high(int32): setError f, tooBig @@ -73,6 +132,9 @@ proc storePrim*(f: var RodFile; s: string) = setError f, ioFailure proc storePrim*[T](f: var RodFile; x: T) = + ## Stores a non-sequence/string `T`. + ## If `T` doesn't support `copyMem` and is an object or tuple then the fields + ## are written -- the user from context will need to know which `T` to load. if f.err != ok: return when supportsCopyMem(T): if writeBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x): @@ -90,6 +152,7 @@ proc storePrim*[T](f: var RodFile; x: T) = {.error: "unsupported type for 'storePrim'".} proc storeSeq*[T](f: var RodFile; s: seq[T]) = + ## Stores a sequence of `T`s, with the len as a prefix for later retrieval. if f.err != ok: return if s.len >= high(int32): setError f, tooBig @@ -102,6 +165,7 @@ proc storeSeq*[T](f: var RodFile; s: seq[T]) = storePrim(f, s[i]) proc loadPrim*(f: var RodFile; s: var string) = + ## Read a string, the length was stored as a prefix if f.err != ok: return var lenPrefix = int32(0) if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix): @@ -113,6 +177,7 @@ proc loadPrim*(f: var RodFile; s: var string) = setError f, ioFailure proc loadPrim*[T](f: var RodFile; x: var T) = + ## Load a non-sequence/string `T`. if f.err != ok: return when supportsCopyMem(T): if readBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x): @@ -130,6 +195,7 @@ proc loadPrim*[T](f: var RodFile; x: var T) = {.error: "unsupported type for 'loadPrim'".} proc loadSeq*[T](f: var RodFile; s: var seq[T]) = + ## `T` must be compatible with `copyMem`, see `loadPrim` if f.err != ok: return var lenPrefix = int32(0) if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix): @@ -140,11 +206,13 @@ proc loadSeq*[T](f: var RodFile; s: var seq[T]) = loadPrim(f, s[i]) proc storeHeader*(f: var RodFile) = + ## stores the header which is described by `cookie`. if f.err != ok: return if f.f.writeBytes(cookie, 0, cookie.len) != cookie.len: setError f, ioFailure proc loadHeader*(f: var RodFile) = + ## Loads the header which is described by `cookie`. if f.err != ok: return var thisCookie: array[cookie.len, byte] if f.f.readBytes(thisCookie, 0, thisCookie.len) != thisCookie.len: @@ -153,12 +221,14 @@ proc loadHeader*(f: var RodFile) = setError f, wrongHeader proc storeSection*(f: var RodFile; s: RodSection) = + ## update `currentSection` and writes the bytes value of s. if f.err != ok: return assert f.currentSection < s f.currentSection = s storePrim(f, s) proc loadSection*(f: var RodFile; expected: RodSection) = + ## read the bytes value of s, sets and error if the section is incorrect. if f.err != ok: return var s: RodSection loadPrim(f, s) @@ -166,11 +236,13 @@ proc loadSection*(f: var RodFile; expected: RodSection) = setError f, wrongSection proc create*(filename: string): RodFile = + ## create the file and open it for writing if not open(result.f, filename, fmWrite): setError result, cannotOpen proc close*(f: var RodFile) = close(f.f) proc open*(filename: string): RodFile = + ## open the file for reading if not open(result.f, filename, fmRead): setError result, cannotOpen From 172a5ab7f82df503c1123a3bfa2f702416733858 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 19 Apr 2021 13:41:50 +0800 Subject: [PATCH 0666/2472] [std/rlocks]add inline pragma (#17773) so that it can be used as the base of new locks module with destructors. --- lib/core/rlocks.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/core/rlocks.nim b/lib/core/rlocks.nim index f9224adb8..5f155262a 100644 --- a/lib/core/rlocks.nim +++ b/lib/core/rlocks.nim @@ -33,15 +33,15 @@ proc deinitRLock*(lock: var RLock) {.inline.} = ## Frees the resources associated with the lock. deinitSys(lock) -proc tryAcquire*(lock: var RLock): bool = +proc tryAcquire*(lock: var RLock): bool {.inline.} = ## Tries to acquire the given lock. Returns `true` on success. result = tryAcquireSys(lock) -proc acquire*(lock: var RLock) = +proc acquire*(lock: var RLock) {.inline.} = ## Acquires the given lock. acquireSys(lock) -proc release*(lock: var RLock) = +proc release*(lock: var RLock) {.inline.} = ## Releases the given lock. releaseSys(lock) From f8038af5ecfa20deaa0b9b1e0d8a5f5915dd9780 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Mon, 19 Apr 2021 02:42:38 -0300 Subject: [PATCH 0667/2472] Documentation only, dom (#17770) * ReSync with Devel * ReSync * https://github.com/timotheecour/Nim/issues/674 * Update lib/js/dom.nim Co-authored-by: flywind Co-authored-by: Andreas Rumpf Co-authored-by: flywind --- lib/js/dom.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/js/dom.nim b/lib/js/dom.nim index a133f1f69..303cc178f 100644 --- a/lib/js/dom.nim +++ b/lib/js/dom.nim @@ -1496,7 +1496,9 @@ proc find*(w: Window, text: cstring, caseSensitive = false, backwards = false) proc focus*(w: Window) proc forward*(w: Window) -proc getComputedStyle*(w: Window, e: Node, pe:Node = nil): Style +proc getComputedStyle*(w: Window, e: Node, pe: Node = nil): Style + ## .. warning:: The returned Style may or may not be read-only at run-time in the browser. getComputedStyle is performance costly. + proc handleEvent*(w: Window, e: Event) proc home*(w: Window) proc moveBy*(w: Window, x, y: int) From cedbc7035d171d1535365c8acb889911b839ce99 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 19 Apr 2021 16:33:56 +0800 Subject: [PATCH 0668/2472] [std/locks]remove workaround for withLock (#17772) Ref #6113 and #6049 The workaround for generics instantiation is unnecessary. It seems to be fixed by another PR I guess. The test still works. So the changes should be harmless. https://github.com/nim-lang/Nim/blob/devel/tests/stdlib/tlocks.nim I also add some inline pragmas. --- lib/core/locks.nim | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/core/locks.nim b/lib/core/locks.nim index 1540e2bcd..674817674 100644 --- a/lib/core/locks.nim +++ b/lib/core/locks.nim @@ -42,16 +42,16 @@ proc deinitLock*(lock: var Lock) {.inline.} = ## Frees the resources associated with the lock. deinitSys(lock) -proc tryAcquire*(lock: var Lock): bool = +proc tryAcquire*(lock: var Lock): bool {.inline.} = ## Tries to acquire the given lock. Returns `true` on success. result = tryAcquireSys(lock) -proc acquire*(lock: var Lock) = +proc acquire*(lock: var Lock) {.inline.} = ## Acquires the given lock. when not defined(js): acquireSys(lock) -proc release*(lock: var Lock) = +proc release*(lock: var Lock) {.inline.} = ## Releases the given lock. when not defined(js): releaseSys(lock) @@ -76,7 +76,6 @@ proc signal*(cond: var Cond) {.inline.} = template withLock*(a: Lock, body: untyped) = ## Acquires the given lock, executes the statements in body and ## releases the lock after the statements finish executing. - mixin acquire, release acquire(a) {.locks: [a].}: try: From dc89b212572fecd449dd469ffb2424c78941c12f Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 19 Apr 2021 16:51:13 +0800 Subject: [PATCH 0669/2472] [std/locks]close #7998(complete condition variables) (#17711) * close #7998 * workaround genode * Update lib/system/syslocks.nim --- lib/core/locks.nim | 9 +++-- lib/genode_cpp/syslocks.h | 5 +++ lib/system/syslocks.nim | 55 ++++++++++++++++++------------- lib/system/sysspawn.nim | 33 ++++++++----------- lib/system/threadlocalstorage.nim | 3 ++ 5 files changed, 60 insertions(+), 45 deletions(-) diff --git a/lib/core/locks.nim b/lib/core/locks.nim index 674817674..92967b9db 100644 --- a/lib/core/locks.nim +++ b/lib/core/locks.nim @@ -66,13 +66,18 @@ proc deinitCond*(cond: var Cond) {.inline.} = deinitSysCond(cond) proc wait*(cond: var Cond, lock: var Lock) {.inline.} = - ## waits on the condition variable `cond`. + ## Waits on the condition variable `cond`. waitSysCond(cond, lock) proc signal*(cond: var Cond) {.inline.} = - ## sends a signal to the condition variable `cond`. + ## Sends a signal to the condition variable `cond`. signalSysCond(cond) +proc broadcast*(cond: var Cond) {.inline.} = + ## Unblocks all threads currently blocked on the + ## specified condition variable `cond`. + broadcastSysCond(cond) + template withLock*(a: Lock, body: untyped) = ## Acquires the given lock, executes the statements in body and ## releases the lock after the statements finish executing. diff --git a/lib/genode_cpp/syslocks.h b/lib/genode_cpp/syslocks.h index c50180903..b5d5ae694 100644 --- a/lib/genode_cpp/syslocks.h +++ b/lib/genode_cpp/syslocks.h @@ -71,6 +71,11 @@ struct Nim::SysCond { _semaphore.up(); } + + void broadcastSysCond() + { + _semaphore.up(); + } }; #endif diff --git a/lib/system/syslocks.nim b/lib/system/syslocks.nim index 51dbfd3a2..ac52d9a8c 100644 --- a/lib/system/syslocks.nim +++ b/lib/system/syslocks.nim @@ -24,7 +24,9 @@ when defined(windows): LockSemaphore: int SpinCount: int - SysCond = Handle + RTL_CONDITION_VARIABLE {.importc: "RTL_CONDITION_VARIABLE", header: "synchapi.h".} = object + thePtr {.importc: "ptr".} : Handle + SysCond = RTL_CONDITION_VARIABLE proc initSysLock(L: var SysLock) {.importc: "InitializeCriticalSection", header: "".} @@ -48,30 +50,29 @@ when defined(windows): proc deinitSys(L: var SysLock) {.importc: "DeleteCriticalSection", header: "".} - proc createEvent(lpEventAttributes: pointer, - bManualReset, bInitialState: int32, - lpName: cstring): SysCond {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "CreateEventA".} + proc initializeConditionVariable( + conditionVariable: var SysCond + ) {.stdcall, noSideEffect, dynlib: "kernel32", importc: "InitializeConditionVariable".} - proc closeHandle(hObject: Handle) {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "CloseHandle".} - proc waitForSingleObject(hHandle: Handle, dwMilliseconds: int32): int32 {. - stdcall, dynlib: "kernel32", importc: "WaitForSingleObject", noSideEffect.} + proc sleepConditionVariableCS( + conditionVariable: var SysCond, + PCRITICAL_SECTION: var SysLock, + dwMilliseconds: int + ): int32 {.stdcall, noSideEffect, dynlib: "kernel32", importc: "SleepConditionVariableCS".} - proc signalSysCond(hEvent: SysCond) {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "SetEvent".} + + proc signalSysCond(hEvent: var SysCond) {.stdcall, noSideEffect, + dynlib: "kernel32", importc: "WakeConditionVariable".} + + proc broadcastSysCond(hEvent: var SysCond) {.stdcall, noSideEffect, + dynlib: "kernel32", importc: "WakeAllConditionVariable".} proc initSysCond(cond: var SysCond) {.inline.} = - cond = createEvent(nil, 0'i32, 0'i32, nil) + initializeConditionVariable(cond) proc deinitSysCond(cond: var SysCond) {.inline.} = - closeHandle(cond) + discard proc waitSysCond(cond: var SysCond, lock: var SysLock) = - releaseSys(lock) - discard waitForSingleObject(cond, -1'i32) - acquireSys(lock) - - proc waitSysCondWindows(cond: var SysCond) = - discard waitForSingleObject(cond, -1'i32) + discard sleepConditionVariableCS(cond, lock, -1'i32) elif defined(genode): const @@ -94,6 +95,8 @@ elif defined(genode): noSideEffect, importcpp.} proc signalSysCond(cond: var SysCond) {. noSideEffect, importcpp.} + proc broadcastSysCond(cond: var SysCond) {. + noSideEffect, importcpp.} else: type @@ -181,7 +184,7 @@ else: releaseSysAux(L) when insideRLocksModule: - let SysLockType_Reentrant{.importc: "PTHREAD_MUTEX_RECURSIVE", + let SysLockType_Reentrant {.importc: "PTHREAD_MUTEX_RECURSIVE", header: "".}: SysLockType proc initSysLockAttr(a: var SysLockAttr) {. importc: "pthread_mutexattr_init", header: "", noSideEffect.} @@ -194,10 +197,12 @@ else: proc deinitSysCondAux(cond: var SysCondObj) {.noSideEffect, importc: "pthread_cond_destroy", header: "".} - proc waitSysCondAux(cond: var SysCondObj, lock: var SysLockObj) {. + proc waitSysCondAux(cond: var SysCondObj, lock: var SysLockObj): cint {. importc: "pthread_cond_wait", header: "", noSideEffect.} proc signalSysCondAux(cond: var SysCondObj) {. importc: "pthread_cond_signal", header: "", noSideEffect.} + proc broadcastSysCondAux(cond: var SysCondObj) {. + importc: "pthread_cond_broadcast", header: "", noSideEffect.} when defined(ios): proc initSysCond(cond: var SysCond, cond_attr: ptr SysCondAttr = nil) = @@ -209,9 +214,11 @@ else: c_free(cond) template waitSysCond(cond: var SysCond, lock: var SysLock) = - waitSysCondAux(cond[], lock[]) + discard waitSysCondAux(cond[], lock[]) template signalSysCond(cond: var SysCond) = signalSysCondAux(cond[]) + template broadcastSysCond(cond: var SysCond) = + broadcastSysCondAux(cond[]) else: template initSysCond(cond: var SysCond, cond_attr: ptr SysCondAttr = nil) = initSysCondAux(cond, cond_attr) @@ -219,8 +226,10 @@ else: deinitSysCondAux(cond) template waitSysCond(cond: var SysCond, lock: var SysLock) = - waitSysCondAux(cond, lock) + discard waitSysCondAux(cond, lock) template signalSysCond(cond: var SysCond) = signalSysCondAux(cond) + template broadcastSysCond(cond: var SysCond) = + broadcastSysCondAux(cond) {.pop.} diff --git a/lib/system/sysspawn.nim b/lib/system/sysspawn.nim index d1f5803f4..c9b9a68f0 100644 --- a/lib/system/sysspawn.nim +++ b/lib/system/sysspawn.nim @@ -20,34 +20,28 @@ when not declared(NimString): type CondVar = object c: SysCond - when defined(posix): - stupidLock: SysLock - counter: int + stupidLock: SysLock + counter: int proc createCondVar(): CondVar = initSysCond(result.c) - when defined(posix): - initSysLock(result.stupidLock) - #acquireSys(result.stupidLock) + initSysLock(result.stupidLock) + #acquireSys(result.stupidLock) proc destroyCondVar(c: var CondVar) {.inline.} = deinitSysCond(c.c) proc await(cv: var CondVar) = - when defined(posix): - acquireSys(cv.stupidLock) - while cv.counter <= 0: - waitSysCond(cv.c, cv.stupidLock) - dec cv.counter - releaseSys(cv.stupidLock) - else: - waitSysCondWindows(cv.c) + acquireSys(cv.stupidLock) + while cv.counter <= 0: + waitSysCond(cv.c, cv.stupidLock) + dec cv.counter + releaseSys(cv.stupidLock) proc signal(cv: var CondVar) = - when defined(posix): - acquireSys(cv.stupidLock) - inc cv.counter - releaseSys(cv.stupidLock) + acquireSys(cv.stupidLock) + inc cv.counter + releaseSys(cv.stupidLock) signalSysCond(cv.c) type @@ -57,8 +51,7 @@ type proc createFastCondVar(): FastCondVar = initSysCond(result.slow.c) - when defined(posix): - initSysLock(result.slow.stupidLock) + initSysLock(result.slow.stupidLock) #acquireSys(result.slow.stupidLock) result.event = false result.slowPath = false diff --git a/lib/system/threadlocalstorage.nim b/lib/system/threadlocalstorage.nim index cbb74c7df..30962ef77 100644 --- a/lib/system/threadlocalstorage.nim +++ b/lib/system/threadlocalstorage.nim @@ -17,6 +17,9 @@ when defined(windows): proc winResumeThread(hThread: SysThread): int32 {. stdcall, dynlib: "kernel32", importc: "ResumeThread".} + proc waitForSingleObject(hHandle: SysThread, dwMilliseconds: int32): int32 {. + stdcall, dynlib: "kernel32", importc: "WaitForSingleObject".} + proc waitForMultipleObjects(nCount: int32, lpHandles: ptr SysThread, bWaitAll: int32, From 83fa0fc843a5757c6fa1af9ac89b85b71c0c7581 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 19 Apr 2021 17:21:35 +0800 Subject: [PATCH 0670/2472] fix #12521(type alias for openArray) (#17593) * fix nim js cmp fails at CT * fix #12521 * address comments --- compiler/ccgtypes.nim | 2 +- tests/openarray/topenarray.nim | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 3d4619764..fa58b5651 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -453,7 +453,7 @@ proc genProcParams(m: BModule, t: PType, rettype, params: var Rope, params.add(~"NIM_NOALIAS ") params.add(param.loc.r) # declare the len field for open arrays: - var arr = param.typ + var arr = param.typ.skipTypes({tyGenericInst}) if arr.kind in {tyVar, tyLent, tySink}: arr = arr.lastSon var j = 0 while arr.kind in {tyOpenArray, tyVarargs}: diff --git a/tests/openarray/topenarray.nim b/tests/openarray/topenarray.nim index 0b6e3afc4..aec5a4cbf 100644 --- a/tests/openarray/topenarray.nim +++ b/tests/openarray/topenarray.nim @@ -23,5 +23,31 @@ proc main = when defined(js): discard # xxx bug #15952: `a` left unchanged else: doAssert a == [1, 0, 10, 20, 5] + block: # bug #12521 + block: + type slice[T] = openArray[T] + + # Proc using that alias + proc testing(sl: slice[int]): seq[int] = + for item in sl: + result.add item + + let mySeq = @[1, 2, 3, 4, 5, 6, 7, 8, 9] + doAssert testing(mySeq) == mySeq + doAssert testing(mySeq[2..^2]) == mySeq[2..^2] + + block: + type slice = openArray[int] + + # Proc using that alias + proc testing(sl: slice): seq[int] = + for item in sl: + result.add item + + let mySeq = @[1, 2, 3, 4, 5, 6, 7, 8, 9] + doAssert testing(mySeq) == mySeq + doAssert testing(mySeq[2..^2]) == mySeq[2..^2] + + main() # static: main() # xxx bug #15952: Error: cannot generate code for: mSlice From 4b0b536419a500dc2e5a987ed9c2a374e0848008 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 19 Apr 2021 11:48:06 +0200 Subject: [PATCH 0671/2472] ic refactoring (#17778) * minor IC documentation update * IC: refactoring: removed the 'shared' type and fields, these were a leftover from an earlier design --- compiler/ic/dce.nim | 18 ++-- compiler/ic/design.rst | 8 +- compiler/ic/ic.nim | 175 +++++++++++++++++++++++-------------- compiler/ic/integrity.nim | 26 +++--- compiler/ic/navigator.nim | 12 +-- compiler/ic/packed_ast.nim | 56 ------------ compiler/ic/rodfiles.nim | 10 +-- 7 files changed, 146 insertions(+), 159 deletions(-) diff --git a/compiler/ic/dce.nim b/compiler/ic/dce.nim index 0918fc379..48ed414e2 100644 --- a/compiler/ic/dce.nim +++ b/compiler/ic/dce.nim @@ -27,7 +27,7 @@ type proc isExportedToC(c: var AliveContext; g: PackedModuleGraph; symId: int32): bool = ## "Exported to C" procs are special (these are marked with '.exportc') because these ## must not be optimized away! - let symPtr = addr g[c.thisModule].fromDisk.sh.syms[symId] + let symPtr = unsafeAddr g[c.thisModule].fromDisk.syms[symId] let flags = symPtr.flags # due to a bug/limitation in the lambda lifting, unused inner procs # are not transformed correctly; issue (#411). However, the whole purpose here @@ -39,7 +39,7 @@ proc isExportedToC(c: var AliveContext; g: PackedModuleGraph; symId: int32): boo # XXX: This used to be a condition to: # (sfExportc in prc.flags and lfExportLib in prc.loc.flags) or if sfCompilerProc in flags: - c.compilerProcs[g[c.thisModule].fromDisk.sh.strings[symPtr.name]] = (c.thisModule, symId) + c.compilerProcs[g[c.thisModule].fromDisk.strings[symPtr.name]] = (c.thisModule, symId) template isNotGeneric(n: NodePos): bool = ithSon(tree, n, genericParamsPos).kind == nkEmpty @@ -47,17 +47,17 @@ proc followLater(c: var AliveContext; g: PackedModuleGraph; module: int; item: i ## Marks a symbol 'item' as used and later in 'followNow' the symbol's body will ## be analysed. if not c.alive[module].containsOrIncl(item): - var body = g[module].fromDisk.sh.syms[item].ast + var body = g[module].fromDisk.syms[item].ast if body != emptyNodeId: - let opt = g[module].fromDisk.sh.syms[item].options - if g[module].fromDisk.sh.syms[item].kind in routineKinds: + let opt = g[module].fromDisk.syms[item].options + if g[module].fromDisk.syms[item].kind in routineKinds: body = NodeId ithSon(g[module].fromDisk.bodies, NodePos body, bodyPos) c.stack.add((module, opt, NodePos(body))) when false: - let nid = g[module].fromDisk.sh.syms[item].name + let nid = g[module].fromDisk.syms[item].name if nid != LitId(0): - let name = g[module].fromDisk.sh.strings[nid] + let name = g[module].fromDisk.strings[nid] if name in ["nimFrame", "callDepthLimitReached"]: echo "I was called! ", name, " body exists: ", body != emptyNodeId, " ", module, " ", item @@ -66,12 +66,12 @@ proc requestCompilerProc(c: var AliveContext; g: PackedModuleGraph; name: string followLater(c, g, module, item) proc loadTypeKind(t: PackedItemId; c: AliveContext; g: PackedModuleGraph; toSkip: set[TTypeKind]): TTypeKind = - template kind(t: ItemId): TTypeKind = g[t.module].fromDisk.sh.types[t.item].kind + template kind(t: ItemId): TTypeKind = g[t.module].fromDisk.types[t.item].kind var t2 = translateId(t, g, c.thisModule, c.decoder.config) result = t2.kind while result in toSkip: - t2 = translateId(g[t2.module].fromDisk.sh.types[t2.item].types[^1], g, t2.module, c.decoder.config) + t2 = translateId(g[t2.module].fromDisk.types[t2.item].types[^1], g, t2.module, c.decoder.config) result = t2.kind proc rangeCheckAnalysis(c: var AliveContext; g: PackedModuleGraph; tree: PackedTree; n: NodePos) = diff --git a/compiler/ic/design.rst b/compiler/ic/design.rst index d8e1315b1..b096e3103 100644 --- a/compiler/ic/design.rst +++ b/compiler/ic/design.rst @@ -7,12 +7,8 @@ The frontend produces a set of `.rod` files. Every `.nim` module produces its own `.rod` file. - The IR must be a faithful representation of the AST in memory. -- The backend can do its own caching but doesn't have to. -- We know by comparing 'nim check compiler/nim' against 'nim c compiler/nim' - that 2/3 of the compiler's runtime is spent in the frontend. Hence we - implement IC for the frontend first and only later for the backend. The - backend will recompile everything until we implement its own caching - mechanisms. +- The backend can do its own caching but doesn't have to. In the + current implementation the backend also caches its results. Advantage of the "set of files" vs the previous global database: - By construction, we either read from the `.rod` file or from the diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim index 6f208a2a4..a23685be7 100644 --- a/compiler/ic/ic.nim +++ b/compiler/ic/ic.nim @@ -51,7 +51,12 @@ type emittedTypeInfo*: seq[string] backendFlags*: set[ModuleBackendFlag] - sh*: Shared + syms*: seq[PackedSym] + types*: seq[PackedType] + strings*: BiTable[string] # we could share these between modules. + numbers*: BiTable[BiggestInt] # we also store floats in here so + # that we can assure that every bit is kept + cfg: PackedConfig PackedEncoder* = object @@ -66,6 +71,50 @@ type symMarker*: IntSet #Table[ItemId, SymId] # ItemId.item -> SymId config*: ConfigRef +proc toString*(tree: PackedTree; n: NodePos; m: PackedModule; nesting: int; + result: var string) = + let pos = n.int + if result.len > 0 and result[^1] notin {' ', '\n'}: + result.add ' ' + + result.add $tree[pos].kind + case tree.nodes[pos].kind + of nkNone, nkEmpty, nkNilLit, nkType: discard + of nkIdent, nkStrLit..nkTripleStrLit: + result.add " " + result.add m.strings[LitId tree.nodes[pos].operand] + of nkSym: + result.add " " + result.add m.strings[m.syms[tree.nodes[pos].operand].name] + of directIntLit: + result.add " " + result.addInt tree.nodes[pos].operand + of externSIntLit: + result.add " " + result.addInt m.numbers[LitId tree.nodes[pos].operand] + of externUIntLit: + result.add " " + result.add $cast[uint64](m.numbers[LitId tree.nodes[pos].operand]) + of nkFloatLit..nkFloat128Lit: + result.add " " + result.add $cast[BiggestFloat](m.numbers[LitId tree.nodes[pos].operand]) + else: + result.add "(\n" + for i in 1..(nesting+1)*2: result.add ' ' + for child in sonsReadonly(tree, n): + toString(tree, child, m, nesting + 1, result) + result.add "\n" + for i in 1..nesting*2: result.add ' ' + result.add ")" + #for i in 1..nesting*2: result.add ' ' + +proc toString*(tree: PackedTree; n: NodePos; m: PackedModule): string = + result = "" + toString(tree, n, m, 0, result) + +proc debug*(tree: PackedTree; m: PackedModule) = + stdout.write toString(tree, NodePos 0, m) + proc isActive*(e: PackedEncoder): bool = e.config != nil proc disable(e: var PackedEncoder) = e.config = nil @@ -134,14 +183,14 @@ proc toLitId(x: FileIndex; c: var PackedEncoder; m: var PackedModule): LitId = result = c.filenames.getOrDefault(x) if result == LitId(0): let p = msgs.toFullPath(c.config, x) - result = getOrIncl(m.sh.strings, p) + result = getOrIncl(m.strings, p) c.filenames[x] = result c.lastFile = x c.lastLit = result assert result != LitId(0) proc toFileIndex*(x: LitId; m: PackedModule; config: ConfigRef): FileIndex = - result = msgs.fileInfoIdx(config, AbsoluteFile m.sh.strings[x]) + result = msgs.fileInfoIdx(config, AbsoluteFile m.strings[x]) proc includesIdentical(m: var PackedModule; config: ConfigRef): bool = for it in mitems(m.includes): @@ -151,7 +200,6 @@ proc includesIdentical(m: var PackedModule; config: ConfigRef): bool = proc initEncoder*(c: var PackedEncoder; m: var PackedModule; moduleSym: PSym; config: ConfigRef; pc: PackedConfig) = ## setup a context for serializing to packed ast - m.sh = Shared() c.thisModule = moduleSym.itemId.module c.config = config m.moduleFlags = moduleSym.flags @@ -179,11 +227,11 @@ proc addImportFileDep*(c: var PackedEncoder; m: var PackedModule; f: FileIndex) m.imports.add toLitId(f, c, m) proc addHidden*(c: var PackedEncoder; m: var PackedModule; s: PSym) = - let nameId = getOrIncl(m.sh.strings, s.name.s) + let nameId = getOrIncl(m.strings, s.name.s) m.hidden.add((nameId, s.itemId.item)) proc addExported*(c: var PackedEncoder; m: var PackedModule; s: PSym) = - let nameId = getOrIncl(m.sh.strings, s.name.s) + let nameId = getOrIncl(m.strings, s.name.s) m.exports.add((nameId, s.itemId.item)) proc addConverter*(c: var PackedEncoder; m: var PackedModule; s: PSym) = @@ -201,12 +249,12 @@ proc addMethod*(c: var PackedEncoder; m: var PackedModule; s: PSym) = m.methods.add s.itemId.item proc addReexport*(c: var PackedEncoder; m: var PackedModule; s: PSym) = - let nameId = getOrIncl(m.sh.strings, s.name.s) + let nameId = getOrIncl(m.strings, s.name.s) m.reexports.add((nameId, PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), item: s.itemId.item))) proc addCompilerProc*(c: var PackedEncoder; m: var PackedModule; s: PSym) = - let nameId = getOrIncl(m.sh.strings, s.name.s) + let nameId = getOrIncl(m.strings, s.name.s) m.compilerProcs.add((nameId, s.itemId.item)) proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule) @@ -225,11 +273,11 @@ proc flush(c: var PackedEncoder; m: var PackedModule) = proc toLitId(x: string; m: var PackedModule): LitId = ## store a string as a literal - result = getOrIncl(m.sh.strings, x) + result = getOrIncl(m.strings, x) proc toLitId(x: BiggestInt; m: var PackedModule): LitId = ## store an integer as a literal - result = getOrIncl(m.sh.numbers, x) + result = getOrIncl(m.numbers, x) proc toPackedInfo(x: TLineInfo; c: var PackedEncoder; m: var PackedModule): PackedLineInfo = PackedLineInfo(line: x.line, col: x.col, file: toLitId(x.fileIndex, c, m)) @@ -299,8 +347,8 @@ proc storeType(t: PType; c: var PackedEncoder; m: var PackedModule): PackedItemI result = PackedItemId(module: toLitId(t.uniqueId.module.FileIndex, c, m), item: t.uniqueId.item) if t.uniqueId.module == c.thisModule and not c.typeMarker.containsOrIncl(t.uniqueId.item): - if t.uniqueId.item >= m.sh.types.len: - setLen m.sh.types, t.uniqueId.item+1 + if t.uniqueId.item >= m.types.len: + setLen m.types, t.uniqueId.item+1 var p = PackedType(kind: t.kind, flags: t.flags, callConv: t.callConv, size: t.size, align: t.align, nonUniqueId: t.itemId.item, @@ -315,7 +363,7 @@ proc storeType(t: PType; c: var PackedEncoder; m: var PackedModule): PackedItemI p.owner = t.owner.safeItemId(c, m) # fill the reserved slot, nothing else: - m.sh.types[t.uniqueId.item] = p + m.types[t.uniqueId.item] = p proc toPackedLib(l: PLib; c: var PackedEncoder; m: var PackedModule): PackedLib = ## the plib hangs off the psym via the .annex field @@ -334,8 +382,8 @@ proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId result = PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), item: s.itemId.item) if s.itemId.module == c.thisModule and not c.symMarker.containsOrIncl(s.itemId.item): - if s.itemId.item >= m.sh.syms.len: - setLen m.sh.syms, s.itemId.item+1 + if s.itemId.item >= m.syms.len: + setLen m.syms, s.itemId.item+1 assert sfForward notin s.flags @@ -363,7 +411,7 @@ proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId p.cname = toLitId(s.cname, m) # fill the reserved slot, nothing else: - m.sh.syms[s.itemId.item] = p + m.syms[s.itemId.item] = p proc addModuleRef(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule) = ## add a remote symbol reference to the tree @@ -387,7 +435,7 @@ proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var Pa typeId: storeTypeLater(n.typ, c, m), info: info) of nkIdent: ir.nodes.add PackedNode(kind: n.kind, flags: n.flags, - operand: int32 getOrIncl(m.sh.strings, n.ident.s), + operand: int32 getOrIncl(m.strings, n.ident.s), typeId: storeTypeLater(n.typ, c, m), info: info) of nkSym: if n.sym.itemId.module == c.thisModule: @@ -405,15 +453,15 @@ proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var Pa typeId: storeTypeLater(n.typ, c, m), info: info) of externIntLit: ir.nodes.add PackedNode(kind: n.kind, flags: n.flags, - operand: int32 getOrIncl(m.sh.numbers, n.intVal), + operand: int32 getOrIncl(m.numbers, n.intVal), typeId: storeTypeLater(n.typ, c, m), info: info) of nkStrLit..nkTripleStrLit: ir.nodes.add PackedNode(kind: n.kind, flags: n.flags, - operand: int32 getOrIncl(m.sh.strings, n.strVal), + operand: int32 getOrIncl(m.strings, n.strVal), typeId: storeTypeLater(n.typ, c, m), info: info) of nkFloatLit..nkFloat128Lit: ir.nodes.add PackedNode(kind: n.kind, flags: n.flags, - operand: int32 getOrIncl(m.sh.numbers, cast[BiggestInt](n.floatVal)), + operand: int32 getOrIncl(m.numbers, cast[BiggestInt](n.floatVal)), typeId: storeTypeLater(n.typ, c, m), info: info) else: let patchPos = ir.prepare(n.kind, n.flags, @@ -499,7 +547,6 @@ proc loadError(err: RodFileError; filename: AbsoluteFile; config: ConfigRef;) = proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef; ignoreConfig = false): RodFileError = - m.sh = Shared() var f = rodfiles.open(filename.string) f.loadHeader() f.loadSection configSection @@ -519,7 +566,7 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef f.loadSection section f.load data - loadTabSection stringsSection, m.sh.strings + loadTabSection stringsSection, m.strings loadSeqSection checkSumsSection, m.includes if not includesIdentical(m, config): @@ -527,7 +574,7 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef loadSeqSection depsSection, m.imports - loadTabSection numbersSection, m.sh.numbers + loadTabSection numbersSection, m.numbers loadSeqSection exportsSection, m.exports loadSeqSection hiddenSection, m.hidden @@ -545,8 +592,8 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef loadSeqSection toReplaySection, m.toReplay.nodes loadSeqSection topLevelSection, m.topLevel.nodes loadSeqSection bodiesSection, m.bodies.nodes - loadSeqSection symsSection, m.sh.syms - loadSeqSection typesSection, m.sh.types + loadSeqSection symsSection, m.syms + loadSeqSection typesSection, m.types loadSeqSection typeInstCacheSection, m.typeInstCache loadSeqSection procInstCacheSection, m.procInstCache @@ -586,13 +633,13 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac f.storeSection section f.store data - storeTabSection stringsSection, m.sh.strings + storeTabSection stringsSection, m.strings storeSeqSection checkSumsSection, m.includes storeSeqSection depsSection, m.imports - storeTabSection numbersSection, m.sh.numbers + storeTabSection numbersSection, m.numbers storeSeqSection exportsSection, m.exports storeSeqSection hiddenSection, m.hidden @@ -610,9 +657,9 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac storeSeqSection topLevelSection, m.topLevel.nodes storeSeqSection bodiesSection, m.bodies.nodes - storeSeqSection symsSection, m.sh.syms + storeSeqSection symsSection, m.syms - storeSeqSection typesSection, m.sh.types + storeSeqSection typesSection, m.types storeSeqSection typeInstCacheSection, m.typeInstCache storeSeqSection procInstCacheSection, m.procInstCache @@ -700,17 +747,17 @@ proc loadNodes*(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; of nkEmpty, nkNilLit, nkType: discard of nkIdent: - result.ident = getIdent(c.cache, g[thisModule].fromDisk.sh.strings[n.litId]) + result.ident = getIdent(c.cache, g[thisModule].fromDisk.strings[n.litId]) of nkSym: result.sym = loadSym(c, g, thisModule, PackedItemId(module: LitId(0), item: tree.nodes[n.int].operand)) of directIntLit: result.intVal = tree.nodes[n.int].operand of externIntLit: - result.intVal = g[thisModule].fromDisk.sh.numbers[n.litId] + result.intVal = g[thisModule].fromDisk.numbers[n.litId] of nkStrLit..nkTripleStrLit: - result.strVal = g[thisModule].fromDisk.sh.strings[n.litId] + result.strVal = g[thisModule].fromDisk.strings[n.litId] of nkFloatLit..nkFloat128Lit: - result.floatVal = cast[BiggestFloat](g[thisModule].fromDisk.sh.numbers[n.litId]) + result.floatVal = cast[BiggestFloat](g[thisModule].fromDisk.numbers[n.litId]) of nkModuleRef: let (n1, n2) = sons2(tree, n) assert n1.kind == nkInt32Lit @@ -767,7 +814,7 @@ proc symHeaderFromPacked(c: var PackedDecoder; g: var PackedModuleGraph; options: s.options, position: if s.kind in {skForVar, skVar, skLet, skTemp}: 0 else: s.position, offset: if s.kind in routineKinds: defaultOffset else: s.offset, - name: getIdent(c.cache, g[si].fromDisk.sh.strings[s.name]) + name: getIdent(c.cache, g[si].fromDisk.strings[s.name]) ) template loadAstBody(p, field) = @@ -785,7 +832,7 @@ proc loadLib(c: var PackedDecoder; g: var PackedModuleGraph; result = nil else: result = PLib(generated: l.generated, isOverriden: l.isOverriden, - kind: l.kind, name: rope g[si].fromDisk.sh.strings[l.name]) + kind: l.kind, name: rope g[si].fromDisk.strings[l.name]) loadAstBody(l, path) proc symBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph; @@ -798,14 +845,14 @@ proc symBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph; loadAstBody(s, ast) result.annex = loadLib(c, g, si, item, s.annex) when hasFFI: - result.cname = g[si].fromDisk.sh.strings[s.cname] + result.cname = g[si].fromDisk.strings[s.cname] if s.kind in {skLet, skVar, skField, skForVar}: result.guard = loadSym(c, g, si, s.guard) result.bitsize = s.bitsize result.alignment = s.alignment result.owner = loadSym(c, g, si, s.owner) - let externalName = g[si].fromDisk.sh.strings[s.externalName] + let externalName = g[si].fromDisk.strings[s.externalName] if externalName != "": result.loc.r = rope externalName result.loc.flags = s.locFlags @@ -818,14 +865,14 @@ proc loadSym(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; s: assert g[si].status in {loaded, storing, stored} if not g[si].symsInit: g[si].symsInit = true - setLen g[si].syms, g[si].fromDisk.sh.syms.len + setLen g[si].syms, g[si].fromDisk.syms.len if g[si].syms[s.item] == nil: - if g[si].fromDisk.sh.syms[s.item].kind != skModule: - result = symHeaderFromPacked(c, g, g[si].fromDisk.sh.syms[s.item], si, s.item) + if g[si].fromDisk.syms[s.item].kind != skModule: + result = symHeaderFromPacked(c, g, g[si].fromDisk.syms[s.item], si, s.item) # store it here early on, so that recursions work properly: g[si].syms[s.item] = result - symBodyFromPacked(c, g, g[si].fromDisk.sh.syms[s.item], si, s.item, result) + symBodyFromPacked(c, g, g[si].fromDisk.syms[s.item], si, s.item, result) else: result = g[si].module assert result != nil @@ -866,13 +913,13 @@ proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; t if not g[si].typesInit: g[si].typesInit = true - setLen g[si].types, g[si].fromDisk.sh.types.len + setLen g[si].types, g[si].fromDisk.types.len if g[si].types[t.item] == nil: - result = typeHeaderFromPacked(c, g, g[si].fromDisk.sh.types[t.item], si, t.item) + result = typeHeaderFromPacked(c, g, g[si].fromDisk.types[t.item], si, t.item) # store it here early on, so that recursions work properly: g[si].types[t.item] = result - typeBodyFromPacked(c, g, g[si].fromDisk.sh.types[t.item], si, t.item, result) + typeBodyFromPacked(c, g, g[si].fromDisk.types[t.item], si, t.item, result) else: result = g[si].types[t.item] assert result.itemId.item > 0 @@ -897,7 +944,7 @@ proc setupLookupTables(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCa let e2 = when e[1] is PackedItemId: e[1] else: PackedItemId(module: LitId(0), item: e[1]) - iface.mgetOrPut(cache.getIdent(m.fromDisk.sh.strings[nameLit]), @[]).add(e2) + iface.mgetOrPut(cache.getIdent(m.fromDisk.strings[nameLit]), @[]).add(e2) for e in m.fromDisk.exports: m.iface.impl(e) @@ -1000,7 +1047,7 @@ proc loadProcBody*(config: ConfigRef, cache: IdentCache; lastFile: FileIndex(-1), config: config, cache: cache) - let pos = g[mId].fromDisk.sh.syms[s.itemId.item].ast + let pos = g[mId].fromDisk.syms[s.itemId.item].ast assert pos != emptyNodeId result = loadProcBody(decoder, g, mId, g[mId].fromDisk.bodies, NodePos pos) @@ -1042,15 +1089,15 @@ proc translateId*(id: PackedItemId; g: PackedModuleGraph; thisModule: int; confi proc checkForHoles(m: PackedModule; config: ConfigRef; moduleId: int) = var bugs = 0 - for i in 1 .. high(m.sh.syms): - if m.sh.syms[i].kind == skUnknown: + for i in 1 .. high(m.syms): + if m.syms[i].kind == skUnknown: echo "EMPTY ID ", i, " module ", moduleId, " ", toFullPath(config, FileIndex(moduleId)) inc bugs assert bugs == 0 when false: var nones = 0 - for i in 1 .. high(m.sh.types): - inc nones, m.sh.types[i].kind == tyNone + for i in 1 .. high(m.types): + inc nones, m.types[i].kind == tyNone assert nones < 1 proc simulateLoadedModule*(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; @@ -1133,13 +1180,13 @@ proc interfaceSymbol*(config: ConfigRef, cache: IdentCache; result = loadSym(decoder, g, int(module), values[0]) proc idgenFromLoadedModule*(m: LoadedModule): IdGenerator = - IdGenerator(module: m.module.itemId.module, symId: int32 m.fromDisk.sh.syms.len, - typeId: int32 m.fromDisk.sh.types.len) + IdGenerator(module: m.module.itemId.module, symId: int32 m.fromDisk.syms.len, + typeId: int32 m.fromDisk.types.len) proc searchForCompilerproc*(m: LoadedModule; name: string): int32 = # slow, linear search, but the results are cached: for it in items(m.fromDisk.compilerProcs): - if m.fromDisk.sh.strings[it[0]] == name: + if m.fromDisk.strings[it[0]] == name: return it[1] return -1 @@ -1149,31 +1196,31 @@ proc rodViewer*(rodfile: AbsoluteFile; config: ConfigRef, cache: IdentCache) = var m: PackedModule let err = loadRodFile(rodfile, m, config, ignoreConfig=true) if err != ok: - config.quitOrRaise "Error: could not load: " & $rodfile.string & " reason: " & $err + config.quitOrRaise "Error: could not load: " & $rodfile.string & " reason: " & $err when true: echo "exports:" for ex in m.exports: - echo " ", m.sh.strings[ex[0]], " local ID: ", ex[1] - assert ex[0] == m.sh.syms[ex[1]].name + echo " ", m.strings[ex[0]], " local ID: ", ex[1] + assert ex[0] == m.syms[ex[1]].name # ex[1] int32 echo "reexports:" for ex in m.reexports: - echo " ", m.sh.strings[ex[0]] + echo " ", m.strings[ex[0]] # reexports*: seq[(LitId, PackedItemId)] echo "hidden: " & $m.hidden.len for ex in m.hidden: - echo " ", m.sh.strings[ex[0]], " local ID: ", ex[1] + echo " ", m.strings[ex[0]], " local ID: ", ex[1] echo "all symbols" - for i in 0..high(m.sh.syms): - if m.sh.syms[i].name != LitId(0): - echo " ", m.sh.strings[m.sh.syms[i].name], " local ID: ", i, " kind ", m.sh.syms[i].kind + for i in 0..high(m.syms): + if m.syms[i].name != LitId(0): + echo " ", m.strings[m.syms[i].name], " local ID: ", i, " kind ", m.syms[i].kind else: - echo " local ID: ", i, " kind ", m.sh.syms[i].kind + echo " local ID: ", i, " kind ", m.syms[i].kind - echo "symbols: ", m.sh.syms.len, " types: ", m.sh.types.len, + echo "symbols: ", m.syms.len, " types: ", m.types.len, " top level nodes: ", m.topLevel.nodes.len, " other nodes: ", m.bodies.nodes.len, - " strings: ", m.sh.strings.len, " numbers: ", m.sh.numbers.len + " strings: ", m.strings.len, " numbers: ", m.numbers.len diff --git a/compiler/ic/integrity.nim b/compiler/ic/integrity.nim index a28f2066a..428c344a3 100644 --- a/compiler/ic/integrity.nim +++ b/compiler/ic/integrity.nim @@ -42,12 +42,12 @@ proc checkType(c: var CheckedContext; typeId: PackedItemId) = if not c.checkedTypes.containsOrIncl(itemId): let oldThisModule = c.thisModule c.thisModule = itemId.module - checkTypeObj c, c.g.packed[itemId.module].fromDisk.sh.types[itemId.item] + checkTypeObj c, c.g.packed[itemId.module].fromDisk.types[itemId.item] c.thisModule = oldThisModule proc checkSym(c: var CheckedContext; s: PackedSym) = if s.name != LitId(0): - assert c.g.packed[c.thisModule].fromDisk.sh.strings.hasLitId s.name + assert c.g.packed[c.thisModule].fromDisk.strings.hasLitId s.name checkType c, s.typ if s.ast != emptyNodeId: checkNode(c, c.g.packed[c.thisModule].fromDisk.bodies, NodePos s.ast) @@ -57,14 +57,14 @@ proc checkSym(c: var CheckedContext; s: PackedSym) = proc checkLocalSym(c: var CheckedContext; item: int32) = let itemId = ItemId(module: c.thisModule, item: item) if not c.checkedSyms.containsOrIncl(itemId): - checkSym c, c.g.packed[c.thisModule].fromDisk.sh.syms[item] + checkSym c, c.g.packed[c.thisModule].fromDisk.syms[item] proc checkForeignSym(c: var CheckedContext; symId: PackedItemId) = let itemId = translateId(symId, c.g.packed, c.thisModule, c.g.config) if not c.checkedSyms.containsOrIncl(itemId): let oldThisModule = c.thisModule c.thisModule = itemId.module - checkSym c, c.g.packed[itemId.module].fromDisk.sh.syms[itemId.item] + checkSym c, c.g.packed[itemId.module].fromDisk.syms[itemId.item] c.thisModule = oldThisModule proc checkNode(c: var CheckedContext; tree: PackedTree; n: NodePos) = @@ -74,15 +74,15 @@ proc checkNode(c: var CheckedContext; tree: PackedTree; n: NodePos) = of nkEmpty, nkNilLit, nkType, nkNilRodNode: discard of nkIdent: - assert c.g.packed[c.thisModule].fromDisk.sh.strings.hasLitId n.litId + assert c.g.packed[c.thisModule].fromDisk.strings.hasLitId n.litId of nkSym: checkLocalSym(c, tree.nodes[n.int].operand) of directIntLit: discard of externIntLit, nkFloatLit..nkFloat128Lit: - assert c.g.packed[c.thisModule].fromDisk.sh.numbers.hasLitId n.litId + assert c.g.packed[c.thisModule].fromDisk.numbers.hasLitId n.litId of nkStrLit..nkTripleStrLit: - assert c.g.packed[c.thisModule].fromDisk.sh.strings.hasLitId n.litId + assert c.g.packed[c.thisModule].fromDisk.strings.hasLitId n.litId of nkModuleRef: let (n1, n2) = sons2(tree, n) assert n1.kind == nkInt32Lit @@ -97,25 +97,25 @@ proc checkTree(c: var CheckedContext; t: PackedTree) = proc checkLocalSymIds(c: var CheckedContext; m: PackedModule; symIds: seq[int32]) = for symId in symIds: - assert symId >= 0 and symId < m.sh.syms.len, $symId & " " & $m.sh.syms.len + assert symId >= 0 and symId < m.syms.len, $symId & " " & $m.syms.len proc checkModule(c: var CheckedContext; m: PackedModule) = # We check that: # - Every symbol references existing types and symbols. # - Every tree node references existing types and symbols. - for i in 0..high(m.sh.syms): + for i in 0..high(m.syms): checkLocalSym c, int32(i) checkTree c, m.toReplay checkTree c, m.topLevel for e in m.exports: - assert e[1] >= 0 and e[1] < m.sh.syms.len - assert e[0] == m.sh.syms[e[1]].name + assert e[1] >= 0 and e[1] < m.syms.len + assert e[0] == m.syms[e[1]].name for e in m.compilerProcs: - assert e[1] >= 0 and e[1] < m.sh.syms.len - assert e[0] == m.sh.syms[e[1]].name + assert e[1] >= 0 and e[1] < m.syms.len + assert e[0] == m.syms[e[1]].name checkLocalSymIds c, m, m.converters checkLocalSymIds c, m, m.methods diff --git a/compiler/ic/navigator.nim b/compiler/ic/navigator.nim index b09275220..b1a237cf7 100644 --- a/compiler/ic/navigator.nim +++ b/compiler/ic/navigator.nim @@ -35,12 +35,12 @@ proc isTracked(current, trackPos: PackedLineInfo, tokenLen: int): bool = proc searchLocalSym(c: var NavContext; s: PackedSym; info: PackedLineInfo): bool = result = s.name != LitId(0) and - isTracked(info, c.trackPos, c.g.packed[c.thisModule].fromDisk.sh.strings[s.name].len) + isTracked(info, c.trackPos, c.g.packed[c.thisModule].fromDisk.strings[s.name].len) proc searchForeignSym(c: var NavContext; s: ItemId; info: PackedLineInfo): bool = - let name = c.g.packed[s.module].fromDisk.sh.syms[s.item].name + let name = c.g.packed[s.module].fromDisk.syms[s.item].name result = name != LitId(0) and - isTracked(info, c.trackPos, c.g.packed[s.module].fromDisk.sh.strings[name].len) + isTracked(info, c.trackPos, c.g.packed[s.module].fromDisk.strings[name].len) const EmptyItemId = ItemId(module: -1'i32, item: -1'i32) @@ -51,7 +51,7 @@ proc search(c: var NavContext; tree: PackedTree): ItemId = case tree.nodes[i].kind of nkSym: let item = tree.nodes[i].operand - if searchLocalSym(c, c.g.packed[c.thisModule].fromDisk.sh.syms[item], tree.nodes[i].info): + if searchLocalSym(c, c.g.packed[c.thisModule].fromDisk.syms[item], tree.nodes[i].info): return ItemId(module: c.thisModule, item: item) of nkModuleRef: if tree.nodes[i].info.line == c.trackPos.line and tree.nodes[i].info.file == c.trackPos.file: @@ -74,7 +74,7 @@ proc isDecl(tree: PackedTree; n: NodePos): bool = proc usage(c: var NavContext; info: PackedLineInfo; isDecl: bool) = var m = "" - var file = c.g.packed[c.thisModule].fromDisk.sh.strings[info.file] + var file = c.g.packed[c.thisModule].fromDisk.strings[info.file] if c.outputSep == ' ': file = os.extractFilename file toLocation(m, file, info.line.int, info.col.int + ColOffset) @@ -102,7 +102,7 @@ proc nav(g: ModuleGraph) = # translate the track position to a packed position: let unpacked = g.config.m.trackPos let mid = unpacked.fileIndex - let fileId = g.packed[int32 mid].fromDisk.sh.strings.getKeyId(toFullPath(g.config, mid)) + let fileId = g.packed[int32 mid].fromDisk.strings.getKeyId(toFullPath(g.config, mid)) if fileId == LitId(0): internalError(g.config, unpacked, "cannot find a valid file ID") diff --git a/compiler/ic/packed_ast.nim b/compiler/ic/packed_ast.nim index 0649b56ce..2700c8253 100644 --- a/compiler/ic/packed_ast.nim +++ b/compiler/ic/packed_ast.nim @@ -98,17 +98,6 @@ type PackedTree* = object ## usually represents a full Nim module nodes*: seq[PackedNode] - #sh*: Shared - - Shared* = ref object # shared between different versions of 'Module'. - # (though there is always exactly one valid - # version of a module) - syms*: seq[PackedSym] - types*: seq[PackedType] - strings*: BiTable[string] # we could share these between modules. - numbers*: BiTable[BiggestInt] # we also store floats in here so - # that we can assure that every bit is kept - #config*: ConfigRef PackedInstantiation* = object key*, sym*: PackedItemId @@ -374,51 +363,6 @@ const externUIntLit* = {nkUIntLit, nkUInt8Lit, nkUInt16Lit, nkUInt32Lit, nkUInt64Lit} directIntLit* = nkInt32Lit -proc toString*(tree: PackedTree; n: NodePos; sh: Shared; nesting: int; - result: var string) = - let pos = n.int - if result.len > 0 and result[^1] notin {' ', '\n'}: - result.add ' ' - - result.add $tree[pos].kind - case tree.nodes[pos].kind - of nkNone, nkEmpty, nkNilLit, nkType: discard - of nkIdent, nkStrLit..nkTripleStrLit: - result.add " " - result.add sh.strings[LitId tree.nodes[pos].operand] - of nkSym: - result.add " " - result.add sh.strings[sh.syms[tree.nodes[pos].operand].name] - of directIntLit: - result.add " " - result.addInt tree.nodes[pos].operand - of externSIntLit: - result.add " " - result.addInt sh.numbers[LitId tree.nodes[pos].operand] - of externUIntLit: - result.add " " - result.add $cast[uint64](sh.numbers[LitId tree.nodes[pos].operand]) - of nkFloatLit..nkFloat128Lit: - result.add " " - result.add $cast[BiggestFloat](sh.numbers[LitId tree.nodes[pos].operand]) - else: - result.add "(\n" - for i in 1..(nesting+1)*2: result.add ' ' - for child in sonsReadonly(tree, n): - toString(tree, child, sh, nesting + 1, result) - result.add "\n" - for i in 1..nesting*2: result.add ' ' - result.add ")" - #for i in 1..nesting*2: result.add ' ' - - -proc toString*(tree: PackedTree; n: NodePos; sh: Shared): string = - result = "" - toString(tree, n, sh, 0, result) - -proc debug*(tree: PackedTree; sh: Shared) = - stdout.write toString(tree, NodePos 0, sh) - when false: proc identIdImpl(tree: PackedTree; n: NodePos): LitId = if n.kind == nkIdent: diff --git a/compiler/ic/rodfiles.nim b/compiler/ic/rodfiles.nim index 86d8ba203..a52bd18b3 100644 --- a/compiler/ic/rodfiles.nim +++ b/compiler/ic/rodfiles.nim @@ -23,8 +23,8 @@ from typetraits import supportsCopyMem ## (`RodFileError`). The file format broken up into sections (`RodSection`) ## and preceeded by a header (see: `cookie`). The precise layout, section ## ordering and data following the section are determined by the user. See -## `ic.loadRoadFile`. -## +## `ic.loadRodFile`. +## ## A basic but "wrong" example of the lifecycle: ## --------------------------------------------- ## 1. `create` or `open` - create a new one or open an existing @@ -45,10 +45,10 @@ from typetraits import supportsCopyMem ## The API is centered around IO and prone to error, each operation checks or ## sets the `RodFile.err` field. A user of this API needs to handle these ## appropriately. -## +## ## API Notes ## ========= -## +## ## Valid inputs for Rod files ## -------------------------- ## ASTs, hopes, dreams, and anything as long as it and any children it may have @@ -61,7 +61,7 @@ from typetraits import supportsCopyMem ## ---------------------------- ## A flag based approach is used where operations no-op in case of a ## preexisting error and set the flag if they encounter one. -## +## ## Misc ## ---- ## * 'Prim' is short for 'primitive', as in a non-sequence type From bae7b5b7795ae8b429e50deba61430ae43a1750f Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 19 Apr 2021 17:48:26 +0800 Subject: [PATCH 0672/2472] fix #17732(forward args... to megatest) (#17780) --- testament/categories.nim | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/testament/categories.nim b/testament/categories.nim index 7b57adb8a..4c3a0b1f5 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -573,7 +573,7 @@ proc quoted(a: string): string = # todo: consider moving to system.nim result.addQuoted(a) -proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) = +proc runJoinedTest(r: var TResults, cat: Category, testsDir: string, options: string) = ## returns a list of tests that have problems #[ xxx create a reusable megatest API after abstracting out testament specific code, @@ -619,7 +619,11 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) = writeFile(megatestFile, megatest) let root = getCurrentDir() - let args = ["c", "--nimCache:" & outDir, "-d:testing", "-d:nimMegatest", "--listCmd", "--path:" & root, megatestFile] + + var args = @["c", "--nimCache:" & outDir, "-d:testing", "-d:nimMegatest", "--listCmd", + "--path:" & root] + args.add options.parseCmdLine + args.add megatestFile var (cmdLine, buf, exitCode) = execCmdEx2(command = compilerPrefix, args = args, input = "") if exitCode != 0: echo "$ " & cmdLine & "\n" & buf @@ -710,7 +714,7 @@ proc processCategory(r: var TResults, cat: Category, if not handled: case cat2 of "megatest": - runJoinedTest(r, cat, testsDir) + runJoinedTest(r, cat, testsDir, options) else: var testsRun = 0 var files: seq[string] From 6de5aa1971b17b0869470d7cdc886ef230103832 Mon Sep 17 00:00:00 2001 From: Clyybber Date: Mon, 19 Apr 2021 13:16:10 +0200 Subject: [PATCH 0673/2472] Rename `=` to `=copy` in stdlib (#17781) --- lib/deprecated/pure/sharedstrings.nim | 2 +- lib/std/channels.nim | 2 +- lib/std/isolation.nim | 2 +- lib/system.nim | 1 + lib/system/widestrs.nim | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/deprecated/pure/sharedstrings.nim b/lib/deprecated/pure/sharedstrings.nim index 29cb0fcc2..4d4ae151b 100644 --- a/lib/deprecated/pure/sharedstrings.nim +++ b/lib/deprecated/pure/sharedstrings.nim @@ -37,7 +37,7 @@ proc `=destroy`*(s: SharedString) = decRef(s.buffer) when false: - proc `=`*(dest: var SharedString; src: SharedString) = + proc `=copy`*(dest: var SharedString; src: SharedString) = incRef(src.buffer) if not dest.buffer.isNil: decRef(dest.buffer) diff --git a/lib/std/channels.nim b/lib/std/channels.nim index 49def0913..5ff141384 100644 --- a/lib/std/channels.nim +++ b/lib/std/channels.nim @@ -443,7 +443,7 @@ proc `=destroy`*[T](c: var Channel[T]) = else: atomicDec(c.d.atomicCounter) -proc `=`*[T](dest: var Channel[T], src: Channel[T]) = +proc `=copy`*[T](dest: var Channel[T], src: Channel[T]) = ## Shares `Channel` by reference counting. if src.d != nil: atomicInc(src.d.atomicCounter) diff --git a/lib/std/isolation.nim b/lib/std/isolation.nim index 8daca233b..7d6ac6092 100644 --- a/lib/std/isolation.nim +++ b/lib/std/isolation.nim @@ -18,7 +18,7 @@ type Isolated*[T] = object ## Isolated data can only be moved, not copied. value: T -proc `=`*[T](dest: var Isolated[T]; src: Isolated[T]) {.error.} +proc `=copy`*[T](dest: var Isolated[T]; src: Isolated[T]) {.error.} proc `=sink`*[T](dest: var Isolated[T]; src: Isolated[T]) {.inline.} = # delegate to value's sink operation diff --git a/lib/system.nim b/lib/system.nim index 532883853..7fd35781b 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -478,6 +478,7 @@ proc `[]`*[I: Ordinal;T](a: T; i: I): T {. proc `[]=`*[I: Ordinal;T,S](a: T; i: I; x: sink S) {.noSideEffect, magic: "ArrPut".} proc `=`*[T](dest: var T; src: T) {.noSideEffect, magic: "Asgn".} +proc `=copy`*[T](dest: var T; src: T) {.noSideEffect, magic: "Asgn".} proc arrGet[I: Ordinal;T](a: T; i: I): T {. noSideEffect, magic: "ArrGet".} diff --git a/lib/system/widestrs.nim b/lib/system/widestrs.nim index 16bc3f3ca..a999c9167 100644 --- a/lib/system/widestrs.nim +++ b/lib/system/widestrs.nim @@ -32,7 +32,7 @@ when defined(nimv2): else: dealloc(a.data) - proc `=`(a: var WideCStringObj; b: WideCStringObj) {.error.} + proc `=copy`(a: var WideCStringObj; b: WideCStringObj) {.error.} proc `=sink`(a: var WideCStringObj; b: WideCStringObj) = a.bytes = b.bytes From 438afb4db93dd9fcc6e94995a56f616a9e3d7826 Mon Sep 17 00:00:00 2001 From: Clyybber Date: Mon, 19 Apr 2021 17:19:21 +0200 Subject: [PATCH 0674/2472] Introduce localErrorNode (#17785) --- compiler/sem.nim | 3 +-- compiler/semdata.nim | 13 +++++++++++++ compiler/semexprs.nim | 6 ++---- compiler/semmagic.nim | 12 ++++-------- compiler/semobjconstr.nim | 6 ++---- compiler/semstmts.nim | 3 +-- compiler/vm.nim | 8 ++++---- 7 files changed, 27 insertions(+), 24 deletions(-) diff --git a/compiler/sem.nim b/compiler/sem.nim index ccc634b4d..4020fc4d6 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -296,8 +296,7 @@ proc fixupTypeAfterEval(c: PContext, evaluated, eOrig: PNode): PNode = result = evaluated let expectedType = eOrig.typ.skipTypes({tyStatic}) if hasCycle(result): - globalError(c.config, eOrig.info, "the resulting AST is cyclic and cannot be processed further") - result = errorNode(c, eOrig) + result = localErrorNode(c, eOrig, "the resulting AST is cyclic and cannot be processed further") else: semmacrosanity.annotateType(result, expectedType, c.config) else: diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 9e7258a69..354d4587d 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -501,6 +501,19 @@ proc errorNode*(c: PContext, n: PNode): PNode = result = newNodeI(nkEmpty, n.info) result.typ = errorType(c) +proc localErrorNode*(c: PContext, n: PNode, info: TLineInfo, msg: TMsgKind, arg: string): PNode = + localError(c.config, info, msg, arg) + result = errorNode(c, n) + +proc localErrorNode*(c: PContext, n: PNode, info: TLineInfo, arg: string): PNode = + localErrorNode(c, n, info, errGenerated, arg) + +proc localErrorNode*(c: PContext, n: PNode, msg: TMsgKind, arg: string): PNode = + localErrorNode(c, n, n.info, msg, arg) + +proc localErrorNode*(c: PContext, n: PNode, arg: string): PNode = + localErrorNode(c, n, n.info, errGenerated, arg) + proc fillTypeS*(dest: PType, kind: TTypeKind, c: PContext) = dest.kind = kind dest.owner = getCurrOwner(c) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 1582cc0fd..353831736 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1044,8 +1044,7 @@ proc buildEchoStmt(c: PContext, n: PNode): PNode = if e != nil: result.add(newSymNode(e)) else: - localError(c.config, n.info, "system needs: echo") - result.add(errorNode(c, n)) + result.add localErrorNode(c, n, "system needs: echo") result.add(n) result = semExpr(c, result) @@ -2621,8 +2620,7 @@ proc semTupleConstr(c: PContext, n: PNode, flags: TExprFlags): PNode = # check if either everything or nothing is tyTypeDesc for i in 1.. high(TSymChoiceRule).int: - localError(c.config, n[2].info, errConstExprExpected) - return errorNode(c, n) + return localErrorNode(c, n, n[2].info, errConstExprExpected) let id = newIdentNode(getIdent(c.cache, sl.strVal), n.info) let s = qualifiedLookUp(c, id, {checkUndeclared}) @@ -253,12 +251,10 @@ proc semBindSym(c: PContext, n: PNode): PNode = proc opBindSym(c: PContext, scope: PScope, n: PNode, isMixin: int, info: PNode): PNode = if n.kind notin {nkStrLit, nkRStrLit, nkTripleStrLit, nkIdent}: - localError(c.config, info.info, errStringOrIdentNodeExpected) - return errorNode(c, n) + return localErrorNode(c, n, info.info, errStringOrIdentNodeExpected) if isMixin < 0 or isMixin > high(TSymChoiceRule).int: - localError(c.config, info.info, errConstExprExpected) - return errorNode(c, n) + return localErrorNode(c, n, info.info, errConstExprExpected) let id = if n.kind == nkIdent: n else: newIdentNode(getIdent(c.cache, n.strVal), info.info) diff --git a/compiler/semobjconstr.nim b/compiler/semobjconstr.nim index dea187ea7..63b4b9a75 100644 --- a/compiler/semobjconstr.nim +++ b/compiler/semobjconstr.nim @@ -379,8 +379,7 @@ proc semObjConstr(c: PContext, n: PNode, flags: TExprFlags): PNode = for child in n: result.add child if t == nil: - localError(c.config, n.info, errGenerated, "object constructor needs an object type") - return errorNode(c, result) + return localErrorNode(c, result, errGenerated, "object constructor needs an object type") t = skipTypes(t, {tyGenericInst, tyAlias, tySink, tyOwned}) if t.kind == tyRef: @@ -391,8 +390,7 @@ proc semObjConstr(c: PContext, n: PNode, flags: TExprFlags): PNode = # multiple times as long as they don't have closures. result.typ.flags.incl tfHasOwned if t.kind != tyObject: - localError(c.config, n.info, errGenerated, "object constructor needs an object type") - return errorNode(c, result) + return localErrorNode(c, result, errGenerated, "object constructor needs an object type") # Check if the object is fully initialized by recursively testing each # field (if this is a case object, initialized fields in two different diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index bedc5a7e5..a10047187 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -725,8 +725,7 @@ proc semForVars(c: PContext, n: PNode; flags: TExprFlags): PNode = if n.len == 3: if n[0].kind == nkVarTuple: if n[0].len-1 != iterAfterVarLent.len: - localError(c.config, n[0].info, errWrongNumberOfVariables) - return errorNode(c, n) + return localErrorNode(c, n, n[0].info, errWrongNumberOfVariables) for i in 0.. Date: Mon, 19 Apr 2021 17:51:00 +0200 Subject: [PATCH 0675/2472] Small privateAccess improvement (#17786) --- lib/std/importutils.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/importutils.nim b/lib/std/importutils.nim index f7a3cbd03..0c0f546b9 100644 --- a/lib/std/importutils.nim +++ b/lib/std/importutils.nim @@ -17,7 +17,7 @@ when defined(nimImportutilsExample): x1: int # private proc initFoo*(): auto = Foo() -proc privateAccess*(t: typedesc[object|ref|ptr]) {.magic: "PrivateAccess".} = +proc privateAccess*(t: typedesc[object|(ref object)|(ptr object)]) {.magic: "PrivateAccess".} = ## Enables access to private fields of `t` in current scope. runnableExamples("-d:nimImportutilsExample"): # here we're importing a module containing: From 3b80f0dc8ef2bc9d2a981a7c92a6355dfc72b7ef Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 19 Apr 2021 22:37:09 +0200 Subject: [PATCH 0676/2472] IC navigator: added support for include files (#17784) * ic fixed navigator crash when track wrong/missed Also fixed an issue with getNimcacheDir not observing the outDir. * closer, but not sure how to test[skip ci][ci skip] * IC navigator: added support for include files * update * make posix happy via expandFilename * update Co-authored-by: Saem Ghani --- compiler/commands.nim | 69 +++++++++++++++++++++----------- compiler/ic/ic.nim | 2 +- compiler/ic/navigator.nim | 28 +++++++++++-- compiler/msgs.nim | 3 ++ compiler/options.nim | 5 ++- testament/categories.nim | 2 +- tests/navigator/minclude.nim | 2 + tests/navigator/tincludefile.nim | 29 ++++++++++++++ tests/navigator/tnav1.nim | 4 +- 9 files changed, 111 insertions(+), 33 deletions(-) create mode 100644 tests/navigator/minclude.nim create mode 100644 tests/navigator/tincludefile.nim diff --git a/compiler/commands.nim b/compiler/commands.nim index e918979fb..5d6e45fb5 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -28,7 +28,7 @@ bootSwitch(usedNoGC, defined(nogc), "--gc:none") import os, msgs, options, nversion, condsyms, strutils, extccomp, platform, wordrecg, parseutils, nimblecmd, parseopt, sequtils, lineinfos, - pathutils, strtabs + pathutils, strtabs, pathnorm from ast import eqTypeFlags, tfGcSafe, tfNoSideEffect @@ -359,31 +359,52 @@ proc processCfgPath(conf: ConfigRef; path: string, info: TLineInfo): AbsoluteDir const errInvalidNumber = "$1 is not a valid number" +proc makeAbsolute(s: string): AbsoluteFile = + if isAbsolute(s): + AbsoluteFile pathnorm.normalizePath(s) + else: + AbsoluteFile pathnorm.normalizePath(os.getCurrentDir() / s) + +proc setTrackingInfo(conf: ConfigRef; dirty, file, line, column: string, + info: TLineInfo) = + ## set tracking info, common code for track, trackDirty, & ideTrack + var ln, col: int + if parseUtils.parseInt(line, ln) <= 0: + localError(conf, info, errInvalidNumber % line) + if parseUtils.parseInt(column, col) <= 0: + localError(conf, info, errInvalidNumber % column) + + let a = makeAbsolute(file) + if dirty == "": + conf.m.trackPos = newLineInfo(conf, a, ln, col) + else: + let dirtyOriginalIdx = fileInfoIdx(conf, a) + if dirtyOriginalIdx.int32 >= 0: + msgs.setDirtyFile(conf, dirtyOriginalIdx, makeAbsolute(dirty)) + conf.m.trackPos = newLineInfo(dirtyOriginalIdx, ln, col) + proc trackDirty(conf: ConfigRef; arg: string, info: TLineInfo) = var a = arg.split(',') if a.len != 4: localError(conf, info, "DIRTY_BUFFER,ORIGINAL_FILE,LINE,COLUMN expected") - var line, column: int - if parseUtils.parseInt(a[2], line) <= 0: - localError(conf, info, errInvalidNumber % a[1]) - if parseUtils.parseInt(a[3], column) <= 0: - localError(conf, info, errInvalidNumber % a[2]) - - let dirtyOriginalIdx = fileInfoIdx(conf, AbsoluteFile a[1]) - if dirtyOriginalIdx.int32 >= 0: - msgs.setDirtyFile(conf, dirtyOriginalIdx, AbsoluteFile a[0]) - - conf.m.trackPos = newLineInfo(dirtyOriginalIdx, line, column) + setTrackingInfo(conf, a[0], a[1], a[2], a[3], info) proc track(conf: ConfigRef; arg: string, info: TLineInfo) = var a = arg.split(',') if a.len != 3: localError(conf, info, "FILE,LINE,COLUMN expected") - var line, column: int - if parseUtils.parseInt(a[1], line) <= 0: - localError(conf, info, errInvalidNumber % a[1]) - if parseUtils.parseInt(a[2], column) <= 0: - localError(conf, info, errInvalidNumber % a[2]) - conf.m.trackPos = newLineInfo(conf, AbsoluteFile a[0], line, column) + setTrackingInfo(conf, "", a[0], a[1], a[2], info) + +proc trackIde(conf: ConfigRef; cmd: IdeCmd, arg: string, info: TLineInfo) = + ## set the tracking info related to an ide cmd, supports optional dirty file + var a = arg.split(',') + case a.len + of 4: + setTrackingInfo(conf, a[0], a[1], a[2], a[3], info) + of 3: + setTrackingInfo(conf, "", a[0], a[1], a[2], info) + else: + localError(conf, info, "[DIRTY_BUFFER,]ORIGINAL_FILE,LINE,COLUMN expected") + conf.ideCmd = cmd proc dynlibOverride(conf: ConfigRef; switch, arg: string, pass: TCmdLinePass, info: TLineInfo) = if pass in {passCmd2, passPP}: @@ -851,17 +872,17 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; expectNoArg(conf, switch, arg, pass, info) conf.ideCmd = ideSug of "def": - expectNoArg(conf, switch, arg, pass, info) - conf.ideCmd = ideDef + expectArg(conf, switch, arg, pass, info) + trackIde(conf, ideDef, arg, info) of "context": expectNoArg(conf, switch, arg, pass, info) conf.ideCmd = ideCon of "usages": - expectNoArg(conf, switch, arg, pass, info) - conf.ideCmd = ideUse + expectArg(conf, switch, arg, pass, info) + trackIde(conf, ideUse, arg, info) of "defusages": - expectNoArg(conf, switch, arg, pass, info) - conf.ideCmd = ideDus + expectArg(conf, switch, arg, pass, info) + trackIde(conf, ideDus, arg, info) of "stdout": processOnOffSwitchG(conf, {optStdout}, arg, pass, info) of "listfullpaths": diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim index a23685be7..d9a8756f1 100644 --- a/compiler/ic/ic.nim +++ b/compiler/ic/ic.nim @@ -29,7 +29,7 @@ type PackedModule* = object ## the parts of a PackedEncoder that are part of the .rod file definedSymbols: string moduleFlags: TSymFlags - includes: seq[(LitId, string)] # first entry is the module filename itself + includes*: seq[(LitId, string)] # first entry is the module filename itself imports: seq[LitId] # the modules this module depends on toReplay*: PackedTree # pragmas and VM specific state to replay. topLevel*: PackedTree # top level statements diff --git a/compiler/ic/navigator.nim b/compiler/ic/navigator.nim index b1a237cf7..cedf41579 100644 --- a/compiler/ic/navigator.nim +++ b/compiler/ic/navigator.nim @@ -98,11 +98,31 @@ proc list(c: var NavContext; tree: PackedTree; sym: ItemId) = usage(c, tree.nodes[i].info, isDecl(tree, parent(NodePos i))) else: discard +proc searchForIncludeFile(g: ModuleGraph; fullPath: string): int = + for i in 0..high(g.packed): + for k in 1..high(g.packed[i].fromDisk.includes): + # we start from 1 because the first "include" file is + # the module's filename. + if os.cmpPaths(g.packed[i].fromDisk.strings[g.packed[i].fromDisk.includes[k][0]], fullPath) == 0: + return i + return -1 + proc nav(g: ModuleGraph) = # translate the track position to a packed position: let unpacked = g.config.m.trackPos - let mid = unpacked.fileIndex - let fileId = g.packed[int32 mid].fromDisk.strings.getKeyId(toFullPath(g.config, mid)) + var mid = unpacked.fileIndex.int + + let fullPath = toFullPath(g.config, unpacked.fileIndex) + + if g.packed[mid].status == undefined: + # check if 'mid' is an include file of some other module: + mid = searchForIncludeFile(g, fullPath) + + if mid < 0: + localError(g.config, unpacked, "unknown file name: " & fullPath) + return + + let fileId = g.packed[mid].fromDisk.strings.getKeyId(fullPath) if fileId == LitId(0): internalError(g.config, unpacked, "cannot find a valid file ID") @@ -114,9 +134,9 @@ proc nav(g: ModuleGraph) = trackPos: PackedLineInfo(line: unpacked.line, col: unpacked.col, file: fileId), outputSep: if isDefined(g.config, "nimIcNavigatorTests"): ' ' else: '\t' ) - var symId = search(c, g.packed[int32 mid].fromDisk.topLevel) + var symId = search(c, g.packed[mid].fromDisk.topLevel) if symId == EmptyItemId: - symId = search(c, g.packed[int32 mid].fromDisk.bodies) + symId = search(c, g.packed[mid].fromDisk.bodies) if symId == EmptyItemId: localError(g.config, unpacked, "no symbol at this position") diff --git a/compiler/msgs.nim b/compiler/msgs.nim index c1e1f9e39..39b10d7df 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -582,6 +582,9 @@ template globalError*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") template globalError*(conf: ConfigRef; info: TLineInfo, arg: string) = liMessage(conf, info, errGenerated, arg, doRaise, instLoc()) +template globalError*(conf: ConfigRef; format: string, params: openArray[string]) = + liMessage(conf, unknownLineInfo, errGenerated, format % params, doRaise, instLoc()) + template localError*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") = liMessage(conf, info, msg, arg, doNothing, instLoc()) diff --git a/compiler/options.nim b/compiler/options.nim index 5475410a4..c1ac6b1e0 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -701,7 +701,10 @@ proc getNimcacheDir*(conf: ConfigRef): AbsoluteDir = result = if not conf.nimcacheDir.isEmpty: conf.nimcacheDir elif conf.backend == backendJs: - conf.projectPath / genSubDir + if conf.outDir.isEmpty: + conf.projectPath / genSubDir + else: + conf.outDir / genSubDir else: AbsoluteDir(getOsCacheDir() / splitFile(conf.projectName).name & nimcacheSuffix(conf)) diff --git a/testament/categories.nim b/testament/categories.nim index 4c3a0b1f5..d05112a25 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -490,7 +490,7 @@ proc icTests(r: var TResults; testsDir: string, cat: Category, options: string; writeOnly = " --incremental:writeonly " readOnly = " --incremental:readonly " incrementalOn = " --incremental:on -d:nimIcIntegrityChecks " - navTestConfig = " --ic:on --defusages -d:nimIcNavigatorTests --hint[Conf]:off --warnings:off " + navTestConfig = " --ic:on -d:nimIcNavigatorTests --hint[Conf]:off --warnings:off " template test(x: untyped) = testSpecWithNimcache(r, makeRawTest(file, x & options, cat), nimcache) diff --git a/tests/navigator/minclude.nim b/tests/navigator/minclude.nim new file mode 100644 index 000000000..f65ebfab9 --- /dev/null +++ b/tests/navigator/minclude.nim @@ -0,0 +1,2 @@ +# An include file. +foo(3) diff --git a/tests/navigator/tincludefile.nim b/tests/navigator/tincludefile.nim new file mode 100644 index 000000000..f35ab2ec9 --- /dev/null +++ b/tests/navigator/tincludefile.nim @@ -0,0 +1,29 @@ +discard """ + cmd: "nim check $options --defusages:$file,12,7 $file" + nimout: '''def tincludefile_temp.nim(11, 10) +usage tincludefile_temp.nim(12, 8) + ''' +""" + + + + +proc foo(x: int) = + echo x + +foo(3) +echo "yes", 1 != 3 + +#!EDIT!# +discard """ + cmd: "nim check $options --defusages:$file/../minclude.nim,2,2 $file" + nimout: '''def tincludefile_temp.nim(10, 6) +usage minclude.nim(2, 1) + ''' +""" + + +proc foo(x: int) = + echo x + +include minclude diff --git a/tests/navigator/tnav1.nim b/tests/navigator/tnav1.nim index d7c6f63e2..b6bbdbf19 100644 --- a/tests/navigator/tnav1.nim +++ b/tests/navigator/tnav1.nim @@ -1,5 +1,5 @@ discard """ - cmd: "nim check $options --track:$file,12,7 $file" + cmd: "nim check $options --defusages:$file,12,7 $file" nimout: '''def tnav1_temp.nim(11, 10) usage tnav1_temp.nim(12, 8) ''' @@ -16,7 +16,7 @@ echo "yes", 1 != 3 #!EDIT!# discard """ - cmd: "nim check $options --track:$file,15,2 $file" + cmd: "nim check $options --defusages:$file,15,2 $file" nimout: '''def tnav1_temp.nim(12, 6) usage tnav1_temp.nim(15, 1) ''' From ad67bcf379dbe5b7b17d85e28620749cd8bcf5e9 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 19 Apr 2021 19:02:52 -0700 Subject: [PATCH 0677/2472] fix https://github.com/nim-lang/RFCs/issues/311 remove unary slice (#16714) --- compiler/suggest.nim | 2 +- lib/system.nim | 8 -------- testament/categories.nim | 2 +- tests/collections/ttables.nim | 2 +- tests/misc/tslices.nim | 11 ++++++----- 5 files changed, 9 insertions(+), 16 deletions(-) diff --git a/compiler/suggest.nim b/compiler/suggest.nim index f3add0849..a5b4ac87d 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -112,7 +112,7 @@ proc getTokenLenFromSource(conf: ConfigRef; ident: string; info: TLineInfo): int if ident[^1] == '=' and ident[0] in linter.Letters: if sourceIdent != "=": result = 0 - elif sourceIdent.len > ident.len and sourceIdent[..ident.high] == ident: + elif sourceIdent.len > ident.len and sourceIdent[0..ident.high] == ident: result = ident.len elif sourceIdent != ident: result = 0 diff --git a/lib/system.nim b/lib/system.nim index 7fd35781b..d73ef6dbe 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -510,14 +510,6 @@ proc `..`*[T, U](a: sink T, b: sink U): HSlice[T, U] {.noSideEffect, inline, mag ## echo a[2 .. 3] # @[30, 40] result = HSlice[T, U](a: a, b: b) -proc `..`*[T](b: sink T): HSlice[int, T] {.noSideEffect, inline, magic: "DotDot".} = - ## Unary `slice`:idx: operator that constructs an interval `[default(int), b]`. - ## - ## .. code-block:: Nim - ## let a = [10, 20, 30, 40, 50] - ## echo a[.. 2] # @[10, 20, 30] - result = HSlice[int, T](a: 0, b: b) - when defined(hotCodeReloading): {.pragma: hcrInline, inline.} else: diff --git a/testament/categories.nim b/testament/categories.nim index d05112a25..d7848d51d 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -339,7 +339,7 @@ proc findMainFile(dir: string): string = var nimFiles = 0 for kind, file in os.walkDir(dir): if kind == pcFile: - if file.endsWith(cfgExt): return file[ .. ^(cfgExt.len+1)] & ".nim" + if file.endsWith(cfgExt): return file[0..^(cfgExt.len+1)] & ".nim" elif file.endsWith(".nim"): if result.len == 0: result = file inc nimFiles diff --git a/tests/collections/ttables.nim b/tests/collections/ttables.nim index c2864b75f..638f4241b 100644 --- a/tests/collections/ttables.nim +++ b/tests/collections/ttables.nim @@ -171,7 +171,7 @@ block tableconstr: # NEW: doAssert 56 in 50..100 - doAssert 56 in ..60 + doAssert 56 in 0..60 block ttables2: diff --git a/tests/misc/tslices.nim b/tests/misc/tslices.nim index d063c5ebf..987b50de1 100644 --- a/tests/misc/tslices.nim +++ b/tests/misc/tslices.nim @@ -11,10 +11,9 @@ verichtetd # Test the new slices. -import strutils - var mystr = "Abgrund" -mystr[..1] = "Zu" +# mystr[..1] = "Zu" # deprecated +mystr[0..1] = "Zu" mystr[4..4] = "5" @@ -23,7 +22,8 @@ type var myarr: array[TEnum, int] = [1, 2, 3, 4, 5, 6] myarr[e1..e3] = myarr[e4..e6] -myarr[..e3] = myarr[e4..e6] +# myarr[..e3] = myarr[e4..e6] # deprecated +myarr[0..e3] = myarr[e4..e6] for x in items(myarr): stdout.write(x) echo() @@ -46,7 +46,8 @@ echo mystr mystr[4..4] = "u" # test full replacement -mystr[.. ^2] = "egerichtet" +# mystr[.. ^2] = "egerichtet" # deprecated +mystr[0 .. ^2] = "egerichtet" echo mystr From 6852d091b33757ab34d1f0c4f9b22d10dbe48049 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 19 Apr 2021 21:54:25 -0700 Subject: [PATCH 0678/2472] changelog: document hash changes (#17792) * changelog: document hash changes * fixup --- changelog.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/changelog.md b/changelog.md index 8fab425f8..3caf5e92d 100644 --- a/changelog.md +++ b/changelog.md @@ -50,6 +50,9 @@ - On POSIX systems, we now ignore `SIGPIPE` signals, use `-d:nimLegacySigpipeHandler` for previous behavior. +- `hashes.hash` now supports `object` and `ref` (can be overloaded in user code). +- `hashes.hash(proc|ptr|ref|pointer)` now calls `hash(int)` and honors `-d:nimIntHash1`, + `hashes.hash(closure)` has also been improved. ## Standard library additions and changes - Added support for parenthesized expressions in `strformat` @@ -235,12 +238,9 @@ level `moveMem`, `copyMem` operations for Orc's copy-on-write string implementation. -- `hashes.hash` now supports `object`, but can be overloaded. - - Added `std/strbasics` for high performance string operations. Added `strip`, `setSlice`, `add(a: var string, b: openArray[char])`. -- `hashes.hash` now supports `object`, but can be overloaded. - Added to `wrapnils` an option-like API via `??.`, `isSome`, `get`. From fb02b569578e4538852cb92b350036548aa5da15 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 20 Apr 2021 00:01:53 -0700 Subject: [PATCH 0679/2472] support skip ci in azure pipelines, and support it properly in github actions pipelines (#17561) * support skip ci in azure pipelines * [skip ci] example of a commit that skips CI * example of a commit that does not skip CI * fix github actions logic for ci skip handling * [skip ci] example of a commit that skips CI for all pipelines * example of a commit that does not skip CI for any pipeline --- .github/workflows/ci_docs.yml | 13 ++++++++++++- .github/workflows/ci_packages.yml | 14 +++++++++++++- azure-pipelines.yml | 22 ++++++++++++++++------ doc/contributing.rst | 7 ++++--- 4 files changed, 45 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci_docs.yml b/.github/workflows/ci_docs.yml index 22d425b10..982efe610 100644 --- a/.github/workflows/ci_docs.yml +++ b/.github/workflows/ci_docs.yml @@ -30,8 +30,9 @@ on: jobs: build: + # see D20210329T004830 if: | - !contains(format('{0} {1}', github.event.head_commit.message, github.event.pull_request.title), '[skip ci]') + !contains(format('{0}', github.event.pull_request.title), '[skip ci]') strategy: fail-fast: false matrix: @@ -50,6 +51,16 @@ jobs: steps: - name: 'Checkout' uses: actions/checkout@v2 + with: + fetch-depth: 2 + + - name: 'Check whether to skip CI' + shell: bash + run: | + # see D20210329T004830 + commitMsg=$(git log --no-merges -1 --pretty=format:"%s") + echo commitMsg: $commitMsg + echo $commitMsg | grep -v '\[skip ci\]' - name: 'Install build dependencies (macOS)' if: runner.os == 'macOS' diff --git a/.github/workflows/ci_packages.yml b/.github/workflows/ci_packages.yml index dd42db0c4..b7af1210c 100644 --- a/.github/workflows/ci_packages.yml +++ b/.github/workflows/ci_packages.yml @@ -3,8 +3,9 @@ on: [push, pull_request] jobs: build: + # see D20210329T004830 if: | - !contains(format('{0} {1}', github.event.head_commit.message, github.event.pull_request.title), '[skip ci]') + !contains(format('{0}', github.event.pull_request.title), '[skip ci]') strategy: fail-fast: false matrix: @@ -19,6 +20,17 @@ jobs: steps: - name: 'Checkout' uses: actions/checkout@v2 + with: + fetch-depth: 2 + + - name: 'Check whether to skip CI' + shell: bash + run: | + # see D20210329T004830 + commitMsg=$(git log --no-merges -1 --pretty=format:"%s") + echo commitMsg: $commitMsg + echo $commitMsg | grep -v '\[skip ci\]' + - name: 'Checkout csources' uses: actions/checkout@v2 with: diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9006cf6e5..55bad609e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -52,10 +52,20 @@ jobs: steps: - bash: git config --global core.autocrlf false displayName: 'Disable auto conversion to CRLF by git (Windows-only)' - condition: eq(variables['Agent.OS'], 'Windows_NT') + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - checkout: self - fetchDepth: 1 + fetchDepth: 2 # see D20210329T004830 + + - bash: | + # D20210329T004830:here refs https://github.com/microsoft/azure-pipelines-agent/issues/2944 + # `--no-merges` is needed to avoid merge commits which occur for PR's. + # $(Build.SourceVersionMessage) is not helpful + # nor is `github.event.head_commit.message` for github actions. + commitMsg=$(git log --no-merges -1 --pretty=format:"%s") + echo commitMsg: $commitMsg + echo $commitMsg | grep -v '\[skip ci\]' # fails if [skip ci] not in commit msg + displayName: 'Check whether to skip CI' - bash: git clone --depth 1 https://github.com/nim-lang/csources displayName: 'Checkout Nim csources' @@ -73,7 +83,7 @@ jobs: echo_run sudo apt-fast install --no-install-recommends -yq \ libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev valgrind libc6-dbg displayName: 'Install dependencies (amd64 Linux)' - condition: and(eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'amd64')) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'amd64')) - bash: | set -e @@ -113,11 +123,11 @@ jobs: echo_run chmod 755 bin/g++ displayName: 'Install dependencies (i386 Linux)' - condition: and(eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'i386')) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'i386')) - bash: brew install boehmgc make sfml displayName: 'Install dependencies (OSX)' - condition: eq(variables['Agent.OS'], 'Darwin') + condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) - bash: | set -e @@ -130,7 +140,7 @@ jobs: echo_run echo '##vso[task.prependpath]$(System.DefaultWorkingDirectory)/dist/mingw64/bin' displayName: 'Install dependencies (Windows)' - condition: eq(variables['Agent.OS'], 'Windows_NT') + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - bash: echo '##vso[task.prependpath]$(System.DefaultWorkingDirectory)/bin' displayName: 'Add build binaries to PATH' diff --git a/doc/contributing.rst b/doc/contributing.rst index 7ee3aa444..3d3da2ce0 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -508,9 +508,10 @@ Continuous Integration (CI) 1. Continuous Integration is by default run on every push in a PR; this clogs the CI pipeline and affects other PR's; if you don't need it (e.g. for WIP or - documentation only changes), add `[ci skip]` to your commit message title. - This convention is supported by `Appveyor - `_ + documentation only changes), add `[skip ci]` to your commit message title. + This convention is supported by our github actions pipelines and our azure pipeline + as well as our former other pipelines: + `Appveyor `_ and `Travis `_. 2. Consider enabling CI (azure, GitHub actions and builds.sr.ht) in your own Nim fork, and From 68e7ed9c57605bc98d3ab7625374113386e62ee7 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 20 Apr 2021 00:02:35 -0700 Subject: [PATCH 0680/2472] important_packages: `allowed_failures` batch (#17757) * important_packages: reserve batch 0 for allowed failures * custom batch name: allowed_failures --- .github/workflows/ci_packages.yml | 2 +- testament/categories.nim | 35 ++++++++++++++++++++++--------- testament/testament.nim | 2 +- 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci_packages.yml b/.github/workflows/ci_packages.yml index b7af1210c..e7073ea23 100644 --- a/.github/workflows/ci_packages.yml +++ b/.github/workflows/ci_packages.yml @@ -11,7 +11,7 @@ jobs: matrix: os: [ubuntu-18.04, macos-10.15] cpu: [amd64] - batch: ["0_3", "1_3", "2_3"] # list of `index_num` + batch: ["allowed_failures", "0_3", "1_3", "2_3"] # list of `index_num` name: '${{ matrix.os }} (batch: ${{ matrix.batch }})' runs-on: ${{ matrix.os }} env: diff --git a/testament/categories.nim b/testament/categories.nim index d7848d51d..19d9e4507 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -14,6 +14,7 @@ import important_packages import std/strformat +from std/sequtils import filterIt const specialCategories = [ @@ -399,8 +400,7 @@ proc testStdlib(r: var TResults, pattern, options: string, cat: Category) = testSpec r, testObj # ----------------------------- nimble ---------------------------------------- -proc listPackages(packageFilter: string): seq[NimblePackage] = - # xxx document `packageFilter`, seems like a bad API (at least should be a regex; a substring match makes no sense) +proc listPackagesAll(): seq[NimblePackage] = var nimbleDir = getEnv("NIMBLE_DIR") if nimbleDir.len == 0: nimbleDir = getHomeDir() / ".nimble" let packageIndex = nimbleDir / "packages_official.json" @@ -409,14 +409,29 @@ proc listPackages(packageFilter: string): seq[NimblePackage] = for a in packageList: if a["name"].str == name: return a for pkg in important_packages.packages.items: - if isCurrentBatch(testamentData0, pkg.name) and packageFilter in pkg.name: - var pkg = pkg - if pkg.url.len == 0: - let pkg2 = findPackage(pkg.name) - if pkg2 == nil: - raise newException(ValueError, "Cannot find package '$#'." % pkg.name) - pkg.url = pkg2["url"].str - result.add pkg + var pkg = pkg + if pkg.url.len == 0: + let pkg2 = findPackage(pkg.name) + if pkg2 == nil: + raise newException(ValueError, "Cannot find package '$#'." % pkg.name) + pkg.url = pkg2["url"].str + result.add pkg + +proc listPackages(packageFilter: string): seq[NimblePackage] = + let pkgs = listPackagesAll() + if packageFilter.len != 0: + # xxx document `packageFilter`, seems like a bad API, + # at least should be a regex; a substring match makes no sense. + result = pkgs.filterIt(packageFilter in it.name) + else: + let pkgs1 = pkgs.filterIt(it.allowFailure) + let pkgs2 = pkgs.filterIt(not it.allowFailure) + if testamentData0.batchArg == "allowed_failures": + result = pkgs1 + else: + for i in 0.. 0 and p.val[0] in {'0'..'9'}: let s = p.val.split("_") doAssert s.len == 2, $(p.val, s) testamentData0.testamentBatch = s[0].parseInt From 4742e6e1fdc6cebd3052875748fa10ad44c8a42e Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 20 Apr 2021 00:25:28 -0700 Subject: [PATCH 0681/2472] followup #16714: add -d:nimLegacyUnarySlice + changelog (#17794) --- changelog.md | 3 +++ lib/system.nim | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/changelog.md b/changelog.md index 3caf5e92d..0537b3d6a 100644 --- a/changelog.md +++ b/changelog.md @@ -54,6 +54,9 @@ - `hashes.hash(proc|ptr|ref|pointer)` now calls `hash(int)` and honors `-d:nimIntHash1`, `hashes.hash(closure)` has also been improved. +- The unary slice `..b` was removed, use `0..b` instead or use `-d:nimLegacyUnarySlice` + for a deprecation period. + ## Standard library additions and changes - Added support for parenthesized expressions in `strformat` diff --git a/lib/system.nim b/lib/system.nim index d73ef6dbe..41749c27a 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -510,6 +510,16 @@ proc `..`*[T, U](a: sink T, b: sink U): HSlice[T, U] {.noSideEffect, inline, mag ## echo a[2 .. 3] # @[30, 40] result = HSlice[T, U](a: a, b: b) +when defined(nimLegacyUnarySlice): + proc `..`*[T](b: sink T): HSlice[int, T] + {.noSideEffect, inline, magic: "DotDot", deprecated: "replace `..b` with `0..b`".} = + ## Unary `slice`:idx: operator that constructs an interval `[default(int), b]`. + ## + ## .. code-block:: Nim + ## let a = [10, 20, 30, 40, 50] + ## echo a[.. 2] # @[10, 20, 30] + result = HSlice[int, T](a: 0, b: b) + when defined(hotCodeReloading): {.pragma: hcrInline, inline.} else: From 54fe44135ee447c167b4a3a2d6ec5ff7855688ea Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 20 Apr 2021 15:24:20 +0200 Subject: [PATCH 0682/2472] updated the documentation (#17796) --- compiler/ic/navigator.nim | 2 +- doc/advopt.txt | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/compiler/ic/navigator.nim b/compiler/ic/navigator.nim index cedf41579..6af2c2ac0 100644 --- a/compiler/ic/navigator.nim +++ b/compiler/ic/navigator.nim @@ -7,7 +7,7 @@ # distribution, for details about the copyright. # -## Supports the "nim check --ic:on --def --track:FILE,LINE,COL" +## Supports the "nim check --ic:on --defusages:FILE,LINE,COL" ## IDE-like features. It uses the set of .rod files to accomplish ## its task. The set must cover a complete Nim project. diff --git a/doc/advopt.txt b/doc/advopt.txt index c7b63d50f..bfc381cff 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -14,8 +14,8 @@ Advanced commands: module dependency graph //dump dump all defined conditionals and search paths see also: --dump.format:json (useful with: `| jq`) - //check checks the project for syntax and semantic - (can be combined with --track and --defusages) + //check checks the project for syntax and semantics + (can be combined with --defusages) Runtime checks (see -x): --objChecks:on|off turn obj conversion checks on|off @@ -28,8 +28,8 @@ Runtime checks (see -x): --infChecks:on|off turn Inf checks on|off Advanced options: - --track:FILE,LINE,COL set the tracking position for (--defusages) - --defusages find the definition and all usages of a symbol + --defusages:FILE,LINE,COL + find the definition and all usages of a symbol -o:FILE, --out:FILE set the output filename --outdir:DIR set the path where the output file will be written --usenimcache will use `outdir=$$nimcache`, whichever it resolves From c776498170a2a08a67b1317d8965482cf266a733 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 20 Apr 2021 21:39:58 +0800 Subject: [PATCH 0683/2472] close #9691 and close #10913('spawn'ed function cannot have a 'typed' or 'untyped' parameter) (#17775) --- compiler/spawn.nim | 12 +++++++++++- tests/parallel/t10913.nim | 20 ++++++++++++++++++++ tests/parallel/t9691.nim | 9 +++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 tests/parallel/t10913.nim create mode 100644 tests/parallel/t9691.nim diff --git a/compiler/spawn.nim b/compiler/spawn.nim index 1635f9f83..205197eaf 100644 --- a/compiler/spawn.nim +++ b/compiler/spawn.nim @@ -192,6 +192,11 @@ proc createCastExpr(argsParam: PSym; objType: PType; idgen: IdGenerator): PNode result.typ = newType(tyPtr, nextTypeId idgen, objType.owner) result.typ.rawAddSon(objType) +template checkMagicProcs(g: ModuleGraph, n: PNode, formal: PNode) = + if (formal.typ.kind == tyVarargs and formal.typ[0].kind in {tyTyped, tyUntyped}) or + formal.typ.kind in {tyTyped, tyUntyped}: + localError(g.config, n.info, "'spawn'ed function cannot have a 'typed' or 'untyped' parameter") + proc setupArgsForConcurrency(g: ModuleGraph; n: PNode; objType: PType; idgen: IdGenerator; owner: PSym; scratchObj: PSym, castExpr, call, @@ -205,6 +210,9 @@ proc setupArgsForConcurrency(g: ModuleGraph; n: PNode; objType: PType; if i < formals.len: if formals[i].typ.kind in {tyVar, tyLent}: localError(g.config, n[i].info, "'spawn'ed function cannot have a 'var' parameter") + + checkMagicProcs(g, n[i], formals[i]) + if formals[i].typ.kind in {tyTypeDesc, tyStatic}: continue #elif containsTyRef(argType): @@ -233,6 +241,9 @@ proc setupArgsForParallelism(g: ModuleGraph; n: PNode; objType: PType; let n = n[i] if i < formals.len and formals[i].typ.kind in {tyStatic, tyTypeDesc}: continue + + checkMagicProcs(g, n, formals[i]) + let argType = skipTypes(if i < formals.len: formals[i].typ else: n.typ, abstractInst) #if containsTyRef(argType): @@ -420,4 +431,3 @@ proc wrapProcForSpawn*(g: ModuleGraph; idgen: IdGenerator; owner: PSym; spawnExp wrapperProc.newSymNode, genAddrOf(scratchObj.newSymNode, idgen), nil, spawnExpr) if spawnKind == srFlowVar: result.add fvField - diff --git a/tests/parallel/t10913.nim b/tests/parallel/t10913.nim new file mode 100644 index 000000000..d8459ecd0 --- /dev/null +++ b/tests/parallel/t10913.nim @@ -0,0 +1,20 @@ +discard """ + matrix: "--threads:on" + errormsg: "'spawn'ed function cannot have a 'typed' or 'untyped' parameter" +""" + +# bug #10913 + +import threadpool + +proc useParallel*[T](unused: T) = + # use a generic T here to show the problem. + {.push experimental: "parallel".} + parallel: + for i in 0..4: + spawn echo "echo in parallel" + sync() + + {.pop.} + +useParallel(1) diff --git a/tests/parallel/t9691.nim b/tests/parallel/t9691.nim new file mode 100644 index 000000000..bbf2b1bc7 --- /dev/null +++ b/tests/parallel/t9691.nim @@ -0,0 +1,9 @@ +discard """ + matrix: "--threads:on" + errormsg: "'spawn'ed function cannot have a 'typed' or 'untyped' parameter" +""" + +# bug #9691 + +import threadpool +spawn echo() From 0b116310bfb691755481293e8f9af11190fe0999 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 20 Apr 2021 16:30:17 +0200 Subject: [PATCH 0684/2472] unit separator (#17730) * use the ASCII Unit Separator so that error messages can be handled precisely by the tooling * updated testament --- compiler/main.nim | 4 +-- compiler/msgs.nim | 61 ++++++++++++++++++-------------- compiler/vm.nim | 10 +++--- testament/important_packages.nim | 2 +- testament/specs.nim | 2 +- testament/testament.nim | 7 ++++ tests/misc/trunner.nim | 4 +-- tests/osproc/treadlines.nim | 4 +-- 8 files changed, 55 insertions(+), 39 deletions(-) diff --git a/compiler/main.nim b/compiler/main.nim index 6eb830164..d66a5f329 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -335,8 +335,8 @@ proc mainCommand*(graph: ModuleGraph) = msgWriteln(conf, $dumpdata, {msgStdout, msgSkipHook}) else: msgWriteln(conf, "-- list of currently defined symbols --", - {msgStdout, msgSkipHook}) - for s in definedSymbolNames(conf.symbols): msgWriteln(conf, s, {msgStdout, msgSkipHook}) + {msgStdout, msgSkipHook, msgNoUnitSep}) + for s in definedSymbolNames(conf.symbols): msgWriteln(conf, s, {msgStdout, msgSkipHook, msgNoUnitSep}) msgWriteln(conf, "-- end of list --", {msgStdout, msgSkipHook}) for it in conf.searchPaths: msgWriteln(conf, it.string) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 39b10d7df..30421fe6a 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -241,12 +241,13 @@ template toFullPath*(conf: ConfigRef; info: TLineInfo): string = template toFullPathConsiderDirty*(conf: ConfigRef; info: TLineInfo): string = string toFullPathConsiderDirty(conf, info.fileIndex) -type FilenameOption* = enum - foAbs # absolute path, e.g.: /pathto/bar/foo.nim - foRelProject # relative to project path, e.g.: ../foo.nim - foMagicSauce # magic sauce, shortest of (foAbs, foRelProject) - foName # lastPathPart, e.g.: foo.nim - foStacktrace # if optExcessiveStackTrace: foAbs else: foName +type + FilenameOption* = enum + foAbs # absolute path, e.g.: /pathto/bar/foo.nim + foRelProject # relative to project path, e.g.: ../foo.nim + foMagicSauce # magic sauce, shortest of (foAbs, foRelProject) + foName # lastPathPart, e.g.: foo.nim + foStacktrace # if optExcessiveStackTrace: foAbs else: foName proc toFilenameOption*(conf: ConfigRef, fileIdx: FileIndex, opt: FilenameOption): string = case opt @@ -295,10 +296,14 @@ proc `??`* (conf: ConfigRef; info: TLineInfo, filename: string): bool = # only for debugging purposes result = filename in toFilename(conf, info) +const + UnitSep = "\31" + type MsgFlag* = enum ## flags altering msgWriteln behavior msgStdout, ## force writing to stdout, even stderr is default msgSkipHook ## skip message hook even if it is present + msgNoUnitSep ## the message is a complete "paragraph". MsgFlags* = set[MsgFlag] proc msgWriteln*(conf: ConfigRef; s: string, flags: MsgFlags = {}) = @@ -310,17 +315,20 @@ proc msgWriteln*(conf: ConfigRef; s: string, flags: MsgFlags = {}) = ## This is used for 'nim dump' etc. where we don't have nimsuggest ## support. #if conf.cmd == cmdIdeTools and optCDebug notin gGlobalOptions: return + let sep = if msgNoUnitSep notin flags: UnitSep else: "" if not isNil(conf.writelnHook) and msgSkipHook notin flags: - conf.writelnHook(s) + conf.writelnHook(s & sep) elif optStdout in conf.globalOptions or msgStdout in flags: if eStdOut in conf.m.errorOutputs: flushDot(conf) - writeLine(stdout, s) + write stdout, s + writeLine(stdout, sep) flushFile(stdout) else: if eStdErr in conf.m.errorOutputs: flushDot(conf) - writeLine(stderr, s) + write stderr, s + writeLine(stderr, sep) # On Windows stderr is fully-buffered when piped, regardless of C std. when defined(windows): flushFile(stderr) @@ -366,7 +374,7 @@ proc msgWrite(conf: ConfigRef; s: string) = flushFile(stdOrr) conf.lastMsgWasDot.incl stdOrr.toStdOrrKind() # subsequent writes need `flushDot` -template styledMsgWriteln*(args: varargs[typed]) = +template styledMsgWriteln(args: varargs[typed]) = if not isNil(conf.writelnHook): callIgnoringStyle(callWritelnHook, nil, args) elif optStdout in conf.globalOptions: @@ -407,7 +415,7 @@ proc quit(conf: ConfigRef; msg: TMsgKind) {.gcsafe.} = styledMsgWriteln(fgRed, """ No stack traceback available To create a stacktrace, rerun compilation with './koch temp $1 ', see $2 for details""" % - [conf.command, "intern.html#debugging-the-compiler".createDocLink]) + [conf.command, "intern.html#debugging-the-compiler".createDocLink], UnitSep) quit 1 proc handleError(conf: ConfigRef; msg: TMsgKind, eh: TErrorHandling, s: string) = @@ -444,10 +452,11 @@ proc writeContext(conf: ConfigRef; lastinfo: TLineInfo) = conf.structuredErrorHook(conf, context.info, instantiationFrom, Severity.Hint) else: - let message = if context.detail == "": - instantiationFrom - else: - instantiationOfFrom.format(context.detail) + let message = + if context.detail == "": + instantiationFrom + else: + instantiationOfFrom.format(context.detail) styledMsgWriteln(styleBright, conf.toFileLineCol(context.info), " ", resetStyle, message) info = context.info @@ -479,11 +488,12 @@ proc sourceLine*(conf: ConfigRef; i: TLineInfo): string = result = conf.m.fileInfos[i.fileIndex.int32].lines[i.line.int-1] -proc writeSurroundingSrc(conf: ConfigRef; info: TLineInfo) = - const indent = " " - msgWriteln(conf, indent & $sourceLine(conf, info)) - if info.col >= 0: - msgWriteln(conf, indent & spaces(info.col) & '^') +proc getSurroundingSrc(conf: ConfigRef; info: TLineInfo): string = + if conf.hasHint(hintSource) and info != unknownLineInfo: + const indent = " " + result = "\n" & indent & $sourceLine(conf, info) + if info.col >= 0: + result.add "\n" & indent & spaces(info.col) & '^' proc formatMsg*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string): string = let title = case msg @@ -545,14 +555,13 @@ proc liMessage*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string, if msg == hintProcessing: msgWrite(conf, ".") else: - styledMsgWriteln(styleBright, loc, resetStyle, color, title, resetStyle, s, KindColor, kindmsg) - if conf.hasHint(hintSource) and info != unknownLineInfo: - conf.writeSurroundingSrc(info) + styledMsgWriteln(styleBright, loc, resetStyle, color, title, resetStyle, s, KindColor, kindmsg, + resetStyle, conf.getSurroundingSrc(info), UnitSep) if hintMsgOrigin in conf.mainPackageNotes: styledMsgWriteln(styleBright, toFileLineCol(info2), resetStyle, " compiler msg initiated here", KindColor, KindFormat % $hintMsgOrigin, - resetStyle) + resetStyle, UnitSep) handleError(conf, msg, eh, s) template rawMessage*(conf: ConfigRef; msg: TMsgKind, args: openArray[string]) = @@ -632,8 +641,8 @@ proc quotedFilename*(conf: ConfigRef; i: TLineInfo): Rope = result = conf.m.fileInfos[i.fileIndex.int32].quotedName template listMsg(title, r) = - msgWriteln(conf, title) - for a in r: msgWriteln(conf, " [$1] $2" % [if a in conf.notes: "x" else: " ", $a]) + msgWriteln(conf, title, {msgNoUnitSep}) + for a in r: msgWriteln(conf, " [$1] $2" % [if a in conf.notes: "x" else: " ", $a], {msgNoUnitSep}) proc listWarnings*(conf: ConfigRef) = listMsg("Warnings:", warnMin..warnMax) proc listHints*(conf: ConfigRef) = listMsg("Hints:", hintMin..hintMax) diff --git a/compiler/vm.nim b/compiler/vm.nim index d588067c7..7a231a482 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -37,7 +37,7 @@ proc stackTraceAux(c: PCtx; x: PStackFrame; pc: int; recursionLimit=100) = while x != nil: inc calls x = x.next - msgWriteln(c.config, $calls & " calls omitted\n") + msgWriteln(c.config, $calls & " calls omitted\n", {msgNoUnitSep}) return stackTraceAux(c, x.next, x.comesFrom, recursionLimit-1) var info = c.debug[pc] @@ -59,12 +59,12 @@ proc stackTraceAux(c: PCtx; x: PStackFrame; pc: int; recursionLimit=100) = if x.prc != nil: for k in 1..max(1, 25-s.len): s.add(' ') s.add(x.prc.name.s) - msgWriteln(c.config, s) + msgWriteln(c.config, s, {msgNoUnitSep}) proc stackTraceImpl(c: PCtx, tos: PStackFrame, pc: int, msg: string, lineInfo: TLineInfo, infoOrigin: InstantiationInfo) {.noinline.} = # noinline to avoid code bloat - msgWriteln(c.config, "stack trace: (most recent call last)") + msgWriteln(c.config, "stack trace: (most recent call last)", {msgNoUnitSep}) stackTraceAux(c, tos, pc) let action = if c.mode == emRepl: doRaise else: doNothing # XXX test if we want 'globalError' for every mode @@ -470,7 +470,7 @@ template handleJmpBack() {.dirty.} = if allowInfiniteLoops in c.features: c.loopIterations = c.config.maxLoopIterationsVM else: - msgWriteln(c.config, "stack trace: (most recent call last)") + msgWriteln(c.config, "stack trace: (most recent call last)", {msgNoUnitSep}) stackTraceAux(c, tos, pc) globalError(c.config, c.debug[pc], errTooManyIterations % $c.config.maxLoopIterationsVM) dec(c.loopIterations) @@ -1163,7 +1163,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = stackTrace(c, tos, pc, "node is not a proc symbol") of opcEcho: let rb = instr.regB - template fn(s) = msgWriteln(c.config, s, {msgStdout}) + template fn(s) = msgWriteln(c.config, s, {msgStdout, msgNoUnitSep}) if rb == 1: fn(regs[ra].node.strVal) else: var outp = "" diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 28d7dc367..ecae42811 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -74,7 +74,7 @@ pkg "gnuplot", "nim c gnuplot.nim" pkg "hts", "nim c -o:htss src/hts.nim" pkg "httpauth" pkg "illwill", "nimble examples" -pkg "inim" +pkg "inim", allowFailure=true pkg "itertools", "nim doc src/itertools.nim" pkg "iterutils" pkg "karax", "nim c -r tests/tester.nim" diff --git a/testament/specs.nim b/testament/specs.nim index b5e6de7c2..043f14ed3 100644 --- a/testament/specs.nim +++ b/testament/specs.nim @@ -202,7 +202,7 @@ proc extractSpec(filename: string; spec: var TSpec): string = # look for """ only in the first section if a >= 0 and b > a and a < 40: - result = s.substr(a+3, b-1).replace("'''", tripleQuote) + result = s.substr(a+3, b-1).multiReplace({"'''": tripleQuote, "\\31": "\31"}) else: #echo "warning: file does not contain spec: " & filename result = "" diff --git a/testament/testament.nim b/testament/testament.nim index f2a8e9055..b79b86e51 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -18,6 +18,11 @@ import compiler/nodejs import lib/stdtest/testutils from lib/stdtest/specialpaths import splitTestFile +proc trimUnitSep(x: var string) = + let L = x.len + if L > 0 and x[^1] == '\31': + setLen x, L-1 + var useColors = true var backendLogging = true var simulate = false @@ -172,6 +177,7 @@ proc callNimCompiler(cmdTemplate, filename, options, nimcache: string, result.nimout = "" while true: if outp.readLine(x): + trimUnitSep x result.nimout.add(x & '\n') if x =~ pegOfInterest: # `err` should contain the last error/warning message @@ -196,6 +202,7 @@ proc callNimCompiler(cmdTemplate, filename, options, nimcache: string, result.msg = matches[0] elif suc.isSuccess: result.err = reSuccess + trimUnitSep result.msg proc callCCompiler(cmdTemplate, filename, options: string, target: TTarget): TSpec = diff --git a/tests/misc/trunner.nim b/tests/misc/trunner.nim index 1895eeb97..df9ee9a48 100644 --- a/tests/misc/trunner.nim +++ b/tests/misc/trunner.nim @@ -235,7 +235,7 @@ tests/newconfig/bar/mfoo.nims""".splitLines var expected = "" for a in files: let b = dir / a - expected.add &"Hint: used config file '{b}' [Conf]\n" + expected.add &"Hint: used config file '{b}' [Conf]\31\n" doAssert outp.endsWith expected, outp & "\n" & expected block: # mfoo2.customext @@ -243,7 +243,7 @@ tests/newconfig/bar/mfoo.nims""".splitLines let cmd = fmt"{nim} e --hint:conf {filename}" let (outp, exitCode) = execCmdEx(cmd, options = {poStdErrToStdOut}) doAssert exitCode == 0 - var expected = &"Hint: used config file '{filename}' [Conf]\n" + var expected = &"Hint: used config file '{filename}' [Conf]\31\n" doAssert outp.endsWith "123" & "\n" & expected diff --git a/tests/osproc/treadlines.nim b/tests/osproc/treadlines.nim index 3a8303321..436dd7a10 100644 --- a/tests/osproc/treadlines.nim +++ b/tests/osproc/treadlines.nim @@ -1,6 +1,6 @@ discard """ - output: '''Error: cannot open 'a.nim' -Error: cannot open 'b.nim' + output: '''Error: cannot open 'a.nim'\31 +Error: cannot open 'b.nim'\31 ''' targets: "c" """ From eb221dcc27b7f8f11ec34046e3165508acdf8beb Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 21 Apr 2021 01:00:44 +0800 Subject: [PATCH 0685/2472] [std/deques] move data instead of copy + destroy (#17800) --- lib/pure/collections/deques.nim | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/pure/collections/deques.nim b/lib/pure/collections/deques.nim index d2d53d993..582bb02a7 100644 --- a/lib/pure/collections/deques.nim +++ b/lib/pure/collections/deques.nim @@ -379,8 +379,7 @@ proc popFirst*[T](deq: var Deque[T]): T {.inline, discardable.} = emptyCheck(deq) dec deq.count - result = deq.data[deq.head] - destroy(deq.data[deq.head]) + result = move deq.data[deq.head] deq.head = (deq.head + 1) and deq.mask proc popLast*[T](deq: var Deque[T]): T {.inline, discardable.} = @@ -398,8 +397,7 @@ proc popLast*[T](deq: var Deque[T]): T {.inline, discardable.} = emptyCheck(deq) dec deq.count deq.tail = (deq.tail - 1) and deq.mask - result = deq.data[deq.tail] - destroy(deq.data[deq.tail]) + result = move deq.data[deq.tail] proc clear*[T](deq: var Deque[T]) {.inline.} = ## Resets the deque so that it is empty. From 603af22b7ca46ac566f8c7c15402028f3f976a4e Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 21 Apr 2021 11:05:16 +0800 Subject: [PATCH 0686/2472] [std/tasks]add gcsafe pragmas (#17799) * [std/tasks]add gcsafe pragmas * Update tasks.nim --- lib/std/tasks.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/std/tasks.nim b/lib/std/tasks.nim index aed36f985..97c4df3f7 100644 --- a/lib/std/tasks.nim +++ b/lib/std/tasks.nim @@ -59,7 +59,7 @@ when compileOption("threads"): type Task* = object ## `Task` contains the callback and its arguments. - callback: proc (args: pointer) {.nimcall.} + callback: proc (args: pointer) {.nimcall, gcsafe.} args: pointer destroy: proc (args: pointer) {.nimcall.} @@ -73,7 +73,7 @@ proc `=destroy`*(t: var Task) {.inline.} = t.destroy(t.args) c_free(t.args) -proc invoke*(task: Task) {.inline.} = +proc invoke*(task: Task) {.inline, gcsafe.} = ## Invokes the `task`. assert task.callback != nil task.callback(task.args) @@ -215,7 +215,7 @@ macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkC result = quote do: `stmtList` - proc `funcName`(args: pointer) {.nimcall.} = + proc `funcName`(args: pointer) {.gcsafe, nimcall.} = let `objTemp` = cast[ptr `scratchObjType`](args) `functionStmtList` @@ -229,7 +229,7 @@ macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkC let funcName = genSym(nskProc, e[0].strVal) result = quote do: - proc `funcName`(args: pointer) {.nimcall.} = + proc `funcName`(args: pointer) {.gcsafe, nimcall.} = `funcCall` Task(callback: `funcName`, args: nil) From 8de053d8708d3547edeee8c4846b355a557f1d18 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 20 Apr 2021 22:00:45 -0700 Subject: [PATCH 0687/2472] fix `hintMsgOrigin` regression + simplify msgs code (#17805) --- compiler/msgs.nim | 5 +---- compiler/semdata.nim | 18 +++++++----------- compiler/semexprs.nim | 2 +- compiler/semmagic.nim | 8 ++++---- compiler/semobjconstr.nim | 13 ++++++------- compiler/semstmts.nim | 8 +++----- 6 files changed, 22 insertions(+), 32 deletions(-) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 30421fe6a..89949ef17 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -14,7 +14,7 @@ import std/private/miscdollars import strutils2 type InstantiationInfo* = typeof(instantiationInfo()) -template instLoc(): InstantiationInfo = instantiationInfo(-2, fullPaths = true) +template instLoc*(): InstantiationInfo = instantiationInfo(-2, fullPaths = true) template toStdOrrKind(stdOrr): untyped = if stdOrr == stdout: stdOrrStdout else: stdOrrStderr @@ -600,9 +600,6 @@ template localError*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") template localError*(conf: ConfigRef; info: TLineInfo, arg: string) = liMessage(conf, info, errGenerated, arg, doNothing, instLoc()) -template localError*(conf: ConfigRef; info: TLineInfo, format: string, params: openArray[string]) = - liMessage(conf, info, errGenerated, format % params, doNothing, instLoc()) - template message*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") = liMessage(conf, info, msg, arg, doNothing, instLoc()) diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 354d4587d..95278893a 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -501,18 +501,14 @@ proc errorNode*(c: PContext, n: PNode): PNode = result = newNodeI(nkEmpty, n.info) result.typ = errorType(c) -proc localErrorNode*(c: PContext, n: PNode, info: TLineInfo, msg: TMsgKind, arg: string): PNode = - localError(c.config, info, msg, arg) - result = errorNode(c, n) +template localErrorNode*(c: PContext, n: PNode, arg: string, info: TLineInfo, kind: TMsgKind = errGenerated): PNode = + liMessage(c.config, info, kind, arg, doNothing, instLoc()) + errorNode(c, n) -proc localErrorNode*(c: PContext, n: PNode, info: TLineInfo, arg: string): PNode = - localErrorNode(c, n, info, errGenerated, arg) - -proc localErrorNode*(c: PContext, n: PNode, msg: TMsgKind, arg: string): PNode = - localErrorNode(c, n, n.info, msg, arg) - -proc localErrorNode*(c: PContext, n: PNode, arg: string): PNode = - localErrorNode(c, n, n.info, errGenerated, arg) +template localErrorNode*(c: PContext, n: PNode, arg: string, kind: TMsgKind = errGenerated): PNode = + let n2 = n + liMessage(c.config, n2.info, kind, arg, doNothing, instLoc()) + errorNode(c, n2) proc fillTypeS*(dest: PType, kind: TTypeKind, c: PContext) = dest.kind = kind diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 353831736..defceb291 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -2620,7 +2620,7 @@ proc semTupleConstr(c: PContext, n: PNode, flags: TExprFlags): PNode = # check if either everything or nothing is tyTypeDesc for i in 1.. high(TSymChoiceRule).int: - return localErrorNode(c, n, n[2].info, errConstExprExpected) + return localErrorNode(c, n, errConstExprExpected, n[2].info) let id = newIdentNode(getIdent(c.cache, sl.strVal), n.info) let s = qualifiedLookUp(c, id, {checkUndeclared}) @@ -251,10 +251,10 @@ proc semBindSym(c: PContext, n: PNode): PNode = proc opBindSym(c: PContext, scope: PScope, n: PNode, isMixin: int, info: PNode): PNode = if n.kind notin {nkStrLit, nkRStrLit, nkTripleStrLit, nkIdent}: - return localErrorNode(c, n, info.info, errStringOrIdentNodeExpected) + return localErrorNode(c, n, errStringOrIdentNodeExpected, info.info) if isMixin < 0 or isMixin > high(TSymChoiceRule).int: - return localErrorNode(c, n, info.info, errConstExprExpected) + return localErrorNode(c, n, errConstExprExpected, info.info) let id = if n.kind == nkIdent: n else: newIdentNode(getIdent(c.cache, n.strVal), info.info) diff --git a/compiler/semobjconstr.nim b/compiler/semobjconstr.nim index 63b4b9a75..01d5a8b28 100644 --- a/compiler/semobjconstr.nim +++ b/compiler/semobjconstr.nim @@ -210,7 +210,7 @@ proc semConstructFields(c: PContext, n: PNode, localError(c.config, constrCtx.initExpr.info, "a case selecting discriminator '$1' with value '$2' " & "appears in the object construction, but the field(s) $3 " & - "are in conflict with this value.", + "are in conflict with this value." % [discriminator.sym.name.s, discriminatorVal.renderTree, fields]) template valuesInConflictError(valsDiff) = @@ -365,11 +365,10 @@ proc defaultConstructionError(c: PContext, t: PType, info: TLineInfo) = assert constrCtx.missingFields.len > 0 localError(c.config, info, "The $1 type doesn't have a default value. The following fields must " & - "be initialized: $2.", - [typeToString(t), listSymbolNames(constrCtx.missingFields)]) + "be initialized: $2." % [typeToString(t), listSymbolNames(constrCtx.missingFields)]) elif objType.kind == tyDistinct: localError(c.config, info, - "The $1 distinct type doesn't have a default value.", [typeToString(t)]) + "The $1 distinct type doesn't have a default value." % typeToString(t)) else: assert false, "Must not enter here." @@ -379,7 +378,7 @@ proc semObjConstr(c: PContext, n: PNode, flags: TExprFlags): PNode = for child in n: result.add child if t == nil: - return localErrorNode(c, result, errGenerated, "object constructor needs an object type") + return localErrorNode(c, result, "object constructor needs an object type") t = skipTypes(t, {tyGenericInst, tyAlias, tySink, tyOwned}) if t.kind == tyRef: @@ -390,7 +389,7 @@ proc semObjConstr(c: PContext, n: PNode, flags: TExprFlags): PNode = # multiple times as long as they don't have closures. result.typ.flags.incl tfHasOwned if t.kind != tyObject: - return localErrorNode(c, result, errGenerated, "object constructor needs an object type") + return localErrorNode(c, result, "object constructor needs an object type") # Check if the object is fully initialized by recursively testing each # field (if this is a case object, initialized fields in two different @@ -404,7 +403,7 @@ proc semObjConstr(c: PContext, n: PNode, flags: TExprFlags): PNode = if constrCtx.missingFields.len > 0: hasError = true localError(c.config, result.info, - "The $1 type requires the following fields to be initialized: $2.", + "The $1 type requires the following fields to be initialized: $2." % [t.sym.name.s, listSymbolNames(constrCtx.missingFields)]) # Since we were traversing the object fields, it's possible that diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index a10047187..22e4295fa 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -725,7 +725,7 @@ proc semForVars(c: PContext, n: PNode; flags: TExprFlags): PNode = if n.len == 3: if n[0].kind == nkVarTuple: if n[0].len-1 != iterAfterVarLent.len: - return localErrorNode(c, n, n[0].info, errWrongNumberOfVariables) + return localErrorNode(c, n, errWrongNumberOfVariables, n[0].info) for i in 0.. 0 and not isException(typ.lastSon): - localError(c.config, n.info, "raised object of type $1 does not inherit from Exception", - [typeToString(typ)]) + localError(c.config, n.info, "raised object of type $1 does not inherit from Exception" % typeToString(typ)) proc addGenericParamListToScope(c: PContext, n: PNode) = if n.kind != nkGenericParams: illFormedAst(n, c.config) @@ -2201,8 +2200,7 @@ proc inferConceptStaticParam(c: PContext, inferred, n: PNode) = if not sameType(res.typ, typ.base): localError(c.config, n.info, "cannot infer the concept parameter '%s', due to a type mismatch. " & - "attempt to equate '%s' and '%s'.", - [inferred.renderTree, $res.typ, $typ.base]) + "attempt to equate '%s' and '%s'." % [inferred.renderTree, $res.typ, $typ.base]) typ.n = res proc semStmtList(c: PContext, n: PNode, flags: TExprFlags): PNode = From 7bce1f8578eafffacd599401e709de279528e68b Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 20 Apr 2021 22:33:56 -0700 Subject: [PATCH 0688/2472] close #13373 document `(ref Foo)(a: 1)` (#17804) --- doc/manual.rst | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/doc/manual.rst b/doc/manual.rst index ea444a0ac..15242cf7c 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -1702,7 +1702,17 @@ has the syntax `T(fieldA: valueA, fieldB: valueB, ...)` where `T` is an `object` type or a `ref object` type: .. code-block:: nim - var student = Student(name: "Anton", age: 5, id: 3) + type + Student = object + name: string + age: int + PStudent = ref Student + var a1 = Student(name: "Anton", age: 5) + var a2 = PStudent(name: "Anton", age: 5) + # this also works directly: + var a3 = (ref Student)(name: "Anton", age: 5) + # not all fields need to be mentioned, and they can be mentioned out of order: + var a4 = Student(age: 5) Note that, unlike tuples, objects require the field names along with their values. For a `ref object` type `system.new` is invoked implicitly. From a9b62de8956af50424767408bbe30631c63b331b Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 21 Apr 2021 07:41:33 +0200 Subject: [PATCH 0689/2472] CIs: attempt to use csources_v1 (#16282) * CIs: attempt to use csources_v1 * also updated the BSDs * also updated azure pipelines * std modules should not itself use the 'std/' import dir... * compiler has to be careful with std/ for v1 booting --- .builds/freebsd.yml | 2 +- .builds/openbsd_0.yml | 2 +- .builds/openbsd_1.yml | 2 +- .github/workflows/ci_docs.yml | 4 +- azure-pipelines.yml | 2 +- ci/build.bat | 2 +- ci/build.sh | 2 +- ci/nsis_build.bat | 2 +- compiler/extccomp.nim | 2 +- compiler/ic/cbackend.nim | 6 +-- compiler/ic/dce.nim | 2 +- compiler/ic/ic.nim | 6 +-- compiler/ic/integrity.nim | 2 +- compiler/ic/navigator.nim | 2 +- compiler/ic/packed_ast.nim | 2 +- compiler/int128.nim | 4 +- compiler/jsgen.nim | 2 +- compiler/lookups.nim | 2 +- compiler/modulegraphs.nim | 2 +- compiler/nilcheck.nim | 2 +- compiler/nim.nim | 2 +- compiler/nimpaths.nim | 2 +- compiler/rodutils.nim | 2 +- compiler/semdata.nim | 2 +- compiler/semtypes.nim | 2 +- compiler/vmdef.nim | 2 +- compiler/vmgen.nim | 2 +- compiler/vmops.nim | 16 +++---- compiler/wordrecg.nim | 80 ++++++++++++++++----------------- lib/pure/cgi.nim | 2 +- lib/pure/collections/deques.nim | 2 +- lib/pure/collections/tables.nim | 2 +- lib/pure/complex.nim | 2 +- lib/pure/cookies.nim | 6 +-- lib/pure/distros.nim | 6 +-- lib/pure/json.nim | 7 ++- lib/pure/marshal.nim | 2 +- lib/pure/math.nim | 2 +- lib/pure/oids.nim | 2 +- lib/pure/options.nim | 2 +- lib/pure/os.nim | 8 ++-- lib/pure/random.nim | 4 +- lib/pure/rationals.nim | 2 +- lib/pure/ropes.nim | 2 +- lib/pure/stats.nim | 2 +- lib/pure/strformat.nim | 4 +- lib/pure/sugar.nim | 2 +- lib/pure/typetraits.nim | 2 +- lib/pure/unittest.nim | 8 ++-- lib/pure/uri.nim | 2 +- lib/std/enumutils.nim | 8 ++-- lib/std/jsonutils.nim | 18 ++++---- lib/std/monotimes.nim | 4 +- lib/std/private/globs.nim | 2 +- lib/std/setutils.nim | 2 +- lib/std/sha1.nim | 4 +- lib/std/sysrand.nim | 10 ++--- lib/std/wrapnils.nim | 4 +- 58 files changed, 142 insertions(+), 143 deletions(-) diff --git a/.builds/freebsd.yml b/.builds/freebsd.yml index 6898a80fd..443be798a 100644 --- a/.builds/freebsd.yml +++ b/.builds/freebsd.yml @@ -15,7 +15,7 @@ environment: tasks: - setup: | cd Nim - git clone --depth 1 -q https://github.com/nim-lang/csources.git + git clone --depth 1 -q https://github.com/nim-lang/csources_v1.git csources gmake -C csources -j $(sysctl -n hw.ncpu) bin/nim c --skipUserCfg --skipParentCfg koch echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv diff --git a/.builds/openbsd_0.yml b/.builds/openbsd_0.yml index 146ad8165..c09f0f08d 100644 --- a/.builds/openbsd_0.yml +++ b/.builds/openbsd_0.yml @@ -18,7 +18,7 @@ environment: tasks: - setup: | cd Nim - git clone --depth 1 -q https://github.com/nim-lang/csources.git + git clone --depth 1 -q https://github.com/nim-lang/csources_v1.git csources gmake -C csources -j $(sysctl -n hw.ncpuonline) bin/nim c koch echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv diff --git a/.builds/openbsd_1.yml b/.builds/openbsd_1.yml index ad3734008..ffe449733 100644 --- a/.builds/openbsd_1.yml +++ b/.builds/openbsd_1.yml @@ -18,7 +18,7 @@ environment: tasks: - setup: | cd Nim - git clone --depth 1 -q https://github.com/nim-lang/csources.git + git clone --depth 1 -q https://github.com/nim-lang/csources_v1.git csources gmake -C csources -j $(sysctl -n hw.ncpuonline) bin/nim c koch echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv diff --git a/.github/workflows/ci_docs.yml b/.github/workflows/ci_docs.yml index 982efe610..c9036d1d4 100644 --- a/.github/workflows/ci_docs.yml +++ b/.github/workflows/ci_docs.yml @@ -85,7 +85,7 @@ jobs: id: csources-version shell: bash run: | - sha=$(git ls-remote https://github.com/nim-lang/csources master | cut -f 1) + sha=$(git ls-remote https://github.com/nim-lang/csources_v1 master | cut -f 1) echo "::set-output name=sha::$sha" - name: 'Get prebuilt csources from cache' @@ -99,7 +99,7 @@ jobs: if: steps.csources-cache.outputs.cache-hit != 'true' uses: actions/checkout@v2 with: - repository: nim-lang/csources + repository: nim-lang/csources_v1 path: csources - name: 'Build 1-stage compiler from csources' diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 55bad609e..5c244648d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -67,7 +67,7 @@ jobs: echo $commitMsg | grep -v '\[skip ci\]' # fails if [skip ci] not in commit msg displayName: 'Check whether to skip CI' - - bash: git clone --depth 1 https://github.com/nim-lang/csources + - bash: git clone --depth 1 https://github.com/nim-lang/csources_v1 csources displayName: 'Checkout Nim csources' - task: NodeTool@0 diff --git a/ci/build.bat b/ci/build.bat index cdce8d3d2..2227168e5 100644 --- a/ci/build.bat +++ b/ci/build.bat @@ -2,7 +2,7 @@ REM Some debug info echo "Running on %CI_RUNNER_ID% (%CI_RUNNER_DESCRIPTION%) with tags %CI_RUNNER_TAGS%." gcc -v -git clone --depth 1 https://github.com/nim-lang/csources.git +git clone --depth 1 https://github.com/nim-lang/csources_v1.git csources cd csources call build64.bat cd .. diff --git a/ci/build.sh b/ci/build.sh index 6321fffba..a89d011ed 100644 --- a/ci/build.sh +++ b/ci/build.sh @@ -1,7 +1,7 @@ sh ci/deps.sh # Build from C sources. -git clone --depth 1 https://github.com/nim-lang/csources.git +git clone --depth 1 https://github.com/nim-lang/csources_v1.git csources cd csources sh build.sh cd .. diff --git a/ci/nsis_build.bat b/ci/nsis_build.bat index 4806810d7..12aff1b72 100644 --- a/ci/nsis_build.bat +++ b/ci/nsis_build.bat @@ -25,7 +25,7 @@ Rem Build csources koch csources -d:release || exit /b rem Grab C sources and nimsuggest -git clone --depth 1 https://github.com/nim-lang/csources.git +git clone --depth 1 https://github.com/nim-lang/csources_v1.git csources set PATH=%CD%\bin;%PATH% diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 8d1432aa8..71a20fc47 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -14,7 +14,7 @@ import ropes, platform, condsyms, options, msgs, lineinfos, pathutils -import std/[os, strutils, osproc, sha1, streams, sequtils, times, strtabs, json] +import os, strutils, osproc, std/sha1, streams, sequtils, times, strtabs, json type TInfoCCProp* = enum # properties of the C compiler: diff --git a/compiler/ic/cbackend.nim b/compiler/ic/cbackend.nim index c7e5c7266..f5811cb3b 100644 --- a/compiler/ic/cbackend.nim +++ b/compiler/ic/cbackend.nim @@ -18,8 +18,8 @@ ## also doing cross-module dependency tracking and DCE that we don't need ## anymore. DCE is now done as prepass over the entire packed module graph. -import std/[packedsets, algorithm, tables] - # std/intsets would give `UnusedImport`, pending https://github.com/nim-lang/Nim/issues/14246 +import std/packedsets, algorithm, tables + import ".."/[ast, options, lineinfos, modulegraphs, cgendata, cgen, pathutils, extccomp, msgs] @@ -70,7 +70,7 @@ proc addFileToLink(config: ConfigRef; m: PSym) = addFileToCompile(config, cf) when defined(debugDce): - import std / [os, packedsets] + import os, std/packedsets proc storeAliveSymsImpl(asymFile: AbsoluteFile; s: seq[int32]) = var f = rodfiles.create(asymFile.string) diff --git a/compiler/ic/dce.nim b/compiler/ic/dce.nim index 48ed414e2..350b17d1b 100644 --- a/compiler/ic/dce.nim +++ b/compiler/ic/dce.nim @@ -9,7 +9,7 @@ ## Dead code elimination (=DCE) for IC. -import std / [intsets, tables] +import intsets, tables import ".." / [ast, options, lineinfos, types] import packed_ast, ic, bitabs diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim index d9a8756f1..6530cb6c2 100644 --- a/compiler/ic/ic.nim +++ b/compiler/ic/ic.nim @@ -7,12 +7,12 @@ # distribution, for details about the copyright. # -import std / [hashes, tables, intsets, sha1] +import hashes, tables, intsets, std/sha1 import packed_ast, bitabs, rodfiles import ".." / [ast, idents, lineinfos, msgs, ropes, options, pathutils, condsyms] #import ".." / [renderer, astalgo] -from std / os import removeFile, isAbsolute +from os import removeFile, isAbsolute type PackedConfig* = object @@ -143,7 +143,7 @@ const debugConfigDiff = defined(debugConfigDiff) when debugConfigDiff: - import std / [hashes, tables, intsets, sha1, strutils, sets] + import hashes, tables, intsets, sha1, strutils, sets proc configIdentical(m: PackedModule; config: ConfigRef): bool = result = m.definedSymbols == definedSymbolsAsString(config) diff --git a/compiler/ic/integrity.nim b/compiler/ic/integrity.nim index 428c344a3..ed367ef61 100644 --- a/compiler/ic/integrity.nim +++ b/compiler/ic/integrity.nim @@ -10,7 +10,7 @@ ## Integrity checking for a set of .rod files. ## The set must cover a complete Nim project. -import std / sets +import sets import ".." / [ast, modulegraphs] import packed_ast, bitabs, ic diff --git a/compiler/ic/navigator.nim b/compiler/ic/navigator.nim index 6af2c2ac0..a1a14885d 100644 --- a/compiler/ic/navigator.nim +++ b/compiler/ic/navigator.nim @@ -11,7 +11,7 @@ ## IDE-like features. It uses the set of .rod files to accomplish ## its task. The set must cover a complete Nim project. -import std / sets +import sets from os import nil from std/private/miscdollars import toLocation diff --git a/compiler/ic/packed_ast.nim b/compiler/ic/packed_ast.nim index 2700c8253..6fadde17c 100644 --- a/compiler/ic/packed_ast.nim +++ b/compiler/ic/packed_ast.nim @@ -12,7 +12,7 @@ ## use this representation directly in all the transformations, ## it is superior. -import std / [hashes, tables, strtabs] +import hashes, tables, strtabs import bitabs import ".." / [ast, options] diff --git a/compiler/int128.nim b/compiler/int128.nim index 8d3cd7113..6ba7c1961 100644 --- a/compiler/int128.nim +++ b/compiler/int128.nim @@ -3,7 +3,7 @@ ## hold all from `low(BiggestInt)` to `high(BiggestUInt)`, This ## type is for that purpose. -from std/math import trunc +from math import trunc type Int128* = object @@ -378,7 +378,7 @@ proc `*`*(lhs, rhs: Int128): Int128 = proc `*=`*(a: var Int128, b: Int128) = a = a * b -import std/bitops +import bitops proc fastLog2*(a: Int128): int = if a.udata[3] != 0: diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 350563d6b..e75ff182f 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -35,7 +35,7 @@ import cgmeth, lowerings, sighashes, modulegraphs, lineinfos, rodutils, transf, injectdestructors, sourcemap -import std/[json, sets, math, tables, intsets, strutils] +import json, sets, math, tables, intsets, strutils from modulegraphs import ModuleGraph, PPassContext diff --git a/compiler/lookups.nim b/compiler/lookups.nim index 7ceadfb96..c4f506a68 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -372,7 +372,7 @@ when false: of 'a'..'z': result = getIdent(c.cache, toLowerAscii(x.s[0]) & x.s.substr(1)) else: result = x -import std/[editdistance, heapqueue] +import std/editdistance, heapqueue type SpellCandidate = object dist: int diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index 9df66269b..c268308d2 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -11,7 +11,7 @@ ## represents a complete Nim project. Single modules can either be kept in RAM ## or stored in a rod-file. -import std / [intsets, tables, hashes, md5] +import intsets, tables, hashes, md5 import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils import ic / [packed_ast, ic] diff --git a/compiler/nilcheck.nim b/compiler/nilcheck.nim index f3ec893f7..9c2d091f7 100644 --- a/compiler/nilcheck.nim +++ b/compiler/nilcheck.nim @@ -8,7 +8,7 @@ # import ast, renderer, intsets, tables, msgs, options, lineinfos, strformat, idents, treetab, hashes -import sequtils, strutils, std / sets +import sequtils, strutils, sets # IMPORTANT: notes not up to date, i'll update this comment again # diff --git a/compiler/nim.nim b/compiler/nim.nim index df06a83a9..f18238f60 100644 --- a/compiler/nim.nim +++ b/compiler/nim.nim @@ -24,7 +24,7 @@ import idents, lineinfos, cmdlinehelper, pathutils, modulegraphs -from std/browsers import openDefaultBrowser +from browsers import openDefaultBrowser from nodejs import findNodeJs when hasTinyCBackend: diff --git a/compiler/nimpaths.nim b/compiler/nimpaths.nim index 71bb9a7d7..a93b488fd 100644 --- a/compiler/nimpaths.nim +++ b/compiler/nimpaths.nim @@ -17,7 +17,7 @@ interpolation variables: Unstable API ]## -import std/[os,strutils] +import os, strutils const docCss* = "$nimr/doc/nimdoc.css" diff --git a/compiler/rodutils.nim b/compiler/rodutils.nim index 353992fca..e13b08e05 100644 --- a/compiler/rodutils.nim +++ b/compiler/rodutils.nim @@ -8,7 +8,7 @@ # ## Serialization utilities for the compiler. -import std/[strutils, math] +import strutils, math # bcc on windows doesn't have C99 functions when defined(windows) and defined(bcc): diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 95278893a..9a17efb1c 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -9,7 +9,7 @@ ## This module contains the data structures for the semantic checking phase. -import std / tables +import tables import intsets, options, ast, astalgo, msgs, idents, renderer, diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 8be210067..c921703b0 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -10,7 +10,7 @@ # this module does the semantic checking of type declarations # included from sem.nim -import std/math +import math const errStringOrIdentNodeExpected = "string or ident node expected" diff --git a/compiler/vmdef.nim b/compiler/vmdef.nim index 068ade4b0..0afb2ed6a 100644 --- a/compiler/vmdef.nim +++ b/compiler/vmdef.nim @@ -10,7 +10,7 @@ ## This module contains the type definitions for the new evaluation engine. ## An instruction is 1-3 int32s in memory, it is a register based VM. -import std / tables +import tables import ast, idents, options, modulegraphs, lineinfos diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index c60e81020..3a6004e8b 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -27,7 +27,7 @@ # solves the opcLdConst vs opcAsgnConst issue. Of course whether we need # this copy depends on the involved types. -import std / tables +import tables import strutils, ast, types, msgs, renderer, vmdef, diff --git a/compiler/vmops.nim b/compiler/vmops.nim index 04356fc76..1d22750e7 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -9,24 +9,24 @@ # Unfortunately this cannot be a module yet: #import vmdeps, vm -from std/math import sqrt, ln, log10, log2, exp, round, arccos, arcsin, +from math import sqrt, ln, log10, log2, exp, round, arccos, arcsin, arctan, arctan2, cos, cosh, hypot, sinh, sin, tan, tanh, pow, trunc, floor, ceil, `mod`, cbrt, arcsinh, arccosh, arctanh, erf, erfc, gamma, lgamma when declared(math.copySign): - from std/math import copySign + from math import copySign when declared(math.signbit): - from std/math import signbit + from math import signbit -from std/os import getEnv, existsEnv, dirExists, fileExists, putEnv, walkDir, +from os import getEnv, existsEnv, dirExists, fileExists, putEnv, walkDir, getAppFilename, raiseOSError, osLastError -from std/md5 import getMD5 -from std/times import cpuTime -from std/hashes import hash -from std/osproc import nil +from md5 import getMD5 +from times import cpuTime +from hashes import hash +from osproc import nil from sighashes import symBodyDigest diff --git a/compiler/wordrecg.nim b/compiler/wordrecg.nim index 827efcfa0..c3625c5f3 100644 --- a/compiler/wordrecg.nim +++ b/compiler/wordrecg.nim @@ -17,58 +17,58 @@ type TSpecialWord* = enum wInvalid = "", wAddr = "addr", wAnd = "and", wAs = "as", wAsm = "asm", - wBind = "bind", wBlock = "block", wBreak = "break", wCase = "case", wCast = "cast", - wConcept = "concept", wConst = "const", wContinue = "continue", wConverter = "converter", + wBind = "bind", wBlock = "block", wBreak = "break", wCase = "case", wCast = "cast", + wConcept = "concept", wConst = "const", wContinue = "continue", wConverter = "converter", wDefer = "defer", wDiscard = "discard", wDistinct = "distinct", wDiv = "div", wDo = "do", - wElif = "elif", wElse = "else", wEnd = "end", wEnum = "enum", wExcept = "except", - wExport = "export", wFinally = "finally", wFor = "for", wFrom = "from", wFunc = "func", - wIf = "if", wImport = "import", wIn = "in", wInclude = "include", wInterface = "interface", + wElif = "elif", wElse = "else", wEnd = "end", wEnum = "enum", wExcept = "except", + wExport = "export", wFinally = "finally", wFor = "for", wFrom = "from", wFunc = "func", + wIf = "if", wImport = "import", wIn = "in", wInclude = "include", wInterface = "interface", wIs = "is", wIsnot = "isnot", wIterator = "iterator", wLet = "let", wMacro = "macro", - wMethod = "method", wMixin = "mixin", wMod = "mod", wNil = "nil", wNot = "not", wNotin = "notin", - wObject = "object", wOf = "of", wOr = "or", wOut = "out", wProc = "proc", wPtr = "ptr", - wRaise = "raise", wRef = "ref", wReturn = "return", wShl = "shl", wShr = "shr", wStatic = "static", - wTemplate = "template", wTry = "try", wTuple = "tuple", wType = "type", wUsing = "using", + wMethod = "method", wMixin = "mixin", wMod = "mod", wNil = "nil", wNot = "not", wNotin = "notin", + wObject = "object", wOf = "of", wOr = "or", wOut = "out", wProc = "proc", wPtr = "ptr", + wRaise = "raise", wRef = "ref", wReturn = "return", wShl = "shl", wShr = "shr", wStatic = "static", + wTemplate = "template", wTry = "try", wTuple = "tuple", wType = "type", wUsing = "using", wVar = "var", wWhen = "when", wWhile = "while", wXor = "xor", wYield = "yield", wColon = ":", wColonColon = "::", wEquals = "=", wDot = ".", wDotDot = "..", wStar = "*", wMinus = "-", - wMagic = "magic", wThread = "thread", wFinal = "final", wProfiler = "profiler", + wMagic = "magic", wThread = "thread", wFinal = "final", wProfiler = "profiler", wMemTracker = "memtracker", wObjChecks = "objchecks", - wIntDefine = "intdefine", wStrDefine = "strdefine", wBoolDefine = "booldefine", + wIntDefine = "intdefine", wStrDefine = "strdefine", wBoolDefine = "booldefine", wCursor = "cursor", wNoalias = "noalias", - wImmediate = "immediate", wConstructor = "constructor", wDestructor = "destructor", + wImmediate = "immediate", wConstructor = "constructor", wDestructor = "destructor", wDelegator = "delegator", wOverride = "override", wImportCpp = "importcpp", - wCppNonPod = "cppNonPod", + wCppNonPod = "cppNonPod", wImportObjC = "importobjc", wImportCompilerProc = "importcompilerproc", - wImportc = "importc", wImportJs = "importjs", wExportc = "exportc", wExportCpp = "exportcpp", + wImportc = "importc", wImportJs = "importjs", wExportc = "exportc", wExportCpp = "exportcpp", wExportNims = "exportnims", wIncompleteStruct = "incompleteStruct", # deprecated wCompleteStruct = "completeStruct", wRequiresInit = "requiresInit", wAlign = "align", wNodecl = "nodecl", wPure = "pure", wSideEffect = "sideEffect", wHeader = "header", wNoSideEffect = "noSideEffect", wGcSafe = "gcsafe", wNoreturn = "noreturn", wNosinks = "nosinks", wMerge = "merge", wLib = "lib", wDynlib = "dynlib", - wCompilerProc = "compilerproc", wCore = "core", wProcVar = "procvar", - wBase = "base", wUsed = "used", wFatal = "fatal", wError = "error", wWarning = "warning", + wCompilerProc = "compilerproc", wCore = "core", wProcVar = "procvar", + wBase = "base", wUsed = "used", wFatal = "fatal", wError = "error", wWarning = "warning", wHint = "hint", wWarningAsError = "warningAsError", wHintAsError = "hintAsError", wLine = "line", wPush = "push", - wPop = "pop", wDefine = "define", wUndef = "undef", wLineDir = "lineDir", + wPop = "pop", wDefine = "define", wUndef = "undef", wLineDir = "lineDir", wStackTrace = "stackTrace", wLineTrace = "lineTrace", wLink = "link", wCompile = "compile", - wLinksys = "linksys", wDeprecated = "deprecated", wVarargs = "varargs", wCallconv = "callconv", - wDebugger = "debugger", wNimcall = "nimcall", wStdcall = "stdcall", wCdecl = "cdecl", + wLinksys = "linksys", wDeprecated = "deprecated", wVarargs = "varargs", wCallconv = "callconv", + wDebugger = "debugger", wNimcall = "nimcall", wStdcall = "stdcall", wCdecl = "cdecl", wSafecall = "safecall", wSyscall = "syscall", wInline = "inline", wNoInline = "noinline", wFastcall = "fastcall", wThiscall = "thiscall", wClosure = "closure", wNoconv = "noconv", - wOn = "on", wOff = "off", wChecks = "checks", wRangeChecks = "rangeChecks", + wOn = "on", wOff = "off", wChecks = "checks", wRangeChecks = "rangeChecks", wBoundChecks = "boundChecks", wOverflowChecks = "overflowChecks", wNilChecks = "nilChecks", - wFloatChecks = "floatChecks", wNanChecks = "nanChecks", wInfChecks = "infChecks", + wFloatChecks = "floatChecks", wNanChecks = "nanChecks", wInfChecks = "infChecks", wStyleChecks = "styleChecks", wStaticBoundchecks = "staticBoundChecks", wNonReloadable = "nonReloadable", wExecuteOnReload = "executeOnReload", - wAssertions = "assertions", wPatterns = "patterns", wTrMacros = "trmacros", + wAssertions = "assertions", wPatterns = "patterns", wTrMacros = "trmacros", wSinkInference = "sinkInference", wWarnings = "warnings", - wHints = "hints", wOptimization = "optimization", wRaises = "raises", + wHints = "hints", wOptimization = "optimization", wRaises = "raises", wWrites = "writes", wReads = "reads", wSize = "size", wEffects = "effects", wTags = "tags", wRequires = "requires", wEnsures = "ensures", wInvariant = "invariant", wAssume = "assume", wAssert = "assert", @@ -76,34 +76,34 @@ type wSafecode = "safecode", wPackage = "package", wNoForward = "noforward", wReorder = "reorder", wNoRewrite = "norewrite", wNoDestroy = "nodestroy", wPragma = "pragma", wCompileTime = "compileTime", wNoInit = "noinit", wPassc = "passc", wPassl = "passl", - wLocalPassc = "localPassC", wBorrow = "borrow", wDiscardable = "discardable", + wLocalPassc = "localPassC", wBorrow = "borrow", wDiscardable = "discardable", wFieldChecks = "fieldChecks", wSubsChar = "subschar", wAcyclic = "acyclic", wShallow = "shallow", wUnroll = "unroll", wLinearScanEnd = "linearScanEnd", wComputedGoto = "computedGoto", wInjectStmt = "injectStmt", wExperimental = "experimental", - wWrite = "write", wGensym = "gensym", wInject = "inject", wDirty = "dirty", + wWrite = "write", wGensym = "gensym", wInject = "inject", wDirty = "dirty", wInheritable = "inheritable", wThreadVar = "threadvar", wEmit = "emit", wAsmNoStackFrame = "asmNoStackFrame", wImplicitStatic = "implicitStatic", wGlobal = "global", wCodegenDecl = "codegenDecl", wUnchecked = "unchecked", wGuard = "guard", wLocks = "locks", wPartial = "partial", wExplain = "explain", wLiftLocals = "liftlocals", - wAuto = "auto", wBool = "bool", wCatch = "catch", wChar = "char", - wClass = "class", wCompl = "compl", wConst_cast = "const_cast", wDefault = "default", - wDelete = "delete", wDouble = "double", wDynamic_cast = "dynamic_cast", + wAuto = "auto", wBool = "bool", wCatch = "catch", wChar = "char", + wClass = "class", wCompl = "compl", wConst_cast = "const_cast", wDefault = "default", + wDelete = "delete", wDouble = "double", wDynamic_cast = "dynamic_cast", wExplicit = "explicit", wExtern = "extern", wFalse = "false", wFloat = "float", - wFriend = "friend", wGoto = "goto", wInt = "int", wLong = "long", wMutable = "mutable", - wNamespace = "namespace", wNew = "new", wOperator = "operator", wPrivate = "private", - wProtected = "protected", wPublic = "public", wRegister = "register", - wReinterpret_cast = "reinterpret_cast", wRestrict = "restrict", wShort = "short", - wSigned = "signed", wSizeof = "sizeof", wStatic_cast = "static_cast", wStruct = "struct", - wSwitch = "switch", wThis = "this", wThrow = "throw", wTrue = "true", wTypedef = "typedef", + wFriend = "friend", wGoto = "goto", wInt = "int", wLong = "long", wMutable = "mutable", + wNamespace = "namespace", wNew = "new", wOperator = "operator", wPrivate = "private", + wProtected = "protected", wPublic = "public", wRegister = "register", + wReinterpret_cast = "reinterpret_cast", wRestrict = "restrict", wShort = "short", + wSigned = "signed", wSizeof = "sizeof", wStatic_cast = "static_cast", wStruct = "struct", + wSwitch = "switch", wThis = "this", wThrow = "throw", wTrue = "true", wTypedef = "typedef", wTypeid = "typeid", wTypeof = "typeof", wTypename = "typename", - wUnion = "union", wPacked = "packed", wUnsigned = "unsigned", wVirtual = "virtual", + wUnion = "union", wPacked = "packed", wUnsigned = "unsigned", wVirtual = "virtual", wVoid = "void", wVolatile = "volatile", wWchar_t = "wchar_t", - wAlignas = "alignas", wAlignof = "alignof", wConstexpr = "constexpr", wDecltype = "decltype", + wAlignas = "alignas", wAlignof = "alignof", wConstexpr = "constexpr", wDecltype = "decltype", wNullptr = "nullptr", wNoexcept = "noexcept", - wThread_local = "thread_local", wStatic_assert = "static_assert", + wThread_local = "thread_local", wStatic_assert = "static_assert", wChar16_t = "char16_t", wChar32_t = "char32_t", wStdIn = "stdin", wStdOut = "stdout", wStdErr = "stderr", @@ -134,14 +134,14 @@ const enumUtilsExist = compiles: when enumUtilsExist: from std/enumutils import genEnumCaseStmt from strutils import normalize - proc findStr*[T: enum](a, b: static[T], s: string, default: T): T = + proc findStr*[T: enum](a, b: static[T], s: string, default: T): T = genEnumCaseStmt(T, s, default, ord(a), ord(b), normalize) else: from strutils import cmpIgnoreStyle - proc findStr*[T: enum](a, b: static[T], s: string, default: T): T {.deprecated.} = + proc findStr*[T: enum](a, b: static[T], s: string, default: T): T {.deprecated.} = # used for compiler bootstrapping only for i in a..b: if cmpIgnoreStyle($i, s) == 0: return i - result = default \ No newline at end of file + result = default \ No newline at end of file diff --git a/lib/pure/cgi.nim b/lib/pure/cgi.nim index fd5089dbb..566482b21 100644 --- a/lib/pure/cgi.nim +++ b/lib/pure/cgi.nim @@ -29,7 +29,7 @@ ## writeLine(stdout, "your password: " & myData["password"]) ## writeLine(stdout, "") -import std/[strutils, os, strtabs, cookies, uri] +import strutils, os, strtabs, cookies, uri export uri.encodeUrl, uri.decodeUrl diff --git a/lib/pure/collections/deques.nim b/lib/pure/collections/deques.nim index 582bb02a7..c378dff1d 100644 --- a/lib/pure/collections/deques.nim +++ b/lib/pure/collections/deques.nim @@ -50,7 +50,7 @@ runnableExamples: import std/private/since -import std/math +import math type Deque*[T] = object diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 9387807c0..8739b60a3 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -196,7 +196,7 @@ runnableExamples: import std/private/since -import std/[hashes, math, algorithm] +import hashes, math, algorithm type KeyValuePair[A, B] = tuple[hcode: Hash, key: A, val: B] diff --git a/lib/pure/complex.nim b/lib/pure/complex.nim index b9371c1e1..5492028a8 100644 --- a/lib/pure/complex.nim +++ b/lib/pure/complex.nim @@ -36,7 +36,7 @@ runnableExamples: {.push checks: off, line_dir: off, stack_trace: off, debugger: off.} # the user does not want to trace a part of the standard library! -import std/math +import math type Complex*[T: SomeFloat] = object diff --git a/lib/pure/cookies.nim b/lib/pure/cookies.nim index 8d9cc0c95..25d701eb4 100644 --- a/lib/pure/cookies.nim +++ b/lib/pure/cookies.nim @@ -9,7 +9,7 @@ ## This module implements helper procs for parsing Cookies. -import std/[strtabs, times, options] +import strtabs, times, options type @@ -25,7 +25,7 @@ proc parseCookies*(s: string): StringTableRef = ## "Set-Cookie" header set by servers. runnableExamples: import std/strtabs - let cookieJar = parseCookies("a=1; foo=bar") + let cookieJar = parseCookies("a=1; foo=bar") assert cookieJar["a"] == "1" assert cookieJar["foo"] == "bar" @@ -46,7 +46,7 @@ proc parseCookies*(s: string): StringTableRef = proc setCookie*(key, value: string, domain = "", path = "", expires = "", noName = false, - secure = false, httpOnly = false, + secure = false, httpOnly = false, maxAge = none(int), sameSite = SameSite.Default): string = ## Creates a command in the format of ## `Set-Cookie: key=value; Domain=...; ...` diff --git a/lib/pure/distros.nim b/lib/pure/distros.nim index 2b119b92c..c1bc44eeb 100644 --- a/lib/pure/distros.nim +++ b/lib/pure/distros.nim @@ -27,11 +27,11 @@ ## ## See `packaging `_ for hints on distributing Nim using OS packages. -from std/strutils import contains, toLowerAscii +from strutils import contains, toLowerAscii when not defined(nimscript): - from std/osproc import execProcess - from std/os import existsEnv + from osproc import execProcess + from os import existsEnv type Distribution* {.pure.} = enum ## the list of known distributions diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 7ee5be9c5..96ee61fd5 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -159,10 +159,9 @@ runnableExamples: a1, a2, a0, a3, a4: int doAssert $(%* Foo()) == """{"a1":0,"a2":0,"a0":0,"a3":0,"a4":0}""" -import - std/[hashes, tables, strutils, lexbase, streams, macros, parsejson] +import hashes, tables, strutils, lexbase, streams, macros, parsejson -import std/options # xxx remove this dependency using same approach as https://github.com/nim-lang/Nim/pull/14563 +import options # xxx remove this dependency using same approach as https://github.com/nim-lang/Nim/pull/14563 import std/private/since export @@ -921,7 +920,7 @@ proc parseJson*(s: Stream, filename: string = ""; rawIntegers = false, rawFloats p.close() when defined(js): - from std/math import `mod` + from math import `mod` from std/jsffi import JSObject, `[]`, to from std/private/jsutils import getProtoName, isInteger, isSafeInteger diff --git a/lib/pure/marshal.nim b/lib/pure/marshal.nim index 936b8fe94..993e0f510 100644 --- a/lib/pure/marshal.nim +++ b/lib/pure/marshal.nim @@ -54,7 +54,7 @@ Please use alternative packages for serialization. It is possible to reimplement this module using generics and type traits. Please contribute a new implementation.""".} -import std/[streams, typeinfo, json, intsets, tables, unicode] +import streams, typeinfo, json, intsets, tables, unicode proc ptrToInt(x: pointer): int {.inline.} = result = cast[int](x) # don't skip alignment diff --git a/lib/pure/math.nim b/lib/pure/math.nim index 03e5040ae..cff946581 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -58,7 +58,7 @@ import std/private/since {.push debugger: off.} # the user does not want to trace a part # of the standard library! -import std/[bitops, fenv] +import bitops, fenv when defined(c) or defined(cpp): proc c_isnan(x: float): bool {.importc: "isnan", header: "".} diff --git a/lib/pure/oids.nim b/lib/pure/oids.nim index fb70047b6..da285a187 100644 --- a/lib/pure/oids.nim +++ b/lib/pure/oids.nim @@ -15,7 +15,7 @@ ## This implementation calls `initRand()` for the first call of ## `genOid`. -import std/[hashes, times, endians, random] +import hashes, times, endians, random from std/private/decode_helpers import handleHexChar type diff --git a/lib/pure/options.nim b/lib/pure/options.nim index 63e3598f0..bd5aa389e 100644 --- a/lib/pure/options.nim +++ b/lib/pure/options.nim @@ -69,7 +69,7 @@ supports pattern matching on `Option`s, with the `Some()` and # xxx pending https://github.com/timotheecour/Nim/issues/376 use `runnableExamples` and `whichModule` -import std/typetraits +import typetraits when (NimMajor, NimMinor) >= (1, 1): type diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 0710b8333..50a88b43b 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -44,7 +44,7 @@ include system/inclrtl import std/private/since -import std/[strutils, pathnorm] +import strutils, pathnorm const weirdTarget = defined(nimscript) or defined(js) @@ -65,9 +65,9 @@ since (1, 1): when weirdTarget: discard elif defined(windows): - import std/[winlean, times] + import winlean, times elif defined(posix): - import std/[posix, times] + import posix, times proc toTime(ts: Timespec): times.Time {.inline.} = result = initTime(ts.tv_sec.int64, ts.tv_nsec.int) @@ -1008,7 +1008,7 @@ proc expandTilde*(path: string): string {. ## ## Windows: this is still supported despite Windows platform not having this ## convention; also, both ``~/`` and ``~\`` are handled. - ## + ## ## .. warning:: `~bob` and `~bob/` are not yet handled correctly. ## ## See also: diff --git a/lib/pure/random.nim b/lib/pure/random.nim index f91d92731..e58623baa 100644 --- a/lib/pure/random.nim +++ b/lib/pure/random.nim @@ -72,7 +72,7 @@ runnableExamples: ## * `list of cryptographic and hashing modules `_ ## in the standard library -import std/[algorithm, math] +import algorithm, math import std/private/since include system/inclrtl @@ -625,7 +625,7 @@ proc shuffle*[T](x: var openArray[T]) = shuffle(state, x) when not defined(nimscript) and not defined(standalone): - import std/times + import times proc initRand(): Rand = ## Initializes a new Rand state with a seed based on the current time. diff --git a/lib/pure/rationals.nim b/lib/pure/rationals.nim index a059651bb..800979cda 100644 --- a/lib/pure/rationals.nim +++ b/lib/pure/rationals.nim @@ -21,7 +21,7 @@ runnableExamples: doAssert r1 * r2 == -3 // 8 doAssert r1 / r2 == -2 // 3 -import std/[math, hashes] +import math, hashes type Rational*[T] = object ## A rational number, consisting of a numerator `num` and a denominator `den`. diff --git a/lib/pure/ropes.nim b/lib/pure/ropes.nim index 42550af1d..1300b4479 100644 --- a/lib/pure/ropes.nim +++ b/lib/pure/ropes.nim @@ -17,7 +17,7 @@ ## runtime efficiency. include system/inclrtl -import std/streams +import streams {.push debugger: off.} # the user does not want to trace a part # of the standard library! diff --git a/lib/pure/stats.nim b/lib/pure/stats.nim index 6e2d5fecd..e46dff212 100644 --- a/lib/pure/stats.nim +++ b/lib/pure/stats.nim @@ -53,7 +53,7 @@ runnableExamples: doAssert statistics.kurtosis() ~= -1.0 doAssert statistics.kurtosisS() ~= -0.7000000000000008 -from std/math import FloatClass, sqrt, pow, round +from math import FloatClass, sqrt, pow, round {.push debugger: off.} # the user does not want to trace a part # of the standard library! diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim index 736b4d501..52731e970 100644 --- a/lib/pure/strformat.nim +++ b/lib/pure/strformat.nim @@ -313,8 +313,8 @@ help with readability, since there is only so much you can cram into single letter DSLs. ]## -import std/[macros, parseutils, unicode] -import std/strutils except format +import macros, parseutils, unicode +import strutils except format proc mkDigit(v: int, typ: char): string {.inline.} = assert(v < 26) diff --git a/lib/pure/sugar.nim b/lib/pure/sugar.nim index 239ed9cf1..9abef55a2 100644 --- a/lib/pure/sugar.nim +++ b/lib/pure/sugar.nim @@ -11,7 +11,7 @@ ## macro system. import std/private/since -import std/macros +import macros proc checkPragma(ex, prag: var NimNode) = since (1, 3): diff --git a/lib/pure/typetraits.nim b/lib/pure/typetraits.nim index 2527dc26e..3e215a0ea 100644 --- a/lib/pure/typetraits.nim +++ b/lib/pure/typetraits.nim @@ -167,7 +167,7 @@ since (1, 3, 5): typeof(block: (for ai in a: ai)) -import std/macros +import macros macro enumLen*(T: typedesc[enum]): int = ## Returns the number of items in the enum `T`. diff --git a/lib/pure/unittest.nim b/lib/pure/unittest.nim index 3cff833e4..80a05db36 100644 --- a/lib/pure/unittest.nim +++ b/lib/pure/unittest.nim @@ -108,15 +108,15 @@ import std/private/since import std/exitprocs -import std/[macros, strutils, streams, times, sets, sequtils] +import macros, strutils, streams, times, sets, sequtils when declared(stdout): - import std/os + import os const useTerminal = not defined(js) when useTerminal: - import std/terminal + import terminal type TestStatus* = enum ## The status of a test when it is done. @@ -747,7 +747,7 @@ macro expect*(exceptions: varargs[typed], body: untyped): untyped = of 2: discard parseInt("Hello World!") of 3: raise newException(IOError, "I can't do that Dave.") else: assert 2 + 2 == 5 - + expect IOError, OSError, ValueError, AssertionDefect: defectiveRobot() diff --git a/lib/pure/uri.nim b/lib/pure/uri.nim index d2a9f3848..cf712fa89 100644 --- a/lib/pure/uri.nim +++ b/lib/pure/uri.nim @@ -39,7 +39,7 @@ runnableExamples: doAssert getDataUri("Nim", "text/plain") == "data:text/plain;charset=utf-8;base64,Tmlt" -import std/[strutils, parseutils, base64] +import strutils, parseutils, base64 import std/private/[since, decode_helpers] diff --git a/lib/std/enumutils.nim b/lib/std/enumutils.nim index 16dab9d1a..6195ae07d 100644 --- a/lib/std/enumutils.nim +++ b/lib/std/enumutils.nim @@ -7,16 +7,16 @@ # distribution, for details about the copyright. # -import std/macros -from std/typetraits import OrdinalEnum, HoleyEnum +import macros +from typetraits import OrdinalEnum, HoleyEnum # xxx `genEnumCaseStmt` needs tests and runnableExamples -macro genEnumCaseStmt*(typ: typedesc, argSym: typed, default: typed, +macro genEnumCaseStmt*(typ: typedesc, argSym: typed, default: typed, userMin, userMax: static[int], normalizer: static[proc(s :string): string]): untyped = # generates a case stmt, which assigns the correct enum field given # a normalized string comparison to the `argSym` input. - # string normalization is done using passed normalizer. + # string normalization is done using passed normalizer. # NOTE: for an enum with fields Foo, Bar, ... we cannot generate # `of "Foo".nimIdentNormalize: Foo`. # This will fail, if the enum is not defined at top level (e.g. in a block). diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim index a5daa9fb4..4c27bc9f2 100644 --- a/lib/std/jsonutils.nim +++ b/lib/std/jsonutils.nim @@ -13,7 +13,7 @@ runnableExamples: let j = a.toJson assert j.jsonTo(typeof(a)).toJson == j -import std/[json,strutils,tables,sets,strtabs,options] +import json, strutils, tables, sets, strtabs, options #[ Future directions: @@ -27,7 +27,7 @@ add a way to customize serialization, for e.g.: objects. ]# -import std/macros +import macros type Joptions* = object @@ -106,7 +106,7 @@ proc hasField[T](obj: T, field: string): bool = return true return false -macro accessField(obj: typed, name: static string): untyped = +macro accessField(obj: typed, name: static string): untyped = newDotExpr(obj, ident(name)) template fromJsonFields(newObj, oldObj, json, discKeys, opt) = @@ -146,7 +146,7 @@ template fromJsonFields(newObj, oldObj, json, discKeys, opt) = json.len == numMatched else: json.len == num and num == numMatched - + checkJson ok, $(json.len, num, numMatched, $T, json) proc fromJson*[T](a: var T, b: JsonNode, opt = Joptions()) @@ -284,7 +284,7 @@ proc toJson*[T](a: T): JsonNode = proc fromJsonHook*[K: string|cstring, V](t: var (Table[K, V] | OrderedTable[K, V]), jsonNode: JsonNode) = ## Enables `fromJson` for `Table` and `OrderedTable` types. - ## + ## ## See also: ## * `toJsonHook proc<#toJsonHook>`_ runnableExamples: @@ -326,7 +326,7 @@ proc toJsonHook*[K: string|cstring, V](t: (Table[K, V] | OrderedTable[K, V])): J proc fromJsonHook*[A](s: var SomeSet[A], jsonNode: JsonNode) = ## Enables `fromJson` for `HashSet` and `OrderedSet` types. - ## + ## ## See also: ## * `toJsonHook proc<#toJsonHook,SomeSet[A]>`_ runnableExamples: @@ -360,7 +360,7 @@ proc toJsonHook*[A](s: SomeSet[A]): JsonNode = proc fromJsonHook*[T](self: var Option[T], jsonNode: JsonNode) = ## Enables `fromJson` for `Option` types. - ## + ## ## See also: ## * `toJsonHook proc<#toJsonHook,Option[T]>`_ runnableExamples: @@ -395,7 +395,7 @@ proc toJsonHook*[T](self: Option[T]): JsonNode = proc fromJsonHook*(a: var StringTableRef, b: JsonNode) = ## Enables `fromJson` for `StringTableRef` type. - ## + ## ## See also: ## * `toJsonHook proc<#toJsonHook,StringTableRef>`_ runnableExamples: @@ -413,7 +413,7 @@ proc fromJsonHook*(a: var StringTableRef, b: JsonNode) = proc toJsonHook*(a: StringTableRef): JsonNode = ## Enables `toJson` for `StringTableRef` type. - ## + ## ## See also: ## * `fromJsonHook proc<#fromJsonHook,StringTableRef,JsonNode>`_ runnableExamples: diff --git a/lib/std/monotimes.nim b/lib/std/monotimes.nim index 78736d719..8f6aa5b66 100644 --- a/lib/std/monotimes.nim +++ b/lib/std/monotimes.nim @@ -39,7 +39,7 @@ See also * `times module `_ ]## -import std/times +import times type MonoTime* = object ## Represents a monotonic timestamp. @@ -77,7 +77,7 @@ when defined(js): {.pop.} elif defined(posix) and not defined(osx): - import std/posix + import posix elif defined(windows): proc QueryPerformanceCounter(res: var uint64) {. diff --git a/lib/std/private/globs.nim b/lib/std/private/globs.nim index b98a7808b..a32f1d1b9 100644 --- a/lib/std/private/globs.nim +++ b/lib/std/private/globs.nim @@ -4,7 +4,7 @@ this can eventually be moved to std/os and `walkDirRec` can be implemented in te to avoid duplication ]## -import std/[os] +import os when defined(windows): from strutils import replace diff --git a/lib/std/setutils.nim b/lib/std/setutils.nim index c7fac0a54..aba5f348a 100644 --- a/lib/std/setutils.nim +++ b/lib/std/setutils.nim @@ -14,7 +14,7 @@ ## * `std/packedsets `_ ## * `std/sets `_ -import std/[typetraits, macros] +import typetraits, macros #[ type SetElement* = char|byte|bool|int16|uint16|enum|uint8|int8 diff --git a/lib/std/sha1.nim b/lib/std/sha1.nim index 9c5efed1f..bda0c6de7 100644 --- a/lib/std/sha1.nim +++ b/lib/std/sha1.nim @@ -26,8 +26,8 @@ runnableExamples("-r:off"): b = parseSecureHash("10DFAEBF6BFDBC7939957068E2EFACEC4972933C") assert a == b, "files don't match" -import std/strutils -from std/endians import bigEndian32, bigEndian64 +import strutils +from endians import bigEndian32, bigEndian64 const Sha1DigestSize = 20 diff --git a/lib/std/sysrand.nim b/lib/std/sysrand.nim index 4a2cdd10b..4c51577ac 100644 --- a/lib/std/sysrand.nim +++ b/lib/std/sysrand.nim @@ -10,7 +10,7 @@ ## .. warning:: This module was added in Nim 1.6. If you are using it for cryptographic purposes, ## keep in mind that so far this has not been audited by any security professionals, ## therefore may not be secure. -## +## ## `std/sysrand` generates random numbers from a secure source provided by the operating system. ## It is a cryptographically secure pseudorandom number generator ## and should be unpredictable enough for cryptographic applications, @@ -52,10 +52,10 @@ runnableExamples: when not defined(js): - import std/os + import os when defined(posix): - import std/posix + import posix const batchImplOS = defined(freebsd) or defined(openbsd) or (defined(macosx) and not defined(ios)) @@ -243,7 +243,7 @@ elif defined(macosx): """ proc getentropy(p: pointer, size: csize_t): cint {.importc: "getentropy", header: sysrandomHeader.} - # getentropy() fills a buffer with random data, which can be used as input + # getentropy() fills a buffer with random data, which can be used as input # for process-context pseudorandom generators like arc4random(3). # The maximum buffer size permitted is 256 bytes. @@ -310,7 +310,7 @@ proc urandom*(dest: var openArray[byte]): bool = proc urandom*(size: Natural): seq[byte] {.inline.} = ## Returns random bytes suitable for cryptographic use. - ## + ## ## .. warning:: The code hasn't been audited by cryptography experts and ## is provided as-is without guarantees. Use at your own risks. For production ## systems we advise you to request an external audit. diff --git a/lib/std/wrapnils.nim b/lib/std/wrapnils.nim index 3ff48fbfe..708faf4cf 100644 --- a/lib/std/wrapnils.nim +++ b/lib/std/wrapnils.nim @@ -24,7 +24,7 @@ runnableExamples: assert (?.f2.x2.x2).x3 == nil # this terminates ?. early -from std/options import Option, isSome, get, option, unsafeGet, UnpackDefect +from options import Option, isSome, get, option, unsafeGet, UnpackDefect export options.get, options.isSome, options.isNone template fakeDot*(a: Option, b): untyped = @@ -59,7 +59,7 @@ func `[]`*[U](a: Option[U]): auto {.inline.} = if a2 != nil: result = option(a2[]) -import std/macros +import macros func replace(n: NimNode): NimNode = if n.kind == nnkDotExpr: From f1ce173283a06197cfde80e9117ba1aa0ed0dd02 Mon Sep 17 00:00:00 2001 From: Joey Date: Tue, 20 Apr 2021 23:42:04 -0600 Subject: [PATCH 0690/2472] Fix Httpclient headers from being modified accidentally (#17808) --- lib/pure/httpclient.nim | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 14bcfd2fb..b60894103 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -962,12 +962,15 @@ proc format(client: HttpClient | AsyncHttpClient, proc override(fallback, override: HttpHeaders): HttpHeaders = # Right-biased map union for `HttpHeaders` - if override.isNil: - return fallback result = newHttpHeaders() # Copy by value result.table[] = fallback.table[] + + if override.isNil: + # Return the copy of fallback so it does not get modified + return result + for k, vs in override.table: result[k] = vs From 2951f89374e9797fa5567b5afd4effb3149c1fd5 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 21 Apr 2021 01:52:17 -0700 Subject: [PATCH 0691/2472] improve errmsg refs #17793 object ctor with generics (#17806) --- compiler/semobjconstr.nim | 5 ++++- compiler/types.nim | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/compiler/semobjconstr.nim b/compiler/semobjconstr.nim index 01d5a8b28..6f3cabcbf 100644 --- a/compiler/semobjconstr.nim +++ b/compiler/semobjconstr.nim @@ -11,6 +11,8 @@ # included from sem.nim +from sugar import dup + type ObjConstrContext = object typ: PType # The constructed type @@ -389,7 +391,8 @@ proc semObjConstr(c: PContext, n: PNode, flags: TExprFlags): PNode = # multiple times as long as they don't have closures. result.typ.flags.incl tfHasOwned if t.kind != tyObject: - return localErrorNode(c, result, "object constructor needs an object type") + return localErrorNode(c, result, + "object constructor needs an object type".dup(addDeclaredLoc(c.config, t))) # Check if the object is fully initialized by recursively testing each # field (if this is a case object, initialized fields in two different diff --git a/compiler/types.nim b/compiler/types.nim index e56fcffe4..ba2cc76ce 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -130,7 +130,7 @@ proc addDeclaredLocMaybe*(result: var string, conf: ConfigRef; sym: PSym) = if optDeclaredLocs in conf.globalOptions and sym != nil: addDeclaredLoc(result, conf, sym) -proc addDeclaredLoc(result: var string, conf: ConfigRef; typ: PType) = +proc addDeclaredLoc*(result: var string, conf: ConfigRef; typ: PType) = let typ = typ.skipTypes(abstractInst - {tyRange}) result.add " [$1" % typ.kind.toHumanStr if typ.sym != nil: From c631648cb31b11d61d249e0745181acb0fcc30cc Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 21 Apr 2021 21:07:36 +0800 Subject: [PATCH 0692/2472] close #9372 add std/tempfiles (#17361) * close #9372 add std/tempfile --- changelog.md | 3 + lib/std/tempfiles.nim | 139 ++++++++++++++++++++++++++++++++++++ tests/stdlib/ttempfiles.nim | 17 +++++ 3 files changed, 159 insertions(+) create mode 100644 lib/std/tempfiles.nim create mode 100644 tests/stdlib/ttempfiles.nim diff --git a/changelog.md b/changelog.md index 0537b3d6a..abd2f8733 100644 --- a/changelog.md +++ b/changelog.md @@ -283,9 +283,12 @@ - Added `hasClosure` to `std/typetraits`. +- Added `std/tempfiles`. + - Added `genasts.genAst` that avoids the problems inherent with `quote do` and can be used as a replacement. + ## Language changes - `nimscript` now handles `except Exception as e`. diff --git a/lib/std/tempfiles.nim b/lib/std/tempfiles.nim new file mode 100644 index 000000000..1e1bbd403 --- /dev/null +++ b/lib/std/tempfiles.nim @@ -0,0 +1,139 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2021 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module creates temporary files and directories. + +import os, random + + +const + maxRetry = 10000 + letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + nimTempPathLength {.intdefine.} = 8 + + +when defined(windows): + import winlean + + var O_RDWR {.importc: "_O_RDWR", header: "".}: cint + + proc c_fdopen( + filehandle: cint, + mode: cstring + ): File {.importc: "_fdopen",header: "".} + + proc open_osfhandle(osh: Handle, mode: cint): cint {. + importc: "_open_osfhandle", header: "".} + + proc close_osfandle(fd: cint): cint {. + importc: "_close", header: "".} +else: + import posix + + proc c_fdopen( + filehandle: cint, + mode: cstring + ): File {.importc: "fdopen",header: "".} + + +proc safeOpen(filename: string): File = + ## Open files exclusively. + when defined(windows): + let dwShareMode = FILE_SHARE_DELETE or FILE_SHARE_READ or FILE_SHARE_WRITE + let dwCreation = CREATE_NEW + let dwFlags = FILE_FLAG_BACKUP_SEMANTICS or FILE_ATTRIBUTE_NORMAL + let handle = createFileW(newWideCString(filename), GENERIC_READ or GENERIC_WRITE, dwShareMode, + nil, dwCreation, dwFlags, Handle(0)) + + if handle == INVALID_HANDLE_VALUE: + raiseOSError(osLastError(), filename) + + let fileHandle = open_osfhandle(handle, O_RDWR) + if fileHandle == -1: + discard closeHandle(handle) + raiseOSError(osLastError(), filename) + + result = c_fdopen(fileHandle, "w+") + if result == nil: + discard close_osfandle(fileHandle) + raiseOSError(osLastError(), filename) + else: + let flags = posix.O_RDWR or posix.O_CREAT or posix.O_EXCL + + let fileHandle = posix.open(filename, flags) + if fileHandle == -1: + raiseOSError(osLastError(), filename) + + result = c_fdopen(fileHandle, "w+") + if result == nil: + discard posix.close(fileHandle) # TODO handles failure when closing file + raiseOSError(osLastError(), filename) + +template randomPathName(length: Natural): string = + var res = newString(length) + var state = initRand() + for i in 0 ..< length: + res[i] = state.sample(letters) + res + +proc createTempFile*(prefix, suffix: string, dir = ""): tuple[fd: File, path: string] = + ## `createTempFile` creates a new temporary file in the directory `dir`. + ## + ## If `dir` is the empty string, the default directory for temporary files + ## (`getTempDir `_) will be used. + ## The temporary file name begins with `prefix` and ends with `suffix`. + ## `createTempFile` returns a file handle to an open file and the path of that file. + ## + ## If failing to create a temporary file, `IOError` will be raised. + ## + ## .. note:: It is the caller's responsibility to remove the file when no longer needed. + ## + var dir = dir + if dir.len == 0: + dir = getTempDir() + + createDir(dir) + + for i in 0 ..< maxRetry: + result.path = dir / (prefix & randomPathName(nimTempPathLength) & suffix) + try: + result.fd = safeOpen(result.path) + except OSError: + continue + return + + raise newException(IOError, "Failed to create a temporary file under directory " & dir) + +proc createTempDir*(prefix, suffix: string, dir = ""): string = + ## `createTempDir` creates a new temporary directory in the directory `dir`. + ## + ## If `dir` is the empty string, the default directory for temporary files + ## (`getTempDir `_) will be used. + ## The temporary directory name begins with `prefix` and ends with `suffix`. + ## `createTempDir` returns the path of that temporary firectory. + ## + ## If failing to create a temporary directory, `IOError` will be raised. + ## + ## .. note:: It is the caller's responsibility to remove the directory when no longer needed. + ## + var dir = dir + if dir.len == 0: + dir = getTempDir() + + createDir(dir) + + for i in 0 ..< maxRetry: + result = dir / (prefix & randomPathName(nimTempPathLength) & suffix) + try: + if not existsOrCreateDir(result): + return + except OSError: + continue + + raise newException(IOError, "Failed to create a temporary directory under directory " & dir) diff --git a/tests/stdlib/ttempfiles.nim b/tests/stdlib/ttempfiles.nim new file mode 100644 index 000000000..29ef2fb2b --- /dev/null +++ b/tests/stdlib/ttempfiles.nim @@ -0,0 +1,17 @@ +import std/[os, tempfiles] + + +doAssert createTempDir("nim", "tmp") != createTempDir("nim", "tmp") + +block: + var t1 = createTempFile("nim", ".tmp") + var t2 = createTempFile("nim", ".tmp") + doAssert t1.path != t2.path + + write(t1.fd, "1234") + doAssert readAll(t2.fd) == "" + + close(t1.fd) + close(t2.fd) + removeFile(t1.path) + removeFile(t2.path) From da1c1a711780e21a372ef70f9080aebe5b9ef987 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 21 Apr 2021 06:26:27 -0700 Subject: [PATCH 0693/2472] `--filenames:abs|canonical|legacyRelProj` for filenames in compiler msgs (replaces `--listfullpaths:on|off`) (#17746) * use canonicalImport for filename_magicSauce * --filenames:abs|canonical|magic * rename: magic => legacyRelProj --- changelog.md | 2 ++ compiler/commands.nim | 18 +++++++++++++++++- compiler/docgen.nim | 15 --------------- compiler/main.nim | 8 +++++--- compiler/msgs.nim | 24 +++++++++--------------- compiler/options.nim | 37 +++++++++++++++++++++++++++++++++---- doc/advopt.txt | 4 +++- drnim/drnim.nim | 3 ++- tests/config.nims | 2 +- 9 files changed, 72 insertions(+), 41 deletions(-) diff --git a/changelog.md b/changelog.md index abd2f8733..30fe237be 100644 --- a/changelog.md +++ b/changelog.md @@ -341,6 +341,8 @@ - Added `--spellSuggest` to show spelling suggestions on typos. +- Added `--filenames:abs|canonical|magic` which replaces --listFullPaths:on|off + - Source+Edit links now appear on top of every docgen'd page when `nim doc --git.url:url ...` is given. diff --git a/compiler/commands.nim b/compiler/commands.nim index 5d6e45fb5..0733e500a 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -141,6 +141,15 @@ proc splitSwitch(conf: ConfigRef; switch: string, cmd, arg: var string, pass: TC elif switch[i] == '[': arg = substr(switch, i) else: invalidCmdLineOption(conf, pass, switch, info) +template switchOn(arg: string): bool = + # xxx use `switchOn` wherever appropriate + case arg.normalize + of "", "on": true + of "off": false + else: + localError(conf, info, errOnOrOffExpectedButXFound % arg) + false + proc processOnOffSwitch(conf: ConfigRef; op: TOptions, arg: string, pass: TCmdLinePass, info: TLineInfo) = case arg.normalize @@ -885,8 +894,15 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; trackIde(conf, ideDus, arg, info) of "stdout": processOnOffSwitchG(conf, {optStdout}, arg, pass, info) + of "filenames": + case arg.normalize + of "abs": conf.filenameOption = foAbs + of "canonical": conf.filenameOption = foCanonical + of "legacyrelproj": conf.filenameOption = foLegacyRelProj + else: localError(conf, info, "expected: abs|canonical|legacyRelProj, got: $1" % arg) of "listfullpaths": - processOnOffSwitchG(conf, {optListFullPaths}, arg, pass, info) + # xxx in future work, use `warningDeprecated` + conf.filenameOption = if switchOn(arg): foAbs else: foCanonical of "spellsuggest": if arg.len == 0: conf.spellSuggestMax = spellSuggestSecretSauce elif arg == "auto": conf.spellSuggestMax = spellSuggestSecretSauce diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 4d9f09ca6..977fcf8ef 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -63,21 +63,6 @@ proc prettyString(a: object): string = for k, v in fieldPairs(a): result.add k & ": " & $v & "\n" -proc canonicalImport*(conf: ConfigRef, file: AbsoluteFile): string = - ##[ - Shows the canonical module import, e.g.: - system, std/tables, fusion/pointers, system/assertions, std/private/asciitables - ]## - var ret = getRelativePathFromConfigPath(conf, file, isTitle = true) - let dir = getNimbleFile(conf, $file).parentDir.AbsoluteDir - if not dir.isEmpty: - let relPath = relativeTo(file, dir) - if not relPath.isEmpty and (ret.isEmpty or relPath.string.len < ret.string.len): - ret = relPath - if ret.isEmpty: - ret = relativeTo(file, conf.projectPath) - result = ret.string.nativeToUnixPath.changeFileExt("") - proc presentationPath*(conf: ConfigRef, file: AbsoluteFile): RelativeFile = ## returns a relative file that will be appended to outDir let file2 = $file diff --git a/compiler/main.nim b/compiler/main.nim index d66a5f329..9c9a789cb 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -362,6 +362,7 @@ proc mainCommand*(graph: ModuleGraph) = rawMessage(conf, errGenerated, "invalid command: " & conf.command) if conf.errorCounter == 0 and conf.cmd notin {cmdTcc, cmdDump, cmdNop}: + # D20210419T170230:here let mem = when declared(system.getMaxMem): formatSize(getMaxMem()) & " peakmem" else: formatSize(getTotalMem()) & " totmem" @@ -370,8 +371,8 @@ proc mainCommand*(graph: ModuleGraph) = elif isDefined(conf, "release"): "Release" else: "Debug" let sec = formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3) - let project = if optListFullPaths in conf.globalOptions: $conf.projectFull else: $conf.projectName - + let project = if conf.filenameOption == foAbs: $conf.projectFull else: $conf.projectName + # xxx honor conf.filenameOption more accurately var output: string if optCompileOnly in conf.globalOptions and conf.cmd != cmdJsonscript: output = $conf.jsonBuildFile @@ -380,7 +381,8 @@ proc mainCommand*(graph: ModuleGraph) = output = "unknownOutput" else: output = $conf.absOutFile - if optListFullPaths notin conf.globalOptions: output = output.AbsoluteFile.extractFilename + if conf.filenameOption != foAbs: output = output.AbsoluteFile.extractFilename + # xxx honor filenameOption more accurately if optProfileVM in conf.globalOptions: echo conf.dump(conf.vmProfileData) rawMessage(conf, hintSuccessX, [ diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 89949ef17..b7b6a582e 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -241,37 +241,30 @@ template toFullPath*(conf: ConfigRef; info: TLineInfo): string = template toFullPathConsiderDirty*(conf: ConfigRef; info: TLineInfo): string = string toFullPathConsiderDirty(conf, info.fileIndex) -type - FilenameOption* = enum - foAbs # absolute path, e.g.: /pathto/bar/foo.nim - foRelProject # relative to project path, e.g.: ../foo.nim - foMagicSauce # magic sauce, shortest of (foAbs, foRelProject) - foName # lastPathPart, e.g.: foo.nim - foStacktrace # if optExcessiveStackTrace: foAbs else: foName - proc toFilenameOption*(conf: ConfigRef, fileIdx: FileIndex, opt: FilenameOption): string = case opt of foAbs: result = toFullPath(conf, fileIdx) of foRelProject: result = toProjPath(conf, fileIdx) - of foMagicSauce: + of foCanonical: + let absPath = toFullPath(conf, fileIdx) + result = canonicalImportAux(conf, absPath.AbsoluteFile) + of foName: result = toProjPath(conf, fileIdx).lastPathPart + of foLegacyRelProj: let absPath = toFullPath(conf, fileIdx) relPath = toProjPath(conf, fileIdx) - result = if (optListFullPaths in conf.globalOptions) or - (relPath.len > absPath.len) or - (relPath.count("..") > 2): + result = if (relPath.len > absPath.len) or (relPath.count("..") > 2): absPath else: relPath - of foName: result = toProjPath(conf, fileIdx).lastPathPart of foStacktrace: if optExcessiveStackTrace in conf.globalOptions: result = toFilenameOption(conf, fileIdx, foAbs) else: result = toFilenameOption(conf, fileIdx, foName) -proc toMsgFilename*(conf: ConfigRef; info: FileIndex): string = - toFilenameOption(conf, info, foMagicSauce) +proc toMsgFilename*(conf: ConfigRef; fileIdx: FileIndex): string = + toFilenameOption(conf, fileIdx, conf.filenameOption) template toMsgFilename*(conf: ConfigRef; info: TLineInfo): string = toMsgFilename(conf, info.fileIndex) @@ -558,6 +551,7 @@ proc liMessage*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string, styledMsgWriteln(styleBright, loc, resetStyle, color, title, resetStyle, s, KindColor, kindmsg, resetStyle, conf.getSurroundingSrc(info), UnitSep) if hintMsgOrigin in conf.mainPackageNotes: + # xxx needs a bit of refactoring to honor `conf.filenameOption` styledMsgWriteln(styleBright, toFileLineCol(info2), resetStyle, " compiler msg initiated here", KindColor, KindFormat % $hintMsgOrigin, diff --git a/compiler/options.nim b/compiler/options.nim index c1ac6b1e0..2aaaf5844 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -13,7 +13,7 @@ import from terminal import isatty from times import utc, fromUnix, local, getTime, format, DateTime - +from std/private/globs import nativeToUnixPath const hasTinyCBackend* = defined(tinyc) useEffectSystem* = true @@ -78,7 +78,6 @@ type # please make sure we have under 32 options optWholeProject # for 'doc': output any dependency optDocInternal # generate documentation for non-exported symbols optMixedMode # true if some module triggered C++ codegen - optListFullPaths # use full paths in toMsgFilename optDeclaredLocs # show declaration locations in messages optNoNimblePath optHotCodeReloading @@ -258,6 +257,14 @@ type stdOrrStdout stdOrrStderr + FilenameOption* = enum + foAbs # absolute path, e.g.: /pathto/bar/foo.nim + foRelProject # relative to project path, e.g.: ../foo.nim + foCanonical # canonical module name + foLegacyRelProj # legacy, shortest of (foAbs, foRelProject) + foName # lastPathPart, e.g.: foo.nim + foStacktrace # if optExcessiveStackTrace: foAbs else: foName + ConfigRef* = ref object ## every global configuration ## fields marked with '*' are subject to ## the incremental compilation mechanisms @@ -270,6 +277,7 @@ type macrosToExpand*: StringTableRef arcToExpand*: StringTableRef m*: MsgConfig + filenameOption*: FilenameOption # how to render paths in compiler messages evalTemplateCounter*: int evalMacroCounter*: int exitcode*: int8 @@ -413,8 +421,7 @@ const optBoundsCheck, optOverflowCheck, optAssert, optWarns, optRefCheck, optHints, optStackTrace, optLineTrace, # consider adding `optStackTraceMsgs` optTrMacros, optStyleCheck, optCursorInference} - DefaultGlobalOptions* = {optThreadAnalysis, - optExcessiveStackTrace, optListFullPaths} + DefaultGlobalOptions* = {optThreadAnalysis, optExcessiveStackTrace} proc getSrcTimestamp(): DateTime = try: @@ -461,6 +468,7 @@ proc newConfigRef*(): ConfigRef = macrosToExpand: newStringTable(modeStyleInsensitive), arcToExpand: newStringTable(modeStyleInsensitive), m: initMsgConfig(), + filenameOption: foAbs, cppDefines: initHashSet[string](), headerFile: "", features: {}, legacyFeatures: {}, foreignPackageNotes: foreignPackageNotesDefault, notes: NotesVerbosity[1], mainPackageNotes: NotesVerbosity[1], @@ -514,6 +522,7 @@ proc newConfigRef*(): ConfigRef = proc newPartialConfigRef*(): ConfigRef = ## create a new ConfigRef that is only good enough for error reporting. + # xxx FACTOR with `newConfigRef` when defined(nimDebugUtils): result = getConfigRef() else: @@ -522,6 +531,7 @@ proc newPartialConfigRef*(): ConfigRef = verbosity: 1, options: DefaultOptions, globalOptions: DefaultGlobalOptions, + filenameOption: foAbs, foreignPackageNotes: foreignPackageNotesDefault, notes: NotesVerbosity[1], mainPackageNotes: NotesVerbosity[1]) @@ -885,6 +895,25 @@ proc findProjectNimFile*(conf: ConfigRef; pkg: string): string = if dir == "": break return "" +proc canonicalImportAux*(conf: ConfigRef, file: AbsoluteFile): string = + ##[ + Shows the canonical module import, e.g.: + system, std/tables, fusion/pointers, system/assertions, std/private/asciitables + ]## + var ret = getRelativePathFromConfigPath(conf, file, isTitle = true) + let dir = getNimbleFile(conf, $file).parentDir.AbsoluteDir + if not dir.isEmpty: + let relPath = relativeTo(file, dir) + if not relPath.isEmpty and (ret.isEmpty or relPath.string.len < ret.string.len): + ret = relPath + if ret.isEmpty: + ret = relativeTo(file, conf.projectPath) + result = ret.string + +proc canonicalImport*(conf: ConfigRef, file: AbsoluteFile): string = + let ret = canonicalImportAux(conf, file) + result = ret.nativeToUnixPath.changeFileExt("") + proc canonDynlibName(s: string): string = let start = if s.startsWith("lib"): 3 else: 0 let ende = strutils.find(s, {'(', ')', '.'}) diff --git a/doc/advopt.txt b/doc/advopt.txt index bfc381cff..539cc620c 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -36,7 +36,9 @@ Advanced options: to after all options have been processed --stdout:on|off output to stdout --colors:on|off turn compiler messages coloring on|off - --listFullPaths:on|off list full paths in messages + --filenames:abs|canonical|legacyRelProj + customize how filenames are rendered in compiler messages, + defaults to `abs` (absolute) --declaredLocs:on|off show declaration locations in messages --spellSuggest|:num show at most `num >= 0` spelling suggestions on typos. if `num` is not specified (or `auto`), return diff --git a/drnim/drnim.nim b/drnim/drnim.nim index d549e1d5b..a591a8ef3 100644 --- a/drnim/drnim.nim +++ b/drnim/drnim.nim @@ -1205,6 +1205,7 @@ proc mainCommand(graph: ModuleGraph) = registerPass graph, semPass compileProject(graph) if conf.errorCounter == 0: + # xxx deduplicate with D20210419T170230 let mem = when declared(system.getMaxMem): formatSize(getMaxMem()) & " peakmem" else: formatSize(getTotalMem()) & " totmem" @@ -1213,7 +1214,7 @@ proc mainCommand(graph: ModuleGraph) = elif isDefined(conf, "release"): "Release" else: "Debug" let sec = formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3) - let project = if optListFullPaths in conf.globalOptions: $conf.projectFull else: $conf.projectName + let project = if conf.filenameOption == foAbs: $conf.projectFull else: $conf.projectName rawMessage(conf, hintSuccessX, [ "loc", loc, "sec", sec, diff --git a/tests/config.nims b/tests/config.nims index 41edf0005..e5d6545f4 100644 --- a/tests/config.nims +++ b/tests/config.nims @@ -6,7 +6,7 @@ switch("path", "$lib/../testament/lib") ## prevent common user config settings to interfere with testament expectations ## Indifidual tests can override this if needed to test for these options. switch("colors", "off") -switch("listFullPaths", "off") +switch("filenames", "legacyRelProj") switch("excessiveStackTrace", "off") switch("spellSuggest", "0") From 7c64e49d452607fa246b54e931c057ed172b264a Mon Sep 17 00:00:00 2001 From: Clyybber Date: Wed, 21 Apr 2021 15:28:42 +0200 Subject: [PATCH 0694/2472] getCustomPragmaVal priority/override fixes (#17725) * Adhere left-to-right rule for custom pragma priority * Improve error message for no custom pragmas * custom pragmas on var/let sym take priority over its type ones * Workaround & bug --- lib/core/macros.nim | 78 ++++++++++++++++---------------- tests/pragmas/tcustom_pragma.nim | 10 ++++ 2 files changed, 50 insertions(+), 38 deletions(-) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index fc68781da..79c3ba28b 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -1536,6 +1536,14 @@ proc getPragmaNodeFromTypeSym(sym: NimNode): NimNode = if pragmaExpr.kind == nnkPragmaExpr: result = pragmaExpr[1] +proc getPragmaNodeFromType(typ: NimNode): NimNode = + case typ.kind + of nnkSym: + result = getPragmaNodeFromTypeSym(typ) + of nnkProcTy: + result = typ[1] + else: error("illegal typ kind for argument: " & $typ.kind, typ) + proc getPragmaNodeFromVarLetSym(sym: NimNode): NimNode = sym.expectKind nnkSym if sym.symKind notin {nskVar, nskLet}: error("expected var/let sym", sym) @@ -1546,73 +1554,64 @@ proc getPragmaNodeFromVarLetSym(sym: NimNode): NimNode = if pragmaExpr.kind == nnkPragmaExpr: result = pragmaExpr[1] -proc getPragmaByName(pragmaExpr: NimNode, name: string): NimNode = +proc getPragmasByName(pragmaExpr: NimNode, name: string): seq[NimNode] = if pragmaExpr.kind == nnkPragma: for it in pragmaExpr: if it.kind in nnkPragmaCallKinds: if eqIdent(it[0], name): - return it + result.add it elif it.kind == nnkSym: if eqIdent(it, name): - return it + result.add it -proc getCustomPragmaNode(sym: NimNode, name: string): NimNode = +proc getCustomPragmaNodes(sym: NimNode, name: string): seq[NimNode] = sym.expectKind nnkSym case sym.symKind of nskField: - result = getPragmaNodeFromObjFieldSym(sym).getPragmaByName(name) + result = getPragmaNodeFromObjFieldSym(sym).getPragmasByName(name) of nskProc: - result = getPragmaNodeFromProcSym(sym).getPragmaByName(name) + result = getPragmaNodeFromProcSym(sym).getPragmasByName(name) of nskType: - result = getPragmaNodeFromTypeSym(sym).getPragmaByName(name) + result = getPragmaNodeFromTypeSym(sym).getPragmasByName(name) of nskParam: # When a typedesc parameter is passed to the macro, it will be of nskParam. let typeInst = getTypeInst(sym) if typeInst.kind == nnkBracketExpr and eqIdent(typeInst[0], "typeDesc"): - result = getPragmaNodeFromTypeSym(typeInst[1]).getPragmaByName(name) + result = getPragmaNodeFromTypeSym(typeInst[1]).getPragmasByName(name) else: error("illegal sym kind for argument: " & $sym.symKind, sym) of nskVar, nskLet: - # I think it is a bad idea to fall back to the typeSym. The API - # explicity requests a var/let symbol, not a type symbol. - result = getPragmaNodeFromVarLetSym(sym).getPragmaByName(name) or - getPragmaNodeFromTypeSym(sym.getTypeInst).getPragmaByName(name) + # This checks the type of the sym too, this is consistent with how + # field expressions are handled too. If this is changed, make sure to + # change it for fields expressions too. + result = getPragmaNodeFromType(sym.getTypeInst).getPragmasByName(name) + result.add getPragmaNodeFromVarLetSym(sym).getPragmasByName(name) else: error("illegal sym kind for argument: " & $sym.symKind, sym) since (1, 5): - export getCustomPragmaNode + export getCustomPragmaNodes proc hasCustomPragma*(n: NimNode, name: string): bool = n.expectKind nnkSym - let pragmaNode = getCustomPragmaNode(n, name) - result = pragmaNode != nil + result = getCustomPragmaNodes(n, name).len > 0 -proc getCustomPragmaNodeSmart(n: NimNode, name: string): NimNode = +proc getCustomPragmaNodesSmart(n: NimNode, name: string): seq[NimNode] = case n.kind of nnkDotExpr: - result = getCustomPragmaNode(n[1], name) + result = getCustomPragmaNodes(n[1], name) of nnkCheckedFieldExpr: expectKind n[0], nnkDotExpr - result = getCustomPragmaNode(n[0][1], name) + result = getCustomPragmaNodes(n[0][1], name) of nnkSym: - result = getCustomPragmaNode(n, name) + result = getCustomPragmaNodes(n, name) of nnkTypeOfExpr: - var typeSym = n.getTypeInst - while typeSym.kind == nnkBracketExpr and typeSym[0].eqIdent "typeDesc": - typeSym = typeSym[1] - case typeSym.kind: - of nnkSym: - result = getCustomPragmaNode(typeSym, name) - of nnkProcTy: - # It is a bad idea to support this. The annotation can't be part - # of a symbol. - let pragmaExpr = typeSym[1] - result = getPragmaByName(pragmaExpr, name) - else: - typeSym.expectKind nnkSym + var typ = n.getTypeInst + while typ.kind == nnkBracketExpr and typ[0].eqIdent "typeDesc": + typ = typ[1] + result = getPragmaNodeFromType(typ).getPragmasByName(name) of nnkBracketExpr: - result = nil #false + discard else: n.expectKind({nnkDotExpr, nnkCheckedFieldExpr, nnkSym, nnkTypeOfExpr}) @@ -1633,7 +1632,7 @@ macro hasCustomPragma*(n: typed, cp: typed{nkSym}): bool = ## var o: MyObj ## assert(o.myField.hasCustomPragma(myAttr)) ## assert(myProc.hasCustomPragma(myAttr)) - result = newLit(getCustomPragmaNodeSmart(n, $cp) != nil) + result = newLit(getCustomPragmaNodesSmart(n, $cp).len > 0) iterator iterOverFormalArgs(f: NimNode): tuple[name, typ, val: NimNode] = f.expectKind nnkFormalParams @@ -1644,7 +1643,7 @@ iterator iterOverFormalArgs(f: NimNode): tuple[name, typ, val: NimNode] = for j in 0.. Date: Wed, 21 Apr 2021 16:03:30 +0200 Subject: [PATCH 0695/2472] Revert localErrorNode param order changes (#17809) * Revert localErrorNode param order changes * Remove unused globalError overload * heh --- compiler/msgs.nim | 3 --- compiler/semdata.nim | 18 ++++++++++++++---- compiler/semexprs.nim | 2 +- compiler/semmagic.nim | 8 ++++---- compiler/semstmts.nim | 2 +- 5 files changed, 20 insertions(+), 13 deletions(-) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index b7b6a582e..f08722cc7 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -585,9 +585,6 @@ template globalError*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") template globalError*(conf: ConfigRef; info: TLineInfo, arg: string) = liMessage(conf, info, errGenerated, arg, doRaise, instLoc()) -template globalError*(conf: ConfigRef; format: string, params: openArray[string]) = - liMessage(conf, unknownLineInfo, errGenerated, format % params, doRaise, instLoc()) - template localError*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") = liMessage(conf, info, msg, arg, doNothing, instLoc()) diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 9a17efb1c..1c1a5159c 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -501,13 +501,23 @@ proc errorNode*(c: PContext, n: PNode): PNode = result = newNodeI(nkEmpty, n.info) result.typ = errorType(c) -template localErrorNode*(c: PContext, n: PNode, arg: string, info: TLineInfo, kind: TMsgKind = errGenerated): PNode = - liMessage(c.config, info, kind, arg, doNothing, instLoc()) +# These mimic localError +template localErrorNode*(c: PContext, n: PNode, info: TLineInfo, msg: TMsgKind, arg: string): PNode = + liMessage(c.config, info, msg, arg, doNothing, instLoc()) errorNode(c, n) -template localErrorNode*(c: PContext, n: PNode, arg: string, kind: TMsgKind = errGenerated): PNode = +template localErrorNode*(c: PContext, n: PNode, info: TLineInfo, arg: string): PNode = + liMessage(c.config, info, errGenerated, arg, doNothing, instLoc()) + errorNode(c, n) + +template localErrorNode*(c: PContext, n: PNode, msg: TMsgKind, arg: string): PNode = let n2 = n - liMessage(c.config, n2.info, kind, arg, doNothing, instLoc()) + liMessage(c.config, n2.info, msg, arg, doNothing, instLoc()) + errorNode(c, n2) + +template localErrorNode*(c: PContext, n: PNode, arg: string): PNode = + let n2 = n + liMessage(c.config, n2.info, errGenerated, arg, doNothing, instLoc()) errorNode(c, n2) proc fillTypeS*(dest: PType, kind: TTypeKind, c: PContext) = diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index defceb291..353831736 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -2620,7 +2620,7 @@ proc semTupleConstr(c: PContext, n: PNode, flags: TExprFlags): PNode = # check if either everything or nothing is tyTypeDesc for i in 1.. high(TSymChoiceRule).int: - return localErrorNode(c, n, errConstExprExpected, n[2].info) + return localErrorNode(c, n, n[2].info, errConstExprExpected) let id = newIdentNode(getIdent(c.cache, sl.strVal), n.info) let s = qualifiedLookUp(c, id, {checkUndeclared}) @@ -251,10 +251,10 @@ proc semBindSym(c: PContext, n: PNode): PNode = proc opBindSym(c: PContext, scope: PScope, n: PNode, isMixin: int, info: PNode): PNode = if n.kind notin {nkStrLit, nkRStrLit, nkTripleStrLit, nkIdent}: - return localErrorNode(c, n, errStringOrIdentNodeExpected, info.info) + return localErrorNode(c, n, info.info, errStringOrIdentNodeExpected) if isMixin < 0 or isMixin > high(TSymChoiceRule).int: - return localErrorNode(c, n, errConstExprExpected, info.info) + return localErrorNode(c, n, info.info, errConstExprExpected) let id = if n.kind == nkIdent: n else: newIdentNode(getIdent(c.cache, n.strVal), info.info) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 22e4295fa..d56c1c1bd 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -725,7 +725,7 @@ proc semForVars(c: PContext, n: PNode; flags: TExprFlags): PNode = if n.len == 3: if n[0].kind == nkVarTuple: if n[0].len-1 != iterAfterVarLent.len: - return localErrorNode(c, n, errWrongNumberOfVariables, n[0].info) + return localErrorNode(c, n, n[0].info, errWrongNumberOfVariables) for i in 0.. Date: Wed, 21 Apr 2021 17:57:54 +0300 Subject: [PATCH 0696/2472] add RST highlighting for command line / shells (also fixes #16858) (#17789) --- config/nimdoc.tex.cfg | 2 + doc/contributing.rst | 177 +++++++++++---------- doc/docstyle.rst | 16 +- doc/nimdoc.css | 31 +++- lib/packages/docutils/highlite.nim | 83 +++++++++- lib/packages/docutils/rst.nim | 49 +++--- lib/packages/docutils/rstast.nim | 4 +- lib/packages/docutils/rstgen.nim | 18 ++- nimdoc/testproject/expected/nimdoc.out.css | 31 +++- tests/stdlib/trstgen.nim | 20 ++- 10 files changed, 291 insertions(+), 140 deletions(-) diff --git a/config/nimdoc.tex.cfg b/config/nimdoc.tex.cfg index 69266f85d..471f20dc1 100644 --- a/config/nimdoc.tex.cfg +++ b/config/nimdoc.tex.cfg @@ -137,6 +137,8 @@ bottomline=false} \newcommand{\spanReference}[1]{#1} \newcommand{\spanOther}[1]{#1} \newcommand{\spantok}[1]{\frame{#1}} +\newcommand{\spanprogram}[1]{\textbf{\underline{#1}}} +\newcommand{\spanoption}[1]{\textbf{#1}} $content \end{document} diff --git a/doc/contributing.rst b/doc/contributing.rst index 3d3da2ce0..81fa9f8eb 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -1,9 +1,10 @@ -.. default-role:: code - ============ Contributing ============ +.. default-role:: code +.. include:: rstcommon.rst + .. contents:: @@ -19,21 +20,23 @@ Writing tests There are 4 types of tests: -1. `runnableExamples` documentation comment tests, ran by `nim doc mymod.nim` +1. `runnableExamples` documentation comment tests, ran by `nim doc mymod.nim`:cmd: These end up in documentation and ensure documentation stays in sync with code. -2. separate test files, e.g.: `tests/stdlib/tos.nim`. - In nim repo, `testament` (see below) runs all `$nim/tests/*/t*.nim` test files; +2. separate test files, e.g.: ``tests/stdlib/tos.nim``. + In nim repo, `testament`:cmd: (see below) runs all + ``$nim/tests/*/t*.nim`` test files; for nimble packages, see https://github.com/nim-lang/nimble#tests. -3. (deprecated) tests in `when isMainModule:` block, ran by `nim r mymod.nim`. - `nimble test` can run those in nimble packages when specified in a +3. (deprecated) tests in `when isMainModule:` block, ran by `nim r mymod.nim`:cmd:. + `nimble test`:cmd: can run those in nimble packages when specified in a `task "test"`. -4. (not preferred) `.. code-block:: nim` RST snippets; these should only be used in rst sources, +4. (not preferred) ``.. code-block:: nim`` RST snippets; + these should only be used in rst sources, in nim sources `runnableExamples` should now always be preferred to those for several reasons (cleaner syntax, syntax highlights, batched testing, and - `rdoccmd` allows customization). + parameter `rdoccmd` allows customization). Not all the tests follow the convention here, feel free to change the ones that don't. Always leave the code cleaner than you found it. @@ -41,8 +44,8 @@ that don't. Always leave the code cleaner than you found it. Stdlib ------ -Each stdlib module (anything under `lib/`, e.g. `lib/pure/os.nim`) should -preferably have a corresponding separate test file, e.g. `tests/stdlib/tos.nim`. +Each stdlib module (anything under ``lib/``, e.g. ``lib/pure/os.nim``) should +preferably have a corresponding separate test file, e.g. ``tests/stdlib/tos.nim``. The old convention was to add a `when isMainModule:` block in the source file, which only gets executed when the tester is building the file. @@ -71,36 +74,36 @@ Sample test: # doAssert with `not` can now be done as follows: doAssert not (1 == 2) -Always refer to a GitHub issue using the following exact syntax: `bug #1234` as shown +Always refer to a GitHub issue using the following exact syntax: ``bug #1234`` as shown above, so that it's consistent and easier to search or for tooling. Some browser extensions (e.g. https://github.com/sindresorhus/refined-github) will even turn those in clickable links when it works. Rationale for using a separate test file instead of `when isMainModule:` block: * allows custom compiler flags or testing options (see details below) -* faster CI since they can be joined in `megatest` (combined into a single test) +* faster CI since they can be joined in ``megatest`` (combined into a single test) * avoids making the parser do un-necessary work when a source file is merely imported * avoids mixing source and test code when reporting line of code statistics or code coverage Compiler -------- -The tests for the compiler use a testing tool called `testament`. They are all -located in `tests/` (e.g.: `tests/destructor/tdestructor3.nim`). +The tests for the compiler use a testing tool called `testament`:cmd:. They are all +located in ``tests/`` (e.g.: ``tests/destructor/tdestructor3.nim``). Each test has its own file. All test files are prefixed with `t`. If you want to create a file for import into another test only, use the prefix `m`. At the beginning of every test is the expected behavior of the test. Possible keys are: -- `cmd`: A compilation command template e.g. `nim $target --threads:on $options $file` +- `cmd`: A compilation command template e.g. `nim $target --threads:on $options $file`:cmd: - `output`: The expected output (stdout + stderr), most likely via `echo` - `exitcode`: Exit code of the test (via `exit(number)`) - `errormsg`: The expected compiler error message - `file`: The file the errormsg was produced at - `line`: The line the errormsg was produced at -For a full spec, see here: `testament/specs.nim` +For a full spec, see here: ``testament/specs.nim`` An example of a test: @@ -124,51 +127,51 @@ Running tests You can run the tests with -:: +.. code-block:: cmd ./koch tests which will run a good subset of tests. Some tests may fail. If you only want to see the output of failing tests, go for -:: - +```cmd ./koch tests --failing all +``` You can also run only a single category of tests. A category is a subdirectory -in the `tests` directory. There are a couple of special categories; for a -list of these, see `testament/categories.nim`, at the bottom. +in the ``tests/`` directory. There are a couple of special categories; for a +list of these, see ``testament/categories.nim``, at the bottom. -:: +.. code:: cmd - ./koch tests c lib # compiles/runs stdlib modules, including `isMainModule` tests + ./koch tests c lib # compiles / runs stdlib modules, including `isMainModule` tests ./koch tests c megatest # runs a set of tests that can be combined into 1 To run a single test: -:: +.. code:: cmd ./koch test run / # e.g.: tuples/ttuples_issues ./koch test run tests/stdlib/tos.nim # can also provide relative path For reproducible tests (to reproduce an environment more similar to the one run by Continuous Integration on travis/appveyor), you may want to disable your -local configuration (e.g. in `~/.config/nim/nim.cfg`) which may affect some +local configuration (e.g. in ``~/.config/nim/nim.cfg``) which may affect some tests; this can also be achieved by using -`export XDG_CONFIG_HOME=pathtoAlternateConfig` before running `./koch` +`export XDG_CONFIG_HOME=pathtoAlternateConfig`:cmd: before running `./koch`:cmd: commands. Comparing tests =============== -Test failures can be grepped using `Failure:`. +Test failures can be grepped using ``Failure:``. The tester can compare two test runs. First, you need to create a reference test. You'll also need to the commit id, because that's what the tester needs to know in order to compare the two. -:: +.. code:: cmd git checkout devel DEVEL_COMMIT=$(git rev-parse HEAD) @@ -176,15 +179,15 @@ the tester needs to know in order to compare the two. Then switch over to your changes and run the tester again. -:: +.. code:: cmd git checkout your-changes ./koch tests -Then you can ask the tester to create a `testresults.html` which will +Then you can ask the tester to create a ``testresults.html`` which will tell you if any new tests passed/failed. -:: +.. code:: cmd ./koch tests --print html $DEVEL_COMMIT @@ -219,14 +222,14 @@ Documentation When contributing new procs, be sure to add documentation, especially if the proc is public. Even private procs benefit from documentation and can be -viewed using `nim doc --docInternal foo.nim`. +viewed using `nim doc --docInternal foo.nim`:cmd:. Documentation begins on the line following the `proc` definition, and is prefixed by `##` on each line. Runnable code examples are also encouraged, to show typical behavior with a few test cases (typically 1 to 3 `assert` statements, depending on complexity). -These `runnableExamples` are automatically run by `nim doc mymodule.nim` -as well as `testament` and guarantee they stay in sync. +These `runnableExamples` are automatically run by `nim doc mymodule.nim`:cmd: +as well as `testament`:cmd: and guarantee they stay in sync. .. code-block:: nim proc addBar*(a: string): string = @@ -238,7 +241,7 @@ as well as `testament` and guarantee they stay in sync. See `parentDir `_ example. The RestructuredText Nim uses has a special syntax for including code snippets -embedded in documentation; these are not run by `nim doc` and therefore are +embedded in documentation; these are not run by `nim doc`:cmd: and therefore are not guaranteed to stay in sync, so `runnableExamples` is almost always preferred: .. code-block:: nim @@ -250,9 +253,9 @@ not guaranteed to stay in sync, so `runnableExamples` is almost always preferred ## echo someProc() # "something" result = "something" # single-hash comments do not produce documentation -The `.. code-block:: nim` followed by a newline and an indentation instructs the -`nim doc` command to produce syntax-highlighted example code with the -documentation (`.. code-block::` is sufficient from inside a nim module). +The ``.. code-block:: nim`` followed by a newline and an indentation instructs the +`nim doc`:cmd: command to produce syntax-highlighted example code with the +documentation (``.. code-block::`` is sufficient from inside a nim module). When forward declaration is used, the documentation should be included with the first appearance of the proc. @@ -298,10 +301,10 @@ example below) from `Nim Index`_ can be used in doc comment this way: Inline monospaced text can be input using \`single backticks\` or \`\`double backticks\`\`. The former are syntactically highlighted, the latter are not. -To avoid accidental highlighting follow this rule in `*.nim` files: +To avoid accidental highlighting follow this rule in ``*.nim`` files: * use single backticks for fragments of code in Nim and other - programming languages, including identifiers, in `*.nim` files. + programming languages, including identifiers, in ``*.nim`` files. For languages other than Nim add a role after final backtick, e.g. for C++ inline highlighting:: @@ -313,18 +316,20 @@ To avoid accidental highlighting follow this rule in `*.nim` files: `SELECT * FROM ;`:code: + Highlight shell commands by ``:cmd:`` role; for command line options use + ``:option:`` role, e.g.: \`--docInternal\`:option:. + * prefer double backticks otherwise: * for file names: \`\`os.nim\`\` * for fragments of strings **not** enclosed by `"` and `"` and not related to code, e.g. text of compiler messages - * for command line options: \`\`--docInternal\`\` * also when code ends with a standalone ``\`` (otherwise a combination of ``\`` and a final \` would get escaped) -.. Note:: `*.rst` files have `:literal:` as their default role. - So for them the rule above is only applicable if the `:nim:` role - is set up manually as the default:: +.. Note:: ``*.rst`` files have ``:literal:`` as their default role. + So for them the rule above is only applicable if the ``:nim:`` role + is set up manually as the default [*]_:: .. role:: nim(code) :language: nim @@ -333,6 +338,8 @@ To avoid accidental highlighting follow this rule in `*.nim` files: The first 2 lines are for other RST implementations, including Github one. + .. [*] this is fulfilled when ``doc/rstcommon.rst`` is included. + Best practices ============== @@ -350,7 +357,7 @@ to avoid name conflicts across packages. # if in nim sources when defined(allocStats): discard # bad, can cause conflicts when defined(nimAllocStats): discard # preferred - # if in a pacakge `cligen`: + # if in a package `cligen`: when defined(debug): discard # bad, can cause conflicts when defined(cligenDebug): discard # preferred @@ -372,7 +379,7 @@ Design with method call syntax chaining in mind # can be called as: `getLines().foo(false)` .. _avoid_quit: -Use exceptions (including assert / doAssert) instead of `quit` +Use exceptions (including `assert` / `doAssert`) instead of `quit` rationale: https://forum.nim-lang.org/t/4089 .. code-block:: nim @@ -382,7 +389,7 @@ rationale: https://forum.nim-lang.org/t/4089 .. _tests_use_doAssert: Use `doAssert` (or `unittest.check`, `unittest.require`), not `assert` in all -tests so they'll be enabled even with `--assertions:off`. +tests so they'll be enabled even with `--assertions:off`:option:. .. code-block:: nim @@ -391,10 +398,10 @@ tests so they'll be enabled even with `--assertions:off`. doAssert foo() # preferred .. _runnableExamples_use_assert: -An exception to the above rule is `runnableExamples` and `code-block` rst blocks +An exception to the above rule is `runnableExamples` and ``code-block`` rst blocks intended to be used as `runnableExamples`, which for brevity use `assert` -instead of `doAssert`. Note that `nim doc -d:danger main` won't pass `-d:danger` to the -`runnableExamples`, but `nim doc --doccmd:-d:danger main` would, and so would the +instead of `doAssert`. Note that `nim doc -d:danger main`:cmd: won't pass `-d:danger`:option: to the +`runnableExamples`, but `nim doc --doccmd:-d:danger main`:cmd: would, and so would the second example below: .. code-block:: nim @@ -437,8 +444,8 @@ https://github.com/nim-lang/Nim/pull/9335 and https://forum.nim-lang.org/t/4089 doAssert foo() == [1, 2] # preferred, except when not possible to do so. -The Git stuff -============= +The `git`:cmd: stuff +==================== General commit rules -------------------- @@ -446,12 +453,12 @@ General commit rules 1. Important, critical bugfixes that have a tiny chance of breaking somebody's code should be backported to the latest stable release branch (currently 1.4.x) and maybe also all the way back to the 1.0.x branch. - The commit message should contain the tag `[backport]` for "backport to all - stable releases" and the tag `[backport:$VERSION]` for backporting to the + The commit message should contain the tag ``[backport]`` for "backport to all + stable releases" and the tag ``[backport:$VERSION]`` for backporting to the given $VERSION. 2. If you introduce changes which affect backward compatibility, - make breaking changes, or have PR which is tagged as `[feature]`, + make breaking changes, or have PR which is tagged as ``[feature]``, the changes should be mentioned in `the changelog `_. @@ -462,29 +469,29 @@ General commit rules your editor reformatted automatically the code or whatever different reason, this should be excluded from the commit. - *Tip:* Never commit everything as is using `git commit -a`, but review - carefully your changes with `git add -p`. + *Tip:* Never commit everything as is using `git commit -a`:cmd:, but review + carefully your changes with `git add -p`:cmd:. 4. Changes should not introduce any trailing whitespace. - Always check your changes for whitespace errors using `git diff --check` - or add the following `pre-commit` hook: + Always check your changes for whitespace errors using `git diff --check`:cmd: + or add the following ``pre-commit`` hook: - .. code-block:: sh + .. code:: cmd #!/bin/sh git diff --check --cached || exit $? 5. Describe your commit and use your common sense. - Example commit message: + Example commit message:: - `Fixes #123; refs #124` + Fixes #123; refs #124 - indicates that issue `#123` is completely fixed (GitHub may automatically - close it when the PR is committed), wheres issue `#124` is referenced + indicates that issue ``#123`` is completely fixed (GitHub may automatically + close it when the PR is committed), wheres issue ``#124`` is referenced (e.g.: partially fixed) and won't close the issue when committed. 6. PR body (not just PR title) should contain references to fixed/referenced github - issues, e.g.: `fix #123` or `refs #123`. This is so that you get proper cross + issues, e.g.: ``fix #123`` or ``refs #123``. This is so that you get proper cross referencing from linked issue to the PR (github won't make those links with just PR title, and commit messages aren't always sufficient to ensure that, e.g. can't be changed after a PR is merged). @@ -492,7 +499,7 @@ General commit rules 7. Commits should be always be rebased against devel (so a fast forward merge can happen) - e.g.: use `git pull --rebase origin devel`. This is to avoid messing up + e.g.: use `git pull --rebase origin devel`:cmd:. This is to avoid messing up git history. Exceptions should be very rare: when rebase gives too many conflicts, simply squash all commits using the script shown in @@ -508,7 +515,7 @@ Continuous Integration (CI) 1. Continuous Integration is by default run on every push in a PR; this clogs the CI pipeline and affects other PR's; if you don't need it (e.g. for WIP or - documentation only changes), add `[skip ci]` to your commit message title. + documentation only changes), add ``[skip ci]`` to your commit message title. This convention is supported by our github actions pipelines and our azure pipeline as well as our former other pipelines: `Appveyor `_ @@ -531,16 +538,16 @@ Debugging CI failures, flaky tests, etc will re-trigger all CI jobs (even successful ones, which can be wasteful). Instead, follow these instructions to only restart the jobs that failed: - * Azure: if on your own fork, it's possible from inside azure console - (e.g. `dev.azure.com/username/username/_build/results?buildId=1430&view=results`) via `rerun failed jobs` on top. - If either on you own fork or in Nim repo, it's possible from inside GitHub UI - under checks tab, see https://github.com/timotheecour/Nim/issues/211#issuecomment-629751569 - * GitHub actions: under "Checks" tab, click "Re-run jobs" in the right. - * builds.sr.ht: create a sourcehut account so you can restart a PR job as illustrated. - builds.sr.ht also allows you to ssh to a CI machine which can help a lot for debugging - issues, see docs in https://man.sr.ht/builds.sr.ht/build-ssh.md and - https://drewdevault.com/2019/08/19/Introducing-shell-access-for-builds.html; see - https://man.sr.ht/tutorials/set-up-account-and-git.md to generate and upload ssh keys. + * Azure: if on your own fork, it's possible from inside azure console + (e.g. ``dev.azure.com/username/username/_build/results?buildId=1430&view=results``) via ``rerun failed jobs`` on top. + If either on you own fork or in Nim repo, it's possible from inside GitHub UI + under checks tab, see https://github.com/timotheecour/Nim/issues/211#issuecomment-629751569 + * GitHub actions: under "Checks" tab, click "Re-run jobs" in the right. + * builds.sr.ht: create a sourcehut account so you can restart a PR job as illustrated. + builds.sr.ht also allows you to ssh to a CI machine which can help a lot for debugging + issues, see docs in https://man.sr.ht/builds.sr.ht/build-ssh.md and + https://drewdevault.com/2019/08/19/Introducing-shell-access-for-builds.html; see + https://man.sr.ht/tutorials/set-up-account-and-git.md to generate and upload ssh keys. Code reviews @@ -554,15 +561,15 @@ Code reviews doesn't help much as it doesn't highlight moves. Instead, you can use something like this, see visual results `here `_: - .. code-block:: sh + .. code:: cmd git fetch origin pull/10431/head && git checkout FETCH_HEAD git diff --color-moved-ws=allow-indentation-change --color-moved=blocks HEAD^ 3. In addition, you can view GitHub-like diffs locally to identify what was changed - within a code block using `diff-highlight` or `diff-so-fancy`, e.g.: + within a code block using `diff-highlight`:cmd: or `diff-so-fancy`:cmd:, e.g.: - .. code-block:: sh + :: # put this in ~/.gitconfig: [core] @@ -651,15 +658,15 @@ to existing modules is acceptable. For two reasons: Conventions ----------- -1. New stdlib modules should go under `Nim/lib/std/`. The rationale is to +1. New stdlib modules should go under ``Nim/lib/std/``. The rationale is to require users to import via `import std/foo` instead of `import foo`, which would cause potential conflicts with nimble packages. Note that this still applies for new modules in existing logical - directories, e.g.: use `lib/std/collections/foo.nim`, - not `lib/pure/collections/foo.nim`. + directories, e.g.: use ``lib/std/collections/foo.nim``, + not ``lib/pure/collections/foo.nim``. 2. New module names should prefer plural form whenever possible, e.g.: - `std/sums.nim` instead of `std/sum.nim`. In particular, this reduces + ``std/sums.nim`` instead of ``std/sum.nim``. In particular, this reduces chances of conflicts between module name and the symbols it defines. Furthermore, module names should use `snake_case` and not use capital letters, which cause issues when going from an OS without case diff --git a/doc/docstyle.rst b/doc/docstyle.rst index 7f3fa8cf2..df1f36dad 100644 --- a/doc/docstyle.rst +++ b/doc/docstyle.rst @@ -6,17 +6,17 @@ General Guidelines * See also `nep1`_ which should probably be merged here. * Authors should document anything that is exported; documentation for private - procs can be useful too (visible via `nim doc --docInternal foo.nim`). + procs can be useful too (visible via `nim doc --docInternal foo.nim`:cmd:). * Within documentation, a period (`.`) should follow each sentence (or sentence fragment) in a comment block. The documentation may be limited to one sentence fragment, but if multiple sentences are within the documentation, each sentence after the first should be complete and in present tense. * Documentation is parsed as a custom ReStructuredText (RST) with partial markdown support. * In nim sources, prefer single backticks to double backticks since it's simpler - and `nim doc` supports it. Likewise with rst files: `nim rst2html` will render those as monospace, and - adding `.. default-role:: code` to an rst file will also make those render as monospace when rendered directly + and `nim doc`:cmd: supports it. Likewise with ``rst`` files: `nim rst2html`:cmd: will render those as monospace, and + adding ``.. default-role:: code`` to an ``rst`` file will also make those render as monospace when rendered directly in tools such as github. -* In nim sources, for links, prefer `[link text](link.html)` to `` `link text`_ `` - since the syntax is simpler and markdown is more common (likewise, `nim rst2html` also supports it in rst files). +* (debatable) In nim sources, for links, prefer ``[link text](link.html)`` to `\`link text\`_`:code: + since the syntax is simpler and markdown is more common (likewise, `nim rst2html`:cmd: also supports it in ``rst`` files). .. code-block:: nim @@ -29,7 +29,7 @@ Module-level documentation -------------------------- Documentation of a module is placed at the top of the module itself. Each line of documentation begins with double hashes (`##`). -Sometimes `##[ multiline docs containing code ]##` is preferable, see `lib/pure/times.nim`. +Sometimes `##[ multiline docs containing code ]##` is preferable, see ``lib/pure/times.nim``. Code samples are encouraged, and should follow the general RST syntax: .. code-block:: Nim @@ -76,11 +76,11 @@ Whenever an example of usage would be helpful to the user, you should include on ## echo execCmdEx("git pull") ## drawOnScreen() runnableExamples: - # `runnableExamples` is usually preferred to `code-block`, when possible. + # `runnableExamples` is usually preferred to ``code-block``, when possible. doAssert addThree(3, 125, 6) == -122 result = x +% y +% z -The command `nim doc` will then correctly syntax highlight the Nim code within the documentation. +The command `nim doc`:cmd: will then correctly syntax highlight the Nim code within the documentation. Types ----- diff --git a/doc/nimdoc.css b/doc/nimdoc.css index ced791d16..6366d33dc 100644 --- a/doc/nimdoc.css +++ b/doc/nimdoc.css @@ -35,6 +35,8 @@ Modified by Boyd Greenfield and narimiran --escapeSequence: #c4891b; --number: #252dbe; --literal: #a4255b; + --program: #6060c0; + --option: #508000; --raw-data: #a4255b; } @@ -63,6 +65,8 @@ Modified by Boyd Greenfield and narimiran --escapeSequence: #bd93f9; --number: #bd93f9; --literal: #f1fa8c; + --program: #9090c0; + --option: #90b010; --raw-data: #8be9fd; } @@ -527,7 +531,6 @@ div.option-list-label { margin-left: -11.5em; margin-right: 0em; min-width: 11.5em; - font-weight: bolder; display: inline-block; vertical-align: top; } @@ -546,7 +549,7 @@ blockquote { border-left: 5px solid #bbc; } -.pre { +.pre, span.tok { font-family: "Source Code Pro", Monaco, Menlo, Consolas, "Courier New", monospace; font-weight: 500; font-size: 0.85em; @@ -557,6 +560,12 @@ blockquote { border-radius: 4px; } +span.tok { + border: 1px solid #808080; + padding-bottom: 0.1em; + margin-right: 0.2em; +} + pre { font-family: "Source Code Pro", Monaco, Menlo, Consolas, "Courier New", monospace; color: var(--text); @@ -844,9 +853,6 @@ span.classifier { span.classifier-delimiter { font-weight: bold; } -span.option { - white-space: nowrap; } - span.problematic { color: #b30000; } @@ -926,6 +932,21 @@ span.Preprocessor { span.Directive { color: #252dbe; } +span.option { + font-weight: bold; + font-family: "Source Code Pro", Monaco, Menlo, Consolas, "Courier New", monospace; + color: var(--option); +} + +span.program { + font-weight: bold; + color: var(--program); + text-decoration: underline; + text-decoration-color: var(--hint); + text-decoration-thickness: 0.05em; + text-underline-offset: 0.15em; +} + span.Command, span.Rule, span.Hyperlink, span.Label, span.Reference, span.Other { color: var(--other); } diff --git a/lib/packages/docutils/highlite.nim b/lib/packages/docutils/highlite.nim index 94cb2ebde..c0f4c9760 100644 --- a/lib/packages/docutils/highlite.nim +++ b/lib/packages/docutils/highlite.nim @@ -37,6 +37,18 @@ ## .. code:: Nim ## for l in ["C", "c++", "jAvA", "Nim", "c#"]: echo getSourceLanguage(l) ## +## There is also a `Cmd` pseudo-language supported, which is a simple generic +## shell/cmdline tokenizer (UNIX shell/Powershell/Windows Command): +## no escaping, no programming language constructs besides variable definition +## at the beginning of line. It supports these operators: +## +## .. code:: Cmd +## & && | || ( ) '' "" ; # for comments +## +## Instead of escaping always use quotes like here +## `nimgrep --ext:'nim|nims' file.name`:cmd: shows how to input ``|``. +## Any argument that contains ``.`` or ``/`` or ``\`` will be treated +## as a file or directory. import strutils @@ -45,7 +57,7 @@ from algorithm import binarySearch type SourceLanguage* = enum langNone, langNim, langCpp, langCsharp, langC, langJava, - langYaml, langPython + langYaml, langPython, langCmd TokenClass* = enum gtEof, gtNone, gtWhitespace, gtDecNumber, gtBinNumber, gtHexNumber, gtOctNumber, gtFloatNumber, gtIdentifier, gtKeyword, gtStringLit, @@ -53,7 +65,7 @@ type gtOperator, gtPunctuation, gtComment, gtLongComment, gtRegularExpression, gtTagStart, gtTagEnd, gtKey, gtValue, gtRawData, gtAssembler, gtPreprocessor, gtDirective, gtCommand, gtRule, gtHyperlink, gtLabel, - gtReference, gtOther + gtReference, gtProgram, gtOption, gtOther GeneralTokenizer* = object of RootObj kind*: TokenClass start*, length*: int @@ -64,14 +76,17 @@ type const sourceLanguageToStr*: array[SourceLanguage, string] = ["none", - "Nim", "C++", "C#", "C", "Java", "Yaml", "Python"] + "Nim", "C++", "C#", "C", "Java", "Yaml", "Python", "Cmd"] tokenClassToStr*: array[TokenClass, string] = ["Eof", "None", "Whitespace", "DecNumber", "BinNumber", "HexNumber", "OctNumber", "FloatNumber", "Identifier", "Keyword", "StringLit", "LongStringLit", "CharLit", "EscapeSequence", "Operator", "Punctuation", "Comment", "LongComment", "RegularExpression", "TagStart", "TagEnd", "Key", "Value", "RawData", "Assembler", "Preprocessor", "Directive", "Command", "Rule", "Hyperlink", - "Label", "Reference", "Other"] + "Label", "Reference", + # start from lower-case if there is a corresponding RST role (see rst.nim) + "program", "option", + "Other"] # The following list comes from doc/keywords.txt, make sure it is # synchronized with this array by running the module itself as a test case. @@ -898,6 +913,65 @@ proc pythonNextToken(g: var GeneralTokenizer) = "with", "yield"] nimNextToken(g, keywords) +proc cmdNextToken(g: var GeneralTokenizer) = + var pos = g.pos + g.start = g.pos + if g.state == low(TokenClass): + g.state = gtProgram + case g.buf[pos] + of ' ', '\t'..'\r': + g.kind = gtWhitespace + while g.buf[pos] in {' ', '\t'..'\r'}: + if g.buf[pos] == '\n': + g.state = gtProgram + inc(pos) + of '\'', '"': + g.kind = gtOption + let q = g.buf[pos] + inc(pos) + while g.buf[pos] notin {q, '\0'}: + inc(pos) + if g.buf[pos] == q: inc(pos) + of '#': + g.kind = gtComment + while g.buf[pos] notin {'\n', '\0'}: + inc(pos) + of '&', '|': + g.kind = gtOperator + inc(pos) + if g.buf[pos] == g.buf[pos-1]: inc(pos) + g.state = gtProgram + of '(': + g.kind = gtOperator + g.state = gtProgram + inc(pos) + of ')': + g.kind = gtOperator + inc(pos) + of ';': + g.state = gtProgram + g.kind = gtOperator + inc(pos) + of '\0': g.kind = gtEof + else: + if g.state == gtProgram: + g.kind = gtProgram + g.state = gtOption + else: + g.kind = gtOption + while g.buf[pos] notin {' ', '\t'..'\r', '&', '|', '(', ')', '\'', '"', '\0'}: + if g.buf[pos] == ';' and g.buf[pos+1] == ' ': + # (check space because ';' can be used inside arguments in Win bat) + break + if g.kind == gtOption and g.buf[pos] in {'/', '\\', '.'}: + g.kind = gtIdentifier # for file/dir name + elif g.kind == gtProgram and g.buf[pos] == '=': + g.kind = gtIdentifier # for env variable setting at beginning of line + g.state = gtProgram + inc(pos) + g.length = pos - g.pos + g.pos = pos + proc getNextToken*(g: var GeneralTokenizer, lang: SourceLanguage) = g.lang = lang case lang @@ -909,6 +983,7 @@ proc getNextToken*(g: var GeneralTokenizer, lang: SourceLanguage) = of langJava: javaNextToken(g) of langYaml: yamlNextToken(g) of langPython: pythonNextToken(g) + of langCmd: cmdNextToken(g) when isMainModule: var keywords: seq[string] diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 7c4b92f88..cb65791ff 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -23,10 +23,10 @@ ## ## Nim can output the result to HTML [#html]_ or Latex [#latex]_. ## -## .. [#html] commands ``nim doc`` for ``*.nim`` files and -## ``nim rst2html`` for ``*.rst`` files +## .. [#html] commands `nim doc`:cmd: for ``*.nim`` files and +## `nim rst2html`:cmd: for ``*.rst`` files ## -## .. [#latex] command ``nim rst2tex`` for ``*.rst``. +## .. [#latex] command `nim rst2tex`:cmd: for ``*.rst``. ## ## If you are new to RST please consider reading the following: ## @@ -78,14 +78,21 @@ ## ## * directives: ``code-block`` [cmp:Sphinx]_, ``title``, ## ``index`` [cmp:Sphinx]_ -## * predefined roles ``:nim:`` (default), ``:c:`` (C programming language), -## ``:python:``, ``:yaml:``, ``:java:``, ``:cpp:`` (C++), ``:csharp`` (C#). -## That is every language that `highlite `_ supports. -## They turn on appropriate syntax highlighting in inline code. +## * predefined roles +## - ``:nim:`` (default), ``:c:`` (C programming language), +## ``:python:``, ``:yaml:``, ``:java:``, ``:cpp:`` (C++), ``:csharp`` (C#). +## That is every language that `highlite `_ supports. +## They turn on appropriate syntax highlighting in inline code. ## -## .. Note:: default role for Nim files is ``:nim:``, -## for ``*.rst`` it's currently ``:literal:``. +## .. Note:: default role for Nim files is ``:nim:``, +## for ``*.rst`` it's currently ``:literal:``. ## +## - generic command line highlighting roles: +## - ``:cmd:`` for commands and common shells syntax +## - ``:program:`` for executable names [cmp:Sphinx]_ +## (one can just use ``:cmd:`` on single word) +## - ``:option:`` for command line options [cmp:Sphinx]_ +## - ``:tok:``, a role for highlighting of programming language tokens ## * ***triple emphasis*** (bold and italic) using \*\*\* ## * ``:idx:`` role for \`interpreted text\` to include the link to this ## text into an index (example: `Nim index`_). @@ -95,11 +102,11 @@ ## //compile compile the project ## //doc generate documentation ## -## Here the dummy `//` will disappear, while options ``compile`` -## and ``doc`` will be left in the final document. +## Here the dummy `//` will disappear, while options `compile`:option: +## and `doc`:option: will be left in the final document. ## ## .. [cmp:Sphinx] similar but different from the directives of -## Python `Sphinx directives`_ extensions +## Python `Sphinx directives`_ and `Sphinx roles`_ extensions ## ## .. _`extra features`: ## @@ -144,7 +151,7 @@ ## ----- ## ## See `Nim DocGen Tools Guide `_ for the details about -## ``nim doc``, ``nim rst2html`` and ``nim rst2tex`` commands. +## `nim doc`:cmd:, `nim rst2html`:cmd: and `nim rst2tex`:cmd: commands. ## ## See `packages/docutils/rstgen module `_ to know how to ## generate HTML or Latex strings to embed them into your documents. @@ -156,6 +163,7 @@ ## .. _RST roles list: https://docutils.sourceforge.io/docs/ref/rst/roles.html ## .. _Nim index: https://nim-lang.org/docs/theindex.html ## .. _Sphinx directives: https://www.sphinx-doc.org/en/master/usage/restructuredtext/directives.html +## .. _Sphinx roles: https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html import os, strutils, rstast, std/enumutils, algorithm, lists, sequtils, @@ -530,7 +538,7 @@ proc defaultRole(options: RstParseOptions): string = # mirror highlite.nim sourceLanguageToStr with substitutions c++ cpp, c# csharp const supportedLanguages = ["nim", "yaml", "python", "java", "c", - "cpp", "csharp"] + "cpp", "csharp", "cmd"] proc whichRoleAux(sym: string): RstNodeKind = let r = sym.toLowerAscii @@ -543,6 +551,7 @@ proc whichRoleAux(sym: string): RstNodeKind = of "sup", "superscript": result = rnSup # literal and code are the same in our implementation of "code": result = rnInlineLiteral + of "program", "option", "tok": result = rnCodeFragment # c++ currently can be spelled only as cpp, c# only as csharp elif r in supportedLanguages: result = rnInlineCode @@ -1113,10 +1122,10 @@ proc toInlineCode(n: PRstNode, language: string): PRstNode = lb.add newLeaf(s) result.add lb -proc toUnknownRole(n: PRstNode, roleName: string): PRstNode = +proc toOtherRole(n: PRstNode, kind: RstNodeKind, roleName: string): PRstNode = let newN = newRstNode(rnInner, n.sons) let newSons = @[newN, newLeaf(roleName)] - result = newRstNode(rnUnknownRole, newSons) + result = newRstNode(kind, newSons) proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode = var newKind = n.kind @@ -1144,8 +1153,8 @@ proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode = # a role: let (roleName, lastIdx) = getRefname(p, p.idx+1) newKind = whichRole(p, roleName) - if newKind == rnUnknownRole: - result = n.toUnknownRole(roleName) + if newKind in {rnUnknownRole, rnCodeFragment}: + result = n.toOtherRole(newKind, roleName) elif newKind == rnInlineCode: result = n.toInlineCode(language=roleName) else: @@ -1417,8 +1426,8 @@ proc parseInline(p: var RstParser, father: PRstNode) = if k == rnInlineCode: n = n.toInlineCode(language=roleName) parseUntil(p, n, "`", false) # bug #17260 - if k == rnUnknownRole: - n = n.toUnknownRole(roleName) + if k in {rnUnknownRole, rnCodeFragment}: + n = n.toOtherRole(k, roleName) father.add(n) elif isInlineMarkupStart(p, "`"): var n = newRstNode(rnInterpretedText) diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim index 00f3f2b35..81e3ba6d9 100644 --- a/lib/packages/docutils/rstast.nim +++ b/lib/packages/docutils/rstast.nim @@ -56,7 +56,9 @@ type # * `file#id '_ rnSubstitutionDef, # a definition of a substitution # Inline markup: - rnInlineCode, + rnInlineCode, # interpreted text with code in a known language + rnCodeFragment, # inline code for highlighting with the specified + # class (which cannot be inferred from context) rnUnknownRole, # interpreted text with an unknown role rnSub, rnSup, rnIdx, rnEmphasis, # "*" diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index 1b9334a77..40ed88954 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -1198,7 +1198,8 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = "$1", result) of rnOptionGroup: renderAux(d, n, - "
    $1
    ", + "
    " & + "$1
    ", "\\item[$1]", result) of rnDescription: renderAux(d, n, "
    $1
    ", @@ -1319,13 +1320,22 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = renderAux(d, n, "|$1|", "|$1|", result) of rnDirective: renderAux(d, n, "", "", result) - of rnUnknownRole: + of rnUnknownRole, rnCodeFragment: var tmp0 = "" var tmp1 = "" renderRstToOut(d, n.sons[0], tmp0) renderRstToOut(d, n.sons[1], tmp1) - dispA(d.target, result, "$1", "\\span$2{$1}", - [tmp0, tmp1]) + var class = tmp1 + # don't allow missing role break latex compilation: + if d.target == outLatex and n.kind == rnUnknownRole: class = "Other" + if n.kind == rnCodeFragment: + dispA(d.target, result, + "" & + "$1", + "\\texttt{\\span$2{$1}}", [tmp0, class]) + else: # rnUnknownRole, not necessarily code/monospace font + dispA(d.target, result, "$1", "\\span$2{$1}", + [tmp0, class]) of rnSub: renderAux(d, n, "$1", "\\rstsub{$1}", result) of rnSup: renderAux(d, n, "$1", "\\rstsup{$1}", result) of rnEmphasis: renderAux(d, n, "$1", "\\emph{$1}", result) diff --git a/nimdoc/testproject/expected/nimdoc.out.css b/nimdoc/testproject/expected/nimdoc.out.css index ced791d16..6366d33dc 100644 --- a/nimdoc/testproject/expected/nimdoc.out.css +++ b/nimdoc/testproject/expected/nimdoc.out.css @@ -35,6 +35,8 @@ Modified by Boyd Greenfield and narimiran --escapeSequence: #c4891b; --number: #252dbe; --literal: #a4255b; + --program: #6060c0; + --option: #508000; --raw-data: #a4255b; } @@ -63,6 +65,8 @@ Modified by Boyd Greenfield and narimiran --escapeSequence: #bd93f9; --number: #bd93f9; --literal: #f1fa8c; + --program: #9090c0; + --option: #90b010; --raw-data: #8be9fd; } @@ -527,7 +531,6 @@ div.option-list-label { margin-left: -11.5em; margin-right: 0em; min-width: 11.5em; - font-weight: bolder; display: inline-block; vertical-align: top; } @@ -546,7 +549,7 @@ blockquote { border-left: 5px solid #bbc; } -.pre { +.pre, span.tok { font-family: "Source Code Pro", Monaco, Menlo, Consolas, "Courier New", monospace; font-weight: 500; font-size: 0.85em; @@ -557,6 +560,12 @@ blockquote { border-radius: 4px; } +span.tok { + border: 1px solid #808080; + padding-bottom: 0.1em; + margin-right: 0.2em; +} + pre { font-family: "Source Code Pro", Monaco, Menlo, Consolas, "Courier New", monospace; color: var(--text); @@ -844,9 +853,6 @@ span.classifier { span.classifier-delimiter { font-weight: bold; } -span.option { - white-space: nowrap; } - span.problematic { color: #b30000; } @@ -926,6 +932,21 @@ span.Preprocessor { span.Directive { color: #252dbe; } +span.option { + font-weight: bold; + font-family: "Source Code Pro", Monaco, Menlo, Consolas, "Courier New", monospace; + color: var(--option); +} + +span.program { + font-weight: bold; + color: var(--program); + text-decoration: underline; + text-decoration-color: var(--hint); + text-decoration-thickness: 0.05em; + text-underline-offset: 0.15em; +} + span.Command, span.Rule, span.Hyperlink, span.Label, span.Reference, span.Other { color: var(--other); } diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index 29747f4e8..0af6ba566 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -40,6 +40,10 @@ proc toHtml(input: string, proc id(str: string): string = """""" & str & "" proc op(str: string): string = """""" & str & "" proc pu(str: string): string = """""" & str & "" +proc optionListLabel(opt: string): string = + """
    """ & + opt & + "
    " suite "YAML syntax highlighting": test "Basics": @@ -1382,10 +1386,10 @@ Test1 check(output.count("") == 2) check(output.count("
    -m
    """ & + check(optionListLabel("-m") & """
    desc
    """ in output) - check("""
    -n
    """ & + check(optionListLabel("-n") & """
    very long desc
    """ in output) @@ -1400,13 +1404,13 @@ Test1 let output = input.toHtml check(output.count("-m""" & + check(optionListLabel("-m") & """
    desc
    """ in output) - check("""
    -n
    """ & + check(optionListLabel("-n") & """
    very long desc
    """ in output) - check("""
    -d
    """ & + check(optionListLabel("-d") & """
    option
    """ in output) check "

    option

    " notin output @@ -1421,13 +1425,13 @@ Test1 let output = input.toHtml check(output.count("compile""" & + check(optionListLabel("compile") & """
    compile1
    """ in output) - check("""
    doc
    """ & + check(optionListLabel("doc") & """
    doc1 cont
    """ in output) - check("""
    -d
    """ & + check(optionListLabel("-d") & """
    option
    """ in output) check "

    option

    " notin output From 4471141a1d68158f3f23c6584d1f0434807c0203 Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 21 Apr 2021 23:53:31 +0800 Subject: [PATCH 0697/2472] fix #7535(Poor error message for spawn when a procedure (without calling it)) (#17774) --- compiler/semexprs.nim | 7 +++++-- tests/parallel/t7535.nim | 11 +++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 tests/parallel/t7535.nim diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 353831736..97f88869e 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -2282,12 +2282,15 @@ proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode = of mSpawn: markUsed(c, n.info, s) when defined(leanCompiler): - localError(c.config, n.info, "compiler was built without 'spawn' support") - result = n + result = localErrorNode(c, n, "compiler was built without 'spawn' support") else: result = setMs(n, s) for i in 1.. 1 and n[1].kind notin nkCallKinds: + return localErrorNode(c, n, n[1].info, "'spawn' takes a call expression; got " & $n[1]) + let typ = result[^1].typ if not typ.isEmptyType: if spawnResult(typ, c.inParallelStmt > 0) == srFlowVar: diff --git a/tests/parallel/t7535.nim b/tests/parallel/t7535.nim new file mode 100644 index 000000000..a6bc0dabe --- /dev/null +++ b/tests/parallel/t7535.nim @@ -0,0 +1,11 @@ +discard """ + matrix: "--threads:on" + errormsg: "'spawn' takes a call expression; got proc (x: uint32) = echo [x]" +""" + +import threadpool + +# bug #7535 +proc print_parallel_nok(r: uint32) = + for x in 0..r: + spawn (proc (x: uint32) = echo x) From fb32fff8dcf2b15a469e5cce07b10d88aa6352ee Mon Sep 17 00:00:00 2001 From: Sivchari <55221074+sivchari@users.noreply.github.com> Date: Thu, 22 Apr 2021 02:47:01 +0900 Subject: [PATCH 0698/2472] js generates spurious >>> on shr (#17767) * js generates spurious >>> on shr * Add shr arithmetic test * fix variables from const to let during testing --- compiler/jsgen.nim | 3 +-- tests/stdlib/tarithmetics.nim | 49 +++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 tests/stdlib/tarithmetics.nim diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index e75ff182f..53ce45e96 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -683,8 +683,7 @@ proc arith(p: PProc, n: PNode, r: var TCompRes, op: TMagic) = var x, y: TCompRes gen(p, n[1], x) gen(p, n[2], y) - let trimmer = unsignedTrimmer(n[1].typ.skipTypes(abstractRange).size) - r.res = "(($1 $2) >>> $3)" % [x.rdLoc, trimmer, y.rdLoc] + r.res = "($1 >>> $2)" % [x.rdLoc, y.rdLoc] of mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mFloatToStr, mCStrToStr, mStrToStr, mEnumToStr: arithAux(p, n, r, op) diff --git a/tests/stdlib/tarithmetics.nim b/tests/stdlib/tarithmetics.nim new file mode 100644 index 000000000..296ccd56e --- /dev/null +++ b/tests/stdlib/tarithmetics.nim @@ -0,0 +1,49 @@ +discard """ + targets: "c cpp js" +""" + +# TODO: in future work move existing arithmetic tests (tests/arithm/*) into this file +# FYI https://github.com/nim-lang/Nim/pull/17767 + +template main = + # put all arithmetic tests + + block tshr: + block: # Signed types + let + a1 = -3 + a2 = -2 + b1 = -4'i8 + b2 = 1'i8 + c1 = -5'i16 + c2 = 1'i16 + d1 = -7i32 + d2 = 1'i32 + e1 = -9'i64 + e2 = 1'i64 + doAssert a1 shr a2 == -1 + doAssert b1 shr b2 == -2 + doAssert c1 shr c2 == -3 + doAssert d1 shr d2 == -4 + doAssert e1 shr e2 == -5 + + block: # Unsigned types + let + a1 = 3'u + a2 = 2'u + b1 = 2'u8 + b2 = 1'u8 + c1 = 5'u16 + c2 = 1'u16 + d1 = 6'u32 + d2 = 1'u32 + e1 = 8'u64 + e2 = 1'u64 + doAssert a1 shr a2 == 0 + doAssert b1 shr b2 == 1 + doAssert c1 shr c2 == 2 + doAssert d1 shr d2 == 3 + doAssert e1 shr e2 == 4 + +static: main() +main() From 53c898de4105049c767804d0d2c520f31a86b874 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 22 Apr 2021 14:08:56 +0800 Subject: [PATCH 0699/2472] fix #17812 (repr fails to compile with ARC/ORC) (#17816) --- lib/system/repr_v2.nim | 2 +- tests/arc/t17812.nim | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 tests/arc/t17812.nim diff --git a/lib/system/repr_v2.nim b/lib/system/repr_v2.nim index d456f4454..618cc2b40 100644 --- a/lib/system/repr_v2.nim +++ b/lib/system/repr_v2.nim @@ -84,7 +84,7 @@ proc repr*(p: pointer): string = proc repr*(p: proc): string = ## repr of a proc as its address - repr(cast[pointer](p)) + repr(cast[ptr pointer](unsafeAddr p)[]) template repr*(x: distinct): string = repr(distinctBase(typeof(x))(x)) diff --git a/tests/arc/t17812.nim b/tests/arc/t17812.nim new file mode 100644 index 000000000..bcd5f3a93 --- /dev/null +++ b/tests/arc/t17812.nim @@ -0,0 +1,29 @@ +discard """ + targets: "c js" + matrix: "--gc:refc; --gc:arc" +""" + +import std/times + +block: # bug #17812 + block: + type + Task = object + cb: proc () + + proc hello() = discard + + + let t = Task(cb: hello) + + doAssert t.repr.len > 0 + + + block: + type MyObj = object + field: DateTime + + + proc `$`(o: MyObj): string = o.repr + + doAssert ($MyObj()).len > 0 From 56859fe8516cd3e8191f9162c6409aeed8493941 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 22 Apr 2021 12:04:13 +0200 Subject: [PATCH 0700/2472] Windows: separate DLLs are not required anymore --- readme.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/readme.md b/readme.md index fa7a2d86d..9a07ae56b 100644 --- a/readme.md +++ b/readme.md @@ -84,10 +84,6 @@ Next, run the appropriate build shell script for your platform: * `build_all.sh` (Linux, Mac) * `build_all.bat` (Windows) -Windows requires a number of other dependencies that you may need to install including -PCRE and OpenSSL. Nim hosts a zip package containing known working versions of the -required DLLs [here](https://nim-lang.org/download/dlls.zip). - Finally, once you have finished the build steps (on Windows, Mac, or Linux) you should add the ``bin`` directory to your PATH. From e4a3feeb92940a9b3c4ce88fc808a0cdff2e4e9c Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 22 Apr 2021 23:07:22 -0700 Subject: [PATCH 0701/2472] add `--processing:dots|filenames|off` to customize `hintProcessing` (#17817) --- changelog.md | 2 ++ compiler/ccgtypes.nim | 2 +- compiler/commands.nim | 10 ++++++++++ compiler/msgs.nim | 2 +- compiler/options.nim | 32 +++++++++++++++--------------- compiler/passaux.nim | 7 ++++--- doc/advopt.txt | 2 ++ drnim/tests/config.nims | 2 +- nimsuggest/config.nims | 2 ++ nimsuggest/tester.nim | 9 ++++++--- nimsuggest/tests/tchk1.nim | 2 +- nimsuggest/tests/tchk_compiles.nim | 2 +- nimsuggest/tests/ttempl_inst.nim | 2 +- testament/testament.nim | 2 +- tests/misc/trunner.nim | 2 +- 15 files changed, 50 insertions(+), 30 deletions(-) create mode 100644 nimsuggest/config.nims diff --git a/changelog.md b/changelog.md index 30fe237be..7b4a2438a 100644 --- a/changelog.md +++ b/changelog.md @@ -343,6 +343,8 @@ - Added `--filenames:abs|canonical|magic` which replaces --listFullPaths:on|off +- Added `--processing:dots|filenames|off` which customizes `hintProcessing` + - Source+Edit links now appear on top of every docgen'd page when `nim doc --git.url:url ...` is given. diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index fa58b5651..767525fd5 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -1018,7 +1018,7 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType; var flags = 0 if not containsGarbageCollectedRef(typ): flags = flags or 1 if not canFormAcycle(typ): flags = flags or 2 - #else MessageOut("can contain a cycle: " & typeToString(typ)) + #else echo("can contain a cycle: " & typeToString(typ)) if flags != 0: m.s[cfsTypeInit3].addf("$1.flags = $2;$n", [nameHcr, rope(flags)]) discard cgsym(m, "TNimType") diff --git a/compiler/commands.nim b/compiler/commands.nim index 0733e500a..278199bc0 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -900,6 +900,16 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; of "canonical": conf.filenameOption = foCanonical of "legacyrelproj": conf.filenameOption = foLegacyRelProj else: localError(conf, info, "expected: abs|canonical|legacyRelProj, got: $1" % arg) + of "processing": + incl(conf.notes, hintProcessing) + incl(conf.mainPackageNotes, hintProcessing) + case arg.normalize + of "dots": conf.hintProcessingDots = true + of "filenames": conf.hintProcessingDots = false + of "off": + excl(conf.notes, hintProcessing) + excl(conf.mainPackageNotes, hintProcessing) + else: localError(conf, info, "expected: dots|filenames|off, got: $1" % arg) of "listfullpaths": # xxx in future work, use `warningDeprecated` conf.filenameOption = if switchOn(arg): foAbs else: foCanonical diff --git a/compiler/msgs.nim b/compiler/msgs.nim index f08722cc7..93ca8d7cc 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -545,7 +545,7 @@ proc liMessage*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string, if conf.structuredErrorHook != nil: conf.structuredErrorHook(conf, info, s & kindmsg, sev) if not ignoreMsgBecauseOfIdeTools(conf, msg): - if msg == hintProcessing: + if msg == hintProcessing and conf.hintProcessingDots: msgWrite(conf, ".") else: styledMsgWriteln(styleBright, loc, resetStyle, color, title, resetStyle, s, KindColor, kindmsg, diff --git a/compiler/options.nim b/compiler/options.nim index 2aaaf5844..7044e64bd 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -287,6 +287,7 @@ type implicitCmd*: bool # whether some flag triggered an implicit `command` selectedGC*: TGCMode # the selected GC (+) exc*: ExceptionSystem + hintProcessingDots*: bool # true for dots, false for filenames verbosity*: int # how verbose the compiler is numberOfProcessors*: int # number of processors lastCmdTime*: float # when caas is enabled, we measure each command @@ -458,20 +459,25 @@ proc isDefined*(conf: ConfigRef; symbol: string): bool when defined(nimDebugUtils): import debugutils +proc initConfigRefCommon(conf: ConfigRef) = + conf.selectedGC = gcRefc + conf.verbosity = 1 + conf.hintProcessingDots = true + conf.options = DefaultOptions + conf.globalOptions = DefaultGlobalOptions + conf.filenameOption = foAbs + conf.foreignPackageNotes = foreignPackageNotesDefault + conf.notes = NotesVerbosity[1] + conf.mainPackageNotes = NotesVerbosity[1] + proc newConfigRef*(): ConfigRef = result = ConfigRef( - selectedGC: gcRefc, cCompiler: ccGcc, - verbosity: 1, - options: DefaultOptions, - globalOptions: DefaultGlobalOptions, macrosToExpand: newStringTable(modeStyleInsensitive), arcToExpand: newStringTable(modeStyleInsensitive), m: initMsgConfig(), - filenameOption: foAbs, cppDefines: initHashSet[string](), - headerFile: "", features: {}, legacyFeatures: {}, foreignPackageNotes: foreignPackageNotesDefault, - notes: NotesVerbosity[1], mainPackageNotes: NotesVerbosity[1], + headerFile: "", features: {}, legacyFeatures: {}, configVars: newStringTable(modeStyleInsensitive), symbols: newStringTable(modeStyleInsensitive), packageCache: newPackageCache(), @@ -513,6 +519,7 @@ proc newConfigRef*(): ConfigRef = vmProfileData: newProfileData(), spellSuggestMax: spellSuggestSecretSauce, ) + initConfigRefCommon(result) setTargetFromSystem(result.target) # enable colors by default on terminals if terminal.isatty(stderr): @@ -522,18 +529,11 @@ proc newConfigRef*(): ConfigRef = proc newPartialConfigRef*(): ConfigRef = ## create a new ConfigRef that is only good enough for error reporting. - # xxx FACTOR with `newConfigRef` when defined(nimDebugUtils): result = getConfigRef() else: - result = ConfigRef( - selectedGC: gcRefc, - verbosity: 1, - options: DefaultOptions, - globalOptions: DefaultGlobalOptions, - filenameOption: foAbs, - foreignPackageNotes: foreignPackageNotesDefault, - notes: NotesVerbosity[1], mainPackageNotes: NotesVerbosity[1]) + result = ConfigRef() + initConfigRefCommon(result) proc cppDefine*(c: ConfigRef; define: string) = c.cppDefines.incl define diff --git a/compiler/passaux.nim b/compiler/passaux.nim index 715b7d676..9abc97be4 100644 --- a/compiler/passaux.nim +++ b/compiler/passaux.nim @@ -19,9 +19,10 @@ type config: ConfigRef proc verboseOpen(graph: ModuleGraph; s: PSym; idgen: IdGenerator): PPassContext = - #MessageOut('compiling ' + s.name.s); - result = VerboseRef(config: graph.config, idgen: idgen) - rawMessage(graph.config, hintProcessing, s.name.s) + let conf = graph.config + result = VerboseRef(config: conf, idgen: idgen) + let path = toFilenameOption(conf, s.position.FileIndex, conf.filenameOption) + rawMessage(conf, hintProcessing, path) proc verboseProcess(context: PPassContext, n: PNode): PNode = result = n diff --git a/doc/advopt.txt b/doc/advopt.txt index 539cc620c..a783ebec5 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -39,6 +39,8 @@ Advanced options: --filenames:abs|canonical|legacyRelProj customize how filenames are rendered in compiler messages, defaults to `abs` (absolute) + --processing:dots|filenames|off + show files as they're being processed by nim compiler --declaredLocs:on|off show declaration locations in messages --spellSuggest|:num show at most `num >= 0` spelling suggestions on typos. if `num` is not specified (or `auto`), return diff --git a/drnim/tests/config.nims b/drnim/tests/config.nims index a6b0e7d88..346b0b4e6 100644 --- a/drnim/tests/config.nims +++ b/drnim/tests/config.nims @@ -3,7 +3,7 @@ switch("path", "$nim/testament/lib") # so we can `import stdtest/foo` in this di ## prevent common user config settings to interfere with testament expectations ## Indifidual tests can override this if needed to test for these options. switch("colors", "off") -switch("listFullPaths", "off") +switch("filenames", "canonical") switch("excessiveStackTrace", "off") # we only want to check the marked parts in the tests: diff --git a/nimsuggest/config.nims b/nimsuggest/config.nims new file mode 100644 index 000000000..ee19f9893 --- /dev/null +++ b/nimsuggest/config.nims @@ -0,0 +1,2 @@ +# xxx not sure why this flag isn't needed: switch("processing", "filenames") +switch("filenames", "canonical") diff --git a/nimsuggest/tester.nim b/nimsuggest/tester.nim index 425430ede..9fcf7eacc 100644 --- a/nimsuggest/tester.nim +++ b/nimsuggest/tester.nim @@ -255,7 +255,6 @@ proc runEpcTest(filename: string): int = options={poStdErrToStdOut, poUsePath, poInteractive, poDaemon}) let outp = p.outputStream - let inp = p.inputStream var report = "" var socket = newSocket() try: @@ -315,8 +314,12 @@ proc runTest(filename: string): int = answer.add '\L' doReport(filename, answer, resp, report) finally: - inp.writeLine("quit") - inp.flush() + try: + inp.writeLine("quit") + inp.flush() + except: + # assume it's SIGPIPE, ie, the child already died + discard close(p) if report.len > 0: echo "==== STDIN ======================================" diff --git a/nimsuggest/tests/tchk1.nim b/nimsuggest/tests/tchk1.nim index c28b88b9b..794392c1f 100644 --- a/nimsuggest/tests/tchk1.nim +++ b/nimsuggest/tests/tchk1.nim @@ -17,7 +17,7 @@ proc main = discard """ $nimsuggest --tester $file >chk $1 -chk;;skUnknown;;;;Hint;;???;;0;;-1;;"tchk1 [Processing]";;0 +chk;;skUnknown;;;;Hint;;???;;0;;-1;;"tests/tchk1.nim [Processing]";;0 chk;;skUnknown;;;;Error;;$file;;12;;0;;"identifier expected, but got \'keyword template\'";;0 chk;;skUnknown;;;;Error;;$file;;14;;0;;"nestable statement requires indentation";;0 chk;;skUnknown;;;;Error;;$file;;12;;0;;"implementation of \'foo\' expected";;0 diff --git a/nimsuggest/tests/tchk_compiles.nim b/nimsuggest/tests/tchk_compiles.nim index 887a947b5..2afa06899 100644 --- a/nimsuggest/tests/tchk_compiles.nim +++ b/nimsuggest/tests/tchk_compiles.nim @@ -4,5 +4,5 @@ discard compiles(2 + "hello") discard """ $nimsuggest --tester $file >chk $1 -chk;;skUnknown;;;;Hint;;???;;0;;-1;;"tchk_compiles [Processing]";;0 +chk;;skUnknown;;;;Hint;;???;;0;;-1;;"tests/tchk_compiles.nim [Processing]";;0 """ diff --git a/nimsuggest/tests/ttempl_inst.nim b/nimsuggest/tests/ttempl_inst.nim index ed04a67ce..da413d99a 100644 --- a/nimsuggest/tests/ttempl_inst.nim +++ b/nimsuggest/tests/ttempl_inst.nim @@ -7,7 +7,7 @@ foo() discard """ $nimsuggest --tester $file >chk $1 -chk;;skUnknown;;;;Hint;;???;;0;;-1;;"ttempl_inst [Processing]";;0 +chk;;skUnknown;;;;Hint;;???;;0;;-1;;"tests/ttempl_inst.nim [Processing]";;0 chk;;skUnknown;;;;Hint;;$file;;4;;3;;"template/generic instantiation from here";;0 chk;;skUnknown;;;;Warning;;$file;;2;;11;;"foo [User]";;0 """ diff --git a/testament/testament.nim b/testament/testament.nim index b79b86e51..f259c7b96 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -104,7 +104,7 @@ proc isSuccess(input: string): bool = # not clear how to do the equivalent of pkg/regex's: re"FOO(.*?)BAR" in pegs # note: this doesn't handle colors, eg: `\e[1m\e[0m\e[32mHint:`; while we # could handle colors, there would be other issues such as handling other flags - # that may appear in user config (eg: `--listFullPaths`). + # that may appear in user config (eg: `--filenames`). # Passing `XDG_CONFIG_HOME= testament args...` can be used to ignore user config # stored in XDG_CONFIG_HOME, refs https://wiki.archlinux.org/index.php/XDG_Base_Directory input.startsWith("Hint: ") and input.endsWith("[SuccessX]") diff --git a/tests/misc/trunner.nim b/tests/misc/trunner.nim index df9ee9a48..505a06cf8 100644 --- a/tests/misc/trunner.nim +++ b/tests/misc/trunner.nim @@ -92,7 +92,7 @@ else: # don't run twice the same test of 5: nimcache / htmldocsDirname else: file.parentDir / htmldocsDirname - var cmd = fmt"{nim} doc --index:on --listFullPaths --hint:successX:on --nimcache:{nimcache} {options[i]} {file}" + var cmd = fmt"{nim} doc --index:on --filenames:abs --hint:successX:on --nimcache:{nimcache} {options[i]} {file}" removeDir(htmldocsDir) let (outp, exitCode) = execCmdEx(cmd) check exitCode == 0 From dbb053492a3c64236f7e8f9358b9c7f297ba241a Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 23 Apr 2021 01:05:23 -0700 Subject: [PATCH 0702/2472] followup #17561, skipping ci now implies green (#17813) * followup #17561, skip ci now implies green; [skip ci] * fixup [skip ci] * fixup test without skip ci * fixup [skip ci] * fixup2 [skip ci] * github actions now automatically supports [skip ci] --- .github/workflows/ci_docs.yml | 11 ----------- .github/workflows/ci_packages.yml | 11 ----------- azure-pipelines.yml | 33 ++++++++++++++++++++----------- ci/funs.sh | 22 +++++++++++++++++++++ 4 files changed, 44 insertions(+), 33 deletions(-) diff --git a/.github/workflows/ci_docs.yml b/.github/workflows/ci_docs.yml index c9036d1d4..750613574 100644 --- a/.github/workflows/ci_docs.yml +++ b/.github/workflows/ci_docs.yml @@ -30,9 +30,6 @@ on: jobs: build: - # see D20210329T004830 - if: | - !contains(format('{0}', github.event.pull_request.title), '[skip ci]') strategy: fail-fast: false matrix: @@ -54,14 +51,6 @@ jobs: with: fetch-depth: 2 - - name: 'Check whether to skip CI' - shell: bash - run: | - # see D20210329T004830 - commitMsg=$(git log --no-merges -1 --pretty=format:"%s") - echo commitMsg: $commitMsg - echo $commitMsg | grep -v '\[skip ci\]' - - name: 'Install build dependencies (macOS)' if: runner.os == 'macOS' run: brew install make diff --git a/.github/workflows/ci_packages.yml b/.github/workflows/ci_packages.yml index e7073ea23..7ebdc5144 100644 --- a/.github/workflows/ci_packages.yml +++ b/.github/workflows/ci_packages.yml @@ -3,9 +3,6 @@ on: [push, pull_request] jobs: build: - # see D20210329T004830 - if: | - !contains(format('{0}', github.event.pull_request.title), '[skip ci]') strategy: fail-fast: false matrix: @@ -23,14 +20,6 @@ jobs: with: fetch-depth: 2 - - name: 'Check whether to skip CI' - shell: bash - run: | - # see D20210329T004830 - commitMsg=$(git log --no-merges -1 --pretty=format:"%s") - echo commitMsg: $commitMsg - echo $commitMsg | grep -v '\[skip ci\]' - - name: 'Checkout csources' uses: actions/checkout@v2 with: diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5c244648d..7e5b5c27d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -7,6 +7,10 @@ pr: include: - '*' +variables: +- name: skipci + value: false + jobs: - job: packages @@ -58,22 +62,22 @@ jobs: fetchDepth: 2 # see D20210329T004830 - bash: | - # D20210329T004830:here refs https://github.com/microsoft/azure-pipelines-agent/issues/2944 - # `--no-merges` is needed to avoid merge commits which occur for PR's. - # $(Build.SourceVersionMessage) is not helpful - # nor is `github.event.head_commit.message` for github actions. - commitMsg=$(git log --no-merges -1 --pretty=format:"%s") - echo commitMsg: $commitMsg - echo $commitMsg | grep -v '\[skip ci\]' # fails if [skip ci] not in commit msg + set -e + . ci/funs.sh + if nimIsCiSkip; then + echo '##vso[task.setvariable variable=skipci]true' + fi displayName: 'Check whether to skip CI' - bash: git clone --depth 1 https://github.com/nim-lang/csources_v1 csources displayName: 'Checkout Nim csources' + condition: and(succeeded(), eq(variables['skipci'], 'false')) - task: NodeTool@0 inputs: versionSpec: '12.x' displayName: 'Install node.js 12.x' + condition: and(succeeded(), eq(variables['skipci'], 'false')) - bash: | set -e @@ -83,7 +87,7 @@ jobs: echo_run sudo apt-fast install --no-install-recommends -yq \ libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev valgrind libc6-dbg displayName: 'Install dependencies (amd64 Linux)' - condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'amd64')) + condition: and(succeeded(), eq(variables['skipci'], 'false'), eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'amd64')) - bash: | set -e @@ -123,11 +127,11 @@ jobs: echo_run chmod 755 bin/g++ displayName: 'Install dependencies (i386 Linux)' - condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'i386')) + condition: and(succeeded(), eq(variables['skipci'], 'false'), eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'i386')) - bash: brew install boehmgc make sfml displayName: 'Install dependencies (OSX)' - condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) + condition: and(succeeded(), eq(variables['skipci'], 'false'), eq(variables['Agent.OS'], 'Darwin')) - bash: | set -e @@ -140,9 +144,10 @@ jobs: echo_run echo '##vso[task.prependpath]$(System.DefaultWorkingDirectory)/dist/mingw64/bin' displayName: 'Install dependencies (Windows)' - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) + condition: and(succeeded(), eq(variables['skipci'], 'false'), eq(variables['Agent.OS'], 'Windows_NT')) - bash: echo '##vso[task.prependpath]$(System.DefaultWorkingDirectory)/bin' + condition: and(succeeded(), eq(variables['skipci'], 'false')) displayName: 'Add build binaries to PATH' - bash: | @@ -155,15 +160,18 @@ jobs: echo_run node -v echo_run echo '##[section]make version' echo_run make -v + condition: and(succeeded(), eq(variables['skipci'], 'false')) displayName: 'System information' - bash: echo '##vso[task.setvariable variable=csources_version]'"$(git -C csources rev-parse HEAD)" + condition: and(succeeded(), eq(variables['skipci'], 'false')) displayName: 'Get csources version' - task: Cache@2 inputs: key: 'csources | "$(Agent.OS)" | $(CPU) | $(csources_version)' path: csources/bin + condition: and(succeeded(), eq(variables['skipci'], 'false')) displayName: 'Restore built csources' - bash: | @@ -192,13 +200,16 @@ jobs: fi echo_run cp csources/bin/nim$ext bin + condition: and(succeeded(), eq(variables['skipci'], 'false')) displayName: 'Build 1-stage compiler from csources' - bash: nim c koch + condition: and(succeeded(), eq(variables['skipci'], 'false')) displayName: 'Build koch' # set result to omit the "bash exited with error code '1'" message - bash: ./koch runCI || echo '##vso[task.complete result=Failed]' + condition: and(succeeded(), eq(variables['skipci'], 'false')) displayName: 'Run CI' env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/ci/funs.sh b/ci/funs.sh index de56ef152..7f82e1ca3 100644 --- a/ci/funs.sh +++ b/ci/funs.sh @@ -1,4 +1,5 @@ # utilities used in CI pipelines to avoid duplication. +# Avoid top-level statements. echo_run () { # echo's a command before running it, which helps understanding logs @@ -6,3 +7,24 @@ echo_run () { echo "$@" "$@" } + +nimGetLastCommit() { + git log --no-merges -1 --pretty=format:"%s" +} + +nimIsCiSkip(){ + # D20210329T004830:here refs https://github.com/microsoft/azure-pipelines-agent/issues/2944 + # `--no-merges` is needed to avoid merge commits which occur for PR's. + # $(Build.SourceVersionMessage) is not helpful + # nor is `github.event.head_commit.message` for github actions. + # Note: `[skip ci]` is now handled automatically for github actions, see https://github.blog/changelog/2021-02-08-github-actions-skip-pull-request-and-push-workflows-with-skip-ci/ + commitMsg=$(nimGetLastCommit) + echo commitMsg: "$commitMsg" + if [[ $commitMsg == *"[skip ci]"* ]]; then + echo "skipci: true" + return 0 + else + echo "skipci: false" + return 1 + fi +} From dce0b3b002fcbac716bada039bed4a02eb501744 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 23 Apr 2021 02:28:42 -0700 Subject: [PATCH 0703/2472] refactor all code that builds csources (#17815) * refactor all code that builds csources * fixup * nim_csourcesDir_v0 + nim_csourcesDir * remove deprecated, unused scripts from ci/ * reuse nimCsourcesHash in ci * simplify CI pipelines by reusing nimBuildCsourcesIfNeeded * simplify ci_docs.yml by reusing nimBuildCsourcesIfNeeded * cleanup * use csources_v1 as destination dir * fixup * remove pushCsources * address comment: remove build.sh support for now * fixup --- .builds/freebsd.yml | 6 +-- .builds/openbsd_0.yml | 8 ++-- .builds/openbsd_1.yml | 8 ++-- .github/workflows/ci.yml.disabled | 25 ++--------- .github/workflows/ci_docs.yml | 51 +++------------------ .github/workflows/ci_packages.yml | 25 ++--------- .gitignore | 3 ++ .travis.yml | 4 +- appveyor.yml.disabled | 8 ++-- azure-pipelines.yml | 50 ++++----------------- build_all.bat | 9 ++-- build_all.sh | 46 +++---------------- ci/build.bat | 14 ------ ci/build.sh | 15 ------- ci/deps.bat | 4 -- ci/deps.sh | 16 ------- ci/funs.sh | 73 ++++++++++++++++++++++++++++++- ci/nsis_build.bat | 59 ------------------------- koch.nim | 36 ++------------- lib/std/private/globs.nim | 2 +- readme.md | 5 ++- tools/ci_generate.nim | 8 ++-- 22 files changed, 136 insertions(+), 339 deletions(-) delete mode 100644 ci/build.bat delete mode 100644 ci/build.sh delete mode 100644 ci/deps.bat delete mode 100644 ci/deps.sh delete mode 100644 ci/nsis_build.bat diff --git a/.builds/freebsd.yml b/.builds/freebsd.yml index 443be798a..c1fb80253 100644 --- a/.builds/freebsd.yml +++ b/.builds/freebsd.yml @@ -14,10 +14,10 @@ environment: CC: /usr/bin/clang tasks: - setup: | + set -e cd Nim - git clone --depth 1 -q https://github.com/nim-lang/csources_v1.git csources - gmake -C csources -j $(sysctl -n hw.ncpu) - bin/nim c --skipUserCfg --skipParentCfg koch + . ci/funs.sh && nimBuildCsourcesIfNeeded + $nim_csources c --skipUserCfg --skipParentCfg koch echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv - test: | cd Nim diff --git a/.builds/openbsd_0.yml b/.builds/openbsd_0.yml index c09f0f08d..e1d85d2a7 100644 --- a/.builds/openbsd_0.yml +++ b/.builds/openbsd_0.yml @@ -17,15 +17,15 @@ environment: CC: /usr/bin/clang tasks: - setup: | + set -e cd Nim - git clone --depth 1 -q https://github.com/nim-lang/csources_v1.git csources - gmake -C csources -j $(sysctl -n hw.ncpuonline) - bin/nim c koch + . ci/funs.sh && nimBuildCsourcesIfNeeded + $nim_csources c koch echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv - test: | cd Nim if ! ./koch runCI; then - nim c -r tools/ci_testresults.nim + nim r tools/ci_testresults.nim exit 1 fi triggers: diff --git a/.builds/openbsd_1.yml b/.builds/openbsd_1.yml index ffe449733..ed2ded056 100644 --- a/.builds/openbsd_1.yml +++ b/.builds/openbsd_1.yml @@ -17,15 +17,15 @@ environment: CC: /usr/bin/clang tasks: - setup: | + set -e cd Nim - git clone --depth 1 -q https://github.com/nim-lang/csources_v1.git csources - gmake -C csources -j $(sysctl -n hw.ncpuonline) - bin/nim c koch + . ci/funs.sh && nimBuildCsourcesIfNeeded + $nim_csources c koch echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv - test: | cd Nim if ! ./koch runCI; then - nim c -r tools/ci_testresults.nim + nim r tools/ci_testresults.nim exit 1 fi triggers: diff --git a/.github/workflows/ci.yml.disabled b/.github/workflows/ci.yml.disabled index 183576456..62daf875a 100644 --- a/.github/workflows/ci.yml.disabled +++ b/.github/workflows/ci.yml.disabled @@ -1,3 +1,5 @@ +# out of date and unused but revivable + name: Continous Integration on: [push, pull_request] @@ -30,11 +32,6 @@ jobs: steps: - name: 'Checkout' uses: actions/checkout@v2 - - name: 'Checkout csources' - uses: actions/checkout@v2 - with: - repository: nim-lang/csources - path: csources - name: 'Install node.js 8.x' uses: actions/setup-node@v1 @@ -91,24 +88,10 @@ jobs: shell: bash run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}" - - name: 'Build csources' + - name: 'Build csourcesAny' shell: bash - run: | - ncpu= - case '${{ runner.os }}' in - 'Linux') - ncpu=$(nproc) - ;; - 'macOS') - ncpu=$(sysctl -n hw.ncpu) - ;; - 'Windows') - ncpu=$NUMBER_OF_PROCESSORS - ;; - esac - [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1 + run: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc ucpu='${{ matrix.cpu }}' - make -C csources -j $ncpu CC=gcc ucpu='${{ matrix.cpu }}' - name: 'Build koch' shell: bash run: nim c koch diff --git a/.github/workflows/ci_docs.yml b/.github/workflows/ci_docs.yml index 750613574..1459d6a31 100644 --- a/.github/workflows/ci_docs.yml +++ b/.github/workflows/ci_docs.yml @@ -70,53 +70,12 @@ jobs: shell: bash run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}" - - name: 'Get current csources version' - id: csources-version + - name: 'Build csourcesAny' shell: bash - run: | - sha=$(git ls-remote https://github.com/nim-lang/csources_v1 master | cut -f 1) - echo "::set-output name=sha::$sha" - - - name: 'Get prebuilt csources from cache' - id: csources-cache - uses: actions/cache@v1 - with: - path: bin - key: '${{ matrix.os }}-${{ steps.csources-version.outputs.sha }}' - - - name: 'Checkout csources' - if: steps.csources-cache.outputs.cache-hit != 'true' - uses: actions/checkout@v2 - with: - repository: nim-lang/csources_v1 - path: csources - - - name: 'Build 1-stage compiler from csources' - shell: bash - run: | - ext= - [[ '${{ runner.os }}' == 'Windows' ]] && ext=.exe - if [[ ! -x bin/nim-csources$ext ]]; then - ncpu= - case '${{ runner.os }}' in - 'Linux') - ncpu=$(nproc) - ;; - 'macOS') - ncpu=$(sysctl -n hw.ncpu) - ;; - 'Windows') - ncpu=$NUMBER_OF_PROCESSORS - ;; - esac - [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1 - - make -C csources -j $ncpu CC=gcc - cp bin/nim{,-csources}$ext - else - echo 'Cache hit, using prebuilt csources' - cp bin/nim{-csources,}$ext - fi + run: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc + # was previously using caching via `actions/cache@v1` but this wasn't + # used in other CI pipelines and it's unclear the added complexity + # was worth the saving; can be revisited if needed. - name: 'Build koch' shell: bash diff --git a/.github/workflows/ci_packages.yml b/.github/workflows/ci_packages.yml index 7ebdc5144..c6355bbcb 100644 --- a/.github/workflows/ci_packages.yml +++ b/.github/workflows/ci_packages.yml @@ -20,16 +20,11 @@ jobs: with: fetch-depth: 2 - - name: 'Checkout csources' - uses: actions/checkout@v2 - with: - repository: nim-lang/csources - path: csources - - name: 'Install node.js 12.x' uses: actions/setup-node@v1 with: node-version: '12.x' + - name: 'Install dependencies (Linux amd64)' if: runner.os == 'Linux' && matrix.cpu == 'amd64' run: | @@ -56,24 +51,10 @@ jobs: shell: bash run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}" - - name: 'Build csources' + - name: 'Build csourcesAny' shell: bash - run: | - ncpu= - case '${{ runner.os }}' in - 'Linux') - ncpu=$(nproc) - ;; - 'macOS') - ncpu=$(sysctl -n hw.ncpu) - ;; - 'Windows') - ncpu=$NUMBER_OF_PROCESSORS - ;; - esac - [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1 + run: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc ucpu='${{ matrix.cpu }}' - make -C csources -j $ncpu CC=gcc ucpu='${{ matrix.cpu }}' - name: 'Build koch' shell: bash run: nim c koch diff --git a/.gitignore b/.gitignore index e9e742404..d22b8316a 100644 --- a/.gitignore +++ b/.gitignore @@ -64,7 +64,10 @@ lib/**/*.html testament.db /tests/**/*.json /tests/**/*.js + /csources +/csources_v1 + /dist/ # /lib/fusion # fusion is now unbundled; `git status` should reveal if it's there so users can act on it diff --git a/.travis.yml b/.travis.yml index 4c4c35334..13e5d151a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,4 @@ +# deprecated, out of date and unused sudo: false language: c @@ -22,9 +23,8 @@ addons: - valgrind before_script: - - git clone --depth 1 https://github.com/nim-lang/csources.git + - . ci/funs.sh && nimBuildCsourcesIfNeeded LD=$CC ucpu=$CPU - export PATH="$PWD/bin${PATH:+:$PATH}" - - make -C csources -j 2 LD=$CC ucpu=$CPU script: - echo "travis_fold:start:nim_c_koch" diff --git a/appveyor.yml.disabled b/appveyor.yml.disabled index 5468ac88a..b13c5d909 100644 --- a/appveyor.yml.disabled +++ b/appveyor.yml.disabled @@ -1,3 +1,4 @@ +# deprecated, out of date and unused version: '{build}' environment: @@ -23,10 +24,9 @@ install: - IF not exist "%MINGW_ARCHIVE%" appveyor DownloadFile "%MINGW_URL%" -FileName "%MINGW_ARCHIVE%" - 7z x -y "%MINGW_ARCHIVE%" -o"%CD%\DIST"> nul - SET PATH=%CD%\DIST\%MINGW_DIR%\BIN;%CD%\BIN;%PATH% - - git clone --depth 1 https://github.com/nim-lang/csources - - cd csources - - build64.bat - - cd .. + # may need adjustments, untested, see similar examples here: + # https://chromium.googlesource.com/external/github.com/swig/swig/+/1e36f51346d95f8b9848e682c2eb986e9cb9b4f4/appveyor.yml + - bash -c ". ci/funs.sh && nimBuildCsourcesIfNeeded" build_script: - openssl version diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7e5b5c27d..8cf69f528 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -69,10 +69,6 @@ jobs: fi displayName: 'Check whether to skip CI' - - bash: git clone --depth 1 https://github.com/nim-lang/csources_v1 csources - displayName: 'Checkout Nim csources' - condition: and(succeeded(), eq(variables['skipci'], 'false')) - - task: NodeTool@0 inputs: versionSpec: '12.x' @@ -163,45 +159,17 @@ jobs: condition: and(succeeded(), eq(variables['skipci'], 'false')) displayName: 'System information' - - bash: echo '##vso[task.setvariable variable=csources_version]'"$(git -C csources rev-parse HEAD)" + - bash: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc ucpu=$(CPU) condition: and(succeeded(), eq(variables['skipci'], 'false')) - displayName: 'Get csources version' + displayName: 'Build csourcesAny' - - task: Cache@2 - inputs: - key: 'csources | "$(Agent.OS)" | $(CPU) | $(csources_version)' - path: csources/bin - condition: and(succeeded(), eq(variables['skipci'], 'false')) - displayName: 'Restore built csources' - - - bash: | - set -e - . ci/funs.sh - ncpu= - ext= - case '$(Agent.OS)' in - 'Linux') - ncpu=$(nproc) - ;; - 'Darwin') - ncpu=$(sysctl -n hw.ncpu) - ;; - 'Windows_NT') - ncpu=$NUMBER_OF_PROCESSORS - ext=.exe - ;; - esac - [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1 - - if [[ -x csources/bin/nim$ext ]]; then - echo_run echo "Found cached compiler, skipping build" - else - echo_run make -C csources -j $ncpu CC=gcc ucpu=$(CPU) koch=no - fi - - echo_run cp csources/bin/nim$ext bin - condition: and(succeeded(), eq(variables['skipci'], 'false')) - displayName: 'Build 1-stage compiler from csources' + # this could be revived if performance justifies it (needs a few updates) + # - task: Cache@2 + # inputs: + # key: 'csourcesAny | "$(Agent.OS)" | $(CPU) | $(csources_version)' + # path: $(nim_csources) + # condition: and(succeeded(), eq(variables['skipci'], 'false')) + # displayName: 'Restore built csourcesAny' - bash: nim c koch condition: and(succeeded(), eq(variables['skipci'], 'false')) diff --git a/build_all.bat b/build_all.bat index 7967e2584..78a7a354a 100644 --- a/build_all.bat +++ b/build_all.bat @@ -1,10 +1,13 @@ @echo off rem build development version of the compiler; can be rerun safely -if not exist csources ( - git clone --depth 1 https://github.com/nim-lang/csources.git +rem TODO: call nimBuildCsourcesIfNeeded or auto-generate this file (from a nim script) +rem to avoid duplication. +if not exist csources_v1 ( + git clone --depth 1 https://github.com/nim-lang/csources_v1.git ) if not exist bin\nim.exe ( - cd csources + cd csources_v1 + git checkout a8a5241f9475099c823cfe1a5e0ca4022ac201ff if PROCESSOR_ARCHITECTURE == AMD64 ( SET ARCH=64 ) diff --git a/build_all.sh b/build_all.sh index c3c6e68b7..49504c8fe 100755 --- a/build_all.sh +++ b/build_all.sh @@ -1,52 +1,16 @@ #! /bin/sh # build development version of the compiler; can be rerun safely. -# arguments can be passed, e.g. `--os freebsd` +# arguments can be passed, e.g.: +# CC=gcc ucpu=amd64 uos=darwin set -u # error on undefined variables set -e # exit on first error -echo_run(){ - echo "$*" - "$@" -} +. ci/funs.sh +nimBuildCsourcesIfNeeded "$@" -[ -d csources ] || echo_run git clone -q --depth 1 https://github.com/nim-lang/csources.git - -nim_csources=bin/nim_csources - -build_nim_csources_via_script(){ - echo_run cd csources - echo_run sh build.sh "$@" -} - -build_nim_csources(){ - # avoid changing dir in case of failure - ( - if [ $# -ne 0 ]; then - # some args were passed (e.g.: `--cpu i386`), need to call build.sh - build_nim_csources_via_script "$@" - else - # no args, use multiple Make jobs (5X faster on 16 cores: 10s instead of 50s) - makeX=make - unamestr=$(uname) - if [ "$unamestr" = 'FreeBSD' ]; then - makeX=gmake - fi - nCPU=$(nproc 2>/dev/null || sysctl -n hw.logicalcpu 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || 1) - which $makeX && echo_run $makeX -C csources -j $((nCPU + 2)) -l $nCPU || build_nim_csources_via_script - fi - ) - # keep $nim_csources in case needed to investigate bootstrap issues - # without having to rebuild from csources - echo_run cp bin/nim $nim_csources -} - -[ -f $nim_csources ] || echo_run build_nim_csources "$@" - -# Note: if fails, may need to `cd csources && git pull` +# Note: if fails, may need to update csourcesAny manually echo_run bin/nim c --skipUserCfg --skipParentCfg koch - echo_run ./koch boot -d:release --skipUserCfg --skipParentCfg echo_run ./koch tools --skipUserCfg --skipParentCfg # Compile Nimble and other tools. - diff --git a/ci/build.bat b/ci/build.bat deleted file mode 100644 index 2227168e5..000000000 --- a/ci/build.bat +++ /dev/null @@ -1,14 +0,0 @@ -REM Some debug info -echo "Running on %CI_RUNNER_ID% (%CI_RUNNER_DESCRIPTION%) with tags %CI_RUNNER_TAGS%." -gcc -v - -git clone --depth 1 https://github.com/nim-lang/csources_v1.git csources -cd csources -call build64.bat -cd .. -set PATH=%CD%\bin;%PATH% -nim -v -nim c koch -koch.exe boot -copy bin/nim bin/nimd -koch.exe boot -d:release diff --git a/ci/build.sh b/ci/build.sh deleted file mode 100644 index a89d011ed..000000000 --- a/ci/build.sh +++ /dev/null @@ -1,15 +0,0 @@ -sh ci/deps.sh - -# Build from C sources. -git clone --depth 1 https://github.com/nim-lang/csources_v1.git csources -cd csources -sh build.sh -cd .. -# Add Nim to the PATH -export PATH=$(pwd)/bin${PATH:+:$PATH} -# Bootstrap. -nim -v -nim c koch -./koch boot -cp bin/nim bin/nimd -./koch boot -d:release diff --git a/ci/deps.bat b/ci/deps.bat deleted file mode 100644 index bda1fe14f..000000000 --- a/ci/deps.bat +++ /dev/null @@ -1,4 +0,0 @@ -nim e install_nimble.nims -nim e tests/test_nimscript.nims -nimble update -nimble install -y zip opengl sdl1 jester@#head niminst diff --git a/ci/deps.sh b/ci/deps.sh deleted file mode 100644 index f0f831a2a..000000000 --- a/ci/deps.sh +++ /dev/null @@ -1,16 +0,0 @@ -# Some debug info -echo "Running on $CI_RUNNER_ID ($CI_RUNNER_DESCRIPTION) with tags $CI_RUNNER_TAGS." - -# Packages -apt-get update -qq -apt-get install -y -qq build-essential git libcurl4-openssl-dev libsdl1.2-dev libgc-dev nodejs - -gcc -v - -export PATH=$(pwd)/bin${PATH:+:$PATH} - -# Nimble deps -nim e install_nimble.nims -nim e tests/test_nimscript.nims -nimble update -nimble install zip opengl sdl1 jester@#head niminst diff --git a/ci/funs.sh b/ci/funs.sh index 7f82e1ca3..7a6c66722 100644 --- a/ci/funs.sh +++ b/ci/funs.sh @@ -1,5 +1,7 @@ -# utilities used in CI pipelines to avoid duplication. +# Utilities used in CI pipelines and tooling to avoid duplication. # Avoid top-level statements. +# Prefer nim scripts whenever possible. +# functions starting with `_` are considered internal, less stable. echo_run () { # echo's a command before running it, which helps understanding logs @@ -28,3 +30,72 @@ nimIsCiSkip(){ return 1 fi } + +nimDefineVars(){ + nim_csourcesDir=csources_v1 # where we clone + nim_csourcesUrl=https://github.com/nim-lang/csources_v1.git + nim_csourcesHash=a8a5241f9475099c823cfe1a5e0ca4022ac201ff + nim_csources=bin/nim_csources_$nim_csourcesHash +} + +_nimNumCpu(){ + # linux: $(nproc) + # FreeBSD | macOS: $(sysctl -n hw.ncpu) + # OpenBSD: $(sysctl -n hw.ncpuonline) + # windows: $NUMBER_OF_PROCESSORS ? + echo $(nproc 2>/dev/null || sysctl -n hw.logicalcpu 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || 1) +} + +_nimBuildCsourcesIfNeeded(){ + # if some systems cannot use make or gmake, we could add support for calling `build.sh` + # but this is slower (not parallel jobs) and would require making build.sh + # understand the arguments passed to the makefile (e.g. `CC=gcc ucpu=amd64 uos=darwin`), + # instead of `--cpu amd64 --os darwin`. + unamestr=$(uname) + # uname values: https://en.wikipedia.org/wiki/Uname + if [ "$unamestr" = 'FreeBSD' ]; then + makeX=gmake + elif [ "$unamestr" = 'OpenBSD' ]; then + makeX=gmake + else + makeX=make + fi + nCPU=$(_nimNumCpu) + echo_run which $makeX + # parallel jobs (5X faster on 16 cores: 10s instead of 50s) + echo_run $makeX -C $nim_csourcesDir -j $((nCPU + 2)) -l $nCPU "$@" + # keep $nim_csources in case needed to investigate bootstrap issues + # without having to rebuild + echo_run cp bin/nim $nim_csources +} + +nimCsourcesHash(){ + nimDefineVars + echo $nim_csourcesHash +} + +nimBuildCsourcesIfNeeded(){ + # goal: allow cachine each tagged version independently + # to avoid rebuilding, so that tools like `git bisect` + # can grab a cached past version without rebuilding. + nimDefineVars + ( + set -e + # avoid polluting caller scope with internal variable definitions. + if test -f "$nim_csources"; then + echo "$nim_csources exists." + else + if test -d "$nim_csourcesDir"; then + echo "$nim_csourcesDir exists." + else + # depth 1: adjust as needed in case useful for `git bisect` + echo_run git clone -q --depth 1 $nim_csourcesUrl "$nim_csourcesDir" + echo_run git -C "$nim_csourcesDir" checkout $nim_csourcesHash + fi + _nimBuildCsourcesIfNeeded "$@" + fi + + echo_run cp $nim_csources bin/nim + echo_run $nim_csources -v + ) +} diff --git a/ci/nsis_build.bat b/ci/nsis_build.bat deleted file mode 100644 index 12aff1b72..000000000 --- a/ci/nsis_build.bat +++ /dev/null @@ -1,59 +0,0 @@ -REM - Run the full testsuite; testament\tester all - -REM - Uncomment the list of changes in news.txt -REM - write a news ticker entry -REM - Update the version - -REM - Generate the full docs; koch web0 -REM - Generate the installers; -REM - Update the version in system.nim -REM - Test the installers -REM - Tag the release -REM - Merge devel into master -REM - Update csources - -set NIMVER=%1 - -Rem Build -docs file: -koch web0 -cd web\upload -7z a -tzip docs-%NIMVER%.zip *.html -move /y docs-%NIMVER%.zip download -cd ..\.. - -Rem Build csources -koch csources -d:release || exit /b - -rem Grab C sources and nimsuggest -git clone --depth 1 https://github.com/nim-lang/csources_v1.git csources - -set PATH=%CD%\bin;%PATH% - -ReM Build Win32 version: - -set PATH=C:\Users\araq\projects\mingw32\bin;%PATH% -cd csources -call build.bat -cd .. -ReM Rebuilding koch is necessary because it uses its pointer size to determine -ReM which mingw link to put in the NSIS installer. -nim c --out:koch_temp koch || exit /b -koch_temp boot -d:release || exit /b -koch_temp nsis -d:release || exit /b -koch_temp zip -d:release || exit /b -dir build -move /y build\nim_%NIMVER%.exe build\nim-%NIMVER%_x32.exe || exit /b -move /y build\nim-%NIMVER%.zip build\nim-%NIMVER%_x32.zip || exit /b - - -ReM Build Win64 version: -set PATH=C:\Users\araq\projects\mingw64\bin;%PATH% -cd csources -call build64.bat -cd .. -nim c --out:koch_temp koch || exit /b -koch_temp boot -d:release || exit /b -koch_temp nsis -d:release || exit /b -koch_temp zip -d:release || exit /b -move /y build\nim_%NIMVER%.exe build\nim-%NIMVER%_x64.exe || exit /b -move /y build\nim-%NIMVER%.zip build\nim-%NIMVER%_x64.zip || exit /b diff --git a/koch.nim b/koch.nim index 49540eb6a..c78356b58 100644 --- a/koch.nim +++ b/koch.nim @@ -89,7 +89,6 @@ Commands for core developers: tests [options] run the testsuite (run a subset of tests by specifying a category, e.g. `tests cat async`) temp options creates a temporary compiler for testing - pushcsource push generated C sources to its repo Web options: --googleAnalytics:UA-... add the given google analytics code to the docs. To build the official docs, use UA-48159761-1 @@ -283,7 +282,7 @@ template doUseCpp(): bool = getEnv("NIM_COMPILE_TO_CPP", "false") == "true" proc boot(args: string) = ## bootstrapping is a process that involves 3 steps: - ## 1. use csources to produce nim1.exe. This nim1.exe is buggy but + ## 1. use csourcesAny to produce nim1.exe. This nim1.exe is buggy but ## rock solid for building a Nim compiler. It shouldn't be used for anything else. ## 2. use nim1.exe to produce nim2.exe. nim2.exe is the one you really need. ## 3. We use nim2.exe to build nim3.exe. nim3.exe is equal to nim2.exe except for timestamps. @@ -315,13 +314,8 @@ proc boot(args: string) = let ret = execCmdEx(nimStart & " --version") doAssert ret.exitCode == 0 let version = ret.output.splitLines[0] - # remove these when csources get updated - template addLib() = + if version.startsWith "Nim Compiler Version 0.20.0": extraOption.add " --lib:lib" # see https://github.com/nim-lang/Nim/pull/14291 - if version.startsWith "Nim Compiler Version 0.19.0": - extraOption.add " -d:nimBoostrapCsources0_19_0" - addLib() - elif version.startsWith "Nim Compiler Version 0.20.0": addLib() # in order to use less memory, we split the build into two steps: # --compileOnly produces a $project.json file and does not run GCC/Clang. @@ -587,29 +581,6 @@ proc runCI(cmd: string) = when defined(posix): execFold("Run nimsuggest tests", "nim c -r nimsuggest/tester") -proc pushCsources() = - if not dirExists("../csources/.git"): - quit "[Error] no csources git repository found" - csource("-d:danger") - let cwd = getCurrentDir() - try: - copyDir("build/c_code", "../csources/c_code") - copyFile("build/build.sh", "../csources/build.sh") - copyFile("build/build.bat", "../csources/build.bat") - copyFile("build/build64.bat", "../csources/build64.bat") - copyFile("build/makefile", "../csources/makefile") - - setCurrentDir("../csources") - for kind, path in walkDir("c_code"): - if kind == pcDir: - exec("git add " & path / "*.c") - exec("git commit -am \"updated csources to version " & NimVersion & "\"") - exec("git push origin master") - exec("git tag -am \"Version $1\" v$1" % NimVersion) - exec("git push origin v$1" % NimVersion) - finally: - setCurrentDir(cwd) - proc testUnixInstall(cmdLineRest: string) = csource("-d:danger" & cmdLineRest) xz(false, cmdLineRest) @@ -723,7 +694,8 @@ when isMainModule: of "tools": buildTools(op.cmdLineRest) bundleNimbleExe(latest, op.cmdLineRest) - of "pushcsource", "pushcsources": pushCsources() + of "pushcsource": + quit "use this instead: https://github.com/nim-lang/csources_v1/blob/master/push_c_code.nim" of "valgrind": valgrind(op.cmdLineRest) of "c2nim": bundleC2nim(op.cmdLineRest) of "drnim": buildDrNim(op.cmdLineRest) diff --git a/lib/std/private/globs.nim b/lib/std/private/globs.nim index a32f1d1b9..55f6a40bd 100644 --- a/lib/std/private/globs.nim +++ b/lib/std/private/globs.nim @@ -52,5 +52,5 @@ proc nativeToUnixPath*(path: string): string = when isMainModule: import sugar - for a in walkDirRecFilter(".", follow = a=>a.path.lastPathPart notin ["nimcache", ".git", ".csources", "bin"]): + for a in walkDirRecFilter(".", follow = a=>a.path.lastPathPart notin ["nimcache", ".git", "csources_v1", "csources", "bin"]): echo a diff --git a/readme.md b/readme.md index 9a07ae56b..37babf711 100644 --- a/readme.md +++ b/readme.md @@ -45,7 +45,7 @@ Compiling the Nim compiler is quite straightforward if you follow these steps: First, the C source of an older version of the Nim compiler is needed to bootstrap the latest version because the Nim compiler itself is written in the Nim programming language. Those C sources are available within the -[``nim-lang/csources``][csources-repo] repository. +[``nim-lang/csources_v1``][csources-v1-repo] repository. Next, to build from source you will need: @@ -217,7 +217,8 @@ Copyright © 2006-2021 Andreas Rumpf, all rights reserved. [nim-bitcoin]: https://blockchain.info/address/1BXfuKM2uvoD6mbx4g5xM3eQhLzkCK77tJ [nimble-repo]: https://github.com/nim-lang/nimble [nimsuggest-repo]: https://github.com/nim-lang/nimsuggest -[csources-repo]: https://github.com/nim-lang/csources +[csources-repo-deprecated]: https://github.com/nim-lang/csources +[csources-v1-repo]: https://github.com/nim-lang/csources_v1 [badge-nim-travisci]: https://img.shields.io/travis/nim-lang/Nim/devel.svg?style=flat-square [badge-nim-irc]: https://img.shields.io/badge/chat-on_irc-blue.svg?style=flat-square [badge-nim-discord]: https://img.shields.io/discord/371759389889003530?color=blue&label=discord&logo=discord&logoColor=gold&style=flat-square diff --git a/tools/ci_generate.nim b/tools/ci_generate.nim index d13f28c33..e2580ff6b 100644 --- a/tools/ci_generate.nim +++ b/tools/ci_generate.nim @@ -33,15 +33,15 @@ environment: CC: /usr/bin/clang tasks: - setup: | + set -e cd Nim - git clone --depth 1 -q https://github.com/nim-lang/csources.git - gmake -C csources -j $(sysctl -n hw.ncpuonline) - bin/nim c koch + . ci/funs.sh && nimBuildCsourcesIfNeeded + $nim_csources c koch echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv - test: | cd Nim if ! ./koch runCI; then - nim c -r tools/ci_testresults.nim + nim r tools/ci_testresults.nim exit 1 fi triggers: From 511b6d2449c65cecc68eccbee1385e02758f0fd4 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 23 Apr 2021 13:50:27 +0200 Subject: [PATCH 0704/2472] typos (#17824) --- compiler/ic/packed_ast.nim | 2 +- doc/destructors.rst | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/compiler/ic/packed_ast.nim b/compiler/ic/packed_ast.nim index 6fadde17c..17beda2c1 100644 --- a/compiler/ic/packed_ast.nim +++ b/compiler/ic/packed_ast.nim @@ -86,7 +86,7 @@ type typeInst*: PackedItemId nonUniqueId*: int32 - PackedNode* = object # 20 bytes + PackedNode* = object # 28 bytes kind*: TNodeKind flags*: TNodeFlags operand*: int32 # for kind in {nkSym, nkSymDef}: SymId diff --git a/doc/destructors.rst b/doc/destructors.rst index 01e2d2ee9..c93fcabe1 100644 --- a/doc/destructors.rst +++ b/doc/destructors.rst @@ -277,7 +277,7 @@ inference. But it has to be enabled via `--sinkInference:on`, either on the command line or via a `push` pragma. To enable it for a section of code, one can -use `{.push sinkInference: on.}`...`{.pop.}`. +use `{.push sinkInference: on.}` ... `{.pop.}`. The `.nosinks`:idx: pragma can be used to disable this inference for a single routine: @@ -661,12 +661,12 @@ There is no copy during assigning operations. y[0] = 'h' # copy The program above shows when the copy operations happen. -When mutating the variable `y`, the Nim compiler creates a fresh copy of `x`, -the variable `y` won't point to the string literal anymore. -Instead it points to the copy of `x` of which the memory can be mutated +When mutating the variable `y`, the Nim compiler creates a fresh copy of `x`, +the variable `y` won't point to the string literal anymore. +Instead it points to the copy of `x` of which the memory can be mutated and the variable `y` becomes a mutable string. -.. Note:: The abstraction fails for `addr x` because whether the address is going to be used for mutations is unknown. +.. Note:: The abstraction fails for `addr x` because whether the address is going to be used for mutations is unknown. Let's look at a silly example demonstrating this behaviour: From 3516f57e172cef22f583f43c7b16fd1f63921c6e Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 23 Apr 2021 04:51:24 -0700 Subject: [PATCH 0705/2472] manual: fix rendering of `Identifiers & Keywords` [skip ci] (#17811) --- doc/manual.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual.rst b/doc/manual.rst index 15242cf7c..e98a0cc09 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -261,8 +261,9 @@ and underscores, with the following restrictions: * begins with a letter * does not end with an underscore `_` -* two immediate following underscores `__` are not allowed:: +* two immediate following underscores `__` are not allowed: +.. code-block:: letter ::= 'A'..'Z' | 'a'..'z' | '\x80'..'\xff' digit ::= '0'..'9' IDENTIFIER ::= letter ( ['_'] (letter | digit) )* From 2abc936d511342265d2ef27c4b079dd49332d65a Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 23 Apr 2021 05:36:38 -0700 Subject: [PATCH 0706/2472] `typeof(voidStmt)` now works (#17807) * `typeof(voidStmt)` now works * remove typeOrVoid * add condsyms, and reference cligen https://github.com/c-blake/cligen/pull/193 * fixup * changelog [skip ci] * fixup --- changelog.md | 2 + compiler/condsyms.nim | 1 + compiler/sem.nim | 1 + compiler/semdata.nim | 3 ++ compiler/semexprs.nim | 4 +- lib/js/asyncjs.nim | 7 +--- testament/lib/stdtest/testutils.nim | 11 +++--- tests/typerel/tvoid.nim | 61 +++++++++++++++++++++++++++++ 8 files changed, 78 insertions(+), 12 deletions(-) diff --git a/changelog.md b/changelog.md index 7b4a2438a..d7b568aa0 100644 --- a/changelog.md +++ b/changelog.md @@ -331,6 +331,8 @@ - Added a new module `std/importutils`, and an API `privateAccess`, which allows access to private fields for an object type in the current scope. +- `typeof(voidStmt)` now works and returns `void`. + ## Compiler changes - Added `--declaredlocs` to show symbol declaration location in messages. diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index bdecd7e53..6a49584c8 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -133,3 +133,4 @@ proc initDefines*(symbols: StringTableRef) = defineSymbol("nimHasCustomLiterals") defineSymbol("nimHasUnifiedTuple") defineSymbol("nimHasIterable") + defineSymbol("nimHasTypeofVoid") diff --git a/compiler/sem.nim b/compiler/sem.nim index 4020fc4d6..b80795354 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -525,6 +525,7 @@ proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext var c = newContext(graph, module) c.idgen = idgen c.enforceVoidContext = newType(tyTyped, nextTypeId(idgen), nil) + c.voidType = newType(tyVoid, nextTypeId(idgen), nil) if c.p != nil: internalError(graph.config, module.info, "sem.myOpen") c.semConstExpr = semConstExpr diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 1c1a5159c..082a4813e 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -90,6 +90,9 @@ type TContext* = object of TPassContext # a context represents the module # that is currently being compiled enforceVoidContext*: PType + # for `if cond: stmt else: foo`, `foo` will be evaluated under + # enforceVoidContext != nil + voidType*: PType # for typeof(stmt) module*: PSym # the module sym belonging to the context currentScope*: PScope # current scope moduleScope*: PScope # scope for modules diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 97f88869e..2e229d861 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -83,7 +83,9 @@ proc semExprCheck(c: PContext, n: PNode, flags: TExprFlags): PNode = proc semExprWithType(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = result = semExprCheck(c, n, flags) - if result.typ == nil or result.typ == c.enforceVoidContext: + if result.typ == nil and efInTypeof in flags: + result.typ = c.voidType + elif result.typ == nil or result.typ == c.enforceVoidContext: localError(c.config, n.info, errExprXHasNoType % renderTree(result, {renderNoComments})) result.typ = errorType(c) diff --git a/lib/js/asyncjs.nim b/lib/js/asyncjs.nim index c62ac633f..861fe3fe2 100644 --- a/lib/js/asyncjs.nim +++ b/lib/js/asyncjs.nim @@ -158,10 +158,6 @@ proc newPromise*(handler: proc(resolve: proc())): Future[void] {.importjs: "(new ## A helper for wrapping callback-based functions ## into promises and async procedures. -template typeOrVoid[T](a: T): type = - # xxx this is useful, make it public in std/typetraits in future work - T - template maybeFuture(T): untyped = # avoids `Future[Future[T]]` when T is Future: T @@ -221,7 +217,8 @@ when defined(nimExperimentalAsyncjsThen): assert witness == 3 template impl(call): untyped = - when typeOrVoid(call) is void: + # see D20210421T014713 + when typeof(block: call) is void: var ret: Future[void] else: var ret = default(maybeFuture(typeof(call))) diff --git a/testament/lib/stdtest/testutils.nim b/testament/lib/stdtest/testutils.nim index 44048adbd..241ab1770 100644 --- a/testament/lib/stdtest/testutils.nim +++ b/testament/lib/stdtest/testutils.nim @@ -91,10 +91,6 @@ template disableVm*(body) = when nimvm: discard else: body -template typeOrVoid[T](a: T): type = - # FACTOR with asyncjs.typeOrVoid - T - macro assertAll*(body) = ## works in VM, unlike `check`, `require` runnableExamples: @@ -106,6 +102,9 @@ macro assertAll*(body) = result = newStmtList() for a in body: result.add genAst(a) do: - # better than: `when not compiles(typeof(a)):` - when typeOrVoid(a) is void: a + # D20210421T014713:here + # xxx pending https://github.com/nim-lang/Nim/issues/12030, + # `typeof` should introduce its own scope, so that this + # is sufficient: `typeof(a)` instead of `typeof(block: a)` + when typeof(block: a) is void: a else: doAssert a diff --git a/tests/typerel/tvoid.nim b/tests/typerel/tvoid.nim index 1e46d016e..0b7b525cc 100644 --- a/tests/typerel/tvoid.nim +++ b/tests/typerel/tvoid.nim @@ -35,3 +35,64 @@ ReturnT[void]() echo ReturnT[string]("abc") nothing() +block: # typeof(stmt) + proc fn1(): auto = + discard + proc fn2(): auto = + 1 + doAssert type(fn1()) is void + doAssert typeof(fn1()) is void + doAssert typeof(fn1()) isnot int + + doAssert type(fn2()) isnot void + doAssert typeof(fn2()) isnot void + when typeof(fn1()) is void: discard + else: doAssert false + + doAssert typeof(1+1) is int + doAssert typeof((discard)) is void + + type A1 = typeof(fn1()) + doAssert A1 is void + type A2 = type(fn1()) + doAssert A2 is void + doAssert A2 is A1 + + when false: + # xxx: MCS/UFCS doesn't work here: Error: expression 'fn1()' has no type (or is ambiguous) + type A3 = fn1().type + proc bar[T](a: T): string = $T + doAssert bar(1) == "int" + doAssert bar(fn1()) == "void" + + proc bar2[T](a: T): bool = T is void + doAssert not bar2(1) + doAssert bar2(fn1()) + + block: + proc bar3[T](a: T): T = a + let a1 = bar3(1) + doAssert compiles(block: + let a1 = bar3(fn2())) + doAssert not compiles(block: + let a2 = bar3(fn1())) + doAssert compiles(block: bar3(fn1())) + doAssert compiles(bar3(fn1())) + doAssert typeof(bar3(fn1())) is void + doAssert not compiles(sizeof(bar3(fn1()))) + + block: + var a = 1 + doAssert typeof((a = 2)) is void + doAssert typeof((a = 2; a = 3)) is void + doAssert typeof(block: + a = 2; a = 3) is void + + block: + var a = 1 + template bad1 = echo (a; a = 2) + doAssert not compiles(bad1()) + + block: + template bad2 = echo (nonexistant; discard) + doAssert not compiles(bad2()) From 80df21d9a368b0ad40f6c36b898861e75926b732 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 23 Apr 2021 14:22:04 -0700 Subject: [PATCH 0707/2472] `--declaredlocs` now shows location for `T` instead of `static` in `proc fn(a: static T)` (#17795) --- compiler/types.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compiler/types.nim b/compiler/types.nim index ba2cc76ce..b57e46444 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -131,7 +131,10 @@ proc addDeclaredLocMaybe*(result: var string, conf: ConfigRef; sym: PSym) = addDeclaredLoc(result, conf, sym) proc addDeclaredLoc*(result: var string, conf: ConfigRef; typ: PType) = - let typ = typ.skipTypes(abstractInst - {tyRange}) + # xxx figure out how to resolve `tyGenericParam`, e.g. for + # proc fn[T](a: T, b: T) = discard + # fn(1.1, "a") + let typ = typ.skipTypes(abstractInst + {tyStatic} - {tyRange}) result.add " [$1" % typ.kind.toHumanStr if typ.sym != nil: result.add " declared in " & toFileLineCol(conf, typ.sym.info) From b667e288e27a09ce4f65166fc66a6b713350a9b6 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 23 Apr 2021 22:54:31 -0700 Subject: [PATCH 0708/2472] move travis, appveyor, ci.yml.disabled to `unmaintained/` (#17828) * move travis, appveyor, ci.yml.disabled to unmaintained * update some mentions of travis, appevyor * fix test --- changelog.md | 2 ++ doc/contributing.rst | 4 ++-- testament/specs.nim | 4 ++-- tests/async/tasyncdial.nim | 1 - tests/stdlib/tmemlinesBuf.nim | 8 +------- tests/stdlib/tnetdial.nim | 1 - tests/system/talloc.nim | 6 +----- tests/system/talloc2.nim | 2 +- .travis.yml => unmaintained/.travis.yml | 0 appveyor.yml.disabled => unmaintained/appveyor.yml | 0 .../workflows/ci.yml.disabled => unmaintained/ci.yml | 0 unmaintained/readme.md | 11 +++++++++++ 12 files changed, 20 insertions(+), 19 deletions(-) rename .travis.yml => unmaintained/.travis.yml (100%) rename appveyor.yml.disabled => unmaintained/appveyor.yml (100%) rename .github/workflows/ci.yml.disabled => unmaintained/ci.yml (100%) create mode 100644 unmaintained/readme.md diff --git a/changelog.md b/changelog.md index d7b568aa0..943b5d6df 100644 --- a/changelog.md +++ b/changelog.md @@ -57,6 +57,8 @@ - The unary slice `..b` was removed, use `0..b` instead or use `-d:nimLegacyUnarySlice` for a deprecation period. +- Moved `.travis.yml`, `appveyor.yml.disabled`, `.github/workflows/ci.yml.disabled` to `unmaintained/`. + ## Standard library additions and changes - Added support for parenthesized expressions in `strformat` diff --git a/doc/contributing.rst b/doc/contributing.rst index 81fa9f8eb..abc81dae6 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -155,7 +155,7 @@ To run a single test: ./koch test run tests/stdlib/tos.nim # can also provide relative path For reproducible tests (to reproduce an environment more similar to the one -run by Continuous Integration on travis/appveyor), you may want to disable your +run by Continuous Integration on github actions/azure pipelines), you may want to disable your local configuration (e.g. in ``~/.config/nim/nim.cfg``) which may affect some tests; this can also be achieved by using `export XDG_CONFIG_HOME=pathtoAlternateConfig`:cmd: before running `./koch`:cmd: @@ -517,7 +517,7 @@ Continuous Integration (CI) the CI pipeline and affects other PR's; if you don't need it (e.g. for WIP or documentation only changes), add ``[skip ci]`` to your commit message title. This convention is supported by our github actions pipelines and our azure pipeline - as well as our former other pipelines: + (using custom logic, which should complete in < 1mn) as well as our former other pipelines: `Appveyor `_ and `Travis `_. diff --git a/testament/specs.nim b/testament/specs.nim index 043f14ed3..38e4ed0db 100644 --- a/testament/specs.nim +++ b/testament/specs.nim @@ -328,9 +328,9 @@ proc parseSpec*(filename: string): TSpec = when defined(unix): result.err = reDisabled of "posix": when defined(posix): result.err = reDisabled - of "travis": + of "travis": # deprecated if isTravis: result.err = reDisabled - of "appveyor": + of "appveyor": # deprecated if isAppVeyor: result.err = reDisabled of "azure": if isAzure: result.err = reDisabled diff --git a/tests/async/tasyncdial.nim b/tests/async/tasyncdial.nim index a755de74e..f0377dfd5 100644 --- a/tests/async/tasyncdial.nim +++ b/tests/async/tasyncdial.nim @@ -3,7 +3,6 @@ discard """ OK AF_INET OK AF_INET6 ''' - disabled: "travis" """ import diff --git a/tests/stdlib/tmemlinesBuf.nim b/tests/stdlib/tmemlinesBuf.nim index ea607525d..3f0bd5182 100644 --- a/tests/stdlib/tmemlinesBuf.nim +++ b/tests/stdlib/tmemlinesBuf.nim @@ -1,8 +1,3 @@ -discard """ -output: "15" -disabled: "appveyor" -""" - import memfiles var inp = memfiles.open("tests/stdlib/tmemlinesBuf.nim") var buffer: string = "" @@ -11,5 +6,4 @@ for line in lines(inp, buffer): lineCount += 1 close(inp) - -echo lineCount +doAssert lineCount == 9, $lineCount # this file's number of lines diff --git a/tests/stdlib/tnetdial.nim b/tests/stdlib/tnetdial.nim index f940bd630..3de00ff99 100644 --- a/tests/stdlib/tnetdial.nim +++ b/tests/stdlib/tnetdial.nim @@ -2,7 +2,6 @@ discard """ cmd: "nim c --threads:on $file" exitcode: 0 output: "OK" - disabled: "travis" """ import os, net, nativesockets, asyncdispatch diff --git a/tests/system/talloc.nim b/tests/system/talloc.nim index 9b970fda0..a81fef481 100644 --- a/tests/system/talloc.nim +++ b/tests/system/talloc.nim @@ -1,8 +1,4 @@ -discard """ -disabled: "appveyor" -""" - -# appveyor is "out of memory" +# was: appveyor is "out of memory" var x: ptr int diff --git a/tests/system/talloc2.nim b/tests/system/talloc2.nim index c498a3f4b..9d1687f34 100644 --- a/tests/system/talloc2.nim +++ b/tests/system/talloc2.nim @@ -5,7 +5,7 @@ joinable: false disabled: 32bit """ # no point to test this on system with smaller address space -# appveyor is "out of memory" +# was: appveyor is "out of memory" const nmax = 2*1024*1024*1024 diff --git a/.travis.yml b/unmaintained/.travis.yml similarity index 100% rename from .travis.yml rename to unmaintained/.travis.yml diff --git a/appveyor.yml.disabled b/unmaintained/appveyor.yml similarity index 100% rename from appveyor.yml.disabled rename to unmaintained/appveyor.yml diff --git a/.github/workflows/ci.yml.disabled b/unmaintained/ci.yml similarity index 100% rename from .github/workflows/ci.yml.disabled rename to unmaintained/ci.yml diff --git a/unmaintained/readme.md b/unmaintained/readme.md new file mode 100644 index 000000000..cddb15cd2 --- /dev/null +++ b/unmaintained/readme.md @@ -0,0 +1,11 @@ +Deprecated, unused and unmaintained files can be moved here when they have some value, +e.g. if someone want to revive them in their project or in this repo. If a deprecated and unused file +is redundant, it should just be removed instead. + +See also https://github.com/nim-lang/graveyard, which is maintained on a best effort basis +and has different goals. + +## table of contents +.travis.yml -> unmaintained/.travis.yml +appveyor.yml.disabled -> unmaintained/appveyor.yml +.github/workflows/ci.yml.disabled -> unmaintained/ci.yml From 6e8f0f8cfcd6d617074be9868027e39daaeba103 Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 25 Apr 2021 01:33:29 +0800 Subject: [PATCH 0709/2472] add colon (#17834) --- compiler/pragmas.nim | 2 +- compiler/semexprs.nim | 2 +- compiler/spawn.nim | 2 +- lib/pure/httpclient.nim | 2 +- lib/pure/selectors.nim | 2 +- tests/parallel/t7535.nim | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 35e75bc7b..3a2d9cede 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -824,7 +824,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, makeExternExport(c, sym, getOptionalStr(c, it, "$1"), it.info) if k == wExportCpp: if c.config.backend != backendCpp: - localError(c.config, it.info, "exportcpp requires `cpp` backend, got " & $c.config.backend) + localError(c.config, it.info, "exportcpp requires `cpp` backend, got: " & $c.config.backend) else: incl(sym.flags, sfMangleCpp) incl(sym.flags, sfUsed) # avoid wrong hints diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 2e229d861..cf8d4777f 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -2291,7 +2291,7 @@ proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode = result[i] = semExpr(c, n[i]) if n.len > 1 and n[1].kind notin nkCallKinds: - return localErrorNode(c, n, n[1].info, "'spawn' takes a call expression; got " & $n[1]) + return localErrorNode(c, n, n[1].info, "'spawn' takes a call expression; got: " & $n[1]) let typ = result[^1].typ if not typ.isEmptyType: diff --git a/compiler/spawn.nim b/compiler/spawn.nim index 205197eaf..e57686cb4 100644 --- a/compiler/spawn.nim +++ b/compiler/spawn.nim @@ -334,7 +334,7 @@ proc wrapProcForSpawn*(g: ModuleGraph; idgen: IdGenerator; owner: PSym; spawnExp result = newNodeI(nkStmtList, n.info) if n.kind notin nkCallKinds: - localError(g.config, n.info, "'spawn' takes a call expression; got " & $n) + localError(g.config, n.info, "'spawn' takes a call expression; got: " & $n) return if optThreadAnalysis in g.config.globalOptions: if {tfThread, tfNoSideEffect} * n[0].typ.flags == {}: diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index b60894103..86708dc40 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -751,7 +751,7 @@ proc parseBody(client: HttpClient | AsyncHttpClient, headers: HttpHeaders, httpError("Got disconnected while trying to read body.") if recvLen != length: httpError("Received length doesn't match expected length. Wanted " & - $length & " got " & $recvLen) + $length & " got: " & $recvLen) else: # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.4 TODO diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim index e78219aec..82550e09b 100644 --- a/lib/pure/selectors.nim +++ b/lib/pure/selectors.nim @@ -321,7 +321,7 @@ else: proc verifySelectParams(timeout: int) = # Timeout of -1 means: wait forever # Anything higher is the time to wait in milliseconds. - doAssert(timeout >= -1, "Cannot select with a negative value, got " & $timeout) + doAssert(timeout >= -1, "Cannot select with a negative value, got: " & $timeout) when defined(linux) and not defined(emscripten): include ioselects/ioselectors_epoll diff --git a/tests/parallel/t7535.nim b/tests/parallel/t7535.nim index a6bc0dabe..052dcdc3a 100644 --- a/tests/parallel/t7535.nim +++ b/tests/parallel/t7535.nim @@ -1,6 +1,6 @@ discard """ matrix: "--threads:on" - errormsg: "'spawn' takes a call expression; got proc (x: uint32) = echo [x]" + errormsg: "'spawn' takes a call expression; got: proc (x: uint32) = echo [x]" """ import threadpool From e67593d31797cd6fa5e360c0f0a30fd02e596308 Mon Sep 17 00:00:00 2001 From: Clyybber Date: Sat, 24 Apr 2021 19:34:23 +0200 Subject: [PATCH 0710/2472] Remove unmaintained dir (#17833) --- unmaintained/.travis.yml | 52 ------------------- unmaintained/appveyor.yml | 37 -------------- unmaintained/ci.yml | 105 -------------------------------------- unmaintained/readme.md | 11 ---- 4 files changed, 205 deletions(-) delete mode 100644 unmaintained/.travis.yml delete mode 100644 unmaintained/appveyor.yml delete mode 100644 unmaintained/ci.yml delete mode 100644 unmaintained/readme.md diff --git a/unmaintained/.travis.yml b/unmaintained/.travis.yml deleted file mode 100644 index 13e5d151a..000000000 --- a/unmaintained/.travis.yml +++ /dev/null @@ -1,52 +0,0 @@ -# deprecated, out of date and unused -sudo: false -language: c - -dist: xenial - -matrix: - include: - - os: linux - env: - - NIM_COMPILE_TO_CPP=false - - CPU=amd64 - -addons: - apt: - # update the list above if more deps are introduced - packages: - - libcurl4-openssl-dev - - libsdl1.2-dev - - libgc-dev - - libsfml-dev - - libc6-dbg - - valgrind - -before_script: - - . ci/funs.sh && nimBuildCsourcesIfNeeded LD=$CC ucpu=$CPU - - export PATH="$PWD/bin${PATH:+:$PATH}" - -script: - - echo "travis_fold:start:nim_c_koch" - - nim c koch - - echo "travis_fold:end:nim_c_koch" - - echo "travis_fold:start:koch_boot" - - ./koch boot - - echo "travis_fold:end:koch_boot" - - echo "travis_fold:start:koch_doc" - - ./koch doc - - echo "travis_fold:end:koch_doc" - -before_deploy: - # Make https://nim-lang.github.io/Nim work the same as https://nim-lang.github.io/Nim/overview.html - - cp -f ./doc/html/overview.html ./doc/html/index.html - -deploy: # https://nim-lang.github.io/Nim - provider: pages - # local-dir *has* to be a relative path from the repo root. - local-dir: "doc/html" - skip-cleanup: true - github-token: "$GITHUB_OAUTH_TOKEN" - keep-history: false - on: - branch: devel diff --git a/unmaintained/appveyor.yml b/unmaintained/appveyor.yml deleted file mode 100644 index b13c5d909..000000000 --- a/unmaintained/appveyor.yml +++ /dev/null @@ -1,37 +0,0 @@ -# deprecated, out of date and unused -version: '{build}' - -environment: - DLLS_URL: https://nim-lang.org/download/dlls.zip - DLLS_ARCHIVE: dlls.zip - MINGW_DIR: mingw64 - MINGW_URL: https://nim-lang.org/download/mingw64.7z - MINGW_ARCHIVE: mingw64.7z - - matrix: - - NIM_TEST_PACKAGES: false - - NIM_TEST_PACKAGES: true - -cache: - - '%MINGW_ARCHIVE%' - - '%DLLS_ARCHIVE%' - - -install: - - ps: Install-Product node 8 # node 8 or later is required to test js async stuff - - IF not exist "%DLLS_ARCHIVE%" appveyor DownloadFile "%DLLS_URL%" -FileName "%DLLS_ARCHIVE%" - - 7z x -y "%DLLS_ARCHIVE%" -o"%CD%\BIN"> nul - - IF not exist "%MINGW_ARCHIVE%" appveyor DownloadFile "%MINGW_URL%" -FileName "%MINGW_ARCHIVE%" - - 7z x -y "%MINGW_ARCHIVE%" -o"%CD%\DIST"> nul - - SET PATH=%CD%\DIST\%MINGW_DIR%\BIN;%CD%\BIN;%PATH% - # may need adjustments, untested, see similar examples here: - # https://chromium.googlesource.com/external/github.com/swig/swig/+/1e36f51346d95f8b9848e682c2eb986e9cb9b4f4/appveyor.yml - - bash -c ". ci/funs.sh && nimBuildCsourcesIfNeeded" - -build_script: - - openssl version - - openssl version -d - - bin\nim c koch - - koch runCI - -deploy: off diff --git a/unmaintained/ci.yml b/unmaintained/ci.yml deleted file mode 100644 index 62daf875a..000000000 --- a/unmaintained/ci.yml +++ /dev/null @@ -1,105 +0,0 @@ -# out of date and unused but revivable - -name: Continous Integration -on: [push, pull_request] - -jobs: - build: - strategy: - matrix: - os: [ubuntu-18.04, macos-10.15, windows-2019] - cpu: [amd64, i386] - cpp: ['false', 'true'] - pkg: ['false', 'true'] - exclude: - - os: ubuntu-18.04 - cpp: 'true' - - os: ubuntu-18.04 - pkg: 'true' - - os: macos-10.15 - cpu: i386 - - os: macos-10.15 - pkg: 'true' - - os: windows-2019 - cpu: i386 - - os: windows-2019 - cpp: 'true' - name: '${{ matrix.os }} (${{ matrix.cpu }}, cpp: ${{ matrix.cpp }}, pkg: ${{ matrix.pkg }})' - runs-on: ${{ matrix.os }} - env: - NIM_COMPILE_TO_CPP: ${{ matrix.cpp }} - NIM_TEST_PACKAGES: ${{ matrix.pkg }} - steps: - - name: 'Checkout' - uses: actions/checkout@v2 - - - name: 'Install node.js 8.x' - uses: actions/setup-node@v1 - with: - node-version: '8.x' - - name: 'Install dependencies (Linux amd64)' - if: runner.os == 'Linux' && matrix.cpu == 'amd64' - run: | - sudo apt-fast update -qq - DEBIAN_FRONTEND='noninteractive' \ - sudo apt-fast install --no-install-recommends -yq \ - libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev \ - valgrind libc6-dbg - - name: 'Install dependencies (Linux i386)' - if: runner.os == 'Linux' && matrix.cpu == 'i386' - run: | - sudo dpkg --add-architecture i386 - - sudo apt-fast update -qq - DEBIAN_FRONTEND='noninteractive' \ - sudo apt-fast install --no-install-recommends --allow-downgrades -yq \ - g++-multilib gcc-multilib libcurl4-openssl-dev:i386 libgc-dev:i386 \ - libsdl1.2-dev:i386 libsfml-dev:i386 libglib2.0-dev:i386 \ - libffi-dev:i386 - - cat << EOF > bin/gcc - #!/bin/bash - - exec $(which gcc) -m32 "\$@" - EOF - cat << EOF > bin/g++ - #!/bin/bash - - exec $(which g++) -m32 "\$@" - EOF - - chmod 755 bin/gcc - chmod 755 bin/g++ - - name: 'Install dependencies (macOS)' - if: runner.os == 'macOS' - run: brew install boehmgc make sfml - - name: 'Install dependencies (Windows)' - if: runner.os == 'Windows' - shell: bash - run: | - mkdir dist - curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z - curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip - 7z x dist/mingw64.7z -odist - 7z x dist/dlls.zip -obin - echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}" - - - name: 'Add build binaries to PATH' - shell: bash - run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}" - - - name: 'Build csourcesAny' - shell: bash - run: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc ucpu='${{ matrix.cpu }}' - - - name: 'Build koch' - shell: bash - run: nim c koch - - name: 'Run CI' - shell: bash - run: ./koch runCI - - - name: 'Show failed tests' - if: failure() - shell: bash - run: nim c -r tools/ci_testresults.nim diff --git a/unmaintained/readme.md b/unmaintained/readme.md deleted file mode 100644 index cddb15cd2..000000000 --- a/unmaintained/readme.md +++ /dev/null @@ -1,11 +0,0 @@ -Deprecated, unused and unmaintained files can be moved here when they have some value, -e.g. if someone want to revive them in their project or in this repo. If a deprecated and unused file -is redundant, it should just be removed instead. - -See also https://github.com/nim-lang/graveyard, which is maintained on a best effort basis -and has different goals. - -## table of contents -.travis.yml -> unmaintained/.travis.yml -appveyor.yml.disabled -> unmaintained/appveyor.yml -.github/workflows/ci.yml.disabled -> unmaintained/ci.yml From ffe4328b3513617be0d27c773d59bea441ec1c65 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 25 Apr 2021 01:25:31 -0700 Subject: [PATCH 0711/2472] `--usenimcache` (implied by `nim r main`) now caches some compile options to avoid recompiling when project was previously compiled with such options. (#17829) * `--usenimcache` (implied by `nim r main`) now caches some compile options to avoid recompiling when project was previously compiled with such options. * works * add test * changelog * use std/with --- changelog.md | 9 +++++++ compiler/extccomp.nim | 17 +++++++------ compiler/main.nim | 51 ++++++++++++++++++++++++++------------- compiler/options.nim | 15 ------------ tests/misc/mbetterrun.nim | 3 +++ tests/misc/trunner.nim | 48 +++++++++++++++++++++++++++--------- 6 files changed, 91 insertions(+), 52 deletions(-) create mode 100644 tests/misc/mbetterrun.nim diff --git a/changelog.md b/changelog.md index 943b5d6df..43ce43352 100644 --- a/changelog.md +++ b/changelog.md @@ -377,7 +377,16 @@ - `--hint:CC` now goes to stderr (like all other hints) instead of stdout. +- json build instructions are now generated in `$nimcache/outFileBasename.json` + instead of `$nimcache/projectName.json`. This allows avoiding recompiling a given project + compiled with different options if the output file differs. +- `--usenimcache` (implied by `nim r main`) now generates an output file that includes a hash of + some of the compilation options, which allows caching generated binaries: + nim r main # recompiles + nim r -d:foo main # recompiles + nim r main # uses cached binary + nim r main arg1 arg2 # ditto (runtime arguments are irrelevant) ## Tool changes diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 71a20fc47..3b415c499 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -934,9 +934,14 @@ proc callCCompiler*(conf: ConfigRef) = script.add("\n") generateScript(conf, script) - template hashNimExe(): string = $secureHashFile(os.getAppFilename()) +proc jsonBuildInstructionsFile*(conf: ConfigRef): AbsoluteFile = + # `outFile` is better than `projectName`, as it allows having different json + # files for a given source file compiled with different options; it also + # works out of the box with `hashMainCompilationParams`. + result = getNimcacheDir(conf) / conf.outFile.changeFileExt("json") + proc writeJsonBuildInstructions*(conf: ConfigRef) = template lit(x: string) = f.write x template str(x: string) = @@ -993,8 +998,7 @@ proc writeJsonBuildInstructions*(conf: ConfigRef) = var buf = newStringOfCap(50) - - let jsonFile = conf.getNimcacheDir / RelativeFile(conf.projectName & ".json") + let jsonFile = conf.jsonBuildInstructionsFile conf.jsonBuildFile = jsonFile let output = conf.absOutFile @@ -1038,8 +1042,7 @@ proc writeJsonBuildInstructions*(conf: ConfigRef) = lit "\L}\L" close(f) -proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile): bool = - let jsonFile = toGeneratedFile(conf, projectfile, "json") +proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile): bool = if not fileExists(jsonFile): return true if not fileExists(conf.absOutFile): return true result = false @@ -1090,11 +1093,9 @@ proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; projectfile: Absol echo "Warning: JSON processing failed: ", getCurrentExceptionMsg() result = true -proc runJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile) = - let jsonFile = toGeneratedFile(conf, projectfile, "json") +proc runJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile) = try: let data = json.parseFile(jsonFile.string) - let output = data["outputFile"].getStr createDir output.parentDir let outputCurrent = $conf.absOutFile diff --git a/compiler/main.nim b/compiler/main.nim index 9c9a789cb..48fbd185c 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -19,7 +19,7 @@ import cgen, json, nversion, platform, nimconf, passaux, depends, vm, modules, - modulegraphs, tables, lineinfos, pathutils, vmprofiler + modulegraphs, tables, lineinfos, pathutils, vmprofiler, std/[sha1, with] import ic / [cbackend, integrity, navigator] from ic / ic import rodViewer @@ -80,15 +80,13 @@ when not defined(leanCompiler): proc commandCompileToC(graph: ModuleGraph) = let conf = graph.config - setOutFile(conf) extccomp.initVars(conf) semanticPasses(graph) if conf.symbolFiles == disabledSf: registerPass(graph, cgenPass) if {optRun, optForceFullMake} * conf.globalOptions == {optRun} or isDefined(conf, "nimBetterRun"): - let proj = changeFileExt(conf.projectFull, "") - if not changeDetectedViaJsonBuildInstructions(conf, proj): + if not changeDetectedViaJsonBuildInstructions(conf, conf.jsonBuildInstructionsFile): # nothing changed graph.config.notes = graph.config.mainPackageNotes return @@ -117,27 +115,20 @@ proc commandCompileToC(graph: ModuleGraph) = writeDepsFile(graph) proc commandJsonScript(graph: ModuleGraph) = - let proj = changeFileExt(graph.config.projectFull, "") - extccomp.runJsonBuildInstructions(graph.config, proj) + extccomp.runJsonBuildInstructions(graph.config, graph.config.jsonBuildInstructionsFile) proc commandCompileToJS(graph: ModuleGraph) = + let conf = graph.config when defined(leanCompiler): - globalError(graph.config, unknownLineInfo, "compiler wasn't built with JS code generator") + globalError(conf, unknownLineInfo, "compiler wasn't built with JS code generator") else: - let conf = graph.config conf.exc = excCpp - - if conf.outFile.isEmpty: - conf.outFile = RelativeFile(conf.projectName & ".js") - - #incl(gGlobalOptions, optSafeCode) - setTarget(graph.config.target, osJS, cpuJS) - #initDefines() - defineSymbol(graph.config.symbols, "ecmascript") # For backward compatibility + setTarget(conf.target, osJS, cpuJS) + defineSymbol(conf.symbols, "ecmascript") # For backward compatibility semanticPasses(graph) registerPass(graph, JSgenPass) compileProject(graph) - if optGenScript in graph.config.globalOptions: + if optGenScript in conf.globalOptions: writeDepsFile(graph) proc interactivePasses(graph: ModuleGraph) = @@ -186,6 +177,31 @@ proc commandView(graph: ModuleGraph) = const PrintRopeCacheStats = false +proc hashMainCompilationParams*(conf: ConfigRef): string = + ## doesn't have to be complete; worst case is a cache hit and recompilation. + var state = newSha1State() + with state: + update os.getAppFilename() # nim compiler + update conf.commandLine # excludes `arguments`, as it should + update $conf.projectFull # so that running `nim r main` from 2 directories caches differently + result = $SecureHash(state.finalize()) + +proc setOutFile*(conf: ConfigRef) = + proc libNameTmpl(conf: ConfigRef): string {.inline.} = + result = if conf.target.targetOS == osWindows: "$1.lib" else: "lib$1.a" + + if conf.outFile.isEmpty: + var base = conf.projectName + if optUseNimcache in conf.globalOptions: + base.add "_" & hashMainCompilationParams(conf) + let targetName = + if conf.backend == backendJs: base & ".js" + elif optGenDynLib in conf.globalOptions: + platform.OS[conf.target.targetOS].dllFrmt % base + elif optGenStaticLib in conf.globalOptions: libNameTmpl(conf) % base + else: base & platform.OS[conf.target.targetOS].exeExt + conf.outFile = RelativeFile targetName + proc mainCommand*(graph: ModuleGraph) = let conf = graph.config let cache = graph.cache @@ -220,6 +236,7 @@ proc mainCommand*(graph: ModuleGraph) = proc compileToBackend() = customizeForBackend(conf.backend) + setOutFile(conf) case conf.backend of backendC: commandCompileToC(graph) of backendCpp: commandCompileToC(graph) diff --git a/compiler/options.nim b/compiler/options.nim index 7044e64bd..209564d0a 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -970,18 +970,3 @@ proc floatInt64Align*(conf: ConfigRef): int16 = # to 4bytes (except with -malign-double) return 4 return 8 - -proc setOutFile*(conf: ConfigRef) = - proc libNameTmpl(conf: ConfigRef): string {.inline.} = - result = if conf.target.targetOS == osWindows: "$1.lib" else: "lib$1.a" - - if conf.outFile.isEmpty: - let base = conf.projectName - let targetName = - if optGenDynLib in conf.globalOptions: - platform.OS[conf.target.targetOS].dllFrmt % base - elif optGenStaticLib in conf.globalOptions: - libNameTmpl(conf) % base - else: - base & platform.OS[conf.target.targetOS].exeExt - conf.outFile = RelativeFile targetName diff --git a/tests/misc/mbetterrun.nim b/tests/misc/mbetterrun.nim new file mode 100644 index 000000000..d4f427af0 --- /dev/null +++ b/tests/misc/mbetterrun.nim @@ -0,0 +1,3 @@ +const mbetterrunVal {.strdefine.} = "" +static: echo "compiling: " & mbetterrunVal +echo "running: " & mbetterrunVal diff --git a/tests/misc/trunner.nim b/tests/misc/trunner.nim index 505a06cf8..2a82ca9ee 100644 --- a/tests/misc/trunner.nim +++ b/tests/misc/trunner.nim @@ -29,13 +29,18 @@ const nimcache = buildDir / "nimcacheTrunner" # instead of `querySetting(nimcacheDir)`, avoids stomping on other parallel tests -proc runCmd(file, options = ""): auto = +proc runNimCmd(file, options = "", rtarg = ""): auto = let fileabs = testsDir / file.unixToNativePath doAssert fileabs.fileExists, fileabs - let cmd = fmt"{nim} {mode} {options} --hints:off {fileabs}" + let cmd = fmt"{nim} {mode} {options} --hints:off {fileabs} {rtarg}" result = execCmdEx(cmd) when false: echo result[0] & "\n" & result[1] # for debugging +proc runNimCmdChk(file, options = "", rtarg = ""): string = + let (ret, status) = runNimCmd(file, options, rtarg = rtarg) + doAssert status == 0, $(file, options) & "\n" & ret + ret + when defined(nimTrunnerFfi): block: # mevalffi when defined(openbsd): @@ -53,8 +58,8 @@ when defined(nimTrunnerFfi): hello world stderr hi stderr """ - let (output, exitCode) = runCmd("vm/mevalffi.nim", fmt"{opt} --experimental:compiletimeFFI") - let expected = fmt""" + let output = runNimCmdChk("vm/mevalffi.nim", fmt"{opt} --experimental:compiletimeFFI") + doAssert output == fmt""" {prefix}foo foo:100 foo:101 @@ -62,12 +67,10 @@ foo:102:103 foo:102:103:104 foo:0.03:asdf:103:105 ret=[s1:foobar s2:foobar age:25 pi:3.14] -""" - doAssert output == expected, output - doAssert exitCode == 0 +""", output else: # don't run twice the same test - import std/[strutils] + import std/strutils template check2(msg) = doAssert msg in output, output block: # tests with various options `nim doc --project --index --docroot` @@ -142,17 +145,16 @@ sub/mmain.idx""", context else: doAssert false block: # mstatic_assert - let (output, exitCode) = runCmd("ccgbugs/mstatic_assert.nim", "-d:caseBad") + let (output, exitCode) = runNimCmd("ccgbugs/mstatic_assert.nim", "-d:caseBad") check2 "sizeof(bool) == 2" check exitCode != 0 block: # ABI checks let file = "misc/msizeof5.nim" block: - let (output, exitCode) = runCmd(file, "-d:checkAbi") - doAssert exitCode == 0, output + discard runNimCmdChk(file, "-d:checkAbi") block: - let (output, exitCode) = runCmd(file, "-d:checkAbi -d:caseBad") + let (output, exitCode) = runNimCmd(file, "-d:checkAbi -d:caseBad") # on platforms that support _StaticAssert natively, errors will show full context, e.g.: # error: static_assert failed due to requirement 'sizeof(unsigned char) == 8' # "backend & Nim disagree on size for: BadImportcType{int64} [declared in mabi_check.nim(1, 6)]" @@ -293,3 +295,25 @@ tests/newconfig/bar/mfoo.nims""".splitLines let (outp, exitCode) = run "echo 1+2; quit(2)" check3 "3" in outp doAssert exitCode == 2 + + block: # nimBetterRun + let file = "misc/mbetterrun.nim" + const nimcache2 = buildDir / "D20210423T185116" + removeDir nimcache2 + # related to `-d:nimBetterRun` + let opt = fmt"-r --usenimcache --nimcache:{nimcache2}" + var ret = "" + for a in @["v1", "v2", "v1", "v3"]: + ret.add runNimCmdChk(file, fmt"{opt} -d:mbetterrunVal:{a}") + ret.add runNimCmdChk(file, fmt"{opt} -d:mbetterrunVal:v2", rtarg = "arg1 arg2") + # rt arguments should not cause a recompilation + doAssert ret == """ +compiling: v1 +running: v1 +compiling: v2 +running: v2 +running: v1 +compiling: v3 +running: v3 +running: v2 +""", ret From 17db15f9b1a03ac93cb6c5c0264f2b534ab8b2ab Mon Sep 17 00:00:00 2001 From: Saem Ghani Date: Sun, 25 Apr 2021 01:30:52 -0700 Subject: [PATCH 0712/2472] fix #17836 (typed macro isNil for proc params) (#17841) thanks @alaviss for the test --- compiler/ast.nim | 4 ++++ compiler/vm.nim | 4 ++-- tests/macros/t17836.nim | 15 +++++++++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 tests/macros/t17836.nim diff --git a/compiler/ast.nim b/compiler/ast.nim index 02a2ae7a4..7090bc7ee 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -1171,6 +1171,7 @@ when defined(useNodeIds): var gNodeId: int proc newNode*(kind: TNodeKind): PNode = + ## new node with unknown line info, no type, and no children result = PNode(kind: kind, info: unknownLineInfo) when defined(useNodeIds): result.id = gNodeId @@ -1180,6 +1181,7 @@ proc newNode*(kind: TNodeKind): PNode = inc gNodeId proc newNodeI*(kind: TNodeKind, info: TLineInfo): PNode = + ## new node with line info, no type, and no children result = PNode(kind: kind, info: info) when defined(useNodeIds): result.id = gNodeId @@ -1189,6 +1191,7 @@ proc newNodeI*(kind: TNodeKind, info: TLineInfo): PNode = inc gNodeId proc newNodeI*(kind: TNodeKind, info: TLineInfo, children: int): PNode = + ## new node with line info, type, and children result = PNode(kind: kind, info: info) if children > 0: newSeq(result.sons, children) @@ -1200,6 +1203,7 @@ proc newNodeI*(kind: TNodeKind, info: TLineInfo, children: int): PNode = inc gNodeId proc newNodeIT*(kind: TNodeKind, info: TLineInfo, typ: PType): PNode = + ## new node with line info, type, and no children result = newNode(kind) result.info = info result.typ = typ diff --git a/compiler/vm.nim b/compiler/vm.nim index 7a231a482..19ccea55b 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -1513,8 +1513,8 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = # reference with the value `nil`, so `isNil` should be false! (node.kind == nkNilLit and nfIsRef notin node.flags) or (not node.typ.isNil and node.typ.kind == tyProc and - node.typ.callConv == ccClosure and node[0].kind == nkNilLit and - node[1].kind == nkNilLit)) + node.typ.callConv == ccClosure and node.safeLen > 0 and + node[0].kind == nkNilLit and node[1].kind == nkNilLit)) of opcNBindSym: # cannot use this simple check # if dynamicBindSym notin c.config.features: diff --git a/tests/macros/t17836.nim b/tests/macros/t17836.nim new file mode 100644 index 000000000..2453637f5 --- /dev/null +++ b/tests/macros/t17836.nim @@ -0,0 +1,15 @@ +import macros + +# Ensure that `isNil` works in the typed macro context when pass procs. + +type + O = object + fn: proc(i: int): int + +var o: O + +macro typedBug(expr: typed) = + doAssert expr[1] != nil + doAssert not expr[1].isNil + +typedBug(o.fn) \ No newline at end of file From 22e06ee95aeb32929a5b912096dfaa662165c162 Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 25 Apr 2021 16:34:53 +0800 Subject: [PATCH 0713/2472] Ref #17831(synchapi.h: No such file or directory) (#17832) * Ref #17831 use windows.h * use header: "windows.h" * Update syslocks.nim * Update lib/system/syslocks.nim --- lib/system/syslocks.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/system/syslocks.nim b/lib/system/syslocks.nim index ac52d9a8c..2f0c8b0ba 100644 --- a/lib/system/syslocks.nim +++ b/lib/system/syslocks.nim @@ -24,9 +24,8 @@ when defined(windows): LockSemaphore: int SpinCount: int - RTL_CONDITION_VARIABLE {.importc: "RTL_CONDITION_VARIABLE", header: "synchapi.h".} = object + SysCond {.importc: "RTL_CONDITION_VARIABLE", header: "".} = object thePtr {.importc: "ptr".} : Handle - SysCond = RTL_CONDITION_VARIABLE proc initSysLock(L: var SysLock) {.importc: "InitializeCriticalSection", header: "".} From 9e6f2d7d186f760d8eb7dc26c863f23bd044b9c9 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 26 Apr 2021 01:16:20 +0800 Subject: [PATCH 0714/2472] bring back jstin after forking a new one (#17844) ref https://github.com/nim-lang/Nim/pull/17765#issuecomment-822019094 --- testament/important_packages.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index ecae42811..64000dbb2 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -77,6 +77,7 @@ pkg "illwill", "nimble examples" pkg "inim", allowFailure=true pkg "itertools", "nim doc src/itertools.nim" pkg "iterutils" +pkg "jstin" pkg "karax", "nim c -r tests/tester.nim" pkg "kdtree", "nimble test", "https://github.com/jblindsay/kdtree" pkg "loopfusion" From 68e522ececdb9f79371f4b60d1fbddcf01736050 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 26 Apr 2021 15:04:52 +0800 Subject: [PATCH 0715/2472] Remove confusing (#17830) --- compiler/renderer.nim | 5 +---- lib/pure/collections/lists.nim | 14 +++++++------- lib/pure/collections/tables.nim | 14 +++++++------- lib/pure/parsecfg.nim | 10 +++++----- lib/pure/unittest.nim | 6 +++--- lib/system.nim | 4 ---- tests/destructor/tnewruntime_misc.nim | 7 ++++++- 7 files changed, 29 insertions(+), 31 deletions(-) diff --git a/compiler/renderer.nim b/compiler/renderer.nim index 4c0de9ed5..8bc3c3306 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -1290,10 +1290,7 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext, fromStmtList = false) = put(g, tkSpaces, Space) infixArgument(g, n, 2) of nkPrefix: - if n.len > 0 and n[0].kind == nkIdent and n[0].ident.s == "": - discard "XXX Remove this hack after 0.20 has been released!" - else: - gsub(g, n, 0) + gsub(g, n, 0) if n.len > 1: let opr = if n[0].kind == nkIdent: n[0].ident elif n[0].kind == nkSym: n[0].sym.name diff --git a/lib/pure/collections/lists.nim b/lib/pure/collections/lists.nim index c1fcb0743..c83be7aa3 100644 --- a/lib/pure/collections/lists.nim +++ b/lib/pure/collections/lists.nim @@ -68,7 +68,7 @@ type ## A node of a doubly linked list. ## ## It consists of a `value` field, and pointers to `next` and `prev`. - next*: (DoublyLinkedNode[T]) + next*: DoublyLinkedNode[T] prev* {.cursor.}: DoublyLinkedNode[T] value*: T DoublyLinkedNode*[T] = ref DoublyLinkedNodeObj[T] @@ -77,23 +77,23 @@ type ## A node of a singly linked list. ## ## It consists of a `value` field, and a pointer to `next`. - next*: (SinglyLinkedNode[T]) + next*: SinglyLinkedNode[T] value*: T SinglyLinkedNode*[T] = ref SinglyLinkedNodeObj[T] SinglyLinkedList*[T] = object ## A singly linked list. - head*: (SinglyLinkedNode[T]) + head*: SinglyLinkedNode[T] tail* {.cursor.}: SinglyLinkedNode[T] DoublyLinkedList*[T] = object ## A doubly linked list. - head*: (DoublyLinkedNode[T]) + head*: DoublyLinkedNode[T] tail* {.cursor.}: DoublyLinkedNode[T] SinglyLinkedRing*[T] = object ## A singly linked ring. - head*: (SinglyLinkedNode[T]) + head*: SinglyLinkedNode[T] tail* {.cursor.}: SinglyLinkedNode[T] DoublyLinkedRing*[T] = object @@ -148,7 +148,7 @@ proc initDoublyLinkedRing*[T](): DoublyLinkedRing[T] = discard -proc newDoublyLinkedNode*[T](value: T): (DoublyLinkedNode[T]) = +proc newDoublyLinkedNode*[T](value: T): DoublyLinkedNode[T] = ## Creates a new doubly linked node with the given `value`. runnableExamples: let n = newDoublyLinkedNode[int](5) @@ -157,7 +157,7 @@ proc newDoublyLinkedNode*[T](value: T): (DoublyLinkedNode[T]) = new(result) result.value = value -proc newSinglyLinkedNode*[T](value: T): (SinglyLinkedNode[T]) = +proc newSinglyLinkedNode*[T](value: T): SinglyLinkedNode[T] = ## Creates a new singly linked node with the given `value`. runnableExamples: let n = newSinglyLinkedNode[int](5) diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 8739b60a3..b71bb0845 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -795,7 +795,7 @@ iterator allValues*[A, B](t: Table[A, B]; key: A): B {.deprecated: # ------------------------------------------------------------------- -proc newTable*[A, B](initialSize = defaultInitialSize): TableRef[A, B] = +proc newTable*[A, B](initialSize = defaultInitialSize): TableRef[A, B] = ## Creates a new ref hash table that is empty. ## ## See also: @@ -810,7 +810,7 @@ proc newTable*[A, B](initialSize = defaultInitialSize): TableRef[A, B] = new(result) result[] = initTable[A, B](initialSize) -proc newTable*[A, B](pairs: openArray[(A, B)]): TableRef[A, B] = +proc newTable*[A, B](pairs: openArray[(A, B)]): TableRef[A, B] = ## Creates a new ref hash table that contains the given `pairs`. ## ## `pairs` is a container consisting of `(key, value)` tuples. @@ -826,7 +826,7 @@ proc newTable*[A, B](pairs: openArray[(A, B)]): TableRef[A, B] = new(result) result[] = toTable[A, B](pairs) -proc newTableFrom*[A, B, C](collection: A, index: proc(x: B): C): TableRef[C, B] = +proc newTableFrom*[A, B, C](collection: A, index: proc(x: B): C): TableRef[C, B] = ## Index the collection with the proc provided. # TODO: As soon as supported, change collection: A to collection: A[B] result = newTable[C, B]() @@ -1790,7 +1790,7 @@ iterator mvalues*[A, B](t: var OrderedTable[A, B]): var B = # --------------------------- OrderedTableRef ------------------------------- # --------------------------------------------------------------------------- -proc newOrderedTable*[A, B](initialSize = defaultInitialSize): OrderedTableRef[A, B] = +proc newOrderedTable*[A, B](initialSize = defaultInitialSize): OrderedTableRef[A, B] = ## Creates a new ordered ref hash table that is empty. ## ## See also: @@ -1805,7 +1805,7 @@ proc newOrderedTable*[A, B](initialSize = defaultInitialSize): OrderedTableR new(result) result[] = initOrderedTable[A, B](initialSize) -proc newOrderedTable*[A, B](pairs: openArray[(A, B)]): OrderedTableRef[A, B] = +proc newOrderedTable*[A, B](pairs: openArray[(A, B)]): OrderedTableRef[A, B] = ## Creates a new ordered ref hash table that contains the given `pairs`. ## ## `pairs` is a container consisting of `(key, value)` tuples. @@ -2610,7 +2610,7 @@ iterator mvalues*[A](t: var CountTable[A]): var int = proc inc*[A](t: CountTableRef[A], key: A, val = 1) -proc newCountTable*[A](initialSize = defaultInitialSize): CountTableRef[A] = +proc newCountTable*[A](initialSize = defaultInitialSize): CountTableRef[A] = ## Creates a new ref count table that is empty. ## ## See also: @@ -2621,7 +2621,7 @@ proc newCountTable*[A](initialSize = defaultInitialSize): CountTableRef[A] = new(result) result[] = initCountTable[A](initialSize) -proc newCountTable*[A](keys: openArray[A]): CountTableRef[A] = +proc newCountTable*[A](keys: openArray[A]): CountTableRef[A] = ## Creates a new ref count table with every member of a container `keys` ## having a count of how many times it occurs in that container. result = newCountTable[A](keys.len) diff --git a/lib/pure/parsecfg.nim b/lib/pure/parsecfg.nim index 7f22a9857..2350cea4a 100644 --- a/lib/pure/parsecfg.nim +++ b/lib/pure/parsecfg.nim @@ -496,17 +496,17 @@ proc next*(c: var CfgParser): CfgEvent {.rtl, extern: "npc$1".} = # ---------------- Configuration file related operations ---------------- type - Config* = OrderedTableRef[string, OrderedTableRef[string, string]] + Config* = OrderedTableRef[string, OrderedTableRef[string, string]] proc newConfig*(): Config = ## Creates a new configuration table. ## Useful when wanting to create a configuration file. - result = newOrderedTable[string, OrderedTableRef[string, string]]() + result = newOrderedTable[string, OrderedTableRef[string, string]]() -proc loadConfig*(stream: Stream, filename: string = "[stream]"): Config = +proc loadConfig*(stream: Stream, filename: string = "[stream]"): Config = ## Loads the specified configuration from stream into a new Config instance. ## `filename` parameter is only used for nicer error messages. - var dict = newOrderedTable[string, OrderedTableRef[string, string]]() + var dict = newOrderedTable[string, OrderedTableRef[string, string]]() var curSection = "" ## Current section, ## the default value of the current section is "", ## which means that the current section is a common @@ -536,7 +536,7 @@ proc loadConfig*(stream: Stream, filename: string = "[stream]"): Config = close(p) result = dict -proc loadConfig*(filename: string): Config = +proc loadConfig*(filename: string): Config = ## Loads the specified configuration file into a new Config instance. let file = open(filename, fmRead) let fileStream = newFileStream(file) diff --git a/lib/pure/unittest.nim b/lib/pure/unittest.nim index 80a05db36..b1c01f8db 100644 --- a/lib/pure/unittest.nim +++ b/lib/pure/unittest.nim @@ -218,7 +218,7 @@ proc resetOutputFormatters* {.since: (1, 1).} = formatters = @[] proc newConsoleOutputFormatter*(outputLevel: OutputLevel = outputLevelDefault, - colorOutput = true): ConsoleOutputFormatter = + colorOutput = true): ConsoleOutputFormatter = ConsoleOutputFormatter( outputLevel: outputLevel, colorOutput: colorOutput @@ -246,7 +246,7 @@ proc colorOutput(): bool = deprecateEnvVarHere() result = false -proc defaultConsoleFormatter*(): ConsoleOutputFormatter = +proc defaultConsoleFormatter*(): ConsoleOutputFormatter = var colorOutput = colorOutput() var outputLevel = nimUnittestOutputLevel.parseEnum[:OutputLevel] when declared(stdout): @@ -315,7 +315,7 @@ proc xmlEscape(s: string): string = else: result.add(c) -proc newJUnitOutputFormatter*(stream: Stream): JUnitOutputFormatter = +proc newJUnitOutputFormatter*(stream: Stream): JUnitOutputFormatter = ## Creates a formatter that writes report to the specified stream in ## JUnit format. ## The ``stream`` is NOT closed automatically when the test are finished, diff --git a/lib/system.nim b/lib/system.nim index 41749c27a..9e9bacf39 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -885,8 +885,6 @@ when defined(nimOwnedEnabled) and not defined(nimscript): proc unown*[T](x: T): T {.magic: "Unown", noSideEffect.} ## Use the expression `x` ignoring its ownership attribute. - # This is only required to make 0.20 compile with the 0.19 line. - template ``*(t: untyped): untyped = owned(t) else: template unown*(x: typed): untyped = x @@ -908,8 +906,6 @@ else: new(r) return r - # This is only required to make 0.20 compile with the 0.19 line. - template ``*(t: untyped): untyped = t template disarm*(x: typed) = ## Useful for `disarming` dangling pointers explicitly for `--newruntime`. diff --git a/tests/destructor/tnewruntime_misc.nim b/tests/destructor/tnewruntime_misc.nim index 1a5b0e3b0..48ea36b7c 100644 --- a/tests/destructor/tnewruntime_misc.nim +++ b/tests/destructor/tnewruntime_misc.nim @@ -25,8 +25,13 @@ putEnv("HEAPTRASHING", "Indeed") let s1 = getAllocStats() + +proc newTableOwned[A, B](initialSize = defaultInitialSize): owned(TableRef[A, B]) = + new(result) + result[] = initTable[A, B](initialSize) + proc main = - var w = newTable[string, owned Node]() + var w = newTableOwned[string, owned Node]() w["key"] = Node(field: "value") echo w["key"][] echo getEnv("HEAPTRASHING") From 01bca8cd4fad857124794144bff09ae0326895bd Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 26 Apr 2021 20:48:25 +0800 Subject: [PATCH 0716/2472] fix gbk encodings on windows (#17843) --- lib/pure/encodings.nim | 1 + tests/stdlib/tencodings.nim | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 tests/stdlib/tencodings.nim diff --git a/lib/pure/encodings.nim b/lib/pure/encodings.nim index 1d8512018..bdc41c6ff 100644 --- a/lib/pure/encodings.nim +++ b/lib/pure/encodings.nim @@ -72,6 +72,7 @@ when defined(windows): (875, "cp875"), # IBM EBCDIC Greek Modern (932, "shift_jis"), # ANSI/OEM Japanese; Japanese (Shift-JIS) (936, "gb2312"), # ANSI/OEM Simplified Chinese (PRC, Singapore); Chinese Simplified (GB2312) + (936, "gbk"), # Alias for GB2312 encoding (949, "ks_c_5601-1987"), # ANSI/OEM Korean (Unified Hangul Code) (950, "big5"), # ANSI/OEM Traditional Chinese (Taiwan; Hong Kong SAR, PRC); Chinese Traditional (Big5) (1026, "IBM1026"), # IBM EBCDIC Turkish (Latin 5) diff --git a/tests/stdlib/tencodings.nim b/tests/stdlib/tencodings.nim new file mode 100644 index 000000000..1da52ba59 --- /dev/null +++ b/tests/stdlib/tencodings.nim @@ -0,0 +1,25 @@ +import std/encodings + +var fromGBK = open("utf-8", "gbk") +var toGBK = open("gbk", "utf-8") + +var fromGB2312 = open("utf-8", "gb2312") +var toGB2312 = open("gb2312", "utf-8") + + +block: + let data = "\215\237\186\243\178\187\214\170\204\236\212\218\203\174\163\172\194\250\180\178\208\199\195\206\209\185\208\199\186\211" + doAssert fromGBK.convert(data) == "醉后不知天在水，满床星梦压星河" + +block: + let data = "万两黄金容易得，知心一个也难求" + doAssert toGBK.convert(data) == "\205\242\193\189\187\198\189\240\200\221\210\215\181\195\163\172\214\170\208\196\210\187\184\246\210\178\196\209\199\243" + + +block: + let data = "\215\212\208\197\200\203\201\250\182\254\176\217\196\234\163\172\187\225\181\177\203\174\187\247\200\253\199\167\192\239" + doAssert fromGB2312.convert(data) == "自信人生二百年，会当水击三千里" + +block: + let data = "谁怕？一蓑烟雨任平生" + doAssert toGB2312.convert(data) == "\203\173\197\194\163\191\210\187\203\242\209\204\211\234\200\206\198\189\201\250" From 01267292d8335b21c69e0e4d840d1fa1078cf1b8 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 26 Apr 2021 11:17:51 -0700 Subject: [PATCH 0717/2472] disable telebot, refs https://github.com/ba0f3/telebot.nim/issues/59 (#17860) --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 64000dbb2..35c0cf0ce 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -139,7 +139,7 @@ pkg "stint", "nim r stint.nim" pkg "strslice" pkg "strunicode", "nim c -r src/strunicode.nim" pkg "synthesis" -pkg "telebot", "nim c -o:tbot -r src/telebot.nim" +pkg "telebot", "nim c -o:tbot -r src/telebot.nim", allowFailure = true # pending https://github.com/ba0f3/telebot.nim/issues/59 pkg "tempdir" pkg "templates" pkg "tensordsl", "nim c -r tests/tests.nim", "https://krux02@bitbucket.org/krux02/tensordslnim.git" From f52aa6bde8ba7b36d0589ee8e5d8319e5152237e Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 26 Apr 2021 11:32:41 -0700 Subject: [PATCH 0718/2472] fix typo in test name undeclared_routime.nim => undeclared_routine.nim (#17861) --- tests/errmsgs/undeclared_routime.nim | 13 ------------- tests/errmsgs/undeclared_routine.nim | 13 +++++++++++++ ...eclared_routime2.nim => undeclared_routine2.nim} | 0 ...eclared_routime3.nim => undeclared_routine3.nim} | 4 ++-- ...eclared_routime4.nim => undeclared_routine4.nim} | 2 +- ...eclared_routime5.nim => undeclared_routine5.nim} | 2 +- ...compiles.nim => undeclared_routine_compiles.nim} | 0 7 files changed, 17 insertions(+), 17 deletions(-) delete mode 100644 tests/errmsgs/undeclared_routime.nim create mode 100644 tests/errmsgs/undeclared_routine.nim rename tests/errmsgs/{undeclared_routime2.nim => undeclared_routine2.nim} (100%) rename tests/errmsgs/{undeclared_routime3.nim => undeclared_routine3.nim} (62%) rename tests/errmsgs/{undeclared_routime4.nim => undeclared_routine4.nim} (69%) rename tests/errmsgs/{undeclared_routime5.nim => undeclared_routine5.nim} (67%) rename tests/errmsgs/{undeclared_routime_compiles.nim => undeclared_routine_compiles.nim} (100%) diff --git a/tests/errmsgs/undeclared_routime.nim b/tests/errmsgs/undeclared_routime.nim deleted file mode 100644 index 7ef12cc0e..000000000 --- a/tests/errmsgs/undeclared_routime.nim +++ /dev/null @@ -1,13 +0,0 @@ -discard """ -cmd: '''nim c --hints:off $file''' -errormsg: "attempting to call routine: 'myiter'" -nimout: '''undeclared_routime.nim(13, 15) Error: attempting to call routine: 'myiter' - found 'undeclared_routime.myiter(a: string)[iterator declared in undeclared_routime.nim(10, 9)]' - found 'undeclared_routime.myiter()[iterator declared in undeclared_routime.nim(11, 9)]' -''' -""" - -iterator myiter(a:string): int = discard -iterator myiter(): int = discard - -let a = myiter(1) diff --git a/tests/errmsgs/undeclared_routine.nim b/tests/errmsgs/undeclared_routine.nim new file mode 100644 index 000000000..126ade52a --- /dev/null +++ b/tests/errmsgs/undeclared_routine.nim @@ -0,0 +1,13 @@ +discard """ +cmd: '''nim c --hints:off $file''' +errormsg: "attempting to call routine: 'myiter'" +nimout: '''undeclared_routine.nim(13, 15) Error: attempting to call routine: 'myiter' + found 'undeclared_routine.myiter(a: string)[iterator declared in undeclared_routine.nim(10, 9)]' + found 'undeclared_routine.myiter()[iterator declared in undeclared_routine.nim(11, 9)]' +''' +""" + +iterator myiter(a:string): int = discard +iterator myiter(): int = discard + +let a = myiter(1) diff --git a/tests/errmsgs/undeclared_routime2.nim b/tests/errmsgs/undeclared_routine2.nim similarity index 100% rename from tests/errmsgs/undeclared_routime2.nim rename to tests/errmsgs/undeclared_routine2.nim diff --git a/tests/errmsgs/undeclared_routime3.nim b/tests/errmsgs/undeclared_routine3.nim similarity index 62% rename from tests/errmsgs/undeclared_routime3.nim rename to tests/errmsgs/undeclared_routine3.nim index 052adfc08..537871900 100644 --- a/tests/errmsgs/undeclared_routime3.nim +++ b/tests/errmsgs/undeclared_routine3.nim @@ -1,8 +1,8 @@ discard """ cmd: '''nim c --hints:off $file''' errormsg: "undeclared field: 'bar'" -nimout: '''undeclared_routime3.nim(13, 10) Error: undeclared field: 'bar' - found 'undeclared_routime3.bar()[declared in undeclared_routime3.nim(12, 9)]' of kind 'iterator' +nimout: '''undeclared_routine3.nim(13, 10) Error: undeclared field: 'bar' + found 'undeclared_routine3.bar()[declared in undeclared_routine3.nim(12, 9)]' of kind 'iterator' ''' """ diff --git a/tests/errmsgs/undeclared_routime4.nim b/tests/errmsgs/undeclared_routine4.nim similarity index 69% rename from tests/errmsgs/undeclared_routime4.nim rename to tests/errmsgs/undeclared_routine4.nim index 674caa421..9012a983f 100644 --- a/tests/errmsgs/undeclared_routime4.nim +++ b/tests/errmsgs/undeclared_routine4.nim @@ -1,7 +1,7 @@ discard """ cmd: '''nim c --hints:off $file''' errormsg: "undeclared field: 'bar'" -nimout: '''undeclared_routime4.nim(10, 10) Error: undeclared field: 'bar' +nimout: '''undeclared_routine4.nim(10, 10) Error: undeclared field: 'bar' ''' """ diff --git a/tests/errmsgs/undeclared_routime5.nim b/tests/errmsgs/undeclared_routine5.nim similarity index 67% rename from tests/errmsgs/undeclared_routime5.nim rename to tests/errmsgs/undeclared_routine5.nim index 4394134ab..0387448a4 100644 --- a/tests/errmsgs/undeclared_routime5.nim +++ b/tests/errmsgs/undeclared_routine5.nim @@ -1,7 +1,7 @@ discard """ cmd: '''nim c --hints:off $file''' errormsg: "undeclared identifier: 'myfun'" -nimout: '''undeclared_routime5.nim(9, 9) Error: undeclared identifier: 'myfun' +nimout: '''undeclared_routine5.nim(9, 9) Error: undeclared identifier: 'myfun' ''' """ diff --git a/tests/errmsgs/undeclared_routime_compiles.nim b/tests/errmsgs/undeclared_routine_compiles.nim similarity index 100% rename from tests/errmsgs/undeclared_routime_compiles.nim rename to tests/errmsgs/undeclared_routine_compiles.nim From 4c9fa9b75cfd6b62ab4d3a8d19fddf7f60c656fd Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 26 Apr 2021 19:14:00 -0700 Subject: [PATCH 0719/2472] fix typo in changelog refs #17746 [skip ci] (#17869) --- changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 43ce43352..94dbbbeb3 100644 --- a/changelog.md +++ b/changelog.md @@ -345,7 +345,7 @@ - Added `--spellSuggest` to show spelling suggestions on typos. -- Added `--filenames:abs|canonical|magic` which replaces --listFullPaths:on|off +- Added `--filenames:abs|canonical|legacyRelProj` which replaces --listFullPaths:on|off - Added `--processing:dots|filenames|off` which customizes `hintProcessing` From fdfd0f35a1c293a813f743694a5bdd199c2a8433 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 27 Apr 2021 10:31:52 +0800 Subject: [PATCH 0720/2472] correct changelog[skip CI] (#17870) --- changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 94dbbbeb3..81a387359 100644 --- a/changelog.md +++ b/changelog.md @@ -57,7 +57,7 @@ - The unary slice `..b` was removed, use `0..b` instead or use `-d:nimLegacyUnarySlice` for a deprecation period. -- Moved `.travis.yml`, `appveyor.yml.disabled`, `.github/workflows/ci.yml.disabled` to `unmaintained/`. +- Removed `.travis.yml`, `appveyor.yml.disabled`, `.github/workflows/ci.yml.disabled`. ## Standard library additions and changes - Added support for parenthesized expressions in `strformat` From 3f58b7face3313a5fd51ab3227eabbcd53da51e2 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 26 Apr 2021 23:23:01 -0700 Subject: [PATCH 0721/2472] add -d:nimLegacyNoHashRef for a transition period which avoids defining hash(ref) (#17858) --- changelog.md | 1 + lib/pure/hashes.nim | 39 ++++++++++++++++++++++----------------- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/changelog.md b/changelog.md index 81a387359..e4b5c39f1 100644 --- a/changelog.md +++ b/changelog.md @@ -51,6 +51,7 @@ for previous behavior. - `hashes.hash` now supports `object` and `ref` (can be overloaded in user code). + For a transition period, use `-d:nimLegacyNoHashRef` to avoid defining `hash(ref)`. - `hashes.hash(proc|ptr|ref|pointer)` now calls `hash(int)` and honors `-d:nimIntHash1`, `hashes.hash(closure)` has also been improved. diff --git a/lib/pure/hashes.nim b/lib/pure/hashes.nim index 2ef0e2454..83b7a8d75 100644 --- a/lib/pure/hashes.nim +++ b/lib/pure/hashes.nim @@ -228,30 +228,35 @@ proc hash*(x: pointer): Hash {.inline.} = let y = cast[int](x) hash(y) # consistent with code expecting scrambled hashes depending on `nimIntHash1`. -proc hash*[T](x: ref[T] | ptr[T]): Hash {.inline.} = +proc hash*[T](x: ptr[T]): Hash {.inline.} = ## Efficient `hash` overload. runnableExamples: var a: array[10, uint8] assert a[0].addr.hash != a[1].addr.hash assert cast[pointer](a[0].addr).hash == a[0].addr.hash - runnableExamples: - type A = ref object - x: int - let a = A(x: 3) - let ha = a.hash - assert ha != A(x: 3).hash # A(x: 3) is a different ref object from `a`. - a.x = 4 - assert ha == a.hash # the hash only depends on the address - runnableExamples: - # you can overload `hash` if you want to customize semantics - type A[T] = ref object - x, y: T - proc hash(a: A): Hash = hash(a.x) - assert A[int](x: 3, y: 4).hash == A[int](x: 3, y: 5).hash - # xxx pending bug #17733, merge as `proc hash*(pointer | ref | ptr): Hash` - # or `proc hash*[T: ref | ptr](x: T): Hash` hash(cast[pointer](x)) +when not defined(nimLegacyNoHashRef): + proc hash*[T](x: ref[T]): Hash {.inline.} = + ## Efficient `hash` overload. + runnableExamples: + type A = ref object + x: int + let a = A(x: 3) + let ha = a.hash + assert ha != A(x: 3).hash # A(x: 3) is a different ref object from `a`. + a.x = 4 + assert ha == a.hash # the hash only depends on the address + runnableExamples: + # you can overload `hash` if you want to customize semantics + type A[T] = ref object + x, y: T + proc hash(a: A): Hash = hash(a.x) + assert A[int](x: 3, y: 4).hash == A[int](x: 3, y: 5).hash + # xxx pending bug #17733, merge as `proc hash*(pointer | ref | ptr): Hash` + # or `proc hash*[T: ref | ptr](x: T): Hash` + hash(cast[pointer](x)) + proc hash*[T: proc](x: T): Hash {.inline.} = ## Efficient hashing of proc vars. Closures are supported too. when T is "closure": From b1c7c994b0bcd39776fce16bdda24358f1fbb0d4 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 27 Apr 2021 16:24:59 +0800 Subject: [PATCH 0722/2472] [std/parsecfg]use runnableExamples (#17868) * [std/parsecfg]use runnableExamples * Apply suggestions from code review Co-authored-by: Timothee Cour --- lib/pure/parsecfg.nim | 191 +++++++++++++++++++++--------------------- 1 file changed, 95 insertions(+), 96 deletions(-) diff --git a/lib/pure/parsecfg.nim b/lib/pure/parsecfg.nim index 2350cea4a..0ee19912c 100644 --- a/lib/pure/parsecfg.nim +++ b/lib/pure/parsecfg.nim @@ -19,37 +19,32 @@ ## :literal: ## ## Here is an example of how to use the configuration file parser: -## -## .. code-block:: nim -## -## import std/[os, parsecfg, strutils, streams] -## -## var f = newFileStream(paramStr(1), fmRead) -## if f != nil: -## var p: CfgParser -## open(p, f, paramStr(1)) -## while true: -## var e = next(p) -## case e.kind -## of cfgEof: break -## of cfgSectionStart: ## a `[section]` has been parsed -## echo("new section: " & e.section) -## of cfgKeyValuePair: -## echo("key-value-pair: " & e.key & ": " & e.value) -## of cfgOption: -## echo("command: " & e.key & ": " & e.value) -## of cfgError: -## echo(e.msg) -## close(p) -## else: -## echo("cannot open: " & paramStr(1)) -## -## -## Examples -## ======== -## +runnableExamples("-r:off"): + import std/[strutils, streams] + + let configFile = "example.ini" + var f = newFileStream(configFile, fmRead) + assert f != nil, "cannot open " & configFile + var p: CfgParser + open(p, f, configFile) + while true: + var e = next(p) + case e.kind + of cfgEof: break + of cfgSectionStart: ## a `[section]` has been parsed + echo "new section: " & e.section + of cfgKeyValuePair: + echo "key-value-pair: " & e.key & ": " & e.value + of cfgOption: + echo "command: " & e.key & ": " & e.value + of cfgError: + echo e.msg + close(p) + +##[ ## Configuration file example -## -------------------------- +]## + ## ## .. code-block:: nim ## @@ -58,60 +53,64 @@ ## name = "hello" ## --threads:on ## [Author] -## name = "lihf8515" -## qq = "10214028" -## email = "lihaifeng@wxm.com" -## +## name = "nim-lang" +## website = "nim-lang.org" + +##[ ## Creating a configuration file -## ----------------------------- -## .. code-block:: nim -## -## import std/parsecfg -## var dict=newConfig() -## dict.setSectionKey("","charset","utf-8") -## dict.setSectionKey("Package","name","hello") -## dict.setSectionKey("Package","--threads","on") -## dict.setSectionKey("Author","name","lihf8515") -## dict.setSectionKey("Author","qq","10214028") -## dict.setSectionKey("Author","email","lihaifeng@wxm.com") -## dict.writeConfig("config.ini") -## +]## + +runnableExamples: + var dict = newConfig() + dict.setSectionKey("","charset", "utf-8") + dict.setSectionKey("Package", "name", "hello") + dict.setSectionKey("Package", "--threads", "on") + dict.setSectionKey("Author", "name", "nim-lang") + dict.setSectionKey("Author", "website", "nim-lang.org") + assert $dict == """ +charset=utf-8 +[Package] +name=hello +--threads:on +[Author] +name=nim-lang +website=nim-lang.org +""" + +##[ ## Reading a configuration file -## ---------------------------- -## .. code-block:: nim -## -## import std/parsecfg -## var dict = loadConfig("config.ini") -## var charset = dict.getSectionValue("","charset") -## var threads = dict.getSectionValue("Package","--threads") -## var pname = dict.getSectionValue("Package","name") -## var name = dict.getSectionValue("Author","name") -## var qq = dict.getSectionValue("Author","qq") -## var email = dict.getSectionValue("Author","email") -## echo pname & "\n" & name & "\n" & qq & "\n" & email -## +]## + +runnableExamples("-r:off"): + let dict = loadConfig("config.ini") + let charset = dict.getSectionValue("","charset") + let threads = dict.getSectionValue("Package","--threads") + let pname = dict.getSectionValue("Package","name") + let name = dict.getSectionValue("Author","name") + let website = dict.getSectionValue("Author","website") + echo pname & "\n" & name & "\n" & website + +##[ ## Modifying a configuration file -## ------------------------------ -## .. code-block:: nim -## -## import std/parsecfg -## var dict = loadConfig("config.ini") -## dict.setSectionKey("Author","name","lhf") -## dict.writeConfig("config.ini") -## +]## + +runnableExamples("-r:off"): + var dict = loadConfig("config.ini") + dict.setSectionKey("Author", "name", "nim-lang") + dict.writeConfig("config.ini") + +##[ ## Deleting a section key in a configuration file -## ---------------------------------------------- -## .. code-block:: nim -## -## import std/parsecfg -## var dict = loadConfig("config.ini") -## dict.delSectionKey("Author","email") -## dict.writeConfig("config.ini") -## +]## + +runnableExamples("-r:off"): + var dict = loadConfig("config.ini") + dict.delSectionKey("Author", "website") + dict.writeConfig("config.ini") + +##[ ## Supported INI File structure -## ---------------------------- -## The examples below are supported: -## +]## # taken from https://docs.python.org/3/library/configparser.html#supported-ini-file-structure runnableExamples: @@ -150,27 +149,27 @@ runnableExamples: ) let section1 = "Simple Values" - doAssert dict.getSectionValue(section1, "key") == "value" - doAssert dict.getSectionValue(section1, "spaces in keys") == "allowed" - doAssert dict.getSectionValue(section1, "spaces in values") == "allowed as well" - doAssert dict.getSectionValue(section1, "spaces around the delimiter") == "obviously" - doAssert dict.getSectionValue(section1, "you can also use") == "to delimit keys from values" + assert dict.getSectionValue(section1, "key") == "value" + assert dict.getSectionValue(section1, "spaces in keys") == "allowed" + assert dict.getSectionValue(section1, "spaces in values") == "allowed as well" + assert dict.getSectionValue(section1, "spaces around the delimiter") == "obviously" + assert dict.getSectionValue(section1, "you can also use") == "to delimit keys from values" let section2 = "All Values Are Strings" - doAssert dict.getSectionValue(section2, "values like this") == "19990429" - doAssert dict.getSectionValue(section2, "or this") == "3.14159265359" - doAssert dict.getSectionValue(section2, "are they treated as numbers") == "no" - doAssert dict.getSectionValue(section2, "integers floats and booleans are held as") == "strings" - doAssert dict.getSectionValue(section2, "can use the API to get converted values directly") == "true" + assert dict.getSectionValue(section2, "values like this") == "19990429" + assert dict.getSectionValue(section2, "or this") == "3.14159265359" + assert dict.getSectionValue(section2, "are they treated as numbers") == "no" + assert dict.getSectionValue(section2, "integers floats and booleans are held as") == "strings" + assert dict.getSectionValue(section2, "can use the API to get converted values directly") == "true" let section3 = "Seletion A" - doAssert dict.getSectionValue(section3, + assert dict.getSectionValue(section3, "space around section name will be ignored", "not an empty value") == "" let section4 = "Sections Can Be Indented" - doAssert dict.getSectionValue(section4, "can_values_be_as_well") == "True" - doAssert dict.getSectionValue(section4, "does_that_mean_anything_special") == "False" - doAssert dict.getSectionValue(section4, "purpose") == "formatting for readability" + assert dict.getSectionValue(section4, "can_values_be_as_well") == "True" + assert dict.getSectionValue(section4, "does_that_mean_anything_special") == "False" + assert dict.getSectionValue(section4, "purpose") == "formatting for readability" import strutils, lexbase, streams, tables import std/private/decode_helpers @@ -564,7 +563,7 @@ proc replace(s: string): string = proc writeConfig*(dict: Config, stream: Stream) = ## Writes the contents of the table to the specified stream. ## - ## **Note:** Comment statement will be ignored. + ## .. note:: Comment statement will be ignored. for section, sectionData in dict.pairs(): if section != "": ## Not general section if not allCharsInSet(section, SymChars): ## Non system character @@ -604,7 +603,7 @@ proc writeConfig*(dict: Config, stream: Stream) = proc `$`*(dict: Config): string = ## Writes the contents of the table to string. ## - ## **Note:** Comment statement will be ignored. + ## .. note:: Comment statement will be ignored. let stream = newStringStream() defer: stream.close() dict.writeConfig(stream) @@ -613,7 +612,7 @@ proc `$`*(dict: Config): string = proc writeConfig*(dict: Config, filename: string) = ## Writes the contents of the table to the specified configuration file. ## - ## **Note:** Comment statement will be ignored. + ## .. note:: Comment statement will be ignored. let file = open(filename, fmWrite) defer: file.close() let fileStream = newFileStream(file) From 93c26041a5484373512069512471bbde943a7242 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 27 Apr 2021 01:42:24 -0700 Subject: [PATCH 0723/2472] fix #17859; rename tests so they run in CI; merge several tests with nim check (#17862) * rename a test so it runs in CI; merge several tests with nim check * continue * continue * continue * rename tests/errmsgs/undeclared_routine_compiles.nim -> tests/errmsgs/tundeclared_routine_compiles.nim --- tests/errmsgs/tundeclared_routine.nim | 44 +++++++++++++++++++ ...s.nim => tundeclared_routine_compiles.nim} | 0 tests/errmsgs/undeclared_routine.nim | 13 ------ tests/errmsgs/undeclared_routine2.nim | 9 ---- tests/errmsgs/undeclared_routine3.nim | 13 ------ tests/errmsgs/undeclared_routine4.nim | 10 ----- tests/errmsgs/undeclared_routine5.nim | 9 ---- 7 files changed, 44 insertions(+), 54 deletions(-) create mode 100644 tests/errmsgs/tundeclared_routine.nim rename tests/errmsgs/{undeclared_routine_compiles.nim => tundeclared_routine_compiles.nim} (100%) delete mode 100644 tests/errmsgs/undeclared_routine.nim delete mode 100644 tests/errmsgs/undeclared_routine2.nim delete mode 100644 tests/errmsgs/undeclared_routine3.nim delete mode 100644 tests/errmsgs/undeclared_routine4.nim delete mode 100644 tests/errmsgs/undeclared_routine5.nim diff --git a/tests/errmsgs/tundeclared_routine.nim b/tests/errmsgs/tundeclared_routine.nim new file mode 100644 index 000000000..2f1320fff --- /dev/null +++ b/tests/errmsgs/tundeclared_routine.nim @@ -0,0 +1,44 @@ +discard """ +cmd: '''nim check --hints:off $file''' +action: reject +nimout: ''' +tundeclared_routine.nim(24, 17) Error: attempting to call routine: 'myiter' + found tundeclared_routine.myiter(a: string) [iterator declared in tundeclared_routine.nim(22, 12)] + found tundeclared_routine.myiter() [iterator declared in tundeclared_routine.nim(23, 12)] +tundeclared_routine.nim(29, 28) Error: invalid pragma: myPragma +tundeclared_routine.nim(36, 13) Error: undeclared field: 'bar3' for type tundeclared_routine.Foo [type declared in tundeclared_routine.nim(33, 8)] + found tundeclared_routine.bar3() [iterator declared in tundeclared_routine.nim(35, 12)] +tundeclared_routine.nim(41, 13) Error: undeclared field: 'bar4' for type tundeclared_routine.Foo [type declared in tundeclared_routine.nim(39, 8)] +tundeclared_routine.nim(44, 15) Error: attempting to call routine: 'bad5' +''' +""" + + + + + +# line 20 +block: + iterator myiter(a:string): int = discard + iterator myiter(): int = discard + let a = myiter(1) + +block: + proc myPragma():int=discard + iterator myPragma():int=discard + proc myfun(a:int): int {.myPragma.} = 1 + let a = myfun(1) + +block: + type Foo = object + var a = Foo() + iterator bar3():int=discard + let a2 = a.bar3 + +block: + type Foo = object + var a = Foo() + let a2 = a.bar4 + +block: + let a = bad5(1) diff --git a/tests/errmsgs/undeclared_routine_compiles.nim b/tests/errmsgs/tundeclared_routine_compiles.nim similarity index 100% rename from tests/errmsgs/undeclared_routine_compiles.nim rename to tests/errmsgs/tundeclared_routine_compiles.nim diff --git a/tests/errmsgs/undeclared_routine.nim b/tests/errmsgs/undeclared_routine.nim deleted file mode 100644 index 126ade52a..000000000 --- a/tests/errmsgs/undeclared_routine.nim +++ /dev/null @@ -1,13 +0,0 @@ -discard """ -cmd: '''nim c --hints:off $file''' -errormsg: "attempting to call routine: 'myiter'" -nimout: '''undeclared_routine.nim(13, 15) Error: attempting to call routine: 'myiter' - found 'undeclared_routine.myiter(a: string)[iterator declared in undeclared_routine.nim(10, 9)]' - found 'undeclared_routine.myiter()[iterator declared in undeclared_routine.nim(11, 9)]' -''' -""" - -iterator myiter(a:string): int = discard -iterator myiter(): int = discard - -let a = myiter(1) diff --git a/tests/errmsgs/undeclared_routine2.nim b/tests/errmsgs/undeclared_routine2.nim deleted file mode 100644 index 3e48b48f4..000000000 --- a/tests/errmsgs/undeclared_routine2.nim +++ /dev/null @@ -1,9 +0,0 @@ -discard """ -cmd: '''nim c --hints:off $file''' -errormsg: "invalid pragma: myPragma" -""" - -proc myPragma():int=discard -iterator myPragma():int=discard -proc myfun(a:int): int {.myPragma.} = 1 -let a = myfun(1) diff --git a/tests/errmsgs/undeclared_routine3.nim b/tests/errmsgs/undeclared_routine3.nim deleted file mode 100644 index 537871900..000000000 --- a/tests/errmsgs/undeclared_routine3.nim +++ /dev/null @@ -1,13 +0,0 @@ -discard """ -cmd: '''nim c --hints:off $file''' -errormsg: "undeclared field: 'bar'" -nimout: '''undeclared_routine3.nim(13, 10) Error: undeclared field: 'bar' - found 'undeclared_routine3.bar()[declared in undeclared_routine3.nim(12, 9)]' of kind 'iterator' -''' -""" - - -type Foo = object -var a = Foo() -iterator bar():int=discard -let a = a.bar diff --git a/tests/errmsgs/undeclared_routine4.nim b/tests/errmsgs/undeclared_routine4.nim deleted file mode 100644 index 9012a983f..000000000 --- a/tests/errmsgs/undeclared_routine4.nim +++ /dev/null @@ -1,10 +0,0 @@ -discard """ -cmd: '''nim c --hints:off $file''' -errormsg: "undeclared field: 'bar'" -nimout: '''undeclared_routine4.nim(10, 10) Error: undeclared field: 'bar' -''' -""" - -type Foo = object -var a = Foo() -let a = a.bar diff --git a/tests/errmsgs/undeclared_routine5.nim b/tests/errmsgs/undeclared_routine5.nim deleted file mode 100644 index 0387448a4..000000000 --- a/tests/errmsgs/undeclared_routine5.nim +++ /dev/null @@ -1,9 +0,0 @@ -discard """ -cmd: '''nim c --hints:off $file''' -errormsg: "undeclared identifier: 'myfun'" -nimout: '''undeclared_routine5.nim(9, 9) Error: undeclared identifier: 'myfun' -''' -""" - - -let a = myfun(1) From 66022423aa21710f57f36e4f3c8c9633567cdb5e Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 27 Apr 2021 17:09:58 +0800 Subject: [PATCH 0724/2472] [std/pegs] increase test coverage (#17871) --- lib/pure/pegs.nim | 144 ------------------------------------- tests/stdlib/tpegs.nim | 158 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 154 insertions(+), 148 deletions(-) diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index 06a77af57..b5d9b1d07 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -2041,147 +2041,3 @@ proc escapePeg*(s: string): string = inQuote = true result.add(c) if inQuote: result.add('\'') - -when isMainModule: - proc pegsTest() = - assert escapePeg("abc''def'") == r"'abc'\x27\x27'def'\x27" - assert match("(a b c)", peg"'(' @ ')'") - assert match("W_HI_Le", peg"\y 'while'") - assert(not match("W_HI_L", peg"\y 'while'")) - assert(not match("W_HI_Le", peg"\y v'while'")) - assert match("W_HI_Le", peg"y'while'") - - assert($ +digits == $peg"\d+") - assert "0158787".match(peg"\d+") - assert "ABC 0232".match(peg"\w+\s+\d+") - assert "ABC".match(peg"\d+ / \w+") - - var accum: seq[string] = @[] - for word in split("00232this02939is39an22example111", peg"\d+"): - accum.add(word) - assert(accum == @["this", "is", "an", "example"]) - - assert matchLen("key", ident) == 3 - - var pattern = sequence(ident, *whitespace, term('='), *whitespace, ident) - assert matchLen("key1= cal9", pattern) == 11 - - var ws = newNonTerminal("ws", 1, 1) - ws.rule = *whitespace - - var expr = newNonTerminal("expr", 1, 1) - expr.rule = sequence(capture(ident), *sequence( - nonterminal(ws), term('+'), nonterminal(ws), nonterminal(expr))) - - var c: Captures - var s = "a+b + c +d+e+f" - assert rawMatch(s, expr.rule, 0, c) == len(s) - var a = "" - for i in 0..c.ml-1: - a.add(substr(s, c.matches[i][0], c.matches[i][1])) - assert a == "abcdef" - #echo expr.rule - - #const filename = "lib/devel/peg/grammar.txt" - #var grammar = parsePeg(newFileStream(filename, fmRead), filename) - #echo "a <- [abc]*?".match(grammar) - assert find("_____abc_______", term("abc"), 2) == 5 - assert match("_______ana", peg"A <- 'ana' / . A") - assert match("abcs%%%", peg"A <- ..A / .A / '%'") - - var matches: array[0..MaxSubpatterns-1, string] - if "abc" =~ peg"{'a'}'bc' 'xyz' / {\ident}": - assert matches[0] == "abc" - else: - assert false - - var g2 = peg"""S <- A B / C D - A <- 'a'+ - B <- 'b'+ - C <- 'c'+ - D <- 'd'+ - """ - assert($g2 == "((A B) / (C D))") - assert match("cccccdddddd", g2) - assert("var1=key; var2=key2".replacef(peg"{\ident}'='{\ident}", "$1<-$2$2") == - "var1<-keykey; var2<-key2key2") - assert("var1=key; var2=key2".replace(peg"{\ident}'='{\ident}", "$1<-$2$2") == - "$1<-$2$2; $1<-$2$2") - assert "var1=key; var2=key2".endsWith(peg"{\ident}'='{\ident}") - - if "aaaaaa" =~ peg"'aa' !. / ({'a'})+": - assert matches[0] == "a" - else: - assert false - - if match("abcdefg", peg"c {d} ef {g}", matches, 2): - assert matches[0] == "d" - assert matches[1] == "g" - else: - assert false - - accum = @[] - for x in findAll("abcdef", peg".", 3): - accum.add(x) - assert(accum == @["d", "e", "f"]) - - for x in findAll("abcdef", peg"^{.}", 3): - assert x == "d" - - if "f(a, b)" =~ peg"{[0-9]+} / ({\ident} '(' {@} ')')": - assert matches[0] == "f" - assert matches[1] == "a, b" - else: - assert false - - assert match("eine übersicht und außerdem", peg"(\letter \white*)+") - # ß is not a lower cased letter?! - assert match("eine übersicht und auerdem", peg"(\lower \white*)+") - assert match("EINE ÜBERSICHT UND AUSSERDEM", peg"(\upper \white*)+") - assert(not match("456678", peg"(\letter)+")) - - assert("var1 = key; var2 = key2".replacef( - peg"\skip(\s*) {\ident}'='{\ident}", "$1<-$2$2") == - "var1<-keykey;var2<-key2key2") - - assert match("prefix/start", peg"^start$", 7) - - if "foo" =~ peg"{'a'}?.*": - assert matches[0].len == 0 - else: assert false - - if "foo" =~ peg"{''}.*": - assert matches[0] == "" - else: assert false - - if "foo" =~ peg"{'foo'}": - assert matches[0] == "foo" - else: assert false - - let empty_test = peg"^\d*" - let str = "XYZ" - - assert(str.find(empty_test) == 0) - assert(str.match(empty_test)) - - proc handleMatches(m: int, n: int, c: openArray[string]): string = - result = "" - - if m > 0: - result.add ", " - - result.add case n: - of 2: toLowerAscii(c[0]) & ": '" & c[1] & "'" - of 1: toLowerAscii(c[0]) & ": ''" - else: "" - - assert("Var1=key1;var2=Key2; VAR3". - replace(peg"{\ident}('='{\ident})* ';'* \s*", - handleMatches) == "var1: 'key1', var2: 'Key2', var3: ''") - - - doAssert "test1".match(peg"""{@}$""") - doAssert "test2".match(peg"""{(!$ .)*} $""") - pegsTest() - static: - pegsTest() diff --git a/tests/stdlib/tpegs.nim b/tests/stdlib/tpegs.nim index 99b7dc6f4..f2c93c2e5 100644 --- a/tests/stdlib/tpegs.nim +++ b/tests/stdlib/tpegs.nim @@ -1,5 +1,5 @@ discard """ - targets: "c js" + targets: "c cpp js" output: ''' PEG AST traversal output ------------------------ @@ -51,8 +51,7 @@ Event parser output ''' """ -import strutils, streams -import pegs +import std/[strutils, streams, pegs] const indent = " " @@ -125,7 +124,7 @@ block: discard of "Sum", "Product": try: - let val = matchStr.parseFloat + let val {.used.} = matchStr.parseFloat except ValueError: if valStack.len > 1 and opStack.len > 0: valStack[^2] = case opStack[^1] @@ -147,3 +146,154 @@ block: echo "-------------------" let pLen = parseArithExpr(txt) doAssert txt.len == pLen + + +import std/importutils + +block: + proc pegsTest() = + privateAccess(NonTerminal) + privateAccess(Captures) + + doAssert escapePeg("abc''def'") == r"'abc'\x27\x27'def'\x27" + doAssert match("(a b c)", peg"'(' @ ')'") + doAssert match("W_HI_Le", peg"\y 'while'") + doAssert(not match("W_HI_L", peg"\y 'while'")) + doAssert(not match("W_HI_Le", peg"\y v'while'")) + doAssert match("W_HI_Le", peg"y'while'") + + doAssert($ +digits == $peg"\d+") + doAssert "0158787".match(peg"\d+") + doAssert "ABC 0232".match(peg"\w+\s+\d+") + doAssert "ABC".match(peg"\d+ / \w+") + + var accum: seq[string] = @[] + for word in split("00232this02939is39an22example111", peg"\d+"): + accum.add(word) + doAssert(accum == @["this", "is", "an", "example"]) + + doAssert matchLen("key", ident) == 3 + + var pattern = sequence(ident, *whitespace, term('='), *whitespace, ident) + doAssert matchLen("key1= cal9", pattern) == 11 + + var ws = newNonTerminal("ws", 1, 1) + ws.rule = *whitespace + + var expr = newNonTerminal("expr", 1, 1) + expr.rule = sequence(capture(ident), *sequence( + nonterminal(ws), term('+'), nonterminal(ws), nonterminal(expr))) + + var c: Captures + var s = "a+b + c +d+e+f" + doAssert rawMatch(s, expr.rule, 0, c) == len(s) + var a = "" + for i in 0..c.ml-1: + a.add(substr(s, c.matches[i][0], c.matches[i][1])) + doAssert a == "abcdef" + #echo expr.rule + + #const filename = "lib/devel/peg/grammar.txt" + #var grammar = parsePeg(newFileStream(filename, fmRead), filename) + #echo "a <- [abc]*?".match(grammar) + doAssert find("_____abc_______", term("abc"), 2) == 5 + doAssert match("_______ana", peg"A <- 'ana' / . A") + doAssert match("abcs%%%", peg"A <- ..A / .A / '%'") + + var matches: array[0..MaxSubpatterns-1, string] + if "abc" =~ peg"{'a'}'bc' 'xyz' / {\ident}": + doAssert matches[0] == "abc" + else: + doAssert false + + var g2 = peg"""S <- A B / C D + A <- 'a'+ + B <- 'b'+ + C <- 'c'+ + D <- 'd'+ + """ + doAssert($g2 == "((A B) / (C D))") + doAssert match("cccccdddddd", g2) + doAssert("var1=key; var2=key2".replacef(peg"{\ident}'='{\ident}", "$1<-$2$2") == + "var1<-keykey; var2<-key2key2") + doAssert("var1=key; var2=key2".replace(peg"{\ident}'='{\ident}", "$1<-$2$2") == + "$1<-$2$2; $1<-$2$2") + doAssert "var1=key; var2=key2".endsWith(peg"{\ident}'='{\ident}") + + if "aaaaaa" =~ peg"'aa' !. / ({'a'})+": + doAssert matches[0] == "a" + else: + doAssert false + + if match("abcdefg", peg"c {d} ef {g}", matches, 2): + doAssert matches[0] == "d" + doAssert matches[1] == "g" + else: + doAssert false + + accum = @[] + for x in findAll("abcdef", peg".", 3): + accum.add(x) + doAssert(accum == @["d", "e", "f"]) + + for x in findAll("abcdef", peg"^{.}", 3): + doAssert x == "d" + + if "f(a, b)" =~ peg"{[0-9]+} / ({\ident} '(' {@} ')')": + doAssert matches[0] == "f" + doAssert matches[1] == "a, b" + else: + doAssert false + + doAssert match("eine übersicht und außerdem", peg"(\letter \white*)+") + # ß is not a lower cased letter?! + doAssert match("eine übersicht und auerdem", peg"(\lower \white*)+") + doAssert match("EINE ÜBERSICHT UND AUSSERDEM", peg"(\upper \white*)+") + doAssert(not match("456678", peg"(\letter)+")) + + doAssert("var1 = key; var2 = key2".replacef( + peg"\skip(\s*) {\ident}'='{\ident}", "$1<-$2$2") == + "var1<-keykey;var2<-key2key2") + + doAssert match("prefix/start", peg"^start$", 7) + + if "foo" =~ peg"{'a'}?.*": + doAssert matches[0].len == 0 + else: doAssert false + + if "foo" =~ peg"{''}.*": + doAssert matches[0] == "" + else: doAssert false + + if "foo" =~ peg"{'foo'}": + doAssert matches[0] == "foo" + else: doAssert false + + let empty_test = peg"^\d*" + let str = "XYZ" + + doAssert(str.find(empty_test) == 0) + doAssert(str.match(empty_test)) + + proc handleMatches(m: int, n: int, c: openArray[string]): string = + result = "" + + if m > 0: + result.add ", " + + result.add case n: + of 2: toLowerAscii(c[0]) & ": '" & c[1] & "'" + of 1: toLowerAscii(c[0]) & ": ''" + else: "" + + doAssert("Var1=key1;var2=Key2; VAR3". + replace(peg"{\ident}('='{\ident})* ';'* \s*", + handleMatches) == "var1: 'key1', var2: 'Key2', var3: ''") + + + doAssert "test1".match(peg"""{@}$""") + doAssert "test2".match(peg"""{(!$ .)*} $""") + pegsTest() + static: + pegsTest() + From a236002e54d68a672c720bd9b6d27ea0ba44edb8 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 27 Apr 2021 02:11:28 -0700 Subject: [PATCH 0725/2472] testament: add `nimoutFull: bool` spec (#17867) * testament: add `nimoutFull: bool` spec * PRTEMP * works * cleanup * add test for #12741 * PRTEMP failing test * remove unrelated changes * changelog --- changelog.md | 3 +++ testament/categories.nim | 3 +++ testament/specs.nim | 3 +++ testament/testament.nim | 18 +++++++++++------- testament/tests/shouldfail/tnimoutfull.nim | 15 +++++++++++++++ tests/config.nims | 8 ++++++-- tests/testament/tshould_not_work.nim | 2 ++ 7 files changed, 43 insertions(+), 9 deletions(-) create mode 100644 testament/tests/shouldfail/tnimoutfull.nim diff --git a/changelog.md b/changelog.md index e4b5c39f1..75d086450 100644 --- a/changelog.md +++ b/changelog.md @@ -399,3 +399,6 @@ - `fusion` is now un-bundled from nim, `./koch fusion` will install it via nimble at a fixed hash. + +- testament: added `nimoutFull: bool` spec to compare full output of compiler + instead of a subset. diff --git a/testament/categories.nim b/testament/categories.nim index 19d9e4507..ffee5eeb3 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -576,6 +576,9 @@ proc isJoinableSpec(spec: TSpec): bool = spec.exitCode == 0 and spec.input.len == 0 and spec.nimout.len == 0 and + spec.nimoutFull == false and + # so that tests can have `nimoutFull: true` with `nimout.len == 0` with + # the meaning that they expect empty output. spec.matrix.len == 0 and spec.outputCheck != ocSubstr and spec.ccodeCheck.len == 0 and diff --git a/testament/specs.nim b/testament/specs.nim index 38e4ed0db..3774778b3 100644 --- a/testament/specs.nim +++ b/testament/specs.nim @@ -88,6 +88,7 @@ type targets*: set[TTarget] matrix*: seq[string] nimout*: string + nimoutFull*: bool # whether nimout is all compiler output or a subset parseErrors*: string # when the spec definition is invalid, this is not empty. unjoinable*: bool unbatchable*: bool @@ -297,6 +298,8 @@ proc parseSpec*(filename: string): TSpec = result.action = actionReject of "nimout": result.nimout = e.value + of "nimoutfull": + result.nimoutFull = parseCfgBool(e.value) of "batchable": result.unbatchable = not parseCfgBool(e.value) of "joinable": diff --git a/testament/testament.nim b/testament/testament.nim index f259c7b96..97f1f01b8 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -371,12 +371,20 @@ proc checkForInlineErrors(r: var TResults, expected, given: TSpec, test: TTest, r.addResult(test, target, "", given.msg, reSuccess) inc(r.passed) +proc nimoutCheck(expected, given: TSpec): bool = + result = true + if expected.nimoutFull: + if expected.nimout != given.nimout: + result = false + elif expected.nimout.len > 0 and not greedyOrderedSubsetLines(expected.nimout, given.nimout): + result = false + proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest, target: TTarget) = if expected.inlineErrors.len > 0: checkForInlineErrors(r, expected, given, test, target) elif strip(expected.msg) notin strip(given.msg): r.addResult(test, target, expected.msg, given.msg, reMsgsDiffer) - elif expected.nimout.len > 0 and not greedyOrderedSubsetLines(expected.nimout, given.nimout): + elif not nimoutCheck(expected, given): r.addResult(test, target, expected.nimout, given.nimout, reMsgsDiffer) elif extractFilename(expected.file) != extractFilename(given.file) and "internal error:" notin expected.msg: @@ -424,10 +432,6 @@ proc codegenCheck(test: TTest, target: TTarget, spec: TSpec, expectedMsg: var st given.err = reCodeNotFound echo getCurrentExceptionMsg() -proc nimoutCheck(test: TTest; expectedNimout: string; given: var TSpec) = - if not greedyOrderedSubsetLines(expectedNimout, given.nimout): - given.err = reMsgsDiffer - proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec, expected: TSpec; r: var TResults) = var expectedmsg: string = "" @@ -436,10 +440,10 @@ proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec, if expected.needsCodegenCheck: codegenCheck(test, target, expected, expectedmsg, given) givenmsg = given.msg - if expected.nimout.len > 0: + if not nimoutCheck(expected, given): + given.err = reMsgsDiffer expectedmsg = expected.nimout givenmsg = given.nimout.strip - nimoutCheck(test, expectedmsg, given) else: givenmsg = "$ " & given.cmd & '\n' & given.nimout if given.err == reSuccess: inc(r.passed) diff --git a/testament/tests/shouldfail/tnimoutfull.nim b/testament/tests/shouldfail/tnimoutfull.nim new file mode 100644 index 000000000..3349ceedf --- /dev/null +++ b/testament/tests/shouldfail/tnimoutfull.nim @@ -0,0 +1,15 @@ +discard """ + targets: "c" + nimout: ''' +msg1 +msg2 +''' + action: compile + nimoutFull: true +""" + +# should fail because `msg3` is not in nimout and `nimoutFill: true` was given +static: + echo "msg1" + echo "msg2" + echo "msg3" diff --git a/tests/config.nims b/tests/config.nims index e5d6545f4..578b69e76 100644 --- a/tests/config.nims +++ b/tests/config.nims @@ -6,9 +6,13 @@ switch("path", "$lib/../testament/lib") ## prevent common user config settings to interfere with testament expectations ## Indifidual tests can override this if needed to test for these options. switch("colors", "off") -switch("filenames", "legacyRelProj") + switch("excessiveStackTrace", "off") -switch("spellSuggest", "0") + +when (NimMajor, NimMinor, NimPatch) >= (1,5,1): + # to make it easier to test against older nim versions, (best effort only) + switch("filenames", "legacyRelProj") + switch("spellSuggest", "0") # for std/unittest switch("define", "nimUnittestOutputLevel:PRINT_FAILURES") diff --git a/tests/testament/tshould_not_work.nim b/tests/testament/tshould_not_work.nim index a0b4d6a36..2777bfe95 100644 --- a/tests/testament/tshould_not_work.nim +++ b/tests/testament/tshould_not_work.nim @@ -21,6 +21,8 @@ Failure: reCodegenFailure max allowed size: 1 FAIL: tests/shouldfail/tnimout.nim c Failure: reMsgsDiffer +FAIL: tests/shouldfail/tnimoutfull.nim c +Failure: reMsgsDiffer FAIL: tests/shouldfail/toutput.nim c Failure: reOutputsDiffer FAIL: tests/shouldfail/toutputsub.nim c From b03d6c9b2f9d117ed85d410313b7b39e49a16964 Mon Sep 17 00:00:00 2001 From: Clyybber Date: Tue, 27 Apr 2021 13:03:26 +0200 Subject: [PATCH 0726/2472] Fix #17712 (#17873) --- compiler/dfa.nim | 7 ++++--- tests/arc/tarcmisc.nim | 11 +++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/compiler/dfa.nim b/compiler/dfa.nim index c7a9d4694..501d39f45 100644 --- a/compiler/dfa.nim +++ b/compiler/dfa.nim @@ -696,9 +696,10 @@ proc skipTrivials(c: var Con, n: PNode): PNode = proc genUse(c: var Con; orig: PNode) = let n = c.skipTrivials(orig) - if n.kind == nkSym and n.sym.kind in InterestingSyms: - c.code.add Instr(n: orig, kind: use) - elif n.kind in nkCallKinds: + if n.kind == nkSym: + if n.sym.kind in InterestingSyms: + c.code.add Instr(n: orig, kind: use) + else: gen(c, n) proc genDef(c: var Con; orig: PNode) = diff --git a/tests/arc/tarcmisc.nim b/tests/arc/tarcmisc.nim index 82751f44c..a52538085 100644 --- a/tests/arc/tarcmisc.nim +++ b/tests/arc/tarcmisc.nim @@ -27,6 +27,7 @@ finalizer aaaaa hello ok +true closed destroying variable: 20 destroying variable: 10 @@ -422,3 +423,13 @@ proc test3 = static: test3() # was buggy test3() + +# bug #17712 +proc t17712 = + var ppv = new int + discard @[ppv] + var el: ref int + el = [ppv][0] + echo el != nil + +t17712() From d881a05bf6b75d9abc48b07b2a093cb96fea0b4c Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 28 Apr 2021 01:14:26 +0800 Subject: [PATCH 0727/2472] [std/encodings] fix iconv headers on OpenBSD (#17872) --- lib/pure/encodings.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pure/encodings.nim b/lib/pure/encodings.nim index bdc41c6ff..893660d87 100644 --- a/lib/pure/encodings.nim +++ b/lib/pure/encodings.nim @@ -282,8 +282,10 @@ else: var errno {.importc, header: "".}: cint - when defined(freebsd) or defined(netbsd): + when defined(bsd): {.pragma: importIconv, cdecl, header: "".} + when defined(openbsd): + {.passL: "-liconv".} else: {.pragma: importIconv, cdecl, dynlib: iconvDll.} From 7637cff9cf677ad7e5952aed7670fe408eded418 Mon Sep 17 00:00:00 2001 From: Michael Krieger Date: Wed, 28 Apr 2021 20:33:08 +0200 Subject: [PATCH 0728/2472] Fix documentation of strutils.stripLineEnd (#17883) It does not return anything. --- lib/pure/strutils.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index bfd53a3a0..5bbd850cc 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -2812,7 +2812,7 @@ func strip*(s: string, leading = true, trailing = true, result = substr(s, first, last) func stripLineEnd*(s: var string) = - ## Returns `s` stripped from one of these suffixes: + ## Strips one of these suffixes from `s` in-place: ## `\r, \n, \r\n, \f, \v` (at most once instance). ## For example, can be useful in conjunction with `osproc.execCmdEx`. ## aka: `chomp`:idx: From 601c050fcb4cc3101265018eb1cc381c6cb9dab9 Mon Sep 17 00:00:00 2001 From: c-blake Date: Thu, 29 Apr 2021 02:55:43 -0400 Subject: [PATCH 0729/2472] Implement https://forum.nim-lang.org/t/7848#50018 (#17874) * Implement https://forum.nim-lang.org/t/7848#50018 with just the same `SuccessX` hint category, build mode on a separate, final line, and no change to how the mode is spelled for -d:release/-d:danger. * Change to add a new BuildMode hint category and keep testament in sync as per comment. * Add "--hint:buildmode:off" to `defaultHintsOff`. * Remove as requested. * As requested for tests clean up. * Address code review. * Address code review. * Mirror db456423116a9b19666f551f4d38aded3964c2e2 --- compiler/lineinfos.nim | 8 +++++--- compiler/main.nim | 8 ++++---- doc/nimc.rst | 1 + drnim/drnim.nim | 8 ++++---- testament/testament.nim | 3 ++- tests/misc/trunner.nim | 2 +- 6 files changed, 17 insertions(+), 13 deletions(-) diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index d8f82aea0..c7057e7db 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -64,7 +64,8 @@ type warnFileChanged = "FileChanged", warnUser = "User", - hintSuccess = "Success", hintSuccessX = "SuccessX", hintCC = "CC", + hintSuccess = "Success", hintSuccessX = "SuccessX", hintBuildMode = "BuildMode", + hintCC = "CC", hintLineTooLong = "LineTooLong", hintXDeclaredButNotUsed = "XDeclaredButNotUsed", hintXCannotRaiseY = "XCannotRaiseY", hintConvToBaseNotNeeded = "ConvToBaseNotNeeded", hintConvFromXtoItselfNotNeeded = "ConvFromXtoItselfNotNeeded", hintExprAlwaysX = "ExprAlwaysX", @@ -142,7 +143,8 @@ const warnUser: "$1", hintSuccess: "operation successful: $#", # keep in sync with `testament.isSuccess` - hintSuccessX: "${loc} lines; ${sec}s; $mem; $build build; proj: $project; out: $output", + hintSuccessX: "$loc lines; ${sec}s; $mem; proj: $project; out: $output", + hintBuildMode: "$1", hintCC: "CC: $1", hintLineTooLong: "line too long", hintXDeclaredButNotUsed: "'$1' is declared but not used", @@ -196,7 +198,7 @@ proc computeNotesVerbosity(): array[0..3, TNoteKinds] = result[1] = result[2] - {warnProveField, warnProveIndex, warnGcUnsafe, hintPath, hintDependency, hintCodeBegin, hintCodeEnd, hintSource, hintGlobalVar, hintGCStats, hintMsgOrigin} - result[0] = result[1] - {hintSuccessX, hintSuccess, hintConf, + result[0] = result[1] - {hintSuccessX, hintBuildMode, hintSuccess, hintConf, hintProcessing, hintPattern, hintExecuting, hintLinking, hintCC} const diff --git a/compiler/main.nim b/compiler/main.nim index 48fbd185c..e7ee021bc 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -384,9 +384,9 @@ proc mainCommand*(graph: ModuleGraph) = when declared(system.getMaxMem): formatSize(getMaxMem()) & " peakmem" else: formatSize(getTotalMem()) & " totmem" let loc = $conf.linesCompiled - let build = if isDefined(conf, "danger"): "Dangerous Release" - elif isDefined(conf, "release"): "Release" - else: "Debug" + let build = if isDefined(conf, "danger"): "Dangerous Release build" + elif isDefined(conf, "release"): "Release build" + else: "***SLOW, DEBUG BUILD***; -d:release makes code run faster." let sec = formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3) let project = if conf.filenameOption == foAbs: $conf.projectFull else: $conf.projectName # xxx honor conf.filenameOption more accurately @@ -406,10 +406,10 @@ proc mainCommand*(graph: ModuleGraph) = "loc", loc, "sec", sec, "mem", mem, - "build", build, "project", project, "output", output, ]) + rawMessage(conf, hintBuildMode, build) when PrintRopeCacheStats: echo "rope cache stats: " diff --git a/doc/nimc.rst b/doc/nimc.rst index aad889591..9db95e81b 100644 --- a/doc/nimc.rst +++ b/doc/nimc.rst @@ -102,6 +102,7 @@ Source The source line that triggered a diagnostic message. StackTrace Success, SuccessX Successful compilation of a library or a binary. +BuildMode Kind of build: debug, release, danger User UserRaw XDeclaredButNotUsed Unused symbols in the code. diff --git a/drnim/drnim.nim b/drnim/drnim.nim index a591a8ef3..1eded533d 100644 --- a/drnim/drnim.nim +++ b/drnim/drnim.nim @@ -1210,19 +1210,19 @@ proc mainCommand(graph: ModuleGraph) = when declared(system.getMaxMem): formatSize(getMaxMem()) & " peakmem" else: formatSize(getTotalMem()) & " totmem" let loc = $conf.linesCompiled - let build = if isDefined(conf, "danger"): "Dangerous Release" - elif isDefined(conf, "release"): "Release" - else: "Debug" + let build = if isDefined(conf, "danger"): "Dangerous Release build" + elif isDefined(conf, "release"): "Release build" + else: "***SLOW, DEBUG BUILD***; -d:release makes code run faster." let sec = formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3) let project = if conf.filenameOption == foAbs: $conf.projectFull else: $conf.projectName rawMessage(conf, hintSuccessX, [ "loc", loc, "sec", sec, "mem", mem, - "build", build, "project", project, "output", "" ]) + rawMessage(conf, hintBuildMode, build) proc processCmdLine(pass: TCmdLinePass, cmd: string; config: ConfigRef) = var p = parseopt.initOptParser(cmd) diff --git a/testament/testament.nim b/testament/testament.nim index 97f1f01b8..9caa3f6b9 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -107,7 +107,8 @@ proc isSuccess(input: string): bool = # that may appear in user config (eg: `--filenames`). # Passing `XDG_CONFIG_HOME= testament args...` can be used to ignore user config # stored in XDG_CONFIG_HOME, refs https://wiki.archlinux.org/index.php/XDG_Base_Directory - input.startsWith("Hint: ") and input.endsWith("[SuccessX]") + input.startsWith("Hint: ") and + (input.endsWith("[SuccessX]") or input.endsWith("[BuildMode]")) proc getFileDir(filename: string): string = result = filename.splitFile().dir diff --git a/tests/misc/trunner.nim b/tests/misc/trunner.nim index 2a82ca9ee..06c828eaa 100644 --- a/tests/misc/trunner.nim +++ b/tests/misc/trunner.nim @@ -21,7 +21,7 @@ proc isDots(a: string): bool = a.startsWith(".") and a.strip(chars = {'.'}) == "" const - defaultHintsOff = "--hint:successx:off --hint:exec:off --hint:link:off --hint:cc:off --hint:conf:off --hint:processing:off --hint:QuitCalled:off" + defaultHintsOff = "--hint:successx:off --hint:buildmode:off --hint:exec:off --hint:link:off --hint:cc:off --hint:conf:off --hint:processing:off --hint:QuitCalled:off" # useful when you want to turn only some hints on, and some common ones off. # pending https://github.com/timotheecour/Nim/issues/453, simplify to: `--hints:off` nim = getCurrentCompilerExe() From 927ae26fad47289ae84de0cb23f0395ca7a79610 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 29 Apr 2021 00:01:00 -0700 Subject: [PATCH 0730/2472] externalToLink: use quoteShell to avoid issues with spaces in paths for {.link.} pragmas (#17875) --- compiler/extccomp.nim | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 3b415c499..1cea9edeb 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -943,6 +943,7 @@ proc jsonBuildInstructionsFile*(conf: ConfigRef): AbsoluteFile = result = getNimcacheDir(conf) / conf.outFile.changeFileExt("json") proc writeJsonBuildInstructions*(conf: ConfigRef) = + # xxx use std/json instead, will result in simpler, more maintainable code. template lit(x: string) = f.write x template str(x: string) = buf.setLen 0 @@ -964,23 +965,18 @@ proc writeJsonBuildInstructions*(conf: ConfigRef) = proc linkfiles(conf: ConfigRef; f: File; buf, objfiles: var string; clist: CfileList; llist: seq[string]) = var pastStart = false + template impl(path) = + let path2 = quoteShell(path) + objfiles.add(' ') + objfiles.add(path2) + if pastStart: lit ",\L" + str path2 + pastStart = true for it in llist: - let objfile = if noAbsolutePaths(conf): it.extractFilename - else: it - let objstr = addFileExt(objfile, CC[conf.cCompiler].objExt) - objfiles.add(' ') - objfiles.add(objstr) - if pastStart: lit ",\L" - str objstr - pastStart = true - + let objfile = if noAbsolutePaths(conf): it.extractFilename else: it + impl(addFileExt(objfile, CC[conf.cCompiler].objExt)) for it in clist: - let objstr = quoteShell(it.obj) - objfiles.add(' ') - objfiles.add(objstr) - if pastStart: lit ",\L" - str objstr - pastStart = true + impl(it.obj) lit "\L" proc depfiles(conf: ConfigRef; f: File; buf: var string) = From 016a8ccd7a8a018b28e8f424085c178d96860c6c Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 29 Apr 2021 15:51:54 +0800 Subject: [PATCH 0731/2472] [std/encodings]move to tests (#17866) --- lib/pure/encodings.nim | 72 ------------------------------------ tests/stdlib/tencodings.nim | 73 +++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 72 deletions(-) diff --git a/lib/pure/encodings.nim b/lib/pure/encodings.nim index 893660d87..89db0a278 100644 --- a/lib/pure/encodings.nim +++ b/lib/pure/encodings.nim @@ -476,75 +476,3 @@ proc convert*(s: string, destEncoding = "UTF-8", result = convert(c, s) finally: close(c) - -when not defined(testing) and isMainModule: - let - orig = "öäüß" - cp1252 = convert(orig, "CP1252", "UTF-8") - ibm850 = convert(cp1252, "ibm850", "CP1252") - current = getCurrentEncoding() - echo "Original string from source code: ", orig - echo "Forced ibm850 encoding: ", ibm850 - echo "Current encoding: ", current - echo "From ibm850 to current: ", convert(ibm850, current, "ibm850") - -when not defined(testing) and isMainModule and defined(windows): - block should_throw_on_unsupported_conversions: - let original = "some string" - - doAssertRaises(EncodingError): - discard convert(original, "utf-8", "utf-32") - - doAssertRaises(EncodingError): - discard convert(original, "utf-8", "unicodeFFFE") - - doAssertRaises(EncodingError): - discard convert(original, "utf-8", "utf-32BE") - - doAssertRaises(EncodingError): - discard convert(original, "unicodeFFFE", "utf-8") - - doAssertRaises(EncodingError): - discard convert(original, "utf-32", "utf-8") - - doAssertRaises(EncodingError): - discard convert(original, "utf-32BE", "utf-8") - - block should_convert_from_utf16_to_utf8: - let original = "\x42\x04\x35\x04\x41\x04\x42\x04" # utf-16 little endian test string "тест" - let result = convert(original, "utf-8", "utf-16") - doAssert(result == "\xd1\x82\xd0\xb5\xd1\x81\xd1\x82") - - block should_convert_from_utf16_to_win1251: - let original = "\x42\x04\x35\x04\x41\x04\x42\x04" # utf-16 little endian test string "тест" - let result = convert(original, "windows-1251", "utf-16") - doAssert(result == "\xf2\xe5\xf1\xf2") - - block should_convert_from_win1251_to_koi8r: - let original = "\xf2\xe5\xf1\xf2" # win1251 test string "тест" - let result = convert(original, "koi8-r", "windows-1251") - doAssert(result == "\xd4\xc5\xd3\xd4") - - block should_convert_from_koi8r_to_win1251: - let original = "\xd4\xc5\xd3\xd4" # koi8r test string "тест" - let result = convert(original, "windows-1251", "koi8-r") - doAssert(result == "\xf2\xe5\xf1\xf2") - - block should_convert_from_utf8_to_win1251: - let original = "\xd1\x82\xd0\xb5\xd1\x81\xd1\x82" # utf-8 test string "тест" - let result = convert(original, "windows-1251", "utf-8") - doAssert(result == "\xf2\xe5\xf1\xf2") - - block should_convert_from_utf8_to_utf16: - let original = "\xd1\x82\xd0\xb5\xd1\x81\xd1\x82" # utf-8 test string "тест" - let result = convert(original, "utf-16", "utf-8") - doAssert(result == "\x42\x04\x35\x04\x41\x04\x42\x04") - - block should_handle_empty_string_for_any_conversion: - let original = "" - var result = convert(original, "utf-16", "utf-8") - doAssert(result == "") - result = convert(original, "utf-8", "utf-16") - doAssert(result == "") - result = convert(original, "windows-1251", "koi8-r") - doAssert(result == "") diff --git a/tests/stdlib/tencodings.nim b/tests/stdlib/tencodings.nim index 1da52ba59..8ca55dbd9 100644 --- a/tests/stdlib/tencodings.nim +++ b/tests/stdlib/tencodings.nim @@ -23,3 +23,76 @@ block: block: let data = "谁怕？一蓑烟雨任平生" doAssert toGB2312.convert(data) == "\203\173\197\194\163\191\210\187\203\242\209\204\211\234\200\206\198\189\201\250" + + +when defined(windows): + block should_throw_on_unsupported_conversions: + let original = "some string" + + doAssertRaises(EncodingError): + discard convert(original, "utf-8", "utf-32") + + doAssertRaises(EncodingError): + discard convert(original, "utf-8", "unicodeFFFE") + + doAssertRaises(EncodingError): + discard convert(original, "utf-8", "utf-32BE") + + doAssertRaises(EncodingError): + discard convert(original, "unicodeFFFE", "utf-8") + + doAssertRaises(EncodingError): + discard convert(original, "utf-32", "utf-8") + + doAssertRaises(EncodingError): + discard convert(original, "utf-32BE", "utf-8") + + block should_convert_from_utf16_to_utf8: + let original = "\x42\x04\x35\x04\x41\x04\x42\x04" # utf-16 little endian test string "тест" + let result = convert(original, "utf-8", "utf-16") + doAssert(result == "\xd1\x82\xd0\xb5\xd1\x81\xd1\x82") + + block should_convert_from_utf16_to_win1251: + let original = "\x42\x04\x35\x04\x41\x04\x42\x04" # utf-16 little endian test string "тест" + let result = convert(original, "windows-1251", "utf-16") + doAssert(result == "\xf2\xe5\xf1\xf2") + + block should_convert_from_win1251_to_koi8r: + let original = "\xf2\xe5\xf1\xf2" # win1251 test string "тест" + let result = convert(original, "koi8-r", "windows-1251") + doAssert(result == "\xd4\xc5\xd3\xd4") + + block should_convert_from_koi8r_to_win1251: + let original = "\xd4\xc5\xd3\xd4" # koi8r test string "тест" + let result = convert(original, "windows-1251", "koi8-r") + doAssert(result == "\xf2\xe5\xf1\xf2") + + block should_convert_from_utf8_to_win1251: + let original = "\xd1\x82\xd0\xb5\xd1\x81\xd1\x82" # utf-8 test string "тест" + let result = convert(original, "windows-1251", "utf-8") + doAssert(result == "\xf2\xe5\xf1\xf2") + + block should_convert_from_utf8_to_utf16: + let original = "\xd1\x82\xd0\xb5\xd1\x81\xd1\x82" # utf-8 test string "тест" + let result = convert(original, "utf-16", "utf-8") + doAssert(result == "\x42\x04\x35\x04\x41\x04\x42\x04") + + block should_handle_empty_string_for_any_conversion: + let original = "" + var result = convert(original, "utf-16", "utf-8") + doAssert(result == "") + result = convert(original, "utf-8", "utf-16") + doAssert(result == "") + result = convert(original, "windows-1251", "koi8-r") + doAssert(result == "") + + +block: + let + orig = "öäüß" + cp1252 = convert(orig, "CP1252", "UTF-8") + ibm850 = convert(cp1252, "ibm850", "CP1252") + current = getCurrentEncoding() + doAssert orig == "\195\182\195\164\195\188\195\159" + doAssert ibm850 == "\148\132\129\225" + doAssert convert(ibm850, current, "ibm850") == orig From 87229e272ecd4012b750215d35e914580cd8475c Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 29 Apr 2021 02:25:08 -0700 Subject: [PATCH 0732/2472] fix #17853 (ascii message separator broke json nim dump) (#17887) --- changelog.md | 4 ++++ compiler/main.nim | 3 ++- compiler/msgs.nim | 1 + compiler/options.nim | 2 +- tests/misc/trunner.nim | 11 +++++++++++ tests/osproc/treadlines.nim | 7 +++++-- 6 files changed, 24 insertions(+), 4 deletions(-) diff --git a/changelog.md b/changelog.md index 75d086450..21f3cb9b5 100644 --- a/changelog.md +++ b/changelog.md @@ -60,6 +60,10 @@ - Removed `.travis.yml`, `appveyor.yml.disabled`, `.github/workflows/ci.yml.disabled`. +- Nim compiler now adds ASCII unit separator `\31` before a newline for every generated + message (potentially multiline), so tooling can tell when messages start and end. + + ## Standard library additions and changes - Added support for parenthesized expressions in `strformat` diff --git a/compiler/main.nim b/compiler/main.nim index e7ee021bc..db5dd439e 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -349,7 +349,8 @@ proc mainCommand*(graph: ModuleGraph) = (key: "warnings", val: warnings), ] - msgWriteln(conf, $dumpdata, {msgStdout, msgSkipHook}) + msgWriteln(conf, $dumpdata, {msgStdout, msgSkipHook, msgNoUnitSep}) + # `msgNoUnitSep` to avoid generating invalid json, refs bug #17853 else: msgWriteln(conf, "-- list of currently defined symbols --", {msgStdout, msgSkipHook, msgNoUnitSep}) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 93ca8d7cc..3abbf9e69 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -291,6 +291,7 @@ proc `??`* (conf: ConfigRef; info: TLineInfo, filename: string): bool = const UnitSep = "\31" + # this needs care to avoid issues similar to https://github.com/nim-lang/Nim/issues/17853 type MsgFlag* = enum ## flags altering msgWriteln behavior diff --git a/compiler/options.nim b/compiler/options.nim index 209564d0a..4773ba284 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -44,7 +44,7 @@ type # please make sure we have under 32 options optImportHidden TOptions* = set[TOption] - TGlobalOption* = enum # **keep binary compatible** + TGlobalOption* = enum gloptNone, optForceFullMake, optWasNimscript, # redundant with `cmdNimscript`, could be removed optListCmd, optCompileOnly, optNoLinking, diff --git a/tests/misc/trunner.nim b/tests/misc/trunner.nim index 06c828eaa..014373dfb 100644 --- a/tests/misc/trunner.nim +++ b/tests/misc/trunner.nim @@ -71,6 +71,7 @@ ret=[s1:foobar s2:foobar age:25 pi:3.14] else: # don't run twice the same test import std/strutils + import std/json template check2(msg) = doAssert msg in output, output block: # tests with various options `nim doc --project --index --docroot` @@ -317,3 +318,13 @@ compiling: v3 running: v3 running: v2 """, ret + + block: # nim dump + let cmd = fmt"{nim} dump --dump.format:json -d:D20210428T161003 --hints:off ." + let (ret, status) = execCmdEx(cmd) + doAssert status == 0 + let j = ret.parseJson + # sanity checks + doAssert "D20210428T161003" in j["defined_symbols"].to(seq[string]) + doAssert j["version"].to(string) == NimVersion + doAssert j["nimExe"].to(string) == getCurrentCompilerExe() diff --git a/tests/osproc/treadlines.nim b/tests/osproc/treadlines.nim index 436dd7a10..bcde19d7f 100644 --- a/tests/osproc/treadlines.nim +++ b/tests/osproc/treadlines.nim @@ -1,15 +1,18 @@ discard """ - output: '''Error: cannot open 'a.nim'\31 + output: ''' +Error: cannot open 'a.nim'\31 Error: cannot open 'b.nim'\31 ''' targets: "c" """ import osproc +from std/os import getCurrentCompilerExe var ps: seq[Process] # compile & run 2 progs in parallel +const nim = getCurrentCompilerExe() for prog in ["a", "b"]: - ps.add startProcess("nim", "", + ps.add startProcess(nim, "", ["r", "--hint[Conf]=off", "--hint[Processing]=off", prog], options = {poUsePath, poDaemon, poStdErrToStdOut}) From 5edddd68d04648553bc02d52cd53d3a1a14102f3 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 29 Apr 2021 19:39:36 +0800 Subject: [PATCH 0733/2472] improve cache line size (#17885) * improve cache line size - 64bit system tend to use cache line of 64 bytes - add align https://trishagee.com/2011/07/22/dissecting_the_disruptor_why_its_so_fast_part_two__magic_cache_line_padding Though I'm not sure, what do you think? @timotheecour * Update lib/pure/concurrency/threadpool.nim --- lib/pure/concurrency/threadpool.nim | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/pure/concurrency/threadpool.nim b/lib/pure/concurrency/threadpool.nim index 9beb39522..f8a5f9ba8 100644 --- a/lib/pure/concurrency/threadpool.nim +++ b/lib/pure/concurrency/threadpool.nim @@ -52,17 +52,14 @@ proc signal(cv: var Semaphore) = release(cv.L) signal(cv.c) -const CacheLineSize = 32 # true for most archs +const CacheLineSize = 64 # true for most archs type Barrier {.compilerproc.} = object entered: int cv: Semaphore # Semaphore takes 3 words at least - when sizeof(int) < 8: - cacheAlign: array[CacheLineSize-4*sizeof(int), byte] - left: int - cacheAlign2: array[CacheLineSize-sizeof(int), byte] - interest: bool # whether the master is interested in the "all done" event + left {.align(CacheLineSize).}: int + interest {.align(CacheLineSize).} : bool # whether the master is interested in the "all done" event proc barrierEnter(b: ptr Barrier) {.compilerproc, inline.} = # due to the signaling between threads, it is ensured we are the only From e4db733d80d0b70192ac7eb4094a46269fe7cd5b Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 29 Apr 2021 04:42:56 -0700 Subject: [PATCH 0734/2472] fix #17888: remove undefined behavior for posix.open; fix tempfiles.createTempFile (#17889) * fix #17888: remove undefined behavior for posix.open; fix tempfiles.createTempFile * fix for tests/async/tasyncfile.nim * hide mode for now * add notice regarding stability --- lib/posix/posix.nim | 6 +++++- lib/pure/streams.nim | 1 + lib/std/tempfiles.nim | 13 ++++++++++--- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 035b4e8fb..45fe00d5d 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -183,7 +183,11 @@ proc dlsym*(a1: pointer, a2: cstring): pointer {.importc, header: "", s proc creat*(a1: cstring, a2: Mode): cint {.importc, header: "", sideEffect.} proc fcntl*(a1: cint | SocketHandle, a2: cint): cint {.varargs, importc, header: "", sideEffect.} -proc open*(a1: cstring, a2: cint): cint {.varargs, importc, header: "", sideEffect.} +proc openImpl(a1: cstring, a2: cint): cint {.varargs, importc: "open", header: "", sideEffect.} +proc open*(a1: cstring, a2: cint, mode: Mode | cint = 0.Mode): cint {.inline.} = + # prevents bug #17888 + openImpl(a1, a2, mode) + proc posix_fadvise*(a1: cint, a2, a3: Off, a4: cint): cint {. importc, header: "".} proc posix_fallocate*(a1: cint, a2, a3: Off): cint {. diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim index eb1c9cc14..3cc5ea038 100644 --- a/lib/pure/streams.nim +++ b/lib/pure/streams.nim @@ -1503,6 +1503,7 @@ when false: of fmReadWrite: flags = O_RDWR or int(O_CREAT) of fmReadWriteExisting: flags = O_RDWR of fmAppend: flags = O_WRONLY or int(O_CREAT) or O_APPEND + static: doAssert false # handle bug #17888 var handle = open(filename, flags) if handle < 0: raise newEOS("posix.open() call failed") result = newFileHandleStream(handle) diff --git a/lib/std/tempfiles.nim b/lib/std/tempfiles.nim index 1e1bbd403..2a6fe7d83 100644 --- a/lib/std/tempfiles.nim +++ b/lib/std/tempfiles.nim @@ -8,6 +8,8 @@ # ## This module creates temporary files and directories. +## +## Experimental API, subject to change. import os, random @@ -44,6 +46,8 @@ else: proc safeOpen(filename: string): File = ## Open files exclusively. + # xxx this should be clarified; it doesn't in particular prevent other processes + # from opening the file, at least currently. when defined(windows): let dwShareMode = FILE_SHARE_DELETE or FILE_SHARE_READ or FILE_SHARE_WRITE let dwCreation = CREATE_NEW @@ -64,9 +68,13 @@ proc safeOpen(filename: string): File = discard close_osfandle(fileHandle) raiseOSError(osLastError(), filename) else: + # xxx we need a `proc toMode(a: FilePermission): Mode`, possibly by + # exposing fusion/filepermissions.fromFilePermissions to stdlib; then we need + # to expose a `perm` param so users can customize this (e.g. the temp file may + # need execute permissions), and figure out how to make the API cross platform. + let mode = Mode(S_IRUSR or S_IWUSR) let flags = posix.O_RDWR or posix.O_CREAT or posix.O_EXCL - - let fileHandle = posix.open(filename, flags) + let fileHandle = posix.open(filename, flags, mode) if fileHandle == -1: raiseOSError(osLastError(), filename) @@ -93,7 +101,6 @@ proc createTempFile*(prefix, suffix: string, dir = ""): tuple[fd: File, path: st ## If failing to create a temporary file, `IOError` will be raised. ## ## .. note:: It is the caller's responsibility to remove the file when no longer needed. - ## var dir = dir if dir.len == 0: dir = getTempDir() From a424075b5eb50681b448dcf73f7559549945ad58 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 29 Apr 2021 04:44:53 -0700 Subject: [PATCH 0735/2472] improve nimsuggest/tester, minor improvements to koch.nim (#17879) * improve nimsuggest/tester * koch improvements --- koch.nim | 21 ++++++++++----------- nimdoc/testproject/testproject.nim | 2 +- nimpretty/tester.nim | 2 +- nimsuggest/tester.nim | 22 ++++++++++++---------- 4 files changed, 24 insertions(+), 23 deletions(-) diff --git a/koch.nim b/koch.nim index c78356b58..12f9e1c14 100644 --- a/koch.nim +++ b/koch.nim @@ -154,7 +154,7 @@ proc bundleNimbleExe(latest: bool, args: string) = proc bundleNimsuggest(args: string) = nimCompileFold("Compile nimsuggest", "nimsuggest/nimsuggest.nim", - options = "-d:release -d:danger " & args) + options = "-d:danger " & args) proc buildVccTool(args: string) = let input = "tools/vccexe/vccexe.nim" @@ -558,28 +558,27 @@ proc runCI(cmd: string) = ## run tests execFold("Test nimscript", "nim e tests/test_nimscript.nims") when defined(windows): - # note: will be over-written below - execFold("Compile tester", "nim c -d:nimCoroutines --os:genode -d:posix --compileOnly testament/testament") + execFold("Compile tester", "nim c --usenimcache -d:nimCoroutines --os:genode -d:posix --compileOnly testament/testament") # main bottleneck here # xxx: even though this is the main bottleneck, we could speedup the rest via batching with `--batch`. # BUG: with initOptParser, `--batch:'' all` interprets `all` as the argument of --batch, pending bug #14343 - execFold("Run tester", "nim c -r -d:nimCoroutines --putenv:NIM_TESTAMENT_REMOTE_NETWORKING:1 -d:nimStrictMode testament/testament $# all -d:nimCoroutines" % batchParam) + execFold("Run tester", "nim c -r --putenv:NIM_TESTAMENT_REMOTE_NETWORKING:1 -d:nimStrictMode testament/testament $# all -d:nimCoroutines" % batchParam) - block CT_FFI: + block: # nimHasLibFFI: when defined(posix): # windows can be handled in future PR's execFold("nimble install -y libffi", "nimble install -y libffi") - const nimFFI = "./bin/nim.ctffi" + const nimFFI = "bin/nim.ctffi" # no need to bootstrap with koch boot (would be slower) let backend = if doUseCpp(): "cpp" else: "c" execFold("build with -d:nimHasLibFFI", "nim $1 -d:release -d:nimHasLibFFI -o:$2 compiler/nim.nim" % [backend, nimFFI]) execFold("test with -d:nimHasLibFFI", "$1 $2 -r testament/testament --nim:$1 r tests/misc/trunner.nim -d:nimTrunnerFfi" % [nimFFI, backend]) - execFold("Run nimdoc tests", "nim c -r nimdoc/tester") - execFold("Run rst2html tests", "nim c -r nimdoc/rsttester") - execFold("Run nimpretty tests", "nim c -r nimpretty/tester.nim") + execFold("Run nimdoc tests", "nim r nimdoc/tester") + execFold("Run rst2html tests", "nim r nimdoc/rsttester") + execFold("Run nimpretty tests", "nim r nimpretty/tester.nim") when defined(posix): - execFold("Run nimsuggest tests", "nim c -r nimsuggest/tester") + execFold("Run nimsuggest tests", "nim r nimsuggest/tester") proc testUnixInstall(cmdLineRest: string) = csource("-d:danger" & cmdLineRest) @@ -605,7 +604,7 @@ proc testUnixInstall(cmdLineRest: string) = execCleanPath("./koch tools") # check the tests work: putEnv("NIM_EXE_NOT_IN_PATH", "NOT_IN_PATH") - execCleanPath("./koch tests --nim:./bin/nim cat megatest", destDir / "bin") + execCleanPath("./koch tests --nim:bin/nim cat megatest", destDir / "bin") else: echo "Version check: failure" finally: diff --git a/nimdoc/testproject/testproject.nim b/nimdoc/testproject/testproject.nim index 6b82acb9b..c7a1cd0fc 100644 --- a/nimdoc/testproject/testproject.nim +++ b/nimdoc/testproject/testproject.nim @@ -341,7 +341,7 @@ when true: # issue #14473 template doit(): untyped = ## doit ## return output only - toSeq([1,2]) + toSeq(["D20210427T172228"]) # make it searcheable at least until we figure out a way to avoid echo echo doit() # using doAssert or similar to avoid echo would "hide" the original bug when true: # issue #14846 diff --git a/nimpretty/tester.nim b/nimpretty/tester.nim index d646b25ce..0a60ce693 100644 --- a/nimpretty/tester.nim +++ b/nimpretty/tester.nim @@ -3,7 +3,7 @@ import strutils, os, sequtils const - dir = "nimpretty/tests/" + dir = "nimpretty/tests" outputdir = dir / "outputdir" var diff --git a/nimsuggest/tester.nim b/nimsuggest/tester.nim index 9fcf7eacc..f19cf5538 100644 --- a/nimsuggest/tester.nim +++ b/nimsuggest/tester.nim @@ -2,6 +2,8 @@ # Every test file can have a #[!]# comment that is deleted from the input # before 'nimsuggest' is invoked to ensure this token doesn't make a # crucial difference for Nim's parser. +# When debugging, to run a single test, use for e.g.: +# `nim r nimsuggest/tester.nim nimsuggest/tests/tsug_accquote.nim` import os, osproc, strutils, streams, re, sexp, net @@ -13,16 +15,16 @@ type disabled: bool const - curDir = when defined(windows): "" else: "" DummyEof = "!EOF!" - -template tpath(): untyped = getAppDir() / "tests" + tpath = "nimsuggest/tests" + # we could also use `stdtest/specialpaths` import std/compilesettings proc parseTest(filename: string; epcMode=false): Test = const cursorMarker = "#[!]#" - let nimsug = curDir & addFileExt("nimsuggest", ExeExt) + let nimsug = "bin" / addFileExt("nimsuggest", ExeExt) + doAssert nimsug.fileExists, nimsug const libpath = querySetting(libPath) result.filename = filename result.dest = getTempDir() / extractFilename(filename) @@ -63,7 +65,7 @@ proc parseTest(filename: string; epcMode=false): Test = elif x.startsWith(">"): # since 'markers' here are not complete yet, we do the $substitutions # afterwards - result.script.add((x.substr(1).replaceWord("$path", tpath()), "")) + result.script.add((x.substr(1).replaceWord("$path", tpath), "")) elif x.len > 0: # expected output line: let x = x % ["file", filename, "lib", libpath] @@ -104,7 +106,7 @@ proc parseCmd(c: string): seq[string] = proc edit(tmpfile: string; x: seq[string]) = if x.len != 3 and x.len != 4: quit "!edit takes two or three arguments" - let f = if x.len >= 4: tpath() / x[3] else: tmpfile + let f = if x.len >= 4: tpath / x[3] else: tmpfile try: let content = readFile(f) let newcontent = content.replace(x[1], x[2]) @@ -121,12 +123,12 @@ proc exec(x: seq[string]) = proc copy(x: seq[string]) = if x.len != 3: quit "!copy takes two arguments" - let rel = tpath() + let rel = tpath copyFile(rel / x[1], rel / x[2]) proc del(x: seq[string]) = if x.len != 2: quit "!del takes one argument" - removeFile(tpath() / x[1]) + removeFile(tpath / x[1]) proc runCmd(cmd, dest: string): bool = result = cmd[0] == '!' @@ -317,7 +319,7 @@ proc runTest(filename: string): int = try: inp.writeLine("quit") inp.flush() - except: + except IOError, OSError: # assume it's SIGPIPE, ie, the child already died discard close(p) @@ -334,7 +336,7 @@ proc main() = failures += runTest(xx) failures += runEpcTest(xx) else: - for x in walkFiles(tpath() / "t*.nim"): + for x in walkFiles(tpath / "t*.nim"): echo "Test ", x let xx = expandFilename x when not defined(windows): From 198b5e3a7a629ea58b1e2a71a3e70f35484f289b Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 29 Apr 2021 04:53:58 -0700 Subject: [PATCH 0736/2472] improve tsets.nim, twrong_setconstr.nim and avoid name clashes with compiler sets (#17876) * improve tests/sets/tsets.nim and avoid name clashes with compiler sets * avoid name clashes in twrong_setconstr.nim and merge into tsets --- tests/ccgbugs/twrong_setconstr.nim | 146 ----------------- tests/sets/tsets.nim | 247 +++++++---------------------- 2 files changed, 59 insertions(+), 334 deletions(-) delete mode 100644 tests/ccgbugs/twrong_setconstr.nim diff --git a/tests/ccgbugs/twrong_setconstr.nim b/tests/ccgbugs/twrong_setconstr.nim deleted file mode 100644 index 8be0b82b5..000000000 --- a/tests/ccgbugs/twrong_setconstr.nim +++ /dev/null @@ -1,146 +0,0 @@ -discard """ - output: "" -""" - -# bug #2880 - -type - TMsgKind* = enum - errUnknown, errIllFormedAstX, errInternal, errCannotOpenFile, errGenerated, - errXCompilerDoesNotSupportCpp, errStringLiteralExpected, - errIntLiteralExpected, errInvalidCharacterConstant, - errClosingTripleQuoteExpected, errClosingQuoteExpected, - errTabulatorsAreNotAllowed, errInvalidToken, errLineTooLong, - errInvalidNumber, errInvalidNumberOctalCode, errNumberOutOfRange, - errNnotAllowedInCharacter, errClosingBracketExpected, errMissingFinalQuote, - errIdentifierExpected, errNewlineExpected, errInvalidModuleName, - errOperatorExpected, errTokenExpected, errStringAfterIncludeExpected, - errRecursiveDependencyX, errOnOrOffExpected, errNoneSpeedOrSizeExpected, - errInvalidPragma, errUnknownPragma, errInvalidDirectiveX, - errAtPopWithoutPush, errEmptyAsm, errInvalidIndentation, - errExceptionExpected, errExceptionAlreadyHandled, - errYieldNotAllowedHere, errYieldNotAllowedInTryStmt, - errInvalidNumberOfYieldExpr, errCannotReturnExpr, errAttemptToRedefine, - errStmtInvalidAfterReturn, errStmtExpected, errInvalidLabel, - errInvalidCmdLineOption, errCmdLineArgExpected, errCmdLineNoArgExpected, - errInvalidVarSubstitution, errUnknownVar, errUnknownCcompiler, - errOnOrOffExpectedButXFound, errOnOffOrListExpectedButXFound, - errNoneBoehmRefcExpectedButXFound, - errNoneSpeedOrSizeExpectedButXFound, errGuiConsoleOrLibExpectedButXFound, - errUnknownOS, errUnknownCPU, errGenOutExpectedButXFound, - errArgsNeedRunOption, errInvalidMultipleAsgn, errColonOrEqualsExpected, - errExprExpected, errUndeclaredIdentifier, errUseQualifier, errTypeExpected, - errSystemNeeds, errExecutionOfProgramFailed, errNotOverloadable, - errInvalidArgForX, errStmtHasNoEffect, errXExpectsTypeOrValue, - errXExpectsArrayType, errIteratorCannotBeInstantiated, errExprXAmbiguous, - errConstantDivisionByZero, errOrdinalTypeExpected, - errOrdinalOrFloatTypeExpected, errOverOrUnderflow, - errCannotEvalXBecauseIncompletelyDefined, errChrExpectsRange0_255, - errDynlibRequiresExportc, errUndeclaredFieldX, errNilAccess, - errIndexOutOfBounds, errIndexTypesDoNotMatch, errBracketsInvalidForType, - errValueOutOfSetBounds, errFieldInitTwice, errFieldNotInit, - errExprXCannotBeCalled, errExprHasNoType, errExprXHasNoType, - errCastNotInSafeMode, errExprCannotBeCastedToX, errCommaOrParRiExpected, - errCurlyLeOrParLeExpected, errSectionExpected, errRangeExpected, - errMagicOnlyInSystem, errPowerOfTwoExpected, - errStringMayNotBeEmpty, errCallConvExpected, errProcOnlyOneCallConv, - errSymbolMustBeImported, errExprMustBeBool, errConstExprExpected, - errDuplicateCaseLabel, errRangeIsEmpty, errSelectorMustBeOfCertainTypes, - errSelectorMustBeOrdinal, errOrdXMustNotBeNegative, errLenXinvalid, - errWrongNumberOfVariables, errExprCannotBeRaised, errBreakOnlyInLoop, - errTypeXhasUnknownSize, errConstNeedsConstExpr, errConstNeedsValue, - errResultCannotBeOpenArray, errSizeTooBig, errSetTooBig, - errBaseTypeMustBeOrdinal, errInheritanceOnlyWithNonFinalObjects, - errInheritanceOnlyWithEnums, errIllegalRecursionInTypeX, - errCannotInstantiateX, errExprHasNoAddress, errXStackEscape, - errVarForOutParamNeeded, - errPureTypeMismatch, errTypeMismatch, errButExpected, errButExpectedX, - errAmbiguousCallXYZ, errWrongNumberOfArguments, - errXCannotBePassedToProcVar, - errXCannotBeInParamDecl, errPragmaOnlyInHeaderOfProc, errImplOfXNotAllowed, - errImplOfXexpected, errNoSymbolToBorrowFromFound, errDiscardValueX, - errInvalidDiscard, errIllegalConvFromXtoY, errCannotBindXTwice, - errInvalidOrderInArrayConstructor, - errInvalidOrderInEnumX, errEnumXHasHoles, errExceptExpected, errInvalidTry, - errOptionExpected, errXisNoLabel, errNotAllCasesCovered, - errUnknownSubstitionVar, errComplexStmtRequiresInd, errXisNotCallable, - errNoPragmasAllowedForX, errNoGenericParamsAllowedForX, - errInvalidParamKindX, errDefaultArgumentInvalid, errNamedParamHasToBeIdent, - errNoReturnTypeForX, errConvNeedsOneArg, errInvalidPragmaX, - errXNotAllowedHere, errInvalidControlFlowX, - errXisNoType, errCircumNeedsPointer, errInvalidExpression, - errInvalidExpressionX, errEnumHasNoValueX, errNamedExprExpected, - errNamedExprNotAllowed, errXExpectsOneTypeParam, - errArrayExpectsTwoTypeParams, errInvalidVisibilityX, errInitHereNotAllowed, - errXCannotBeAssignedTo, errIteratorNotAllowed, errXNeedsReturnType, - errNoReturnTypeDeclared, - errInvalidCommandX, errXOnlyAtModuleScope, - errXNeedsParamObjectType, - errTemplateInstantiationTooNested, errInstantiationFrom, - errInvalidIndexValueForTuple, errCommandExpectsFilename, - errMainModuleMustBeSpecified, - errXExpected, - errTIsNotAConcreteType, - errInvalidSectionStart, errGridTableNotImplemented, errGeneralParseError, - errNewSectionExpected, errWhitespaceExpected, errXisNoValidIndexFile, - errCannotRenderX, errVarVarTypeNotAllowed, errInstantiateXExplicitly, - errOnlyACallOpCanBeDelegator, errUsingNoSymbol, - errMacroBodyDependsOnGenericTypes, - errDestructorNotGenericEnough, - errInlineIteratorsAsProcParams, - errXExpectsTwoArguments, - errXExpectsObjectTypes, errXcanNeverBeOfThisSubtype, errTooManyIterations, - errCannotInterpretNodeX, errFieldXNotFound, errInvalidConversionFromTypeX, - errAssertionFailed, errCannotGenerateCodeForX, errXRequiresOneArgument, - errUnhandledExceptionX, errCyclicTree, errXisNoMacroOrTemplate, - errXhasSideEffects, errIteratorExpected, errLetNeedsInit, - errThreadvarCannotInit, errWrongSymbolX, errIllegalCaptureX, - errXCannotBeClosure, errXMustBeCompileTime, - errCannotInferTypeOfTheLiteral, - errCannotInferReturnType, - errGenericLambdaNotAllowed, - errCompilerDoesntSupportTarget, - errUser, - warnCannotOpenFile, - warnOctalEscape, warnXIsNeverRead, warnXmightNotBeenInit, - warnDeprecated, warnConfigDeprecated, - warnSmallLshouldNotBeUsed, warnUnknownMagic, warnRedefinitionOfLabel, - warnUnknownSubstitutionX, warnLanguageXNotSupported, - warnFieldXNotSupported, warnCommentXIgnored, - warnNilStatement, warnTypelessParam, - warnDifferentHeaps, warnWriteToForeignHeap, warnUnsafeCode, - warnEachIdentIsTuple - warnProveInit, warnProveField, warnProveIndex, warnGcUnsafe, warnGcUnsafe2, - warnUninit, warnGcMem, warnDestructor, warnLockLevel, warnResultShadowed, - warnUser, - hintSuccess, hintSuccessX, - hintLineTooLong, hintXDeclaredButNotUsed, hintConvToBaseNotNeeded, - hintConvFromXtoItselfNotNeeded, hintExprAlwaysX, hintQuitCalled, - hintProcessing, hintCodeBegin, hintCodeEnd, hintConf, hintPath, - hintConditionAlwaysTrue, hintName, hintPattern, - hintExecuting, hintLinking, hintDependency, - hintSource, hintStackTrace, hintGCStats, - hintUser - -const - warnMin = warnCannotOpenFile - hintMax = high(TMsgKind) - -type - TNoteKind = range[warnMin..hintMax] # "notes" are warnings or hints - TNoteKinds = set[TNoteKind] - -const - NotesVerbosityConst: array[0..0, TNoteKinds] = [ - {low(TNoteKind)..high(TNoteKind)} - {hintGCStats}] - fuckyou = NotesVerbosityConst[0] - -var - gNotesFromConst: TNoteKinds = NotesVerbosityConst[0] - gNotesFromConst2: TNoteKinds = fuckyou - -if hintGCStats in gNotesFromConst: - echo "hintGCStats in gNotesFromConst A" - -if hintGCStats in gNotesFromConst2: - echo "hintGCStats in gNotesFromConst B" diff --git a/tests/sets/tsets.nim b/tests/sets/tsets.nim index dbbeed543..9ccf7a0c2 100644 --- a/tests/sets/tsets.nim +++ b/tests/sets/tsets.nim @@ -1,207 +1,57 @@ -discard """ -output: ''' -Ha ein F ist in s! -false -''' -""" -# Test the handling of sets +# Test builtin sets -import - strutils +# xxx these tests are not very good, this should be revisited. + +when defined nimTestsTsetsGenerate: + # to generate enums for this test + var ret: string + for i in 0..<276: + ret.add "k" & $i & ", " + echo ret proc testSets(s: var set[char]) = s = {'A', 'B', 'C', 'E'..'G'} + {'Z'} + s # test sets if the first element is different from 0: -type - TAZ = range['a'..'z'] - TAZset = set[TAZ] +block: + type + TAZ = range['a'..'z'] + TAZset = set[TAZ] + FakeTokType = enum + k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10, k11, k12, k13, k14, k15, k16, k17, k18, k19, k20, k21, k22, k23, k24, k25, k26, k27, k28, k29, k30, k31, k32, k33, k34, k35, k36, k37, k38, k39, k40, k41, k42, k43, k44, k45, k46, k47, k48, k49, k50, k51, k52, k53, k54, k55, k56, k57, k58, k59, k60, k61, k62, k63, k64, k65, k66, k67, k68, k69, k70, k71, k72, k73, k74, k75, k76, k77, k78, k79, k80, k81, k82, k83, k84, k85, k86, k87, k88, k89, k90, k91, k92, k93, k94, k95, k96, k97, k98, k99, k100, k101, k102, k103, k104, k105, k106, k107, k108, k109, k110, k111, k112, k113, k114, k115, k116, k117, k118, k119, k120, k121, k122, k123, k124, k125, k126, k127, k128, k129, k130, k131, k132, k133, k134, k135, k136, k137, k138, k139, k140, k141, k142, k143, k144, k145, k146, k147, k148, k149, k150, k151, k152, k153, k154, k155, k156, k157, k158, k159, k160, k161, k162, k163, k164, k165, k166, k167, k168, k169, k170, k171, k172, k173, k174, k175, k176, k177, k178, k179, k180, k181, k182, k183, k184, k185, k186, k187, k188, k189, k190, k191, k192, k193, k194, k195, k196, k197, k198, k199, k200, k201, k202, k203, k204, k205, k206, k207, k208, k209, k210, k211, k212, k213, k214, k215, k216, k217, k218, k219, k220, k221, k222, k223, k224, k225, k226, k227, k228, k229, k230, k231, k232, k233, k234, k235, k236, k237, k238, k239, k240, k241, k242, k243, k244, k245, k246, k247, k248, k249 + FakeTokTypeRange = range[k2..k101] + FakeTokTypes = set[FakeTokTypeRange] - TTokType* = enum - tkInvalid, tkEof, - tkSymbol, - tkAddr, tkAnd, tkAs, tkAsm, tkBlock, tkBreak, tkCase, tkCast, tkConst, - tkContinue, tkConverter, tkDiscard, tkDiv, tkElif, tkElse, tkEnd, tkEnum, - tkExcept, tkException, tkFinally, tkFor, tkFrom, tkGeneric, tkIf, tkImplies, - tkImport, tkIn, tkInclude, tkIs, tkIsnot, tkIterator, tkLambda, tkMacro, - tkMethod, tkMod, tkNil, tkNot, tkNotin, tkObject, tkOf, tkOr, tkOut, tkProc, - tkPtr, tkRaise, tkRecord, tkRef, tkReturn, tkShl, tkShr, tkTemplate, tkTry, - tkType, tkVar, tkWhen, tkWhere, tkWhile, tkWith, tkWithout, tkXor, tkYield, - tkIntLit, tkInt8Lit, tkInt16Lit, tkInt32Lit, tkInt64Lit, tkFloatLit, - tkFloat32Lit, tkFloat64Lit, tkStrLit, tkRStrLit, tkTripleStrLit, tkCharLit, - tkRCharLit, tkParLe, tkParRi, tkBracketLe, tkBracketRi, tkCurlyLe, - tkCurlyRi, tkBracketDotLe, tkBracketDotRi, - tkCurlyDotLe, tkCurlyDotRi, - tkParDotLe, tkParDotRi, - tkComma, tkSemiColon, tkColon, tkEquals, tkDot, tkDotDot, tkHat, tkOpr, - tkComment, tkAccent, tkInd, tkSad, tkDed, - tkSpaces, tkInfixOpr, tkPrefixOpr, tkPostfixOpr - TTokTypeRange = range[tkSymbol..tkDed] - TTokTypes* = set[TTokTypeRange] + const toktypes: FakeTokTypes = {FakeTokTypeRange(k2)..pred(k64), k72..k74} -const - toktypes: TTokTypes = {TTokTypeRange(tkSymbol)..pred(tkIntLit), - tkStrLit..tkTripleStrLit} - -var - s: set[char] - a: TAZset -s = {'0'..'9'} -testSets(s) -if 'F' in s: write(stdout, "Ha ein F ist in s!\n") -else: write(stdout, "BUG: F ist nicht in s!\n") -a = {} #{'a'..'z'} -for x in low(TAZ) .. high(TAZ): - incl(a, x) - if x in a: discard - else: write(stdout, "BUG: something not in a!\n") - -for x in low(TTokTypeRange) .. high(TTokTypeRange): - if x in tokTypes: - discard - #writeLine(stdout, "the token '$1' is in the set" % repr(x)) - -#OUT Ha ein F ist in s! + var + s: set[char] + a: TAZset + s = {'0'..'9'} + testSets(s) + doAssert 'F' in s + a = {} #{'a'..'z'} + for x in low(TAZ) .. high(TAZ): + incl(a, x) + doAssert x in a + for x in low(FakeTokTypeRange) .. high(FakeTokTypeRange): + if x in tokTypes: + discard type - TMsgKind* = enum - errUnknown, errIllFormedAstX, errInternal, errCannotOpenFile, errGenerated, - errXCompilerDoesNotSupportCpp, errStringLiteralExpected, - errIntLiteralExpected, errInvalidCharacterConstant, - errClosingTripleQuoteExpected, errClosingQuoteExpected, - errTabulatorsAreNotAllowed, errInvalidToken, errLineTooLong, - errInvalidNumber, errNumberOutOfRange, errNnotAllowedInCharacter, - errClosingBracketExpected, errMissingFinalQuote, errIdentifierExpected, - errNewlineExpected, - errInvalidModuleName, - errOperatorExpected, errTokenExpected, errStringAfterIncludeExpected, - errRecursiveDependencyX, errOnOrOffExpected, errNoneSpeedOrSizeExpected, - errInvalidPragma, errUnknownPragma, errInvalidDirectiveX, - errAtPopWithoutPush, errEmptyAsm, errInvalidIndentation, - errExceptionExpected, errExceptionAlreadyHandled, - errYieldNotAllowedHere, errYieldNotAllowedInTryStmt, - errInvalidNumberOfYieldExpr, errCannotReturnExpr, errAttemptToRedefine, - errStmtInvalidAfterReturn, errStmtExpected, errInvalidLabel, - errInvalidCmdLineOption, errCmdLineArgExpected, errCmdLineNoArgExpected, - errInvalidVarSubstitution, errUnknownVar, errUnknownCcompiler, - errOnOrOffExpectedButXFound, errNoneBoehmRefcExpectedButXFound, - errNoneSpeedOrSizeExpectedButXFound, errGuiConsoleOrLibExpectedButXFound, - errUnknownOS, errUnknownCPU, errGenOutExpectedButXFound, - errArgsNeedRunOption, errInvalidMultipleAsgn, errColonOrEqualsExpected, - errExprExpected, errUndeclaredIdentifier, errUseQualifier, errTypeExpected, - errSystemNeeds, errExecutionOfProgramFailed, errNotOverloadable, - errInvalidArgForX, errStmtHasNoEffect, errXExpectsTypeOrValue, - errXExpectsArrayType, errIteratorCannotBeInstantiated, errExprXAmbiguous, - errConstantDivisionByZero, errOrdinalTypeExpected, - errOrdinalOrFloatTypeExpected, errOverOrUnderflow, - errCannotEvalXBecauseIncompletelyDefined, errChrExpectsRange0_255, - errDynlibRequiresExportc, errUndeclaredFieldX, errNilAccess, - errIndexOutOfBounds, errIndexTypesDoNotMatch, errBracketsInvalidForType, - errValueOutOfSetBounds, errFieldInitTwice, errFieldNotInit, - errExprXCannotBeCalled, errExprHasNoType, errExprXHasNoType, - errCastNotInSafeMode, errExprCannotBeCastedToX, errCommaOrParRiExpected, - errCurlyLeOrParLeExpected, errSectionExpected, errRangeExpected, - errMagicOnlyInSystem, errPowerOfTwoExpected, - errStringMayNotBeEmpty, errCallConvExpected, errProcOnlyOneCallConv, - errSymbolMustBeImported, errExprMustBeBool, errConstExprExpected, - errDuplicateCaseLabel, errRangeIsEmpty, errSelectorMustBeOfCertainTypes, - errSelectorMustBeOrdinal, errOrdXMustNotBeNegative, errLenXinvalid, - errWrongNumberOfVariables, errExprCannotBeRaised, errBreakOnlyInLoop, - errTypeXhasUnknownSize, errConstNeedsConstExpr, errConstNeedsValue, - errResultCannotBeOpenArray, errSizeTooBig, errSetTooBig, - errBaseTypeMustBeOrdinal, errInheritanceOnlyWithNonFinalObjects, - errInheritanceOnlyWithEnums, errIllegalRecursionInTypeX, - errCannotInstantiateX, errExprHasNoAddress, errXStackEscape, - errVarForOutParamNeeded, - errPureTypeMismatch, errTypeMismatch, errButExpected, errButExpectedX, - errAmbiguousCallXYZ, errWrongNumberOfArguments, - errXCannotBePassedToProcVar, - errXCannotBeInParamDecl, errPragmaOnlyInHeaderOfProc, errImplOfXNotAllowed, - errImplOfXexpected, errNoSymbolToBorrowFromFound, errDiscardValueX, - errInvalidDiscard, errIllegalConvFromXtoY, errCannotBindXTwice, - errInvalidOrderInArrayConstructor, - errInvalidOrderInEnumX, errEnumXHasHoles, errExceptExpected, errInvalidTry, - errOptionExpected, errXisNoLabel, errNotAllCasesCovered, - errUnknownSubstitionVar, errComplexStmtRequiresInd, errXisNotCallable, - errNoPragmasAllowedForX, errNoGenericParamsAllowedForX, - errInvalidParamKindX, errDefaultArgumentInvalid, errNamedParamHasToBeIdent, - errNoReturnTypeForX, errConvNeedsOneArg, errInvalidPragmaX, - errXNotAllowedHere, errInvalidControlFlowX, - errXisNoType, errCircumNeedsPointer, errInvalidExpression, - errInvalidExpressionX, errEnumHasNoValueX, errNamedExprExpected, - errNamedExprNotAllowed, errXExpectsOneTypeParam, - errArrayExpectsTwoTypeParams, errInvalidVisibilityX, errInitHereNotAllowed, - errXCannotBeAssignedTo, errIteratorNotAllowed, errXNeedsReturnType, - errNoReturnTypeDeclared, - errInvalidCommandX, errXOnlyAtModuleScope, - errXNeedsParamObjectType, - errTemplateInstantiationTooNested, errInstantiationFrom, - errInvalidIndexValueForTuple, errCommandExpectsFilename, - errMainModuleMustBeSpecified, - errXExpected, - errTIsNotAConcreteType, - errInvalidSectionStart, errGridTableNotImplemented, errGeneralParseError, - errNewSectionExpected, errWhitespaceExpected, errXisNoValidIndexFile, - errCannotRenderX, errVarVarTypeNotAllowed, errInstantiateXExplicitly, - errOnlyACallOpCanBeDelegator, errUsingNoSymbol, - errMacroBodyDependsOnGenericTypes, - errDestructorNotGenericEnough, - errInlineIteratorsAsProcParams, - errXExpectsTwoArguments, - errXExpectsObjectTypes, errXcanNeverBeOfThisSubtype, errTooManyIterations, - errCannotInterpretNodeX, errFieldXNotFound, errInvalidConversionFromTypeX, - errAssertionFailed, errCannotGenerateCodeForX, errXRequiresOneArgument, - errUnhandledExceptionX, errCyclicTree, errXisNoMacroOrTemplate, - errXhasSideEffects, errIteratorExpected, errLetNeedsInit, - errThreadvarCannotInit, errWrongSymbolX, errIllegalCaptureX, - errXCannotBeClosure, errXMustBeCompileTime, - errCannotInferTypeOfTheLiteral, - errCannotInferReturnType, - errGenericLambdaNotAllowed, - errCompilerDoesntSupportTarget, - errUser, - warnCannotOpenFile, - warnOctalEscape, warnXIsNeverRead, warnXmightNotBeenInit, - warnDeprecated, warnConfigDeprecated, - warnSmallLshouldNotBeUsed, warnUnknownMagic, warnRedefinitionOfLabel, - warnUnknownSubstitutionX, warnLanguageXNotSupported, - warnFieldXNotSupported, warnCommentXIgnored, - warnNilStatement, warnTypelessParam, - warnDifferentHeaps, warnWriteToForeignHeap, warnUnsafeCode, - warnEachIdentIsTuple, - warnProveInit, warnProveField, warnProveIndex, warnGcUnsafe, warnGcUnsafe2, - warnUninit, warnGcMem, warnDestructor, warnLockLevel, warnResultShadowed, - warnUser, - hintSuccess, hintSuccessX, - hintLineTooLong, hintXDeclaredButNotUsed, hintConvToBaseNotNeeded, - hintConvFromXtoItselfNotNeeded, hintExprAlwaysX, hintQuitCalled, - hintProcessing, hintCodeBegin, hintCodeEnd, hintConf, hintPath, - hintConditionAlwaysTrue, hintName, hintPattern, - hintUser + FakeMsgKind* = enum + k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10, k11, k12, k13, k14, k15, k16, k17, k18, k19, k20, k21, k22, k23, k24, k25, k26, k27, k28, k29, k30, k31, k32, k33, k34, k35, k36, k37, k38, k39, k40, k41, k42, k43, k44, k45, k46, k47, k48, k49, k50, k51, k52, k53, k54, k55, k56, k57, k58, k59, k60, k61, k62, k63, k64, k65, k66, k67, k68, k69, k70, k71, k72, k73, k74, k75, k76, k77, k78, k79, k80, k81, k82, k83, k84, k85, k86, k87, k88, k89, k90, k91, k92, k93, k94, k95, k96, k97, k98, k99, k100, k101, k102, k103, k104, k105, k106, k107, k108, k109, k110, k111, k112, k113, k114, k115, k116, k117, k118, k119, k120, k121, k122, k123, k124, k125, k126, k127, k128, k129, k130, k131, k132, k133, k134, k135, k136, k137, k138, k139, k140, k141, k142, k143, k144, k145, k146, k147, k148, k149, k150, k151, k152, k153, k154, k155, k156, k157, k158, k159, k160, k161, k162, k163, k164, k165, k166, k167, k168, k169, k170, k171, k172, k173, k174, k175, k176, k177, k178, k179, k180, k181, k182, k183, k184, k185, k186, k187, k188, k189, k190, k191, k192, k193, k194, k195, k196, k197, k198, k199, k200, k201, k202, k203, k204, k205, k206, k207, k208, k209, k210, k211, k212, k213, k214, k215, k216, k217, k218, k219, k220, k221, k222, k223, k224, k225, k226, k227, k228, k229, k230, k231, k232, k233, k234, k235, k236, k237, k238, k239, k240, k241, k242, k243, k244, k245, k246, k247, k248, k249, k250, k251, k252, k253, k254, k255, k256, k257, k258, k259, k260, k261, k262, k263, k264, k265, k266, k267, k268, k269, k270, k271, k272, k273, k274, k275, -const - fatalMin* = errUnknown - fatalMax* = errInternal - errMin* = errUnknown - errMax* = errUser - warnMin* = warnCannotOpenFile - warnMax* = pred(hintSuccess) - hintMin* = hintSuccess - hintMax* = high(TMsgKind) +doAssert pred(k260) == k259 type - TNoteKind* = range[warnMin..hintMax] # "notes" are warnings or hints - TNoteKinds* = set[TNoteKind] + FakeMsgKind2 = range[k230..high(FakeMsgKind)] + FakeMsgKind3 = set[FakeMsgKind2] -var - gNotes*: TNoteKinds = {low(TNoteKind)..high(TNoteKind)} - - {warnUninit, warnProveField, warnProveIndex, warnGcUnsafe} +var gNotes: FakeMsgKind3 = {low(FakeMsgKind2)..high(FakeMsgKind2)} - {k233, k235} - -#import compiler.msgs - -echo warnUninit in gNotes +doAssert k233 notin gNotes # 7555 doAssert {-1.int8, -2, -2}.card == 2 @@ -218,3 +68,24 @@ type Foo = enum let x = { Foo1, Foo2 } # bug #8425 + +block: + # bug #2880 + type + FakeMsgKind = enum + k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10, k11, k12, k13, k14, k15, k16, k17, k18, k19, k20, k21, k22, k23, k24, k25, k26, k27, k28, k29, k30, k31, k32, k33, k34, k35, k36, k37, k38, k39, k40, k41, k42, k43, k44, k45, k46, k47, k48, k49, k50, k51, k52, k53, k54, k55, k56, k57, k58, k59, k60, k61, k62, k63, k64, k65, k66, k67, k68, k69, k70, k71, k72, k73, k74, k75, k76, k77, k78, k79, k80, k81, k82, k83, k84, k85, k86, k87, k88, k89, k90, k91, k92, k93, k94, k95, k96, k97, k98, k99, k100 + + type + FakeMsgKind2 = range[k50..high(FakeMsgKind)] + FakeMsgKind2s = set[FakeMsgKind2] + + const + a1: array[0..0, FakeMsgKind2s] = [{low(FakeMsgKind2)..high(FakeMsgKind2)} - {k99}] + a2 = a1[0] + + var + s1: FakeMsgKind2s = a1[0] + s2: FakeMsgKind2s = a2 + + doAssert k99 notin s1 + doAssert k99 notin s2 From 5439cfc317cd678438bd6c154b84c2bf625504ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=B6hlich=20A?= Date: Thu, 29 Apr 2021 14:11:05 +0200 Subject: [PATCH 0737/2472] Fix `insert` calling wrong function (#17856) The `insert` method is calling `tryInsertID`, which ignores the `pkName` parameter. Calling `tryInsert` instead should be correct. --- lib/impure/db_postgres.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/impure/db_postgres.nim b/lib/impure/db_postgres.nim index 034a94852..cb9e5414a 100644 --- a/lib/impure/db_postgres.nim +++ b/lib/impure/db_postgres.nim @@ -524,7 +524,7 @@ proc insert*(db: DbConn, query: SqlQuery, pkName: string, {.tags: [WriteDbEffect], since: (1, 3).} = ## executes the query (typically "INSERT") and returns the ## generated ID - result = tryInsertID(db, query, args) + result = tryInsert(db, query, pkName, args) if result < 0: dbError(db) proc execAffectedRows*(db: DbConn, query: SqlQuery, From e61381a293ab4986427c5ce8968c6c031c83f34f Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Thu, 29 Apr 2021 18:16:14 +0300 Subject: [PATCH 0738/2472] follow-up #17692: more inline syntax highlighting (#17837) --- doc/apis.rst | 5 +- doc/backends.rst | 131 +++++++++++++------------ doc/manual.rst | 47 ++++----- doc/nimc.rst | 240 +++++++++++++++++++++++++++------------------- doc/rstcommon.rst | 45 +++++++++ 5 files changed, 287 insertions(+), 181 deletions(-) diff --git a/doc/apis.rst b/doc/apis.rst index d01e75d78..e8313749d 100644 --- a/doc/apis.rst +++ b/doc/apis.rst @@ -1,9 +1,10 @@ -.. default-role:: code - ================= API naming design ================= +.. default-role:: code +.. include:: rstcommon.rst + The API is designed to be **easy to use** and consistent. Ease of use is measured by the number of calls to achieve a concrete high-level action. diff --git a/doc/backends.rst b/doc/backends.rst index 896b0f834..377e899b0 100644 --- a/doc/backends.rst +++ b/doc/backends.rst @@ -1,5 +1,3 @@ -.. default-role:: code - ================================ Nim Backend Integration ================================ @@ -7,6 +5,10 @@ :Author: Puppet Master :Version: |nimversion| +.. default-role:: code +.. include:: rstcommon.rst +.. no syntax highlighting here by default: + .. contents:: "Heresy grows from idleness." -- Unknown. @@ -15,8 +17,9 @@ Introduction ============ The `Nim Compiler User Guide `_ documents the typical -compiler invocation, using the `compile` or `c` command to transform a -`.nim` file into one or more `.c` files which are then compiled with the +compiler invocation, using the `compile`:option: +or `c`:option: command to transform a +``.nim`` file into one or more ``.c`` files which are then compiled with the platform's C compiler into a static binary. However, there are other commands to compile to C++, Objective-C, or JavaScript. This document tries to concentrate in a single place all the backend and interfacing options. @@ -25,7 +28,7 @@ The Nim compiler supports mainly two backend families: the C, C++ and Objective-C targets and the JavaScript target. `The C like targets <#backends-the-c-like-targets>`_ creates source files that can be compiled into a library or a final executable. `The JavaScript target -<#backends-the-javascript-target>`_ can generate a `.js` file which you +<#backends-the-javascript-target>`_ can generate a ``.js`` file which you reference from an HTML file or create a `standalone Node.js program `_. @@ -42,20 +45,22 @@ The C like targets The commands to compile to either C, C++ or Objective-C are: - //compileToC, cc compile project with C code generator - //compileToCpp, cpp compile project to C++ code - //compileToOC, objc compile project to Objective C code +//compileToC, cc compile project with C code generator +//compileToCpp, cpp compile project to C++ code +//compileToOC, objc compile project to Objective C code The most significant difference between these commands is that if you look -into the `nimcache` directory you will find `.c`, `.cpp` or `.m` +into the ``nimcache`` directory you will find ``.c``, ``.cpp`` or ``.m`` files, other than that all of them will produce a native binary for your project. This allows you to take the generated code and place it directly into a project using any of these languages. Here are some typical command- -line invocations:: +line invocations: - $ nim c hallo.nim - $ nim cpp hallo.nim - $ nim objc hallo.nim +.. code:: cmd + + nim c hallo.nim + nim cpp hallo.nim + nim objc hallo.nim The compiler commands select the target backend, but if needed you can `specify additional switches for cross-compilation @@ -66,11 +71,11 @@ or compiler/linker commands. The JavaScript target --------------------- -Nim can also generate `JavaScript`:idx: code through the `js` command. +Nim can also generate `JavaScript`:idx: code through the `js`:option: command. Nim targets JavaScript 1.5 which is supported by any widely used browser. Since JavaScript does not have a portable means to include another module, -Nim just generates a long `.js` file. +Nim just generates a long ``.js`` file. Features or modules that the JavaScript platform does not support are not available. This includes: @@ -88,10 +93,12 @@ To compensate, the standard library has modules `catered to the JS backend and more support will come in the future (for instance, Node.js bindings to get OS info). -To compile a Nim module into a `.js` file use the `js` command; the -default is a `.js` file that is supposed to be referenced in an `.html` +To compile a Nim module into a ``.js`` file use the `js`:option: command; the +default is a ``.js`` file that is supposed to be referenced in an ``.html`` file. However, you can also run the code with `nodejs`:idx: -(``_):: +(``_): + +.. code:: cmd nim js -d:nodejs -r examples/hallo.nim @@ -150,7 +157,7 @@ interface. C invocation example ~~~~~~~~~~~~~~~~~~~~ -Create a `logic.c` file with the following content: +Create a ``logic.c`` file with the following content: .. code-block:: c int addTwoIntegers(int a, int b) @@ -158,7 +165,7 @@ Create a `logic.c` file with the following content: return a + b; } -Create a `calculator.nim` file with the following content: +Create a ``calculator.nim`` file with the following content: .. code-block:: nim @@ -168,26 +175,28 @@ Create a `calculator.nim` file with the following content: when isMainModule: echo addTwoIntegers(3, 7) -With these two files in place, you can run `nim c -r calculator.nim` and -the Nim compiler will compile the `logic.c` file in addition to -`calculator.nim` and link both into an executable, which outputs `10` when +With these two files in place, you can run `nim c -r calculator.nim`:cmd: and +the Nim compiler will compile the ``logic.c`` file in addition to +``calculator.nim`` and link both into an executable, which outputs `10` when run. Another way to link the C file statically and get the same effect would -be to remove the line with the `compile` pragma and run the following typical -Unix commands:: +be to remove the line with the `compile` pragma and run the following +typical Unix commands: - $ gcc -c logic.c - $ ar rvs mylib.a logic.o - $ nim c --passL:mylib.a -r calculator.nim +.. code:: cmd -Just like in this example we pass the path to the `mylib.a` library (and we -could as well pass `logic.o`) we could be passing switches to link any other + gcc -c logic.c + ar rvs mylib.a logic.o + nim c --passL:mylib.a -r calculator.nim + +Just like in this example we pass the path to the ``mylib.a`` library (and we +could as well pass ``logic.o``) we could be passing switches to link any other static C library. JavaScript invocation example ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Create a `host.html` file with the following content: +Create a ``host.html`` file with the following content: .. code-block:: @@ -201,7 +210,7 @@ Create a `host.html` file with the following content: -Create a `calculator.nim` file with the following content (or reuse the one +Create a ``calculator.nim`` file with the following content (or reuse the one from the previous section): .. code-block:: nim @@ -212,7 +221,7 @@ from the previous section): echo addTwoIntegers(3, 7) Compile the Nim code to JavaScript with `nim js -o:calculator.js -calculator.nim` and open `host.html` in a browser. If the browser supports +calculator.nim`:cmd: and open ``host.html`` in a browser. If the browser supports javascript, you should see the value `10` in the browser's console. Use the `dom module `_ for specific DOM querying and modification procs or take a look at `karax `_ for how to @@ -237,7 +246,7 @@ Also, C code requires you to specify a forward declaration for functions or the compiler will assume certain types for the return value and parameters which will likely make your program crash at runtime. -The Nim compiler can generate a C interface header through the `--header` +The Nim compiler can generate a C interface header through the `--header`:option: command-line switch. The generated header will contain all the exported symbols and the `NimMain` proc which you need to call before any other Nim code. @@ -246,7 +255,7 @@ Nim code. Nim invocation example from C ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Create a `fib.nim` file with the following content: +Create a ``fib.nim`` file with the following content: .. code-block:: nim @@ -256,7 +265,7 @@ Create a `fib.nim` file with the following content: else: result = fib(a - 1) + fib(a - 2) -Create a `maths.c` file with the following content: +Create a ``maths.c`` file with the following content: .. code-block:: c @@ -273,36 +282,40 @@ Create a `maths.c` file with the following content: Now you can run the following Unix like commands to first generate C sources from the Nim code, then link them into a static binary along your main C -program:: +program: - $ nim c --noMain --noLinking --header:fib.h fib.nim - $ gcc -o m -I$HOME/.cache/nim/fib_d -Ipath/to/nim/lib $HOME/.cache/nim/fib_d/*.c maths.c +.. code:: cmd + + nim c --noMain --noLinking --header:fib.h fib.nim + gcc -o m -I$HOME/.cache/nim/fib_d -Ipath/to/nim/lib $HOME/.cache/nim/fib_d/*.c maths.c The first command runs the Nim compiler with three special options to avoid -generating a `main()` function in the generated files, avoid linking the +generating a `main()`:c: function in the generated files, avoid linking the object files into a final binary, and explicitly generate a header file for C -integration. All the generated files are placed into the `nimcache` -directory. That's why the next command compiles the `maths.c` source plus -all the `.c` files from `nimcache`. In addition to this path, you also -have to tell the C compiler where to find Nim's `nimbase.h` header file. +integration. All the generated files are placed into the ``nimcache`` +directory. That's why the next command compiles the ``maths.c`` source plus +all the ``.c`` files from ``nimcache``. In addition to this path, you also +have to tell the C compiler where to find Nim's ``nimbase.h`` header file. -Instead of depending on the generation of the individual `.c` files you can -also ask the Nim compiler to generate a statically linked library:: +Instead of depending on the generation of the individual ``.c`` files you can +also ask the Nim compiler to generate a statically linked library: - $ nim c --app:staticLib --noMain --header fib.nim - $ gcc -o m -Inimcache -Ipath/to/nim/lib libfib.nim.a maths.c +.. code:: cmd + + nim c --app:staticLib --noMain --header fib.nim + gcc -o m -Inimcache -Ipath/to/nim/lib libfib.nim.a maths.c The Nim compiler will handle linking the source files generated in the -`nimcache` directory into the `libfib.nim.a` static library, which you can +``nimcache`` directory into the ``libfib.nim.a`` static library, which you can then link into your C program. Note that these commands are generic and will vary for each system. For instance, on Linux systems you will likely need to -use `-ldl` too to link in required dlopen functionality. +use `-ldl`:option: too to link in required dlopen functionality. Nim invocation example from JavaScript ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Create a `mhost.html` file with the following content: +Create a ``mhost.html`` file with the following content: .. code-block:: @@ -313,7 +326,7 @@ Create a `mhost.html` file with the following content: -Create a `fib.nim` file with the following content (or reuse the one +Create a ``fib.nim`` file with the following content (or reuse the one from the previous section): .. code-block:: nim @@ -324,9 +337,9 @@ from the previous section): else: result = fib(a - 1) + fib(a - 2) -Compile the Nim code to JavaScript with `nim js -o:fib.js fib.nim` and -open `mhost.html` in a browser. If the browser supports javascript, you -should see an alert box displaying the text `Fib for 9 is 34`. As mentioned +Compile the Nim code to JavaScript with `nim js -o:fib.js fib.nim`:cmd: and +open ``mhost.html`` in a browser. If the browser supports javascript, you +should see an alert box displaying the text ``Fib for 9 is 34``. As mentioned earlier, JavaScript doesn't require an initialization call to `NimMain` or a similar function and you can call the exported Nim proc directly. @@ -337,7 +350,7 @@ Nimcache naming logic The `nimcache`:idx: directory is generated during compilation and will hold either temporary or final files depending on your backend target. The default name for the directory depends on the used backend and on your OS but you can -use the `--nimcache` `compiler switch +use the `--nimcache`:option: `compiler switch `_ to change it. @@ -362,8 +375,8 @@ painless. Most C functions accepting a Nim string converted to a `cstring` will likely not need to keep this string around and by the time they return the string won't be needed anymore. However, for the rare cases where a Nim string has to be preserved and made available to the C backend -as a `cstring`, you will need to manually prevent the string data from being -freed with `GC_ref `_ and `GC_unref +as a `cstring`, you will need to manually prevent the string data +from being freed with `GC_ref `_ and `GC_unref `_. A similar thing happens with C code invoking Nim code which returns a @@ -399,7 +412,7 @@ Again, if you are wrapping a library which *mallocs* and *frees* data structures, you need to expose the appropriate *free* function to Nim so you can clean it up. And of course, once cleaned you should avoid accessing it from Nim (or C for that matter). Typically C data structures have their own -`malloc_structure` and `free_structure` specific functions, so wrapping +`malloc_structure`:c: and `free_structure`:c: specific functions, so wrapping these for the Nim side should be enough. diff --git a/doc/manual.rst b/doc/manual.rst index e98a0cc09..50e9b5be5 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -127,9 +127,9 @@ compiler may instead choose to allow the program to die with a fatal error. echo "invalid index" The current implementation allows to switch between these different behaviors -via ``--panics:on|off``. When panics are turned on, the program dies with a +via `--panics:on|off`:option:. When panics are turned on, the program dies with a panic, if they are turned off the runtime errors are turned into -exceptions. The benefit of ``--panics:on`` is that it produces smaller binary +exceptions. The benefit of `--panics:on`:option: is that it produces smaller binary code and the compiler has more freedom to optimize the code. An `unchecked runtime error`:idx: is an error that is not guaranteed to be @@ -678,7 +678,7 @@ defined here.) These keywords are also operators: `and or not xor shl shr div mod in notin is isnot of as from`. -`.`:tok: `=`:tok:, `:`:tok:, `::`:tok: are not available as general operators; they +`.`:tok:, `=`:tok:, `:`:tok:, `::`:tok: are not available as general operators; they are used for other notational purposes. `*:` is as a special case treated as the two tokens `*`:tok: and `:`:tok: @@ -697,7 +697,7 @@ The following strings denote other tokens:: The `slice`:idx: operator `..`:tok: takes precedence over other tokens that -contain a dot: `{..}`:tok: are the three tokens `{`:tok:, `..`:tok:, `}`:tok: +contain a dot: `{..}` are the three tokens `{`:tok:, `..`:tok:, `}`:tok: and not the two tokens `{.`:tok:, `.}`:tok:. @@ -1460,7 +1460,7 @@ The notation `x[i]` can be used to access the i-th element of `x`. Arrays are always bounds checked (statically or at runtime). These checks can be disabled via pragmas or invoking the compiler with the -``--boundChecks:off`` command-line switch. +`--boundChecks:off`:option: command-line switch. An array constructor can have explicit indexes for readability: @@ -4093,7 +4093,7 @@ Multi-methods -------------- **Note:** Starting from Nim 0.20, to use multi-methods one must explicitly pass -``--multimethods:on`` when compiling. +`--multimethods:on`:option: when compiling. In a multi-method, all parameters that have an object type are used for the dispatching: @@ -4805,7 +4805,7 @@ And so is: The reason for this is that `DivByZeroDefect` inherits from `Defect` and -with ``--panics:on`` Defects become unrecoverable errors. +with `--panics:on`:option: Defects become unrecoverable errors. (Since version 1.4 of the language.) @@ -5604,7 +5604,7 @@ However, this means that the method call syntax is not available for **Note**: The Nim compiler prior to version 1 was more lenient about this -requirement. Use the ``--useVersion:0.19`` switch for a transition period. +requirement. Use the `--useVersion:0.19`:option: switch for a transition period. @@ -6960,7 +6960,7 @@ with the project: {.compile: "myfile.cpp".} **Note**: Nim computes a SHA1 checksum and only recompiles the file if it -has changed. One can use the ``-f`` command-line option to force +has changed. One can use the `-f`:option: command-line option to force the recompilation of the file. Since 1.4 the `compile` pragma is also available with this syntax: @@ -6983,7 +6983,7 @@ The `link` pragma can be used to link an additional file with the project: PassC pragma ------------ The `passc` pragma can be used to pass additional parameters to the C -compiler like one would using the command-line switch ``--passc``: +compiler like one would using the command-line switch `--passc`:option:\: .. code-block:: Nim {.passc: "-Wall -Werror".} @@ -7011,7 +7011,7 @@ the pragma resides in: PassL pragma ------------ The `passL` pragma can be used to pass additional parameters to the linker -like one would be using the command-line switch ``--passL``: +like one would be using the command-line switch `--passL`:option:\: .. code-block:: Nim {.passL: "-lSDLmain -lSDL".} @@ -7047,8 +7047,8 @@ Example: embedsC() -`nimbase.h` defines `NIM_EXTERNC` C macro that can be used for -`extern "C"`:cpp: code to work with both `nim c` and `nim cpp`, e.g.: +``nimbase.h`` defines `NIM_EXTERNC`:c: C macro that can be used for +`extern "C"`:cpp: code to work with both `nim c`:cmd: and `nim cpp`:cmd:, e.g.: .. code-block:: Nim proc foobar() {.importc:"$1".} @@ -7124,7 +7124,7 @@ pragmas this allows *sloppy* interfacing with libraries written in C++: proc run(device: IrrlichtDevice): bool {. header: irr, importcpp: "#.run(@)".} -The compiler needs to be told to generate C++ (command ``cpp``) for +The compiler needs to be told to generate C++ (command `cpp`:option:) for this to work. The conditional symbol `cpp` is defined when the compiler emits C++ code. @@ -7368,7 +7368,7 @@ allows *sloppy* interfacing with libraries written in Objective C: g.greet(12, 34) g.free() -The compiler needs to be told to generate Objective C (command ``objc``) for +The compiler needs to be told to generate Objective C (command `objc`:option:) for this to work. The conditional symbol ``objc`` is defined when the compiler emits Objective C code. @@ -7414,7 +7414,7 @@ will generate this code: The `.cppNonPod` pragma should be used for non-POD `importcpp` types so that they work properly (in particular regarding constructor and destructor) for -`.threadvar` variables. This requires ``--tlsEmulation:off``. +`.threadvar` variables. This requires `--tlsEmulation:off`:option:. .. code-block:: nim type Foo {.cppNonPod, importcpp, header: "funs.h".} = object @@ -7458,12 +7458,13 @@ pragma description :: nim c -d:FooBar=42 foobar.nim -In the above example, providing the ``-d`` flag causes the symbol +In the above example, providing the `-d`:option: flag causes the symbol `FooBar` to be overwritten at compile-time, printing out 42. If the -``-d:FooBar=42`` were to be omitted, the default value of 5 would be +`-d:FooBar=42`:option: were to be omitted, the default value of 5 would be used. To see if a value was provided, `defined(FooBar)` can be used. -The syntax ``-d:flag`` is actually just a shortcut for ``-d:flag=true``. +The syntax `-d:flag`:option: is actually just a shortcut for +`-d:flag=true`:option:. User-defined pragmas ==================== @@ -7800,7 +7801,7 @@ strings, because they are precompiled. because of order of initialization problems. **Note**: A `dynlib` import can be overridden with -the ``--dynlibOverride:name`` command-line option. The +the `--dynlibOverride:name`:option: command-line option. The `Compiler User Guide `_ contains further information. @@ -7815,14 +7816,14 @@ conjunction with the `exportc` pragma: proc exportme(): int {.cdecl, exportc, dynlib.} This is only useful if the program is compiled as a dynamic library via the -``--app:lib`` command-line option. +`--app:lib`:option: command-line option. Threads ======= -To enable thread support the ``--threads:on`` command-line switch needs to +To enable thread support the `--threads:on`:option: command-line switch needs to be used. The system_ module then contains several threading primitives. See the `threads `_ and `channels `_ modules for the low-level thread API. There are also high-level parallelism constructs @@ -7864,7 +7865,7 @@ any of its parameters contain a `ref` or `closure` type. This enforces the *no heap sharing restriction*. Routines that are imported from C are always assumed to be `gcsafe`. -To disable the GC-safety checking the ``--threadAnalysis:off`` command-line +To disable the GC-safety checking the `--threadAnalysis:off`:option: command-line switch can be used. This is a temporary workaround to ease the porting effort from old code to the new threading model. diff --git a/doc/nimc.rst b/doc/nimc.rst index 9db95e81b..47f7243cc 100644 --- a/doc/nimc.rst +++ b/doc/nimc.rst @@ -1,5 +1,3 @@ -.. default-role:: code - =================================== Nim Compiler User Guide =================================== @@ -7,8 +5,12 @@ :Author: Andreas Rumpf :Version: |nimversion| +.. default-role:: code +.. include:: rstcommon.rst .. contents:: +.. + "Look at you, hacker. A pathetic creature of meat and bone, panting and sweating as you run through my corridors. How can you challenge a perfect, immortal machine?" @@ -32,6 +34,9 @@ Command-line switches --------------------- Basic command-line switches are: +.. no syntax highlighting in the below included files at the moment +.. default-role:: code + Usage: .. include:: basicopt.txt @@ -43,11 +48,12 @@ Advanced command-line switches are: .. include:: advopt.txt +.. include:: rstcommon.rst List of warnings ---------------- -Each warning can be activated individually with `--warning[NAME]:on|off` or +Each warning can be activated individually with `--warning[NAME]:on|off`:option: or in a `push` pragma. ========================== ============================================ @@ -71,7 +77,7 @@ User Some user-defined warning. List of hints ------------- -Each hint can be activated individually with `--hint[NAME]:on|off` or in a +Each hint can be activated individually with `--hint[NAME]:on|off`:option: or in a `push` pragma. ========================== ============================================ @@ -132,22 +138,22 @@ Level Description Compile-time symbols -------------------- -Through the `-d:x` or `--define:x` switch you can define compile-time +Through the `-d:x`:option: or `--define:x`:option: switch you can define compile-time symbols for conditional compilation. The defined switches can be checked in source code with the `when statement `_ and `defined proc `_. The typical use of this switch is -to enable builds in release mode (`-d:release`) where optimizations are -enabled for better performance. Another common use is the `-d:ssl` switch to +to enable builds in release mode (`-d:release`:option:) where optimizations are +enabled for better performance. Another common use is the `-d:ssl`:option: switch to activate SSL sockets. -Additionally, you may pass a value along with the symbol: `-d:x=y` +Additionally, you may pass a value along with the symbol: `-d:x=y`:option: which may be used in conjunction with the `compile-time define pragmas`_ to override symbols during build time. Compile-time symbols are completely **case insensitive** and underscores are -ignored too. `--define:FOO` and `--define:foo` are identical. +ignored too. `--define:FOO`:option: and `--define:foo`:option: are identical. Compile-time symbols starting with the `nim` prefix are reserved for the implementation and should not be used elsewhere. @@ -156,28 +162,47 @@ implementation and should not be used elsewhere. Configuration files ------------------- -**Note:** The *project file name* is the name of the `.nim` file that is +**Note:** The *project file name* is the name of the ``.nim`` file that is passed as a command-line argument to the compiler. -The `nim` executable processes configuration files in the following +The `nim`:cmd: executable processes configuration files in the following directories (in this order; later files overwrite previous settings): -1) `$nim/config/nim.cfg`, `/etc/nim/nim.cfg` (UNIX) or ``\config\nim.cfg`` (Windows). This file can be skipped with the `--skipCfg` command line option. -2) If environment variable `XDG_CONFIG_HOME` is defined, `$XDG_CONFIG_HOME/nim/nim.cfg` or `~/.config/nim/nim.cfg` (POSIX) or `%APPDATA%/nim/nim.cfg` (Windows). This file can be skipped with the `--skipUserCfg` command line option. -3) `$parentDir/nim.cfg` where `$parentDir` stands for any parent directory of the project file's path. These files can be skipped with the `--skipParentCfg` command-line option. -4) `$projectDir/nim.cfg` where `$projectDir` stands for the project file's path. This file can be skipped with the `--skipProjCfg` command-line option. -5) A project can also have a project-specific configuration file named `$project.nim.cfg` that resides in the same directory as `$project.nim`. This file can be skipped with the `--skipProjCfg` command-line option. +1) ``$nim/config/nim.cfg``, ``/etc/nim/nim.cfg`` (UNIX) or + ``\config\nim.cfg`` (Windows). + This file can be skipped with the `--skipCfg`:option: command line option. +2) If environment variable `XDG_CONFIG_HOME` is defined, + ``$XDG_CONFIG_HOME/nim/nim.cfg`` or ``~/.config/nim/nim.cfg`` (POSIX) or + ``%APPDATA%/nim/nim.cfg`` (Windows). + This file can be skipped with the `--skipUserCfg`:option: command line + option. +3) ``$parentDir/nim.cfg`` where ``$parentDir`` stands for any parent + directory of the project file's path. + These files can be skipped with the `--skipParentCfg`:option: + command-line option. +4) ``$projectDir/nim.cfg`` where ``$projectDir`` stands for the project + file's path. + This file can be skipped with the `--skipProjCfg`:option: + command-line option. +5) A project can also have a project-specific configuration file named + ``$project.nim.cfg`` that resides in the same directory as ``$project.nim``. + This file can be skipped with the `--skipProjCfg`:option: + command-line option. Command-line settings have priority over configuration file settings. The default build of a project is a `debug build`:idx:. To compile a -`release build`:idx: define the `release` symbol:: +`release build`:idx: define the `release` symbol: + +.. code:: cmd nim c -d:release myproject.nim - To compile a `dangerous release build`:idx: define the `danger` symbol:: +To compile a `dangerous release build`:idx: define the `danger` symbol: + +.. code:: cmd nim c -d:danger myproject.nim @@ -189,10 +214,10 @@ Nim has the concept of a global search path (PATH) that is queried to determine where to find imported modules or include files. If multiple files are found an ambiguity error is produced. -`nim dump` shows the contents of the PATH. +`nim dump`:cmd: shows the contents of the PATH. However before the PATH is used the current directory is checked for the -file's existence. So if PATH contains `$lib` and `$lib/bar` and the +file's existence. So if PATH contains ``$lib`` and ``$lib/bar`` and the directory structure looks like this:: $lib/x.nim @@ -202,26 +227,26 @@ directory structure looks like this:: other.nim And `main` imports `x`, `foo/x` is imported. If `other` imports `x` -then both `$lib/x.nim` and `$lib/bar/x.nim` match but `$lib/x.nim` is used +then both ``$lib/x.nim`` and ``$lib/bar/x.nim`` match but ``$lib/x.nim`` is used as it is the first match. Generated C code directory -------------------------- The generated files that Nim produces all go into a subdirectory called -`nimcache`. Its full path is +``nimcache``. Its full path is -- `$XDG_CACHE_HOME/nim/$projectname(_r|_d)` or `~/.cache/nim/$projectname(_r|_d)` +- ``$XDG_CACHE_HOME/nim/$projectname(_r|_d)`` or ``~/.cache/nim/$projectname(_r|_d)`` on Posix -- `$HOME/nimcache/$projectname(_r|_d)` on Windows. +- ``$HOME\nimcache\$projectname(_r|_d)`` on Windows. The `_r` suffix is used for release builds, `_d` is for debug builds. This makes it easy to delete all generated files. -The `--nimcache` +The `--nimcache`:option: `compiler switch <#compiler-usage-commandminusline-switches>`_ can be used to -to change the `nimcache` directory. +to change the ``nimcache`` directory. However, the generated C code is not platform-independent. C code generated for Linux does not compile on Windows, for instance. The comment on top of the @@ -231,34 +256,40 @@ C file lists the OS, CPU, and CC the file has been compiled for. Compiler Selection ================== -To change the compiler from the default compiler (at the command line):: +To change the compiler from the default compiler (at the command line): + +.. code:: cmd nim c --cc:llvm_gcc --compile_only myfile.nim -This uses the configuration defined in ``config\nim.cfg`` for `lvm_gcc`. +This uses the configuration defined in ``config\nim.cfg`` for `llvm_gcc`:cmd:. If nimcache already contains compiled code from a different compiler for the same project, -add the `-f` flag to force all files to be recompiled. +add the `-f`:option: flag to force all files to be recompiled. The default compiler is defined at the top of ``config\nim.cfg``. -Changing this setting affects the compiler used by `koch` to (re)build Nim. +Changing this setting affects the compiler used by `koch`:cmd: to (re)build Nim. -To use the `CC` environment variable, use `nim c --cc:env myfile.nim`. To use the -`CXX` environment variable, use `nim cpp --cc:env myfile.nim`. `--cc:env` is available -since Nim version 1.4. +To use the `CC` environment variable, use `nim c --cc:env myfile.nim`:cmd:. +To use the `CXX` environment variable, use `nim cpp --cc:env myfile.nim`:cmd:. +`--cc:env`:option: is available since Nim version 1.4. Cross-compilation ================= -To cross compile, use for example:: +To cross compile, use for example: + +.. code:: cmd nim c --cpu:i386 --os:linux --compileOnly --genScript myproject.nim -Then move the C code and the compile script `compile_myproject.sh` to your +Then move the C code and the compile script `compile_myproject.sh`:cmd: to your Linux i386 machine and run the script. -Another way is to make Nim invoke a cross compiler toolchain:: +Another way is to make Nim invoke a cross compiler toolchain: + +.. code:: cmd nim c --cpu:arm --os:linux myproject.nim @@ -274,17 +305,22 @@ configuration file should contain something like:: Cross-compilation for Windows ============================= -To cross-compile for Windows from Linux or macOS using the MinGW-w64 toolchain:: +To cross-compile for Windows from Linux or macOS using the MinGW-w64 toolchain: + +.. code:: cmd nim c -d:mingw myproject.nim -Use `--cpu:i386` or `--cpu:amd64` to switch the CPU architecture. +Use `--cpu:i386`:option: or `--cpu:amd64`:option: to switch the CPU architecture. -The MinGW-w64 toolchain can be installed as follows:: +The MinGW-w64 toolchain can be installed as follows: - Ubuntu: apt install mingw-w64 - CentOS: yum install mingw32-gcc | mingw64-gcc - requires EPEL - OSX: brew install mingw-w64 +.. code:: cmd + + apt install mingw-w64 # Ubuntu + yum install mingw32-gcc + yum install mingw64-gcc # CentOS - requires EPEL + brew install mingw-w64 # OSX Cross-compilation for Android @@ -298,7 +334,7 @@ The first one is to treat Android as a simple Linux and use directly on android as if it was Linux. These programs are console-only programs that can't be distributed in the Play Store. -Use regular `nim c` inside termux to make Android terminal programs. +Use regular `nim c`:cmd: inside termux to make Android terminal programs. Normal Android apps are written in Java, to use Nim inside an Android app you need a small Java stub that calls out to a native library written in @@ -306,16 +342,16 @@ Nim using the `NDK `_. You can also use `native-activity `_ to have the Java stub be auto-generated for you. -Use `nim c -c --cpu:arm --os:android -d:androidNDK --noMain:on` to +Use `nim c -c --cpu:arm --os:android -d:androidNDK --noMain:on`:cmd: to generate the C source files you need to include in your Android Studio project. Add the generated C files to CMake build script in your Android project. Then do the final compile with Android Studio which uses Gradle to call CMake to compile the project. -Because Nim is part of a library it can't have its own c style `main()` -so you would need to define your own `android_main` and init the Java +Because Nim is part of a library it can't have its own C-style `main()`:c: +so you would need to define your own `android_main`:c: and init the Java environment, or use a library like SDL2 or GLFM to do it. After the Android -stuff is done, it's very important to call `NimMain()` in order to +stuff is done, it's very important to call `NimMain()`:c: in order to initialize Nim's garbage collector and to run the top level statements of your program. @@ -334,14 +370,14 @@ Normal languages for iOS development are Swift and Objective C. Both of these use LLVM and can be compiled into object files linked together with C, C++ or Objective C code produced by Nim. -Use `nim c -c --os:ios --noMain:on` to generate C files and include them in +Use `nim c -c --os:ios --noMain:on`:cmd: to generate C files and include them in your XCode project. Then you can use XCode to compile, link, package and sign everything. -Because Nim is part of a library it can't have its own c style `main()` so you +Because Nim is part of a library it can't have its own C-style `main()`:c: so you would need to define `main` that calls `autoreleasepool` and `UIApplicationMain` to do it, or use a library like SDL2 or GLFM. After -the iOS setup is done, it's very important to call `NimMain()` to +the iOS setup is done, it's very important to call `NimMain()`:c: to initialize Nim's garbage collector and to run the top-level statements of your program. @@ -358,17 +394,16 @@ so you need to clean those files manually to do a clean build. Cross-compilation for Nintendo Switch ===================================== -Simply add --os:nintendoswitch -to your usual `nim c` or `nim cpp` command and set the `passC` -and `passL` command line switches to something like: +Simply add `--os:nintendoswitch`:option: +to your usual `nim c`:cmd: or `nim cpp`:cmd: command and set the `passC`:option: +and `passL`:option: command line switches to something like: -.. code-block:: console +.. code-block:: cmd nim c ... --passC="-I$DEVKITPRO/libnx/include" ... --passL="-specs=$DEVKITPRO/libnx/switch.specs -L$DEVKITPRO/libnx/lib -lnx" -or setup a nim.cfg file like so: +or setup a ``nim.cfg`` file like so:: -.. code-block:: Nim #nim.cfg --passC="-I$DEVKITPRO/libnx/include" --passL="-specs=$DEVKITPRO/libnx/switch.specs -L$DEVKITPRO/libnx/lib -lnx" @@ -377,12 +412,14 @@ The DevkitPro setup must be the same as the default with their new installer `here for Mac/Linux `_ or `here for Windows `_. -For example, with the above-mentioned config:: +For example, with the above-mentioned config: + +.. code:: cmd nim c --os:nintendoswitch switchhomebrew.nim -This will generate a file called `switchhomebrew.elf` which can then be turned into -an nro file with the `elf2nro` tool in the DevkitPro release. Examples can be found at +This will generate a file called ``switchhomebrew.elf`` which can then be turned into +an nro file with the `elf2nro`:cmd: tool in the DevkitPro release. Examples can be found at `the nim-libnx github repo `_. There are a few things that don't work because the DevkitPro libraries don't support them. @@ -402,16 +439,20 @@ DLL generation Nim supports the generation of DLLs. However, there must be only one instance of the GC per process/address space. This instance is contained in -`nimrtl.dll`. This means that every generated Nim DLL depends -on `nimrtl.dll`. To generate the "nimrtl.dll" file, use the command:: +``nimrtl.dll``. This means that every generated Nim DLL depends +on ``nimrtl.dll``. To generate the "nimrtl.dll" file, use the command: + +.. code:: cmd nim c -d:release lib/nimrtl.nim -To link against `nimrtl.dll` use the command:: +To link against ``nimrtl.dll`` use the command: + +.. code:: cmd nim c -d:useNimRtl myprog.nim -**Note**: Currently the creation of `nimrtl.dll` with thread support has +**Note**: Currently the creation of ``nimrtl.dll`` with thread support has never been tested and is unlikely to work! @@ -427,20 +468,20 @@ Define Effect ====================== ========================================================= `release` Turns on the optimizer. More aggressive optimizations are possible, e.g.: - `--passC:-ffast-math` (but see issue #10305) + `--passC:-ffast-math`:option: (but see issue #10305) `danger` Turns off all runtime checks and turns on the optimizer. -`useFork` Makes `osproc` use `fork` instead of `posix_spawn`. -`useNimRtl` Compile and link against `nimrtl.dll`. +`useFork` Makes `osproc` use `fork`:c: instead of `posix_spawn`:c:. +`useNimRtl` Compile and link against ``nimrtl.dll``. `useMalloc` Makes Nim use C's `malloc`:idx: instead of Nim's own memory manager, albeit prefixing each allocation with its size to support clearing memory on reallocation. - This only works with `gc:none`, `gc:arc` and - `--gc:orc`. + This only works with `--gc:none`:option:, + `--gc:arc`:option: and `--gc:orc`:option:. `useRealtimeGC` Enables support of Nim's GC for *soft* realtime systems. See the documentation of the `gc `_ for further information. `logGC` Enable GC logging to stdout. -`nodejs` The JS target is actually `node.js`. +`nodejs` The JS target is actually ``node.js``. `ssl` Enables OpenSSL support for the sockets module. `memProfiler` Enables memory profiling for the native GC. `uClibc` Use uClibc instead of libc. (Relevant for Unix-like OSes) @@ -448,16 +489,16 @@ Define Effect what's in the Nim file with what's in the C header. This may become enabled by default in the future. `tempDir` This symbol takes a string as its value, like - `--define:tempDir:/some/temp/path` to override the - temporary directory returned by `os.getTempDir()`. + `--define:tempDir:/some/temp/path`:option: to override + the temporary directory returned by `os.getTempDir()`. The value **should** end with a directory separator character. (Relevant for the Android platform) `useShPath` This symbol takes a string as its value, like - `--define:useShPath:/opt/sh/bin/sh` to override the - path for the `sh` binary, in cases where it is not - located in the default location `/bin/sh`. -`noSignalHandler` Disable the crash handler from `system.nim`. -`globalSymbols` Load all `{.dynlib.}` libraries with the `RTLD_GLOBAL` + `--define:useShPath:/opt/sh/bin/sh`:option: to override + the path for the `sh`:cmd: binary, in cases where it is + not located in the default location ``/bin/sh``. +`noSignalHandler` Disable the crash handler from ``system.nim``. +`globalSymbols` Load all `{.dynlib.}` libraries with the `RTLD_GLOBAL`:c: flag on Posix systems to resolve symbols in subsequently loaded libraries. ====================== ========================================================= @@ -474,20 +515,21 @@ generator and are subject to change. LineDir option -------------- -The `lineDir` option can be turned on or off. If turned on the -generated C code contains `#line` directives. This may be helpful for +The `--lineDir`:option: option can be turned on or off. If turned on the +generated C code contains `#line`:c: directives. This may be helpful for debugging with GDB. StackTrace option ----------------- -If the `stackTrace` option is turned on, the generated C contains code to +If the `--stackTrace`:option: option is turned on, the generated C contains code to ensure that proper stack traces are given if the program crashes or some uncaught exception is raised. LineTrace option ---------------- -The `lineTrace` option implies the `stackTrace` option. If turned on, +The `--lineTrace`:option: option implies the `stackTrace`:option: option. +If turned on, the generated C contains code to ensure that proper stack traces with line number information are given if the program crashes or an uncaught exception is raised. @@ -497,11 +539,13 @@ DynlibOverride ============== By default Nim's `dynlib` pragma causes the compiler to generate -`GetProcAddress` (or their Unix counterparts) -calls to bind to a DLL. With the `dynlibOverride` command line switch this -can be prevented and then via `--passL` the static library can be linked +`GetProcAddress`:cpp: (or their Unix counterparts) +calls to bind to a DLL. With the `dynlibOverride`:option: command line switch this +can be prevented and then via `--passL`:option: the static library can be linked against. For instance, to link statically against Lua this command might work -on Linux:: +on Linux: + +.. code:: cmd nim c --dynlibOverride:lua --passL:liblua.lib program.nim @@ -509,8 +553,8 @@ on Linux:: Backend language options ======================== -The typical compiler usage involves using the `compile` or `c` command to -transform a `.nim` file into one or more `.c` files which are then +The typical compiler usage involves using the `compile`:option: or `c`:option: +command to transform a ``.nim`` file into one or more ``.c`` files which are then compiled with the platform's C compiler into a static binary. However, there are other commands to compile to C++, Objective-C, or JavaScript. More details can be read in the `Nim Backend Integration document `_. @@ -520,7 +564,7 @@ Nim documentation tools ======================= Nim provides the `doc`:idx: command to generate HTML -documentation from `.nim` source files. Only exported symbols will appear in +documentation from ``.nim`` source files. Only exported symbols will appear in the output. For more details `see the docgen documentation `_. Nim idetools integration @@ -558,19 +602,21 @@ embedded microprocessors with only a few kilobytes of memory. A good start is to use the `any` operating target together with the `malloc` memory allocator and the `arc` garbage collector. For example: -`nim c --os:any --gc:arc -d:useMalloc [...] x.nim` +.. code:: cmd -- `--gc:arc` will enable the reference counting memory management instead + nim c --os:any --gc:arc -d:useMalloc [...] x.nim + +- `--gc:arc`:option: will enable the reference counting memory management instead of the default garbage collector. This enables Nim to use heap memory which is required for strings and seqs, for example. -- The `--os:any` target makes sure Nim does not depend on any specific +- The `--os:any`:option: target makes sure Nim does not depend on any specific operating system primitives. Your platform should support only some basic ANSI C library `stdlib` and `stdio` functions which should be available on almost any platform. -- The `-d:useMalloc` option configures Nim to use only the standard C memory - manage primitives `malloc()`, `free()`, `realloc()`. +- The `-d:useMalloc`:option: option configures Nim to use only the standard C memory + manage primitives `malloc()`:c:, `free()`:c:, `realloc()`:c:. If your platform does not provide these functions it should be trivial to provide an implementation for them and link these to your program. @@ -580,13 +626,13 @@ additional flags to both the Nim compiler and the C compiler and/or linker to optimize the build for size. For example, the following flags can be used when targeting a gcc compiler: -`--opt:size --passC:-flto --passL:-flto` +`--opt:size --passC:-flto --passL:-flto`:option: -The `--opt:size` flag instructs Nim to optimize code generation for small -size (with the help of the C compiler), the `flto` flags enable link-time +The `--opt:size`:option: flag instructs Nim to optimize code generation for small +size (with the help of the C compiler), the `-flto`:option: flags enable link-time optimization in the compiler and linker. -Check the `Cross-compilation` section for instructions on how to compile the +Check the `Cross-compilation`_ section for instructions on how to compile the program for your target. Nim for realtime systems @@ -603,7 +649,7 @@ The Nim programming language has no concept of Posix's signal handling mechanisms. However, the standard library offers some rudimentary support for signal handling, in particular, segmentation faults are turned into fatal errors that produce a stack trace. This can be disabled with the -`-d:noSignalHandler` switch. +`-d:noSignalHandler`:option: switch. Optimizing for Nim diff --git a/doc/rstcommon.rst b/doc/rstcommon.rst index c1503fa96..7ecc7b8cb 100644 --- a/doc/rstcommon.rst +++ b/doc/rstcommon.rst @@ -1,6 +1,51 @@ +.. + Usage of this file: + Add this in the beginning of *.rst file:: + + .. default-role:: code + .. include:: rstcommon.rst + + It's the current trick for brevity and compatibility with both Github and + rst2html.py, considering that Github cannot highlight Nim in + RST files anyway and it does not include files. + This way interpreted text is displayed with monospaced font in Github + and it's displayed an Nim code in both rst2html.py + (note ".. default-role:: Nim" above) and `nim rst2html`. + + For files that are user manual and consist of stuff like cmdline + option description, use 'code' as a **real** default role: + + .. include:: rstcommon.rst + .. default-role:: code + +.. define language roles explicitly (for compatibility with rst2html.py): .. role:: nim(code) :language: nim .. default-role:: nim +.. role:: c(code) + :language: c + +.. role:: cpp(code) + :language: cpp + +.. role:: yaml(code) + :language: yaml + +.. role:: python(code) + :language: python + +.. role:: java(code) + :language: java + +.. role:: csharp(code) + :language: csharp + +.. role:: cmd(code) + +.. role:: program(code) + +.. role:: option(code) + From 16405083485967a395b8d677833bc26040881b21 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Thu, 29 Apr 2021 18:31:15 +0300 Subject: [PATCH 0739/2472] more strict RST inline markup parsing (#17827) * more strict RST inline markup parsing * add test for unexpected bonus * introduce `roPreferMarkdown` --- compiler/docgen.nim | 7 +- lib/packages/docutils/rst.nim | 52 ++++++++--- tests/stdlib/trst.nim | 161 ++++++++++++++++++++++++++++++++++ 3 files changed, 207 insertions(+), 13 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 977fcf8ef..78fb88f9b 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -178,7 +178,8 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, result.outDir = conf.outDir.string initRstGenerator(result[], (if conf.cmd != cmdRst2tex: outHtml else: outLatex), conf.configVars, filename.string, - {roSupportRawDirective, roSupportMarkdown, roNimFile}, + {roSupportRawDirective, roSupportMarkdown, + roPreferMarkdown, roNimFile}, docgenFindFile, compilerMsgHandler) if conf.configVars.hasKey("doc.googleAnalytics"): @@ -1380,7 +1381,9 @@ proc commandRstAux(cache: IdentCache, conf: ConfigRef; d.isPureRst = true var rst = parseRst(readFile(filen.string), filen.string, line=LineRstInit, column=ColRstInit, - d.hasToc, {roSupportRawDirective, roSupportMarkdown}, conf) + d.hasToc, + {roSupportRawDirective, roSupportMarkdown, roPreferMarkdown}, + conf) var modDesc = newStringOfCap(30_000) renderRstToOut(d[], rst, modDesc) d.modDesc = rope(modDesc) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index cb65791ff..6ad466716 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -11,9 +11,9 @@ ## rst ## ================================== ## -## ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -## Nim-flavored reStructuredText -## ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +## ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +## Nim-flavored reStructuredText and Markdown +## ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ## ## This module implements a `reStructuredText`:idx: (RST) parser. ## A large subset is implemented with some limitations_ and @@ -177,6 +177,8 @@ type roSupportRawDirective, ## support the ``raw`` directive (don't support ## it for sandboxing) roSupportMarkdown, ## support additional features of Markdown + roPreferMarkdown, ## parse as Markdown (keeping RST as "extension" + ## to Markdown) -- implies `roSupportMarkdown` roNimFile ## set for Nim files where default interpreted ## text role should be :nim: @@ -277,6 +279,7 @@ type line*, col*, baseIndent*: int skipPounds*: bool adornmentLine*: bool + escapeNext*: bool proc getThing(L: var Lexer, tok: var Token, s: set[char]) = tok.kind = tkWord @@ -314,10 +317,18 @@ proc getPunctAdornment(L: var Lexer, tok: var Token) = tok.col = L.col var pos = L.bufpos let c = L.buf[pos] - while true: + if not L.escapeNext and (c != '\\' or L.adornmentLine): + while true: + tok.symbol.add(L.buf[pos]) + inc pos + if L.buf[pos] != c: break + elif L.escapeNext: tok.symbol.add(L.buf[pos]) inc pos - if L.buf[pos] != c: break + else: # not L.escapeNext and c == '\\' and not L.adornmentLine + tok.symbol.add '\\' + inc pos + L.escapeNext = true inc L.col, pos - L.bufpos L.bufpos = pos if tok.symbol == "\\": tok.kind = tkPunct @@ -429,7 +440,9 @@ proc getTokens(buffer: string, skipPounds: bool, tokens: var TokenSeq): int = while true: inc length setLen(tokens, length) + let toEscape = L.escapeNext rawGetTok(L, tokens[length - 1]) + if toEscape: L.escapeNext = false if tokens[length - 1].kind == tkEof: break if tokens[0].kind == tkWhite: # BUGFIX @@ -981,16 +994,24 @@ proc expect(p: var RstParser, tok: string) = if currentTok(p).symbol == tok: inc p.idx else: rstMessage(p, meExpected, tok) -proc isInlineMarkupEnd(p: RstParser, markup: string): bool = +proc isInlineMarkupEnd(p: RstParser, markup: string, exact: bool): bool = # rst rules: https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#inline-markup-recognition-rules - result = currentTok(p).symbol == markup + if exact: + result = currentTok(p).symbol == markup + else: + result = currentTok(p).symbol.endsWith markup + if (not result) and markup == "``": + # check that escaping may have splitted `` to 2 tokens ` and ` + result = currentTok(p).symbol == "`" and prevTok(p).symbol == "`" if not result: return # Rule 2: result = prevTok(p).kind notin {tkIndent, tkWhite} if not result: return # Rule 7: result = nextTok(p).kind in {tkIndent, tkWhite, tkEof} or - markup in ["``", "`"] and nextTok(p).kind in {tkIndent, tkWhite, tkWord, tkEof} or + (roPreferMarkdown in p.s.options and + markup in ["``", "`"] and + nextTok(p).kind in {tkIndent, tkWhite, tkWord, tkEof}) or nextTok(p).symbol[0] in {'\'', '\"', ')', ']', '}', '>', '-', '/', '\\', ':', '.', ',', ';', '!', '?', '_'} if not result: return @@ -1130,7 +1151,8 @@ proc toOtherRole(n: PRstNode, kind: RstNodeKind, roleName: string): PRstNode = proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode = var newKind = n.kind var newSons = n.sons - if isInlineMarkupEnd(p, "_") or isInlineMarkupEnd(p, "__"): + if isInlineMarkupEnd(p, "_", exact=true) or + isInlineMarkupEnd(p, "__", exact=true): inc p.idx if p.tok[p.idx-2].symbol == "`" and p.tok[p.idx-3].symbol == ">": var a = newRstNode(rnInner) @@ -1215,7 +1237,7 @@ proc parseWordOrRef(p: var RstParser, father: PRstNode) = inc p.idx while currentTok(p).kind in {tkWord, tkPunct}: if currentTok(p).kind == tkPunct: - if isInlineMarkupEnd(p, "_"): + if isInlineMarkupEnd(p, "_", exact=true): isRef = true break if not validRefnamePunct(currentTok(p).symbol): @@ -1253,7 +1275,15 @@ proc parseUntil(p: var RstParser, father: PRstNode, postfix: string, while true: case currentTok(p).kind of tkPunct: - if isInlineMarkupEnd(p, postfix): + if isInlineMarkupEnd(p, postfix, exact=false): + let l = currentTok(p).symbol.len + if l > postfix.len: + # handle cases like *emphasis with stars****. (It's valid RST!) + father.add newLeaf(currentTok(p).symbol[0 ..< l - postfix.len]) + elif postfix == "``" and currentTok(p).symbol == "`" and + prevTok(p).symbol == "`": + # handle cases like ``literal\`` - delete ` already added after \ + father.sons.setLen(father.sons.len - 1) inc p.idx break else: diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index 2398b92a8..fef80dfc7 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -4,6 +4,10 @@ discard """ [Suite] RST indentation [Suite] RST include directive + +[Suite] RST escaping + +[Suite] RST inline markup ''' """ @@ -267,3 +271,160 @@ And this should **NOT** be visible in `docs.html` """ doAssert "Visible" == rstTohtml(input, {}, defaultConfig()) removeFile("other.rst") + +suite "RST escaping": + test "backspaces": + check("""\ this""".toAst == dedent""" + rnLeaf 'this' + """) + + check("""\\ this""".toAst == dedent""" + rnInner + rnLeaf '\' + rnLeaf ' ' + rnLeaf 'this' + """) + + check("""\\\ this""".toAst == dedent""" + rnInner + rnLeaf '\' + rnLeaf 'this' + """) + + check("""\\\\ this""".toAst == dedent""" + rnInner + rnLeaf '\' + rnLeaf '\' + rnLeaf ' ' + rnLeaf 'this' + """) + +suite "RST inline markup": + test "end-string has repeating symbols": + check("*emphasis content****".toAst == dedent""" + rnEmphasis + rnLeaf 'emphasis' + rnLeaf ' ' + rnLeaf 'content' + rnLeaf '***' + """) + + check("""*emphasis content\****""".toAst == dedent""" + rnEmphasis + rnLeaf 'emphasis' + rnLeaf ' ' + rnLeaf 'content' + rnLeaf '*' + rnLeaf '**' + """) # exact configuration of leafs with * is not really essential, + # only total number of * is essential + + check("**strong content****".toAst == dedent""" + rnStrongEmphasis + rnLeaf 'strong' + rnLeaf ' ' + rnLeaf 'content' + rnLeaf '**' + """) + + check("""**strong content*\****""".toAst == dedent""" + rnStrongEmphasis + rnLeaf 'strong' + rnLeaf ' ' + rnLeaf 'content' + rnLeaf '*' + rnLeaf '*' + rnLeaf '*' + """) + + check("``lit content`````".toAst == dedent""" + rnInlineLiteral + rnLeaf 'lit' + rnLeaf ' ' + rnLeaf 'content' + rnLeaf '```' + """) + + + test """interpreted text can be ended with \` """: + let output = (".. default-role:: literal\n" & """`\``""").toAst + check(output.endsWith """ + rnParagraph + rnInlineLiteral + rnLeaf '`'""" & "\n") + + let output2 = """`\``""".toAst + check(output2 == dedent""" + rnInlineCode + rnDirArg + rnLeaf 'nim' + [nil] + rnLiteralBlock + rnLeaf '`' + """) + + let output3 = """`proc \`+\``""".toAst + check(output3 == dedent""" + rnInlineCode + rnDirArg + rnLeaf 'nim' + [nil] + rnLiteralBlock + rnLeaf 'proc `+`' + """) + + test """inline literals can contain \ anywhere""": + check("""``\``""".toAst == dedent""" + rnInlineLiteral + rnLeaf '\' + """) + + check("""``\\``""".toAst == dedent""" + rnInlineLiteral + rnLeaf '\' + rnLeaf '\' + """) + + check("""``\```""".toAst == dedent""" + rnInlineLiteral + rnLeaf '\' + rnLeaf '`' + """) + + check("""``\\```""".toAst == dedent""" + rnInlineLiteral + rnLeaf '\' + rnLeaf '\' + rnLeaf '`' + """) + + check("""``\````""".toAst == dedent""" + rnInlineLiteral + rnLeaf '\' + rnLeaf '`' + rnLeaf '`' + """) + + test "references with _ at the end": + check(dedent""" + .. _lnk: https + + lnk_""".toAst == + dedent""" + rnHyperlink + rnInner + rnLeaf 'lnk' + rnInner + rnLeaf 'https' + """) + + test "not a hyper link": + check(dedent""" + .. _lnk: https + + lnk___""".toAst == + dedent""" + rnInner + rnLeaf 'lnk' + rnLeaf '___' + """) From 20248a68fd867ce64822698b316f1a8b6300d7ca Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 30 Apr 2021 02:00:33 -0700 Subject: [PATCH 0740/2472] gitutils: add diffStrings, diffFiles, and use it in testament to compare expected vs gotten (#17892) * gitutils: add diffStrings, diffFiles, and use it in testament to compare expected vs gotten * refactor with createTempDir * cleanup * refacotr * PRTEMP fake test spec changes to show effect of diffStrings * add runnableExamples for experimental/diff + cross-reference with gitutils * Revert "PRTEMP fake test spec changes to show effect of diffStrings" This reverts commit 57dc8d642dce6c1127c98b7cbc9edbfe747d4047. --- lib/experimental/diff.nim | 45 +++++++++++++++++---------------- lib/std/private/gitutils.nim | 40 +++++++++++++++++++++++++++++- lib/std/tempfiles.nim | 48 +++++++++++++++++++----------------- nimdoc/rsttester.nim | 3 ++- nimdoc/tester.nim | 3 ++- nimpretty/tester.nim | 5 ++-- testament/categories.nim | 4 +-- testament/testament.nim | 3 ++- 8 files changed, 99 insertions(+), 52 deletions(-) diff --git a/lib/experimental/diff.nim b/lib/experimental/diff.nim index ba4e8ad64..ea0a5cfb8 100644 --- a/lib/experimental/diff.nim +++ b/lib/experimental/diff.nim @@ -10,28 +10,31 @@ ## This module implements an algorithm to compute the ## `diff`:idx: between two sequences of lines. ## -## A basic example of `diffInt` on 2 arrays of integers: -## -## .. code:: Nim -## -## import experimental/diff -## echo diffInt([0, 1, 2, 3, 4, 5, 6, 7, 8], [-1, 1, 2, 3, 4, 5, 666, 7, 42]) -## -## Another short example of `diffText` to diff strings: -## -## .. code:: Nim -## -## import experimental/diff -## # 2 samples of text for testing (from "The Call of Cthulhu" by Lovecraft) -## let txt0 = """I have looked upon all the universe has to hold of horror, -## even skies of spring and flowers of summer must ever be poison to me.""" -## let txt1 = """I have looked upon all your code has to hold of bugs, -## even skies of spring and flowers of summer must ever be poison to me.""" -## -## echo diffText(txt0, txt1) -## ## - To learn more see `Diff on Wikipedia. `_ +runnableExamples: + assert diffInt( + [0, 1, 2, 3, 4, 5, 6, 7, 8], + [-1, 1, 2, 3, 4, 5, 666, 7, 42]) == + @[Item(startA: 0, startB: 0, deletedA: 1, insertedB: 1), + Item(startA: 6, startB: 6, deletedA: 1, insertedB: 1), + Item(startA: 8, startB: 8, deletedA: 1, insertedB: 1)] + +runnableExamples: + # 2 samples of text (from "The Call of Cthulhu" by Lovecraft) + let txt0 = """ +abc +def ghi +jkl2""" + let txt1 = """ +bacx +abc +def ghi +jkl""" + assert diffText(txt0, txt1) == + @[Item(startA: 0, startB: 0, deletedA: 0, insertedB: 1), + Item(startA: 2, startB: 3, deletedA: 1, insertedB: 1)] + # code owner: Arne Döring # # This is based on C# code written by Matthias Hertel, http://www.mathertel.de @@ -309,7 +312,7 @@ proc diffText*(textA, textB: string): seq[Item] = ## `textB` B-version of the text (usually the new one) ## ## Returns a seq of Items that describe the differences. - + # See also `gitutils.diffStrings`. # prepare the input-text and convert to comparable numbers. var h = initTable[string, int]() # TextA.len + TextB.len <- probably wrong initial size # The A-Version of the data (original data) to be compared. diff --git a/lib/std/private/gitutils.nim b/lib/std/private/gitutils.nim index bf5e7cb1f..5bcd9e377 100644 --- a/lib/std/private/gitutils.nim +++ b/lib/std/private/gitutils.nim @@ -4,7 +4,7 @@ internal API for now, API subject to change # xxx move other git utilities here; candidate for stdlib. -import std/[os, osproc, strutils] +import std/[os, osproc, strutils, tempfiles] const commitHead* = "HEAD" @@ -38,3 +38,41 @@ proc isGitRepo*(dir: string): bool = # usually a series of ../), so we know that it's safe to unconditionally # remove trailing whitespaces from the result. result = status == 0 and output.strip() == "" + +proc diffFiles*(path1, path2: string): tuple[output: string, same: bool] = + ## Returns a human readable diff of files `path1`, `path2`, the exact form of + ## which is implementation defined. + # This could be customized, e.g. non-git diff with `diff -uNdr`, or with + # git diff options (e.g. --color-moved, --word-diff). + # in general, `git diff` has more options than `diff`. + var status = 0 + (result.output, status) = execCmdEx("git diff --no-index $1 $2" % [path1.quoteShell, path2.quoteShell]) + doAssert (status == 0) or (status == 1) + result.same = status == 0 + +proc diffStrings*(a, b: string): tuple[output: string, same: bool] = + ## Returns a human readable diff of `a`, `b`, the exact form of which is + ## implementation defined. + ## See also `experimental.diff`. + runnableExamples: + let a = "ok1\nok2\nok3\n" + let b = "ok1\nok2 alt\nok3\nok4\n" + let (c, same) = diffStrings(a, b) + doAssert not same + let (c2, same2) = diffStrings(a, a) + doAssert same2 + runnableExamples("-r:off"): + let a = "ok1\nok2\nok3\n" + let b = "ok1\nok2 alt\nok3\nok4\n" + echo diffStrings(a, b).output + + template tmpFileImpl(prefix, str): auto = + let path = genTempPath(prefix, "") + writeFile(path, str) + path + let patha = tmpFileImpl("diffStrings_a_", a) + let pathb = tmpFileImpl("diffStrings_b_", b) + defer: + removeFile(patha) + removeFile(pathb) + result = diffFiles(patha, pathb) diff --git a/lib/std/tempfiles.nim b/lib/std/tempfiles.nim index 2a6fe7d83..91a3ce7f3 100644 --- a/lib/std/tempfiles.nim +++ b/lib/std/tempfiles.nim @@ -90,25 +90,33 @@ template randomPathName(length: Natural): string = res[i] = state.sample(letters) res +proc getTempDirImpl(dir: string): string {.inline.} = + result = dir + if result.len == 0: + result = getTempDir() + +proc genTempPath*(prefix, suffix: string, dir = ""): string = + ## Generates a path name in `dir`. + ## + ## If `dir` is empty, (`getTempDir `_) will be used. + ## The path begins with `prefix` and ends with `suffix`. + let dir = getTempDirImpl(dir) + result = dir / (prefix & randomPathName(nimTempPathLength) & suffix) + proc createTempFile*(prefix, suffix: string, dir = ""): tuple[fd: File, path: string] = - ## `createTempFile` creates a new temporary file in the directory `dir`. + ## Creates a new temporary file in the directory `dir`. ## - ## If `dir` is the empty string, the default directory for temporary files - ## (`getTempDir `_) will be used. - ## The temporary file name begins with `prefix` and ends with `suffix`. - ## `createTempFile` returns a file handle to an open file and the path of that file. + ## This generates a path name using `genTempPath(prefix, suffix, dir)` and + ## returns a file handle to an open file and the path of that file, possibly after + ## retrying to ensure it doesn't already exist. ## ## If failing to create a temporary file, `IOError` will be raised. ## ## .. note:: It is the caller's responsibility to remove the file when no longer needed. - var dir = dir - if dir.len == 0: - dir = getTempDir() - + let dir = getTempDirImpl(dir) createDir(dir) - for i in 0 ..< maxRetry: - result.path = dir / (prefix & randomPathName(nimTempPathLength) & suffix) + result.path = genTempPath(prefix, suffix, dir) try: result.fd = safeOpen(result.path) except OSError: @@ -118,25 +126,19 @@ proc createTempFile*(prefix, suffix: string, dir = ""): tuple[fd: File, path: st raise newException(IOError, "Failed to create a temporary file under directory " & dir) proc createTempDir*(prefix, suffix: string, dir = ""): string = - ## `createTempDir` creates a new temporary directory in the directory `dir`. + ## Creates a new temporary directory in the directory `dir`. ## - ## If `dir` is the empty string, the default directory for temporary files - ## (`getTempDir `_) will be used. - ## The temporary directory name begins with `prefix` and ends with `suffix`. - ## `createTempDir` returns the path of that temporary firectory. + ## This generates a dir name using `genTempPath(prefix, suffix, dir)`, creates + ## the directory and returns it, possibly after retrying to ensure it doesn't + ## already exist. ## ## If failing to create a temporary directory, `IOError` will be raised. ## ## .. note:: It is the caller's responsibility to remove the directory when no longer needed. - ## - var dir = dir - if dir.len == 0: - dir = getTempDir() - + let dir = getTempDirImpl(dir) createDir(dir) - for i in 0 ..< maxRetry: - result = dir / (prefix & randomPathName(nimTempPathLength) & suffix) + result = genTempPath(prefix, suffix, dir) try: if not existsOrCreateDir(result): return diff --git a/nimdoc/rsttester.nim b/nimdoc/rsttester.nim index 6d41ffb86..daca3dfc7 100644 --- a/nimdoc/rsttester.nim +++ b/nimdoc/rsttester.nim @@ -1,4 +1,5 @@ import os, strutils +from std/private/gitutils import diffFiles const baseDir = "nimdoc/rst2html" @@ -19,7 +20,7 @@ proc testRst2Html(fixup = false) = exec("$1 rst2html $2" % [nimExe, sourceFile]) let producedHtml = expectedHtml.replace('\\', '/').replace("/expected/", "/source/htmldocs/") if readFile(expectedHtml) != readFile(producedHtml): - discard execShellCmd("diff -uNdr " & expectedHtml & " " & producedHtml) + echo diffFiles(expectedHtml, producedHtml).output inc failures if fixup: copyFile(producedHtml, expectedHtml) diff --git a/nimdoc/tester.nim b/nimdoc/tester.nim index 526295222..9daa0bb51 100644 --- a/nimdoc/tester.nim +++ b/nimdoc/tester.nim @@ -3,6 +3,7 @@ # to change expected results (after carefully verifying everything), use -d:fixup import strutils, os +from std/private/gitutils import diffFiles var failures = 0 @@ -40,7 +41,7 @@ proc testNimDoc(prjDir, docsDir: string; switches: NimSwitches; fixup = false) = inc failures elif readFile(expected) != readFile(produced): echo "FAILURE: files differ: ", produced - discard execShellCmd("diff -uNdr " & expected & " " & produced) + echo diffFiles(expected, produced).output inc failures if fixup: copyFile(produced, expected) diff --git a/nimpretty/tester.nim b/nimpretty/tester.nim index 0a60ce693..b1f15aee6 100644 --- a/nimpretty/tester.nim +++ b/nimpretty/tester.nim @@ -1,6 +1,7 @@ # Small program that runs the test cases import strutils, os, sequtils +from std/private/gitutils import diffFiles const dir = "nimpretty/tests" @@ -26,7 +27,7 @@ proc test(infile, ext: string) = let produced = dir / nimFile.changeFileExt(ext) if readFile(expected) != readFile(produced): echo "FAILURE: files differ: ", nimFile - discard execShellCmd("diff -uNdr " & expected & " " & produced) + echo diffFiles(expected, produced).output failures += 1 else: echo "SUCCESS: files identical: ", nimFile @@ -43,7 +44,7 @@ proc testTogether(infiles: seq[string]) = let produced = dir / "outputdir" / infile if readFile(expected) != readFile(produced): echo "FAILURE: files differ: ", nimFile - discard execShellCmd("diff -uNdr " & expected & " " & produced) + echo diffFiles(expected, produced).output failures += 1 else: echo "SUCCESS: files identical: ", nimFile diff --git a/testament/categories.nim b/testament/categories.nim index ffee5eeb3..f1dee3570 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -665,8 +665,8 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string, options: st if buf != outputExpected: writeFile(outputExceptedFile, outputExpected) - discard execShellCmd("diff -uNdr $1 $2" % [outputExceptedFile, outputGottenFile]) - echo failString & "megatest output different!" + echo diffFiles(outputGottenFile, outputExceptedFile).output + echo failString & "megatest output different, see $1 vs $2" % [outputGottenFile, outputExceptedFile] # outputGottenFile, outputExceptedFile not removed on purpose for debugging. quit 1 else: diff --git a/testament/testament.nim b/testament/testament.nim index 9caa3f6b9..0607ac41d 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -17,6 +17,7 @@ from std/sugar import dup import compiler/nodejs import lib/stdtest/testutils from lib/stdtest/specialpaths import splitTestFile +from std/private/gitutils import diffStrings proc trimUnitSep(x: var string) = let L = x.len @@ -307,7 +308,7 @@ proc addResult(r: var TResults, test: TTest, target: TTarget, maybeStyledEcho styleBright, expected, "\n" maybeStyledEcho fgYellow, "Gotten:" maybeStyledEcho styleBright, given, "\n" - + echo diffStrings(expected, given).output if backendLogging and (isAppVeyor or isAzure): let (outcome, msg) = From 76f93877cdb2a2baf28eed6edda5bcc96f4ef404 Mon Sep 17 00:00:00 2001 From: Clyybber Date: Fri, 30 Apr 2021 17:17:29 +0200 Subject: [PATCH 0741/2472] Small parser loop fix (#17904) * Fix loop on error with nim check in semiStmtList * Add test --- compiler/parser.nim | 1 + tests/parser/t15667.nim | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/compiler/parser.nim b/compiler/parser.nim index fd9d06de5..9cddf8e4f 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -556,6 +556,7 @@ proc semiStmtList(p: var Parser, result: PNode) = let a = complexOrSimpleStmt(p) if a.kind == nkEmpty: parMessage(p, errExprExpected, p.tok) + getTok(p) else: result.add a dec p.inSemiStmtList diff --git a/tests/parser/t15667.nim b/tests/parser/t15667.nim index d15178789..fcb862825 100644 --- a/tests/parser/t15667.nim +++ b/tests/parser/t15667.nim @@ -7,6 +7,7 @@ t15667.nim(28, 5) Error: invalid indentation, maybe you forgot a '=' at t15667.n t15667.nim(33, 5) Error: invalid indentation, maybe you forgot a '=' at t15667.nim(31, 25) ? t15667.nim(42, 5) Error: invalid indentation, maybe you forgot a '=' at t15667.nim(38, 12) ? t15667.nim(56, 5) Error: invalid indentation, maybe you forgot a '=' at t15667.nim(55, 13) ? +t15667.nim(61, 48) Error: expression expected, but found ',' ''' """ @@ -16,7 +17,6 @@ t15667.nim(56, 5) Error: invalid indentation, maybe you forgot a '=' at t15667.n - # line 20 block: proc fn1() @@ -56,3 +56,6 @@ block: runnableExamples: discard discard + +# semiStmtList loop issue +proc bar(k:static bool):SomeNumber = (when k: 3, else: 3.0) From 3192995ac97ad0648b6d1e5a5a1a42d403b37e72 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 30 Apr 2021 09:54:47 -0700 Subject: [PATCH 0742/2472] close #16646; `since` now works with bootstrap nim post csources_v1 (#17895) * revive #16627 now that csources_v1 was merged * use dedent in rst.nim, refs https://github.com/nim-lang/Nim/pull/17257#discussion_r589025683 * fix test and improve rendering of a rst warning --- compiler/semtypes.nim | 4 ++-- lib/packages/docutils/rst.nim | 5 ++--- tests/stdlib/trstgen.nim | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index c921703b0..6a59624cd 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -256,8 +256,8 @@ proc semRangeAux(c: PContext, n: PNode, prev: PType): PType = else: result.n.add semConstExpr(c, range[i]) - if (result.n[0].kind in {nkFloatLit..nkFloat64Lit} and classify(result.n[0].floatVal) == fcNan) or - (result.n[1].kind in {nkFloatLit..nkFloat64Lit} and classify(result.n[1].floatVal) == fcNan): + if (result.n[0].kind in {nkFloatLit..nkFloat64Lit} and result.n[0].floatVal.isNaN) or + (result.n[1].kind in {nkFloatLit..nkFloat64Lit} and result.n[1].floatVal.isNaN): localError(c.config, n.info, "NaN is not a valid start or end for a range") if weakLeValue(result.n[0], result.n[1]) == impNo: diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 6ad466716..223e57bc4 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -2250,11 +2250,10 @@ proc parseEnumList(p: var RstParser): PRstNode = let n = p.line + p.tok[j].line let msg = "\n" & """ not enough indentation on line $2 - (should be at column $3 if it's a continuation of enum. list), + (should be at column $3 if it's a continuation of enum. list), or no blank line after line $1 (if it should be the next paragraph), or no escaping \ at the beginning of line $1 - (if lines $1..$2 are a normal paragraph, not enum. list)""". - unindent(8) + (if lines $1..$2 are a normal paragraph, not enum. list)""".dedent let c = p.col + requiredIndent + ColRstOffset rstMessage(p, mwRstStyle, msg % [$(n-1), $n, $c], p.tok[j].line, p.tok[j].col) diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index 0af6ba566..cbdb23cd6 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -849,7 +849,7 @@ Test1 let output8 = input8.toHtml(warnings = warnings8) check(warnings8[].len == 1) check("input(6, 1) Warning: RST style: \n" & - " not enough indentation on line 6" in warnings8[0]) + "not enough indentation on line 6" in warnings8[0]) doAssert output8 == "Paragraph.
      " & "
    1. stringA
      2. \n
    2. stringB
      3. \n
    \n

    C. string1 string2

    \n" From d32ab61e61a9c8875cda926fb76e12dc8e8b9a23 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 30 Apr 2021 12:40:39 -0700 Subject: [PATCH 0743/2472] fix #17905: hash(closure) was not being tested (#17906) --- lib/pure/hashes.nim | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/lib/pure/hashes.nim b/lib/pure/hashes.nim index 83b7a8d75..2dab498d7 100644 --- a/lib/pure/hashes.nim +++ b/lib/pure/hashes.nim @@ -257,13 +257,6 @@ when not defined(nimLegacyNoHashRef): # or `proc hash*[T: ref | ptr](x: T): Hash` hash(cast[pointer](x)) -proc hash*[T: proc](x: T): Hash {.inline.} = - ## Efficient hashing of proc vars. Closures are supported too. - when T is "closure": - result = hash((rawProc(x), rawEnv(x))) - else: - result = hash(pointer(x)) - proc hash*(x: float): Hash {.inline.} = ## Efficient hashing of floats. let y = x + 0.0 # for denormalization @@ -516,10 +509,10 @@ proc hashIgnoreCase*(sBuf: string, sPos, ePos: int): Hash = h = h !& ord(c) result = !$h -proc hash*[T: tuple | object](x: T): Hash = +proc hash*[T: tuple | object | proc](x: T): Hash {.inline.} = ## Efficient `hash` overload. - ## `hash` must be defined for each component of `x`. runnableExamples: + # for `tuple|object`, `hash` must be defined for each component of `x`. type Obj = object x: int y: string @@ -530,9 +523,24 @@ proc hash*[T: tuple | object](x: T): Hash = # you can define custom hashes for objects (even if they're generic): proc hash(a: Obj2): Hash = hash((a.x)) assert hash(Obj2[float](x: 520, y: "Nim")) == hash(Obj2[float](x: 520, y: "Nim2")) - for f in fields(x): - result = result !& hash(f) - result = !$result + runnableExamples: + # proc and closure examples + proc fn1() = discard + var a = 0 + proc fn2() = a.inc + assert hash(fn1) != hash(fn2) + const fn1b = fn1 + assert hash(fn1b) == hash(fn1) + let fn2b = fn2 + assert hash(fn2b) == hash(fn2) + when T is "closure": + result = hash((rawProc(x), rawEnv(x))) + elif T is (proc): + result = hash(pointer(x)) + else: + for f in fields(x): + result = result !& hash(f) + result = !$result proc hash*[A](x: openArray[A]): Hash = ## Efficient hashing of arrays and sequences. From d0485e326ab56af367246227dedc088839678b6b Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 30 Apr 2021 13:27:53 -0700 Subject: [PATCH 0744/2472] bring back std/ prefix within compiler and ensure it works in bootstrap + bsd (#17902) * [WIP] bring back std/ prefix within compiler and ensure it works in bootstrap + bsd * refs https://github.com/nim-lang/Nim/pull/16282#discussion_r616846863 * sounds very similar to https://github.com/nim-lang/Nim/pull/14291 * more: vmops * update tools/ci_generate.nim * auto-generate freebsd.yml as well, to avoid duplication with openbsd.yml * cleanup * undo temporary CI removal --- .builds/freebsd.yml | 8 ++++++- .builds/openbsd_0.yml | 7 ++++-- .builds/openbsd_1.yml | 7 ++++-- compiler/ic/dce.nim | 2 +- compiler/vmops.nim | 16 ++++++------- koch.nim | 4 +++- tools/ci_generate.nim | 54 ++++++++++++++++++++++++++++--------------- 7 files changed, 65 insertions(+), 33 deletions(-) diff --git a/.builds/freebsd.yml b/.builds/freebsd.yml index c1fb80253..8f336771a 100644 --- a/.builds/freebsd.yml +++ b/.builds/freebsd.yml @@ -1,3 +1,5 @@ +## DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim` + # see https://man.sr.ht/builds.sr.ht/compatibility.md#freebsd image: freebsd/latest packages: @@ -8,9 +10,12 @@ packages: - devel/sfml - www/node - devel/gmake + + sources: - https://github.com/nim-lang/Nim environment: + NIM_TESTAMENT_BATCH: "0_1" CC: /usr/bin/clang tasks: - setup: | @@ -20,9 +25,10 @@ tasks: $nim_csources c --skipUserCfg --skipParentCfg koch echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv - test: | + set -e cd Nim if ! ./koch runCI; then - nim c -r tools/ci_testresults.nim + nim r tools/ci_testresults.nim exit 1 fi triggers: diff --git a/.builds/openbsd_0.yml b/.builds/openbsd_0.yml index e1d85d2a7..c19cd63c6 100644 --- a/.builds/openbsd_0.yml +++ b/.builds/openbsd_0.yml @@ -1,4 +1,4 @@ -## do not edit directly; auto-generated by `nim r tools/ci_generate.nim` +## DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim` image: openbsd/latest packages: @@ -10,6 +10,8 @@ packages: - sfml - sdl2 - libffi + + sources: - https://github.com/nim-lang/Nim environment: @@ -20,9 +22,10 @@ tasks: set -e cd Nim . ci/funs.sh && nimBuildCsourcesIfNeeded - $nim_csources c koch + $nim_csources c --skipUserCfg --skipParentCfg koch echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv - test: | + set -e cd Nim if ! ./koch runCI; then nim r tools/ci_testresults.nim diff --git a/.builds/openbsd_1.yml b/.builds/openbsd_1.yml index ed2ded056..06f5f2ae5 100644 --- a/.builds/openbsd_1.yml +++ b/.builds/openbsd_1.yml @@ -1,4 +1,4 @@ -## do not edit directly; auto-generated by `nim r tools/ci_generate.nim` +## DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim` image: openbsd/latest packages: @@ -10,6 +10,8 @@ packages: - sfml - sdl2 - libffi + + sources: - https://github.com/nim-lang/Nim environment: @@ -20,9 +22,10 @@ tasks: set -e cd Nim . ci/funs.sh && nimBuildCsourcesIfNeeded - $nim_csources c koch + $nim_csources c --skipUserCfg --skipParentCfg koch echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv - test: | + set -e cd Nim if ! ./koch runCI; then nim r tools/ci_testresults.nim diff --git a/compiler/ic/dce.nim b/compiler/ic/dce.nim index 350b17d1b..0a436a5d1 100644 --- a/compiler/ic/dce.nim +++ b/compiler/ic/dce.nim @@ -9,7 +9,7 @@ ## Dead code elimination (=DCE) for IC. -import intsets, tables +import std/[intsets, tables] import ".." / [ast, options, lineinfos, types] import packed_ast, ic, bitabs diff --git a/compiler/vmops.nim b/compiler/vmops.nim index 1d22750e7..04356fc76 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -9,24 +9,24 @@ # Unfortunately this cannot be a module yet: #import vmdeps, vm -from math import sqrt, ln, log10, log2, exp, round, arccos, arcsin, +from std/math import sqrt, ln, log10, log2, exp, round, arccos, arcsin, arctan, arctan2, cos, cosh, hypot, sinh, sin, tan, tanh, pow, trunc, floor, ceil, `mod`, cbrt, arcsinh, arccosh, arctanh, erf, erfc, gamma, lgamma when declared(math.copySign): - from math import copySign + from std/math import copySign when declared(math.signbit): - from math import signbit + from std/math import signbit -from os import getEnv, existsEnv, dirExists, fileExists, putEnv, walkDir, +from std/os import getEnv, existsEnv, dirExists, fileExists, putEnv, walkDir, getAppFilename, raiseOSError, osLastError -from md5 import getMD5 -from times import cpuTime -from hashes import hash -from osproc import nil +from std/md5 import getMD5 +from std/times import cpuTime +from std/hashes import hash +from std/osproc import nil from sighashes import symBodyDigest diff --git a/koch.nim b/koch.nim index 12f9e1c14..15355ec36 100644 --- a/koch.nim +++ b/koch.nim @@ -534,7 +534,9 @@ proc runCI(cmd: string) = echo "runCI: ", cmd echo hostInfo() # boot without -d:nimHasLibFFI to make sure this still works - kochExecFold("Boot in release mode", "boot -d:release -d:nimStrictMode") + # `--lib:lib` is needed for bootstrap on openbsd, for reasons described in + # https://github.com/nim-lang/Nim/pull/14291 (`getAppFilename` bugsfor older nim on openbsd). + kochExecFold("Boot in release mode", "boot -d:release -d:nimStrictMode --lib:lib") when false: # debugging: when you need to run only 1 test in CI, use something like this: execFold("debugging test", "nim r tests/stdlib/tosproc.nim") diff --git a/tools/ci_generate.nim b/tools/ci_generate.nim index e2580ff6b..6517df34a 100644 --- a/tools/ci_generate.nim +++ b/tools/ci_generate.nim @@ -1,6 +1,6 @@ ##[ avoid code duplication in CI pipelines. -For now, this is only used for openbsd, but there is a lot of other code +For now, this is only used for openbsd + freebsd, but there is a lot of other code duplication that could be removed. ## usage @@ -10,22 +10,14 @@ nim r tools/ci_generate.nim ``` ]## -import std/strformat +import std/[strformat, os] -proc genCIopenbsd(batch: int, num: int): string = +proc genCiBsd(header: string, batch: int, num: int): string = result = fmt""" -## do not edit directly; auto-generated by `nim r tools/ci_generate.nim` +## DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim` + +{header} -image: openbsd/latest -packages: -- gmake -- sqlite3 -- node -- boehm-gc -- pcre -- sfml -- sdl2 -- libffi sources: - https://github.com/nim-lang/Nim environment: @@ -36,9 +28,10 @@ tasks: set -e cd Nim . ci/funs.sh && nimBuildCsourcesIfNeeded - $nim_csources c koch + $nim_csources c --skipUserCfg --skipParentCfg koch echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv - test: | + set -e cd Nim if ! ./koch runCI; then nim r tools/ci_testresults.nim @@ -51,14 +44,39 @@ triggers: """ proc main()= + let dir = ".builds" # not too large to be resource friendly, refs bug #17107 let num = 2 # if you reduce this, make sure to remove files that shouldn't be generated, # or better, do the cleanup logic here e.g.: `rm .builds/openbsd_*` + let headerFreebsd = """ +# see https://man.sr.ht/builds.sr.ht/compatibility.md#freebsd +image: freebsd/latest +packages: +- databases/sqlite3 +- devel/boehm-gc-threaded +- devel/pcre +- devel/sdl20 +- devel/sfml +- www/node +- devel/gmake +""" + + let headerOpenbsd = """ +image: openbsd/latest +packages: +- gmake +- sqlite3 +- node +- boehm-gc +- pcre +- sfml +- sdl2 +- libffi +""" for i in 0.. Date: Fri, 30 Apr 2021 22:39:25 +0200 Subject: [PATCH 0745/2472] Document the difference between toFloat/toInt and type conversion (#17894) --- lib/system.nim | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/system.nim b/lib/system.nim index 9e9bacf39..174630317 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1472,7 +1472,7 @@ type # these work for most platforms: PInt32* = ptr int32 ## An alias for `ptr int32`. proc toFloat*(i: int): float {.noSideEffect, inline.} = - ## Converts an integer `i` into a `float`. + ## Converts an integer `i` into a `float`. Same as `float(i)`. ## ## If the conversion fails, `ValueError` is raised. ## However, on most platforms the conversion cannot fail. @@ -1494,7 +1494,8 @@ proc toInt*(f: float): int {.noSideEffect.} = ## ## Conversion rounds `f` half away from 0, see ## `Round half away from zero - ## `_. + ## `_, + ## as opposed to a type conversion which rounds towards zero. ## ## Note that some floating point numbers (e.g. infinity or even 1e19) ## cannot be accurately converted. From abb8a73134597297b2c14567f7f8d72f6b723d24 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Sat, 1 May 2021 08:20:33 +0300 Subject: [PATCH 0746/2472] Fix nim-lang/nimforum#285 - punctuation after URL (#17908) * Fix nim-lang/nimforum#285 - punctuation after URL * keep only one leaf in a rnStandaloneHyperlink * add more complex URL --- lib/packages/docutils/rst.nim | 28 ++++++++++++-------- tests/stdlib/trst.nim | 50 +++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 11 deletions(-) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 223e57bc4..249c310b9 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -1213,23 +1213,29 @@ proc isUrl(p: RstParser, i: int): bool = p.tok[i+3].kind == tkWord and p.tok[i].symbol in ["http", "https", "ftp", "telnet", "file"] +proc parseUrl(p: var RstParser): PRstNode = + ## https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#standalone-hyperlinks + result = newRstNode(rnStandaloneHyperlink) + var lastIdx = p.idx + while p.tok[lastIdx].kind in {tkWord, tkPunct, tkOther}: + inc lastIdx + dec lastIdx + # standalone URL can not end with punctuation in RST + while lastIdx >= p.idx and p.tok[lastIdx].kind == tkPunct and + p.tok[lastIdx].symbol != "/": + dec lastIdx + var s = "" + for i in p.idx .. lastIdx: s.add p.tok[i].symbol + result.add s + p.idx = lastIdx + 1 + proc parseWordOrRef(p: var RstParser, father: PRstNode) = ## Parses a normal word or may be a reference or URL. if nextTok(p).kind != tkPunct: # <- main path, a normal word father.add newLeaf(p) inc p.idx elif isUrl(p, p.idx): # URL http://something - var n = newRstNode(rnStandaloneHyperlink) - while true: - case currentTok(p).kind - of tkWord, tkAdornment, tkOther: discard - of tkPunct: - if nextTok(p).kind notin {tkWord, tkAdornment, tkOther, tkPunct}: - break - else: break - n.add(newLeaf(p)) - inc p.idx - father.add(n) + father.add parseUrl(p) else: # check for reference (probably, long one like some.ref.with.dots_ ) var saveIdx = p.idx diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index fef80dfc7..905e5143a 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -428,3 +428,53 @@ suite "RST inline markup": rnLeaf 'lnk' rnLeaf '___' """) + + test "no punctuation in the end of a standalone URI is allowed": + check(dedent""" + [see (http://no.org)], end""".toAst == + dedent""" + rnInner + rnLeaf '[' + rnLeaf 'see' + rnLeaf ' ' + rnLeaf '(' + rnStandaloneHyperlink + rnLeaf 'http://no.org' + rnLeaf ')' + rnLeaf ']' + rnLeaf ',' + rnLeaf ' ' + rnLeaf 'end' + """) + + # but `/` at the end is OK + check( + dedent""" + See http://no.org/ end""".toAst == + dedent""" + rnInner + rnLeaf 'See' + rnLeaf ' ' + rnStandaloneHyperlink + rnLeaf 'http://no.org/' + rnLeaf ' ' + rnLeaf 'end' + """) + + # a more complex URL with some made-up ending '&='. + # Github Markdown would include final &= and + # so would rst2html.py in contradiction with RST spec. + check( + dedent""" + See https://www.google.com/url?sa=t&source=web&cd=&cad=rja&url=https%3A%2F%2Fnim-lang.github.io%2FNim%2Frst.html%23features&usg=AO&= end""".toAst == + dedent""" + rnInner + rnLeaf 'See' + rnLeaf ' ' + rnStandaloneHyperlink + rnLeaf 'https://www.google.com/url?sa=t&source=web&cd=&cad=rja&url=https%3A%2F%2Fnim-lang.github.io%2FNim%2Frst.html%23features&usg=AO' + rnLeaf '&' + rnLeaf '=' + rnLeaf ' ' + rnLeaf 'end' + """) From 1f1d85bb9c614d93ce38becdcc421dda00264a75 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 30 Apr 2021 22:24:41 -0700 Subject: [PATCH 0747/2472] reuse config/build_config.txt for all bootstrap scripts (posix + windows + ci); use build_all.bat in 1 CI, fix bug in build_all.bat (#17899) * reuse config/build_config.txt for all bootstrap scripts (posix + windows + ci) * ci_docs: use build_all.bat in CI (just in that pipeline) to ensure it keeps working * fixup * fix pre-existing bug in build_all.bat * fixup * cp => copy /y * auto-generate build_all.bat, build_all.sh * fixup --- .github/workflows/ci_docs.yml | 10 +++++- build_all.bat | 35 ++++++++++++------- build_all.sh | 9 ++--- ci/build_autogen.bat | 26 ++++++++++++++ ci/funs.sh | 8 ++--- config/build_config.txt | 4 +++ tools/ci_generate.nim | 65 ++++++++++++++++++++++++++++++++++- 7 files changed, 134 insertions(+), 23 deletions(-) create mode 100644 ci/build_autogen.bat create mode 100644 config/build_config.txt diff --git a/.github/workflows/ci_docs.yml b/.github/workflows/ci_docs.yml index 1459d6a31..dfcff66e0 100644 --- a/.github/workflows/ci_docs.yml +++ b/.github/workflows/ci_docs.yml @@ -70,13 +70,21 @@ jobs: shell: bash run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}" - - name: 'Build csourcesAny' + - name: 'Build csourcesAny (posix)' + # this would work on windows and other CI use this on windows, + # but we ensure here that `ci/build_autogen.bat` keeps working on windows. + if: runner.os != 'Windows' shell: bash run: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc # was previously using caching via `actions/cache@v1` but this wasn't # used in other CI pipelines and it's unclear the added complexity # was worth the saving; can be revisited if needed. + - name: 'Build csourcesAny (windows)' + if: runner.os == 'Windows' + shell: cmd + run: ci/build_autogen.bat + - name: 'Build koch' shell: bash run: nim c koch diff --git a/build_all.bat b/build_all.bat index 78a7a354a..1bff761b3 100644 --- a/build_all.bat +++ b/build_all.bat @@ -1,20 +1,29 @@ @echo off -rem build development version of the compiler; can be rerun safely -rem TODO: call nimBuildCsourcesIfNeeded or auto-generate this file (from a nim script) -rem to avoid duplication. -if not exist csources_v1 ( - git clone --depth 1 https://github.com/nim-lang/csources_v1.git +rem DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim` +rem Build development version of the compiler; can be rerun safely +rem bare bones version of ci/funs.sh adapted for windows. + +rem Read in some common shared variables (shared with other tools), +rem see https://stackoverflow.com/questions/3068929/how-to-read-file-contents-into-a-variable-in-a-batch-file +for /f "delims== tokens=1,2" %%G in (config/build_config.txt) do set %%G=%%H +SET nim_csources=bin\nim_csources_%nim_csourcesHash%.exe +echo "building from csources: %nim_csources%" + +if not exist %nim_csourcesDir% ( + git clone -q --depth 1 %nim_csourcesUrl% %nim_csourcesDir% ) -if not exist bin\nim.exe ( - cd csources_v1 - git checkout a8a5241f9475099c823cfe1a5e0ca4022ac201ff - if PROCESSOR_ARCHITECTURE == AMD64 ( + +if not exist %nim_csources% ( + cd %nim_csourcesDir% + git checkout %nim_csourcesHash% + echo "%PROCESSOR_ARCHITECTURE%" + if "%PROCESSOR_ARCHITECTURE%"=="AMD64" ( SET ARCH=64 ) CALL build.bat cd .. + copy /y bin\nim.exe %nim_csources% ) -bin\nim.exe c --skipUserCfg --skipParentCfg koch -koch.exe boot -d:release --skipUserCfg --skipParentCfg -koch.exe tools --skipUserCfg --skipParentCfg - + bin\nim.exe c --skipUserCfg --skipParentCfg --hints:off koch + koch boot -d:release --skipUserCfg --skipParentCfg --hints:off + koch tools --skipUserCfg --skipParentCfg --hints:off diff --git a/build_all.sh b/build_all.sh index 49504c8fe..90665672f 100755 --- a/build_all.sh +++ b/build_all.sh @@ -1,4 +1,5 @@ #! /bin/sh +rem DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim` # build development version of the compiler; can be rerun safely. # arguments can be passed, e.g.: @@ -10,7 +11,7 @@ set -e # exit on first error . ci/funs.sh nimBuildCsourcesIfNeeded "$@" -# Note: if fails, may need to update csourcesAny manually -echo_run bin/nim c --skipUserCfg --skipParentCfg koch -echo_run ./koch boot -d:release --skipUserCfg --skipParentCfg -echo_run ./koch tools --skipUserCfg --skipParentCfg # Compile Nimble and other tools. +echo_run bin/nim c --skipUserCfg --skipParentCfg --hints:off koch +echo_run ./koch boot -d:release --skipUserCfg --skipParentCfg --hints:off +echo_run ./koch tools --skipUserCfg --skipParentCfg --hints:off + diff --git a/ci/build_autogen.bat b/ci/build_autogen.bat new file mode 100644 index 000000000..8b1113175 --- /dev/null +++ b/ci/build_autogen.bat @@ -0,0 +1,26 @@ +@echo off +rem DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim` +rem Build development version of the compiler; can be rerun safely +rem bare bones version of ci/funs.sh adapted for windows. + +rem Read in some common shared variables (shared with other tools), +rem see https://stackoverflow.com/questions/3068929/how-to-read-file-contents-into-a-variable-in-a-batch-file +for /f "delims== tokens=1,2" %%G in (config/build_config.txt) do set %%G=%%H +SET nim_csources=bin\nim_csources_%nim_csourcesHash%.exe +echo "building from csources: %nim_csources%" + +if not exist %nim_csourcesDir% ( + git clone -q --depth 1 %nim_csourcesUrl% %nim_csourcesDir% +) + +if not exist %nim_csources% ( + cd %nim_csourcesDir% + git checkout %nim_csourcesHash% + echo "%PROCESSOR_ARCHITECTURE%" + if "%PROCESSOR_ARCHITECTURE%"=="AMD64" ( + SET ARCH=64 + ) + CALL build.bat + cd .. + copy /y bin\nim.exe %nim_csources% +) diff --git a/ci/funs.sh b/ci/funs.sh index 7a6c66722..5158f38d5 100644 --- a/ci/funs.sh +++ b/ci/funs.sh @@ -32,9 +32,7 @@ nimIsCiSkip(){ } nimDefineVars(){ - nim_csourcesDir=csources_v1 # where we clone - nim_csourcesUrl=https://github.com/nim-lang/csources_v1.git - nim_csourcesHash=a8a5241f9475099c823cfe1a5e0ca4022ac201ff + . config/build_config.txt nim_csources=bin/nim_csources_$nim_csourcesHash } @@ -88,9 +86,11 @@ nimBuildCsourcesIfNeeded(){ if test -d "$nim_csourcesDir"; then echo "$nim_csourcesDir exists." else - # depth 1: adjust as needed in case useful for `git bisect` + # Note: using git tags would allow fetching just what's needed, unlike git hashes, e.g. + # via `git clone -q --depth 1 --branch $tag $nim_csourcesUrl`. echo_run git clone -q --depth 1 $nim_csourcesUrl "$nim_csourcesDir" echo_run git -C "$nim_csourcesDir" checkout $nim_csourcesHash + # if needed we could also add: `git reset --hard $nim_csourcesHash` fi _nimBuildCsourcesIfNeeded "$@" fi diff --git a/config/build_config.txt b/config/build_config.txt new file mode 100644 index 000000000..2484d7a52 --- /dev/null +++ b/config/build_config.txt @@ -0,0 +1,4 @@ +nim_comment="key-value pairs for windows/posix bootstrapping build scripts" +nim_csourcesDir=csources_v1 +nim_csourcesUrl=https://github.com/nim-lang/csources_v1.git +nim_csourcesHash=a8a5241f9475099c823cfe1a5e0ca4022ac201ff diff --git a/tools/ci_generate.nim b/tools/ci_generate.nim index 6517df34a..2e6b4b3de 100644 --- a/tools/ci_generate.nim +++ b/tools/ci_generate.nim @@ -12,9 +12,10 @@ nim r tools/ci_generate.nim import std/[strformat, os] +const doNotEdit = "DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim`" proc genCiBsd(header: string, batch: int, num: int): string = result = fmt""" -## DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim` +## {doNotEdit} {header} @@ -43,6 +44,64 @@ triggers: to: Andreas Rumpf """ +proc genBuildExtras(echoRun, koch, nim: string): string = + result = fmt""" +{echoRun} {nim} c --skipUserCfg --skipParentCfg --hints:off koch +{echoRun} {koch} boot -d:release --skipUserCfg --skipParentCfg --hints:off +{echoRun} {koch} tools --skipUserCfg --skipParentCfg --hints:off +""" + +proc genWindowsScript(buildAll: bool): string = + result = fmt""" +@echo off +rem {doNotEdit} +rem Build development version of the compiler; can be rerun safely +rem bare bones version of ci/funs.sh adapted for windows. + +rem Read in some common shared variables (shared with other tools), +rem see https://stackoverflow.com/questions/3068929/how-to-read-file-contents-into-a-variable-in-a-batch-file +for /f "delims== tokens=1,2" %%G in (config/build_config.txt) do set %%G=%%H +SET nim_csources=bin\nim_csources_%nim_csourcesHash%.exe +echo "building from csources: %nim_csources%" + +if not exist %nim_csourcesDir% ( + git clone -q --depth 1 %nim_csourcesUrl% %nim_csourcesDir% +) + +if not exist %nim_csources% ( + cd %nim_csourcesDir% + git checkout %nim_csourcesHash% + echo "%PROCESSOR_ARCHITECTURE%" + if "%PROCESSOR_ARCHITECTURE%"=="AMD64" ( + SET ARCH=64 + ) + CALL build.bat + cd .. + copy /y bin\nim.exe %nim_csources% +) +""" + + if buildAll: + result.add genBuildExtras("", "koch", r"bin\nim.exe") + +proc genPosixScript(): string = + result = fmt""" +#! /bin/sh +rem {doNotEdit} + +# build development version of the compiler; can be rerun safely. +# arguments can be passed, e.g.: +# CC=gcc ucpu=amd64 uos=darwin + +set -u # error on undefined variables +set -e # exit on first error + +. ci/funs.sh +nimBuildCsourcesIfNeeded "$@" + +{genBuildExtras("echo_run", "./koch", "bin/nim")} +""" + proc main()= let dir = ".builds" # not too large to be resource friendly, refs bug #17107 @@ -74,9 +133,13 @@ packages: - sdl2 - libffi """ + for i in 0.. Date: Fri, 30 Apr 2021 22:26:52 -0700 Subject: [PATCH 0748/2472] misc fixes: remove `forceConst` (obsolete by static), add more runnableExamples to system (#17896) * misc fixes * add runnableExamples for compileOption * add runnableExamples for runnableExamples * move tconsteval => tconst * cleanup --- lib/std/compilesettings.nim | 2 +- lib/system.nim | 84 ++++++++++++++++++------------------- tests/vm/tconst.nim | 35 ++++++++++++++++ tests/vm/tconsteval.nim | 31 -------------- tests/vm/tvmops.nim | 14 +------ 5 files changed, 78 insertions(+), 88 deletions(-) create mode 100644 tests/vm/tconst.nim delete mode 100644 tests/vm/tconsteval.nim diff --git a/lib/std/compilesettings.nim b/lib/std/compilesettings.nim index 725b68acd..273cfdb57 100644 --- a/lib/std/compilesettings.nim +++ b/lib/std/compilesettings.nim @@ -8,7 +8,7 @@ # ## This module allows querying the compiler about -## diverse configuration settings. +## diverse configuration settings. See also `compileOption`. # Note: Only add new enum values at the end to ensure binary compatibility with # other Nim compiler versions! diff --git a/lib/system.nim b/lib/system.nim index 174630317..c2c3306d4 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -55,35 +55,61 @@ type include "system/basic_types" +proc runnableExamples*(rdoccmd = "", body: untyped) {.magic: "RunnableExamples".} = + ## A section you should use to mark `runnable example`:idx: code with. + ## + ## - In normal debug and release builds code within + ## a `runnableExamples` section is ignored. + ## - The documentation generator is aware of these examples and considers them + ## part of the `##` doc comment. As the last step of documentation + ## generation each runnableExample is put in its own file `$file_examples$i.nim`, + ## compiled and tested. The collected examples are + ## put into their own module to ensure the examples do not refer to + ## non-exported symbols. + runnableExamples: + proc timesTwo*(x: int): int = + ## This proc doubles a number. + runnableExamples: + # at module scope + const exported* = 123 + assert timesTwo(5) == 10 + block: # at block scope + defer: echo "done" + runnableExamples "-d:foo -b:cpp": + import std/compilesettings + assert querySetting(backend) == "cpp" + assert defined(foo) + runnableExamples "-r:off": ## this one is only compiled + import std/browsers + openDefaultBrowser "https://forum.nim-lang.org/" + 2 * x + proc compileOption*(option: string): bool {. - magic: "CompileOption", noSideEffect.} + magic: "CompileOption", noSideEffect.} = ## Can be used to determine an `on|off` compile-time option. ## ## See also: ## * `compileOption <#compileOption,string,string>`_ for enum options ## * `defined <#defined,untyped>`_ ## * `std/compilesettings module `_ - ## - ## Example: - ## - ## .. code-block:: Nim - ## when compileOption("floatchecks"): - ## echo "compiled with floating point NaN and Inf checks" + runnableExamples("--floatChecks:off"): + static: doAssert not compileOption("floatchecks") + {.push floatChecks: on.} + static: doAssert compileOption("floatchecks") + # floating point NaN and Inf checks enabled in this scope + {.pop.} proc compileOption*(option, arg: string): bool {. - magic: "CompileOptionArg", noSideEffect.} + magic: "CompileOptionArg", noSideEffect.} = ## Can be used to determine an enum compile-time option. ## ## See also: ## * `compileOption <#compileOption,string>`_ for `on|off` options ## * `defined <#defined,untyped>`_ ## * `std/compilesettings module `_ - ## - ## Example: - ## - ## .. code-block:: Nim - ## when compileOption("opt", "size") and compileOption("gc", "boehm"): - ## echo "compiled with optimization for size and uses Boehm's GC" + runnableExamples: + when compileOption("opt", "size") and compileOption("gc", "boehm"): + discard "compiled with optimization for size and uses Boehm's GC" {.push warning[GcMem]: off, warning[Uninit]: off.} {.push hints: off.} @@ -139,36 +165,6 @@ else: OrdinalImpl[T] {.magic: Ordinal.} Ordinal* = OrdinalImpl | uint | uint64 -proc runnableExamples*(rdoccmd = "", body: untyped) {.magic: "RunnableExamples".} - ## A section you should use to mark `runnable example`:idx: code with. - ## - ## - In normal debug and release builds code within - ## a `runnableExamples` section is ignored. - ## - The documentation generator is aware of these examples and considers them - ## part of the `##` doc comment. As the last step of documentation - ## generation each runnableExample is put in its own file `$file_examples$i.nim`, - ## compiled and tested. The collected examples are - ## put into their own module to ensure the examples do not refer to - ## non-exported symbols. - ## - ## Usage: - ## - ## .. code-block:: Nim - ## proc double*(x: int): int = - ## ## This proc doubles a number. - ## runnableExamples: - ## ## at module scope - ## assert double(5) == 10 - ## block: ## at block scope - ## defer: echo "done" - ## result = 2 * x - ## runnableExamples "-d:foo -b:cpp": - ## import std/compilesettings - ## doAssert querySetting(backend) == "cpp" - ## runnableExamples "-r:off": ## this one is only compiled - ## import std/browsers - ## openDefaultBrowser "https://forum.nim-lang.org/" - when defined(nimHasDeclaredMagic): proc declared*(x: untyped): bool {.magic: "Declared", noSideEffect, compileTime.} ## Special compile-time procedure that checks whether `x` is diff --git a/tests/vm/tconst.nim b/tests/vm/tconst.nim new file mode 100644 index 000000000..3ebbec043 --- /dev/null +++ b/tests/vm/tconst.nim @@ -0,0 +1,35 @@ +discard """ + targets: "c cpp js" +""" + +import std/strutils + +template forceConst(a: untyped): untyped = + ## Force evaluation at CT, but `static(a)` is simpler + const ret = a + ret + +proc isNimVm(): bool = + when nimvm: result = true + else: result = false + +block: + doAssert forceConst(isNimVm()) + doAssert not isNimVm() + doAssert forceConst(isNimVm()) == static(isNimVm()) + doAssert forceConst(isNimVm()) == isNimVm().static + +template main() = + # xxx merge more const related tests here + block: + const + a = """ + Version $1| + Compiled at: $2, $3 + """ % [NimVersion & spaces(44-len(NimVersion)), CompileDate, CompileTime] + let b = $a + doAssert CompileTime in b + doAssert NimVersion in b + +static: main() +main() diff --git a/tests/vm/tconsteval.nim b/tests/vm/tconsteval.nim deleted file mode 100644 index 2e0fcb888..000000000 --- a/tests/vm/tconsteval.nim +++ /dev/null @@ -1,31 +0,0 @@ -discard """ -action: compile -""" - -import strutils - -const - HelpText = """ -+-----------------------------------------------------------------+ -| Maintenance program for Nim | -| Version $1| -| (c) 2012 Andreas Rumpf | -+-----------------------------------------------------------------+ -Compiled at: $2, $3 - -Usage: - koch [options] command [options for command] -Options: - --force, -f, -B, -b forces rebuild - --help, -h shows this help and quits -Possible Commands: - boot [options] bootstraps with given command line options - clean cleans Nim project; removes generated files - web generates the website - csource [options] builds the C sources for installation - zip builds the installation ZIP package - inno builds the Inno Setup installer -""" % [NimVersion & spaces(44-len(NimVersion)), - CompileDate, CompileTime] - -echo HelpText diff --git a/tests/vm/tvmops.nim b/tests/vm/tvmops.nim index c8febd982..632dcbee7 100644 --- a/tests/vm/tvmops.nim +++ b/tests/vm/tvmops.nim @@ -9,16 +9,6 @@ import os import math import strutils -template forceConst(a: untyped): untyped = - ## Force evaluation at CT, useful for example here: - ## `callFoo(forceConst(getBar1()), getBar2())` - ## instead of: - ## block: - ## const a = getBar1() - ## `callFoo(a, getBar2())` - const ret = a - ret - static: # TODO: add more tests block: #getAppFilename, gorgeEx, gorge @@ -51,6 +41,6 @@ static: block: # Check against bugs like #9176 - doAssert getCurrentCompilerExe() == forceConst(getCurrentCompilerExe()) + doAssert getCurrentCompilerExe() == getCurrentCompilerExe().static if false: #pending #9176 - doAssert gorgeEx("unexistant") == forceConst(gorgeEx("unexistant")) + doAssert gorgeEx("unexistant") == gorgeEx("unexistant").static From 9f75e8abc153bafdc5d3a34f001d20a92d36e220 Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 1 May 2021 13:27:50 +0800 Subject: [PATCH 0749/2472] [std/base64] uses runnableExamples (#17882) * [std/base64] uses runnableExamples * Update lib/pure/base64.nim Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- lib/pure/base64.nim | 62 ++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 32 deletions(-) diff --git a/lib/pure/base64.nim b/lib/pure/base64.nim index d5c1fda7a..daba786f2 100644 --- a/lib/pure/base64.nim +++ b/lib/pure/base64.nim @@ -16,44 +16,42 @@ ## Each Base64 digit represents exactly 6 bits of data. Three 8-bit ## bytes (i.e., a total of 24 bits) can therefore be represented by ## four 6-bit Base64 digits. -## -## Basic usage -## =========== -## + +##[ +# Basic usage ## Encoding data -## ------------- -## -## .. code-block:: Nim -## import std/base64 -## let encoded = encode("Hello World") -## assert encoded == "SGVsbG8gV29ybGQ=" +]## + +runnableExamples: + let encoded = encode("Hello World") + assert encoded == "SGVsbG8gV29ybGQ=" + ## ## Apart from strings you can also encode lists of integers or characters: ## -## .. code-block:: Nim -## import std/base64 -## let encodedInts = encode([1,2,3]) -## assert encodedInts == "AQID" -## let encodedChars = encode(['h','e','y']) -## assert encodedChars == "aGV5" -## -## + +runnableExamples: + let encodedInts = encode([1,2,3]) + assert encodedInts == "AQID" + let encodedChars = encode(['h','e','y']) + assert encodedChars == "aGV5" + +##[ ## Decoding data -## ------------- -## -## .. code-block:: Nim -## import std/base64 -## let decoded = decode("SGVsbG8gV29ybGQ=") -## assert decoded == "Hello World" -## +]## + +runnableExamples: + let decoded = decode("SGVsbG8gV29ybGQ=") + assert decoded == "Hello World" + +##[ ## URL Safe Base64 -## --------------- -## -## .. code-block:: Nim -## import std/base64 -## doAssert encode("c\xf7>", safe = true) == "Y_c-" -## doAssert encode("c\xf7>", safe = false) == "Y/c+" -## +]## + +runnableExamples: + assert encode("c\xf7>", safe = true) == "Y_c-" + assert encode("c\xf7>", safe = false) == "Y/c+" + ## See also ## ======== ## From a55c7e9679e8df0ffc7eaff1e2d5626ec5a9129a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yanis=20Zafir=C3=B3pulos?= <1265028+drkameleon@users.noreply.github.com> Date: Sat, 1 May 2021 09:37:02 +0200 Subject: [PATCH 0750/2472] WIP: Added missing functions to jsre module (#17881) * added missing functions: `replace`, `replaceAll`, `split`, `match` * added `startsWith` & `endsWith` * Update lib/js/jsre.nim Co-authored-by: Juan Carlos --- lib/js/jsre.nim | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/lib/js/jsre.nim b/lib/js/jsre.nim index 7d51db646..a23fccaa8 100644 --- a/lib/js/jsre.nim +++ b/lib/js/jsre.nim @@ -30,6 +30,15 @@ func newRegExp*(pattern: cstring): RegExp {.importjs: "new RegExp(@)".} func compile*(self: RegExp; pattern: cstring; flags: cstring) {.importjs: "#.compile(@)".} ## Recompiles a regular expression during execution of a script. +func replace*(pattern: cstring; self: RegExp; replacement: cstring): cstring {.importjs: "#.replace(#, #)".} + ## Returns a new string with some or all matches of a pattern replaced by given replacement + +func split*(pattern: cstring; self: RegExp): seq[cstring] {.importjs: "#.split(#)".} + ## Divides a string into an ordered list of substrings and returns the array + +func match*(pattern: cstring; self: RegExp): seq[cstring] {.importjs: "#.match(#)".} + ## Returns an array of matches of a RegExp against given string + func exec*(self: RegExp; pattern: cstring): seq[cstring] {.importjs: "#.exec(#)".} ## Executes a search for a match in its string parameter. @@ -51,6 +60,20 @@ func contains*(pattern: cstring; self: RegExp): bool = assert "xabc".contains jsregex asm "`result` = `self`.test(`pattern`);" +func startsWith*(pattern: cstring; self: RegExp): bool = + ## Tests if string starts with given RegExp + runnableExamples: + let jsregex: RegExp = newRegExp(r"abc", r"i") + assert "abcd".startsWith jsregex + pattern.contains(newRegExp(("^" & $(self.source)).cstring, self.flags)) + +func endsWith*(pattern: cstring; self: RegExp): bool = + ## Tests if string ends with given RegExp + runnableExamples: + let jsregex: RegExp = newRegExp(r"bcd", r"i") + assert "abcd".endsWith jsregex + pattern.contains(newRegExp(($(self.source) & "$").cstring, self.flags)) + runnableExamples: let jsregex: RegExp = newRegExp(r"\s+", r"i") @@ -61,3 +84,10 @@ runnableExamples: jsregex.compile(r"[0-9]", r"i") assert "0123456789abcd".contains jsregex assert $jsregex == "/[0-9]/i" + jsregex.compile(r"abc", r"i") + assert "abcd".startsWith jsregex + assert "dabc".endsWith jsregex + jsregex.compile(r"\d", r"i") + assert "do1ne".split(jsregex) == @["do".cstring, "ne".cstring] + jsregex.compile(r"[lw]", r"i") + assert "hello world".replace(jsregex,"X") == "heXlo world" From fb86271556b4ea4485195febaa02b972149ca088 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 1 May 2021 11:10:40 +0200 Subject: [PATCH 0751/2472] =?UTF-8?q?system.nim=20cleanup=20some=20exporte?= =?UTF-8?q?d=20constants=20which=20should=20never=20have=20be=E2=80=A6=20(?= =?UTF-8?q?#17909)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * system.nim cleanup some exported constants which should never have been exported --- compiler/commands.nim | 5 +---- compiler/options.nim | 2 ++ lib/pure/coro.nim | 2 ++ lib/pure/os.nim | 2 +- lib/system.nim | 22 ++-------------------- lib/system/ansi_c.nim | 2 +- lib/system/coro_detection.nim | 20 ++++++++++++++++++++ lib/system/excpt.nim | 2 +- lib/system/io.nim | 2 +- tests/cpp/tcovariancerules.nim | 2 +- 10 files changed, 32 insertions(+), 29 deletions(-) create mode 100644 lib/system/coro_detection.nim diff --git a/compiler/commands.nim b/compiler/commands.nim index 278199bc0..32f56ad0e 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -34,9 +34,6 @@ from ast import eqTypeFlags, tfGcSafe, tfNoSideEffect # but some have deps to imported modules. Yay. bootSwitch(usedTinyC, hasTinyCBackend, "-d:tinyc") -bootSwitch(usedNativeStacktrace, - defined(nativeStackTrace) and nativeStackTraceSupported, - "-d:nativeStackTrace") bootSwitch(usedFFI, hasFFI, "-d:nimHasLibFFI") type @@ -101,7 +98,7 @@ proc writeVersionInfo(conf: ConfigRef; pass: TCmdLinePass) = msgWriteln(conf, "git hash: " & gitHash, {msgStdout}) msgWriteln(conf, "active boot switches:" & usedRelease & usedDanger & - usedTinyC & useLinenoise & usedNativeStacktrace & + usedTinyC & useLinenoise & usedFFI & usedBoehm & usedMarkAndSweep & usedGoGC & usedNoGC, {msgStdout}) msgQuit(0) diff --git a/compiler/options.nim b/compiler/options.nim index 4773ba284..36e0e64da 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -21,6 +21,8 @@ const hasFFI* = defined(nimHasLibFFI) copyrightYear* = "2021" + nimEnableCovariance* = defined(nimEnableCovariance) + type # please make sure we have under 32 options # (improves code efficiency a lot!) TOption* = enum # **keep binary compatible** diff --git a/lib/pure/coro.nim b/lib/pure/coro.nim index e3ed6aadd..f4495a536 100644 --- a/lib/pure/coro.nim +++ b/lib/pure/coro.nim @@ -21,6 +21,8 @@ ## ## Unstable API. +import system/coro_detection + when not nimCoroutines and not defined(nimdoc): when defined(noNimCoroutines): {.error: "Coroutines can not be used with -d:noNimCoroutines".} diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 50a88b43b..287fbe125 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -1910,7 +1910,7 @@ proc copyFileToDir*(source, dir: string, options = {cfSymlinkFollow}) copyFile(source, dir / source.lastPathPart, options) when not declared(ENOENT) and not defined(windows): - when NoFakeVars: + when defined(nimscript): when not defined(haiku): const ENOENT = cint(2) # 2 on most systems including Solaris else: diff --git a/lib/system.nim b/lib/system.nim index c2c3306d4..e9c036b8a 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -537,7 +537,7 @@ const include "system/inclrtl" -const NoFakeVars* = defined(nimscript) ## `true` if the backend doesn't support \ +const NoFakeVars = defined(nimscript) ## `true` if the backend doesn't support \ ## "fake variables" like `var EBADF {.importc.}: cint`. const notJSnotNims = not defined(js) and not defined(nimscript) @@ -1132,7 +1132,6 @@ const const hasThreadSupport = compileOption("threads") and not defined(nimscript) hasSharedHeap = defined(boehmgc) or defined(gogc) # don't share heaps; every thread has its own - nimEnableCovariance* = defined(nimEnableCovariance) # or true when hasThreadSupport and defined(tcc) and not compileOption("tlsEmulation"): # tcc doesn't support TLS @@ -1920,24 +1919,7 @@ include "system/gc_interface" # we have to compute this here before turning it off in except.nim anyway ... const NimStackTrace = compileOption("stacktrace") -template coroutinesSupportedPlatform(): bool = - when defined(sparc) or defined(ELATE) or defined(boehmgc) or defined(gogc) or - defined(nogc) or defined(gcRegions) or defined(gcMarkAndSweep): - false - else: - true - -when defined(nimCoroutines): - # Explicit opt-in. - when not coroutinesSupportedPlatform(): - {.error: "Coroutines are not supported on this architecture and/or garbage collector.".} - const nimCoroutines* = true -elif defined(noNimCoroutines): - # Explicit opt-out. - const nimCoroutines* = false -else: - # Autodetect coroutine support. - const nimCoroutines* = false +import system/coro_detection {.push checks: off.} # obviously we cannot generate checking operations here :-) diff --git a/lib/system/ansi_c.nim b/lib/system/ansi_c.nim index 7e156eaab..e5f26e070 100644 --- a/lib/system/ansi_c.nim +++ b/lib/system/ansi_c.nim @@ -74,7 +74,7 @@ elif defined(haiku): SIGPIPE* = cint(7) SIG_DFL* = cast[CSighandlerT](0) else: - when NoFakeVars: + when defined(nimscript): {.error: "SIGABRT not ported to your platform".} else: var diff --git a/lib/system/coro_detection.nim b/lib/system/coro_detection.nim new file mode 100644 index 000000000..f6c1b5c15 --- /dev/null +++ b/lib/system/coro_detection.nim @@ -0,0 +1,20 @@ +## Coroutine detection logic + +template coroutinesSupportedPlatform(): bool = + when defined(sparc) or defined(ELATE) or defined(boehmgc) or defined(gogc) or + defined(nogc) or defined(gcRegions) or defined(gcMarkAndSweep): + false + else: + true + +when defined(nimCoroutines): + # Explicit opt-in. + when not coroutinesSupportedPlatform(): + {.error: "Coroutines are not supported on this architecture and/or garbage collector.".} + const nimCoroutines* = true +elif defined(noNimCoroutines): + # Explicit opt-out. + const nimCoroutines* = false +else: + # Autodetect coroutine support. + const nimCoroutines* = false diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index da034e57e..8ba3a1601 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -146,7 +146,7 @@ proc closureIterSetupExc(e: ref Exception) {.compilerproc, inline.} = # some platforms have native support for stack traces: const - nativeStackTraceSupported* = (defined(macosx) or defined(linux)) and + nativeStackTraceSupported = (defined(macosx) or defined(linux)) and not NimStackTrace hasSomeStackTrace = NimStackTrace or defined(nimStackTraceOverride) or (defined(nativeStackTrace) and nativeStackTraceSupported) diff --git a/lib/system/io.nim b/lib/system/io.nim index 300e7ea3f..594c78209 100644 --- a/lib/system/io.nim +++ b/lib/system/io.nim @@ -252,7 +252,7 @@ proc write*(f: File, s: string) {.tags: [WriteIOEffect], benign.} = raiseEIO("cannot write string to file") {.pop.} -when NoFakeVars: +when defined(nimscript): when defined(windows): const IOFBF = cint(0) diff --git a/tests/cpp/tcovariancerules.nim b/tests/cpp/tcovariancerules.nim index 49fe8015b..5ac7d8bac 100644 --- a/tests/cpp/tcovariancerules.nim +++ b/tests/cpp/tcovariancerules.nim @@ -31,7 +31,7 @@ import macros macro skipElse(n: untyped): untyped = n[0] template acceptWithCovariance(x, otherwise): untyped = - when nimEnableCovariance: + when defined nimEnableCovariance: x else: reject(x) From ee6d56141c11cf9273d3671eb9dd54097361cc1b Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 1 May 2021 02:12:13 -0700 Subject: [PATCH 0752/2472] fix #17911 rawProc for cpp (#17912) --- lib/pure/hashes.nim | 18 ++++++++++------ lib/system.nim | 13 ++++++++---- tests/stdlib/thashes.nim | 30 ++++++++++++++++++++++---- tests/stdlib/tsystem.nim | 46 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 14 deletions(-) create mode 100644 tests/stdlib/tsystem.nim diff --git a/lib/pure/hashes.nim b/lib/pure/hashes.nim index 2dab498d7..f61fbdf2b 100644 --- a/lib/pure/hashes.nim +++ b/lib/pure/hashes.nim @@ -524,15 +524,21 @@ proc hash*[T: tuple | object | proc](x: T): Hash {.inline.} = proc hash(a: Obj2): Hash = hash((a.x)) assert hash(Obj2[float](x: 520, y: "Nim")) == hash(Obj2[float](x: 520, y: "Nim2")) runnableExamples: - # proc and closure examples + # proc proc fn1() = discard - var a = 0 - proc fn2() = a.inc - assert hash(fn1) != hash(fn2) const fn1b = fn1 assert hash(fn1b) == hash(fn1) - let fn2b = fn2 - assert hash(fn2b) == hash(fn2) + + # closure + proc outer = + var a = 0 + proc fn2() = a.inc + assert fn2 is "closure" + let fn2b = fn2 + assert hash(fn2b) == hash(fn2) + assert hash(fn2) != hash(fn1) + outer() + when T is "closure": result = hash((rawProc(x), rawEnv(x))) elif T is (proc): diff --git a/lib/system.nim b/lib/system.nim index e9c036b8a..aecda4a70 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2414,17 +2414,22 @@ when notJSnotNims: proc rawProc*[T: proc](x: T): pointer {.noSideEffect, inline.} = ## Retrieves the raw proc pointer of the closure `x`. This is - ## useful for interfacing closures with C. + ## useful for interfacing closures with C/C++, hash compuations, etc. when T is "closure": + #[ + The conversion from function pointer to `void*` is a tricky topic, but this + should work at least for c++ >= c++11, e.g. for `dlsym` support. + refs: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=57869, + https://stackoverflow.com/questions/14125474/casts-between-pointer-to-function-and-pointer-to-object-in-c-and-c + ]# {.emit: """ - `result` = `x`.ClP_0; + `result` = (void*)`x`.ClP_0; """.} else: {.error: "Only closure function and iterator are allowed!".} proc rawEnv*[T: proc](x: T): pointer {.noSideEffect, inline.} = - ## Retrieves the raw environment pointer of the closure `x`. This is - ## useful for interfacing closures with C. + ## Retrieves the raw environment pointer of the closure `x`. See also `rawProc`. when T is "closure": {.emit: """ `result` = `x`.ClE_0; diff --git a/tests/stdlib/thashes.nim b/tests/stdlib/thashes.nim index 66857d3ca..46576ef12 100644 --- a/tests/stdlib/thashes.nim +++ b/tests/stdlib/thashes.nim @@ -184,8 +184,8 @@ proc main() = # pending fix proposed in https://github.com/nim-lang/Nim/issues/15952#issuecomment-786312417 discard do: - assert a[0].addr.hash != a[1].addr.hash - assert cast[pointer](a[0].addr).hash == a[0].addr.hash + doAssert a[0].addr.hash != a[1].addr.hash + doAssert cast[pointer](a[0].addr).hash == a[0].addr.hash block: # hash(ref) type A = ref object @@ -193,9 +193,31 @@ proc main() = let a = A(x: 3) disableVm: # xxx Error: VM does not support 'cast' from tyRef to tyPointer let ha = a.hash - assert ha != A(x: 3).hash # A(x: 3) is a different ref object from `a`. + doAssert ha != A(x: 3).hash # A(x: 3) is a different ref object from `a`. a.x = 4 - assert ha == a.hash # the hash only depends on the address + doAssert ha == a.hash # the hash only depends on the address + + block: # hash(proc) + proc fn(a: int): auto = a*2 + doAssert fn isnot "closure" + doAssert fn is (proc) + const fn2 = fn + let fn3 = fn + whenVMorJs: discard + do: + doAssert hash(fn2) == hash(fn) + doAssert hash(fn3) == hash(fn) + + block: # hash(closure) + proc outer() = + var a = 0 + proc inner() = a.inc + doAssert inner is "closure" + let inner2 = inner + whenVMorJs: discard + do: + doAssert hash(inner2) == hash(inner) + outer() static: main() main() diff --git a/tests/stdlib/tsystem.nim b/tests/stdlib/tsystem.nim new file mode 100644 index 000000000..f8b172b44 --- /dev/null +++ b/tests/stdlib/tsystem.nim @@ -0,0 +1,46 @@ +discard """ + targets: "c cpp js" +""" + +# TODO: in future work move existing `system` tests here, where they belong + +import stdtest/testutils + +template main = + block: # closure + proc outer() = + var a = 0 + proc inner1 = a.inc + proc inner2 = discard + doAssert inner1 is "closure" + doAssert inner2 isnot "closure" + doAssert inner1 is (proc) + doAssert inner2 is (proc) + let inner1b = inner1 + doAssert inner1b is "closure" + doAssert inner1b == inner1 + outer() + + block: # rawProc, rawProc, bug #17911 + proc outer() = + var a = 0 + var b = 0 + proc inner1() = a.inc + proc inner2() = a += 2 + proc inner3() = b.inc + let inner1b = inner1 + doAssert inner2 != inner1 + doAssert inner3 != inner1 + whenVMorJs: discard + do: + doAssert rawProc(inner1b) == rawProc(inner1) + doAssert rawProc(inner2) != rawProc(inner1) + doAssert rawProc(inner3) != rawProc(inner1) + + doAssert rawEnv(inner1b) == rawEnv(inner1) + doAssert rawEnv(inner2) == rawEnv(inner1) # because both use `a` + # doAssert rawEnv(inner3) != rawEnv(inner1) # because `a` vs `b` # this doesn't hold + outer() + +static: main() +main() From 13b57524d35a18df57bec2a7d6a2faa7ce712e9f Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 1 May 2021 10:43:20 -0700 Subject: [PATCH 0753/2472] ref #17913; fix typo in tools/ci_generate.nim generating build_all.sh (#17915) --- build_all.sh | 2 +- tools/ci_generate.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build_all.sh b/build_all.sh index 90665672f..37374aa7c 100755 --- a/build_all.sh +++ b/build_all.sh @@ -1,5 +1,5 @@ #! /bin/sh -rem DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim` +# DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim` # build development version of the compiler; can be rerun safely. # arguments can be passed, e.g.: diff --git a/tools/ci_generate.nim b/tools/ci_generate.nim index 2e6b4b3de..f08cd5409 100644 --- a/tools/ci_generate.nim +++ b/tools/ci_generate.nim @@ -87,7 +87,7 @@ if not exist %nim_csources% ( proc genPosixScript(): string = result = fmt""" #! /bin/sh -rem {doNotEdit} +# {doNotEdit} # build development version of the compiler; can be rerun safely. # arguments can be passed, e.g.: From 78e2d299dffc9508430778fcff1dcaee2c8d39fd Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 1 May 2021 15:26:41 -0700 Subject: [PATCH 0754/2472] typo: nonexistant => nonexistent (#17918) * typo: nonexistant => nonexistent * fix test (ordering differs because of https://github.com/nim-lang/Nim/issues/17910) --- lib/pure/osproc.nim | 2 +- lib/pure/sugar.nim | 2 +- nimdoc/testproject/expected/testproject.html | 14 +++++++------- nimdoc/testproject/expected/testproject.idx | 2 +- nimdoc/testproject/expected/theindex.html | 4 ++-- nimdoc/testproject/testproject.nim | 2 +- tests/generics/t13525.nim | 8 ++++---- tests/importalls/mt6.nim | 2 +- tests/lexer/tcustom_numeric_literals.nim | 4 ++-- tests/nimdoc/trunnableexamples.nim | 4 ++-- tests/stdlib/tos.nim | 8 ++++---- tests/stdlib/tosproc.nim | 4 ++-- tests/stdlib/trstgen.nim | 4 ++-- tests/typerel/tvoid.nim | 2 +- tests/types/titerable.nim | 10 +++++----- tests/vm/tvmops.nim | 4 ++-- 16 files changed, 38 insertions(+), 38 deletions(-) diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 6aefb8d6c..1ac37661b 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -1595,7 +1595,7 @@ proc execCmdEx*(command: string, options: set[ProcessOption] = { ## import std/[strutils, strtabs] ## stripLineEnd(result[0]) ## portable way to remove trailing newline, if any ## doAssert result == ("12", 0) - ## doAssert execCmdEx("ls --nonexistant").exitCode != 0 + ## doAssert execCmdEx("ls --nonexistent").exitCode != 0 ## when defined(posix): ## assert execCmdEx("echo $FO", env = newStringTable({"FO": "B"})) == ("B\n", 0) ## assert execCmdEx("echo $PWD", workingDir = "/") == ("/\n", 0) diff --git a/lib/pure/sugar.nim b/lib/pure/sugar.nim index 9abef55a2..7cfa932e6 100644 --- a/lib/pure/sugar.nim +++ b/lib/pure/sugar.nim @@ -186,7 +186,7 @@ macro dumpToString*(x: untyped): string = assert dumpToString(a + x) == "a + x: 1 + x = 11" template square(x): untyped = x * x assert dumpToString(square(x)) == "square(x): x * x = 100" - assert not compiles dumpToString(1 + nonexistant) + assert not compiles dumpToString(1 + nonexistent) import std/strutils assert "failedAssertImpl" in dumpToString(assert true) # example with a statement result = newCall(bindSym"dumpToStringImpl") diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html index 1ea9e172a..70c5b695a 100644 --- a/nimdoc/testproject/expected/testproject.html +++ b/nimdoc/testproject/expected/testproject.html @@ -210,6 +210,11 @@ window.addEventListener('DOMContentLoaded', main);
  • z11()
    • + + -
      asyncFun1
    • - -
      proc c_nonexistant(frmt: cstring): cint {.importc: "nonexistant",
+
+
      proc c_nonexistent(frmt: cstring): cint {.importc: "nonexistent",
     header: "<stdio.h>", varargs, discardable.}
      
 
      
 
diff --git a/nimdoc/testproject/expected/testproject.idx b/nimdoc/testproject/expected/testproject.idx
index 1d8be99da..a8cdc3be2 100644
--- a/nimdoc/testproject/expected/testproject.idx
+++ b/nimdoc/testproject/expected/testproject.idx
@@ -35,7 +35,7 @@ z17	testproject.html#z17	testproject: z17()
 p1	testproject.html#p1	testproject: p1()	
 addfBug14485	testproject.html#addfBug14485	testproject: addfBug14485()	
 c_printf	testproject.html#c_printf,cstring	testproject: c_printf(frmt: cstring): cint	
-c_nonexistant	testproject.html#c_nonexistant,cstring	testproject: c_nonexistant(frmt: cstring): cint	
+c_nonexistent	testproject.html#c_nonexistent,cstring	testproject: c_nonexistent(frmt: cstring): cint	
 low	testproject.html#low,T	testproject: low[T: Ordinal | enum | range](x: T): T	
 low2	testproject.html#low2,T	testproject: low2[T: Ordinal | enum | range](x: T): T	
 tripleStrLitTest	testproject.html#tripleStrLitTest	testproject: tripleStrLitTest()	
diff --git a/nimdoc/testproject/expected/theindex.html b/nimdoc/testproject/expected/theindex.html
index 81370d115..e6efee986 100644
--- a/nimdoc/testproject/expected/theindex.html
+++ b/nimdoc/testproject/expected/theindex.html
@@ -142,9 +142,9 @@ window.addEventListener('DOMContentLoaded', main);
 
    • Shapes.Circle
      •
    -
    c_nonexistant:
    @@ -597,7 +597,7 @@ cz1 cz2

    Example:

    -
    discard "in cz2"
    +
    discard "in cz2"
    @@ -648,7 +648,7 @@ cz8

    Example:

    -
    doAssert 1 + 1 == 2
    +
    doAssert 1 + 1 == 2
    @@ -657,7 +657,7 @@ cz8

    Example: cmd: -d:foobar

    -
    discard 1
    cz10 +
    discard 1
    cz10 @@ -666,7 +666,7 @@ cz8

    Example:

    -
    discard 1
    +
    discard 1
    @@ -675,7 +675,7 @@ cz8

    Example:

    -
    discard 1
    +
    discard 1
    @@ -700,7 +700,7 @@ cz13 cz17 rest

    Example:

    -
    discard 1
    rest +
    discard 1
    rest @@ -709,21 +709,21 @@ cz17 rest cp1

    Example:

    -
    doAssert 1 == 1 # regular comments work here
    c4 +
    doAssert 1 == 1 # regular comments work here
    c4

    Example:

    -
    # c5 regular comments before 1st token work
-# regular comment
-#[
+
    # c5 regular comments before 1st token work
+# regular comment
+#[
 nested regular comment
-]#
-doAssert 2 == 2 # c8
-## this is a non-nested doc comment
+]#
+doAssert 2 == 2 # c8
+## this is a non-nested doc comment
 
-##[
+##[
 this is a nested doc comment
-]##
-discard "c9"
-# also work after
    
+]##    
+discard "c9"
+# also work after
    @@ -732,9 +732,9 @@ this is a nested doc comment Some proc

    Example:

    -
    discard "foo() = " & $[1]
-#[
-0: let's also add some broken html to make sure this won't break in future
+
    discard "foo() = " & $[1]
+#[
+0: let's also add some broken html to make sure this won't break in future
 1: </span>
 2: </span>
 3: </span
@@ -782,7 +782,7 @@ the c printf. etc.
 
 
    low2(2) # => -9223372036854775808
    
 
    Example:
    
-
    discard "in low2"
    
+
    discard "in low2"
    
 
 
 
@@ -791,41 +791,41 @@ the c printf. etc.
 
 
 
    Example:
    
-
    ## mullitline string litterals are tricky as their indentation can span
-## below that of the runnableExamples
-let s1a = """
+
    ## mullitline string litterals are tricky as their indentation can span
+## below that of the runnableExamples
+let s1a = """
 should appear at indent 0
   at indent 2
 at indent 0
-"""
-# make sure this works too
-let s1b = """start at same line
+"""
+# make sure this works too
+let s1b = """start at same line
   at indent 2
 at indent 0
-""" # comment after
-let s2 = """sandwich """
-let s3 = """"""
-when false:
-  let s5 = """
-        in s5 """
+""" # comment after
+let s2 = """sandwich """
+let s3 = """"""
+when false:
+  let s5 = """
+        in s5 """
 
-let s3b = ["""
+let s3b = ["""
 %!? #[...] # inside a multiline ...
-""", "foo"]
+""", "foo"]
 
-## make sure handles trailing spaces
-let s4 = """ 
-"""
+## make sure handles trailing spaces
+let s4 = """ 
+"""
 
-let s5 = """ x
-"""
-let s6 = """ ""
-"""
-let s7 = """"""""""
-let s8 = ["""""""""", """
-  """ ]
-discard
-# should be in
    
+let s5 = """ x
+"""
+let s6 = """ ""
+"""
+let s7 = """"""""""
+let s8 = ["""""""""", """
+  """ ]
+discard
+# should be in
    
 
 
 
@@ -896,9 +896,9 @@ foo3
 
 
 
    Example:
    
-
    # ok1
-assert 1 == 1
-# ok2
    
+
    # ok1
+assert 1 == 1
+# ok2
    
 
 
 
@@ -914,7 +914,7 @@ foo1
 
 foo2
 
    Example:
    
-
    discard # bar
    
+
    discard # bar
    
 
 
 
@@ -935,9 +935,9 @@ foo2
 
 
 
    Example:
    
-
    discard 1
    cz16 after
+
    discard 1
    cz16 after
 
    Example:
    
-
    doAssert 2 == 1 + 1
    
+
    doAssert 2 == 1 + 1
    
 
 
 
@@ -979,20 +979,20 @@ This does nothing
 
 
 
    Example:
    
-
    import std/strutils
-## issue #8871 preserve formatting
-## line doc comment
-# bar
-doAssert "'foo" == "'foo"
-##[
+
    import std/strutils
+## issue #8871 preserve formatting
+## line doc comment
+# bar
+doAssert "'foo" == "'foo"
+##[
 foo
 bar
-]##
+]##
 
-doAssert: not "foo".startsWith "ba"
-block:
-  discard 0xff # elu par cette crapule
-# should be in
    should be still in
+doAssert: not "foo".startsWith "ba"
+block:
+  discard 0xff # elu par cette crapule
+# should be in
    should be still in
 
 
 
@@ -1012,13 +1012,13 @@ cz15
 
    Example:
    
 
    discard
    
 
    Example:
    
-
    discard 3
    
+
    discard 3
    
 
    Example:
    
-
    discard 4
    ok5 ok5b
+
    discard 4
    ok5 ok5b
 
    Example:
    
-
    assert true
    
+
    assert true
    
 
    Example:
    
-
    discard 1
    in or out?
+
    discard 1
    in or out?
 
 
 
@@ -1027,7 +1027,7 @@ cz15
 
 
 
    Example:
    
-
    discard 2
    
+
    discard 2
    
 
 
 

From 2ce592a209f4cd7f59c009a01e4ac4037e63a7bd Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Fri, 14 May 2021 15:33:17 -0700
Subject: [PATCH 0808/2472] refs #18011 disable some newly failing tests on cpp
 windows; refs #17946 increase timeout for tchannels (#18012)

---
 lib/system.nim                        | 8 ++++----
 tests/cpp/temitlist.nim               | 4 +++-
 tests/cpp/tempty_generic_obj.nim      | 4 +++-
 tests/exception/tcpp_imported_exc.nim | 4 +++-
 tests/stdlib/tchannels.nim            | 2 +-
 5 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/lib/system.nim b/lib/system.nim
index 9317c3440..be30665dd 100644
--- a/lib/system.nim
+++ b/lib/system.nim
@@ -1317,10 +1317,10 @@ proc del*[T](x: var seq[T], i: Natural) {.noSideEffect.} =
   ##
   ## See also:
   ## * `delete <#delete,seq[T],Natural>`_ for preserving the order
-  ##
-  ## .. code-block:: Nim
-  ##  var i = @[1, 2, 3, 4, 5]
-  ##  i.del(2) # => @[1, 2, 5, 4]
+  runnableExamples:
+    var a = @[10, 11, 12, 13, 14]
+    a.del(2)
+    assert a == @[10, 11, 14, 13]
   let xl = x.len - 1
   movingCopy(x[i], x[xl])
   setLen(x, xl)
diff --git a/tests/cpp/temitlist.nim b/tests/cpp/temitlist.nim
index 852537518..d446176fa 100644
--- a/tests/cpp/temitlist.nim
+++ b/tests/cpp/temitlist.nim
@@ -1,7 +1,9 @@
 discard """
   targets: "cpp"
-  output: '''6.0
+  output: '''
+6.0
 0'''
+disabled: "windows" # pending bug #18011
 """
 
 # bug #4730
diff --git a/tests/cpp/tempty_generic_obj.nim b/tests/cpp/tempty_generic_obj.nim
index 913c1ec3c..6125190b4 100644
--- a/tests/cpp/tempty_generic_obj.nim
+++ b/tests/cpp/tempty_generic_obj.nim
@@ -1,7 +1,9 @@
 discard """
   targets: "cpp"
-  output: '''int
+  output: '''
+int
 float'''
+disabled: "windows" # pending bug #18011
 """
 
 import typetraits
diff --git a/tests/exception/tcpp_imported_exc.nim b/tests/exception/tcpp_imported_exc.nim
index 8b96ca635..5195b48e7 100644
--- a/tests/exception/tcpp_imported_exc.nim
+++ b/tests/exception/tcpp_imported_exc.nim
@@ -1,6 +1,7 @@
 discard """
 targets: "cpp"
-output: '''caught as std::exception
+output: '''
+caught as std::exception
 expected
 finally1
 finally2
@@ -12,6 +13,7 @@ finally 2
 expected
 cpp exception caught
 '''
+disabled: "windows" # pending bug #18011
 """
 
 type
diff --git a/tests/stdlib/tchannels.nim b/tests/stdlib/tchannels.nim
index d609eec78..492faf500 100644
--- a/tests/stdlib/tchannels.nim
+++ b/tests/stdlib/tchannels.nim
@@ -1,5 +1,5 @@
 discard """
-  timeout:  20.0 # but typically < 1s (in isolation but other tests running in parallel can affect this since based on epochTime)
+  timeout:  60.0 # but typically < 1s (in isolation but other tests running in parallel can affect this since based on epochTime)
   disabled: "freebsd"
   matrix: "--gc:arc --threads:on; --gc:arc --threads:on -d:danger"
 """

From 4857c462d5a9dc68519ab3c6996523bdcccb883f Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Sat, 15 May 2021 06:50:39 +0200
Subject: [PATCH 0809/2472] Revert "fix #14873 properly by skipping `abi` field
 in importc type (#17944)" (#17992)

This reverts commit 98c29c01eb91a0c6ce7da09380a272eebe6bca6f.
---
 lib/core/locks.nim        |  5 +++++
 lib/system/syslocks.nim   |  8 ++++++++
 tests/stdlib/uselocks.nim | 18 +-----------------
 3 files changed, 14 insertions(+), 17 deletions(-)

diff --git a/lib/core/locks.nim b/lib/core/locks.nim
index baec45829..92967b9db 100644
--- a/lib/core/locks.nim
+++ b/lib/core/locks.nim
@@ -28,6 +28,11 @@ type
 
 {.push stackTrace: off.}
 
+
+proc `$`*(lock: Lock): string =
+  # workaround bug #14873
+  result = "()"
+
 proc initLock*(lock: var Lock) {.inline.} =
   ## Initializes the given lock.
   when not defined(js):
diff --git a/lib/system/syslocks.nim b/lib/system/syslocks.nim
index 9e8da610d..2f0c8b0ba 100644
--- a/lib/system/syslocks.nim
+++ b/lib/system/syslocks.nim
@@ -102,18 +102,26 @@ else:
     SysLockObj {.importc: "pthread_mutex_t", pure, final,
                header: """#include 
                           #include """.} = object
+      when defined(linux) and defined(amd64):
+        abi: array[40 div sizeof(clong), clong]
 
     SysLockAttr {.importc: "pthread_mutexattr_t", pure, final
                header: """#include 
                           #include """.} = object
+      when defined(linux) and defined(amd64):
+        abi: array[4 div sizeof(cint), cint]  # actually a cint
 
     SysCondObj {.importc: "pthread_cond_t", pure, final,
                header: """#include 
                           #include """.} = object
+      when defined(linux) and defined(amd64):
+        abi: array[48 div sizeof(clonglong), clonglong]
 
     SysCondAttr {.importc: "pthread_condattr_t", pure, final
                header: """#include 
                           #include """.} = object
+      when defined(linux) and defined(amd64):
+        abi: array[4 div sizeof(cint), cint]  # actually a cint
 
     SysLockType = distinct cint
 
diff --git a/tests/stdlib/uselocks.nim b/tests/stdlib/uselocks.nim
index e82de8adf..e9d23f9d9 100644
--- a/tests/stdlib/uselocks.nim
+++ b/tests/stdlib/uselocks.nim
@@ -11,21 +11,5 @@ proc use* (m: var MyType): int =
     result = 3
 
 block:
-  # bug #14873
   var l: Lock
-  doAssert ($l).len > 0
-    # on posix, "()", on windows, something else, but that shouldn't be part of the spec
-    # what matters is that `$` doesn't cause the codegen bug mentioned
-
-when true: # intentional
-  # bug https://github.com/nim-lang/Nim/issues/14873#issuecomment-784241605
-  type
-    Test = object
-      path: string # Removing this makes both cases work.
-      lock: Lock
-  # A: This is not fine.
-  var a = Test()
-  proc main(): void =
-    # B: This is fine.
-    var b = Test()
-  main()
+  doAssert $l == "()"

From 1568ae23c66c572891207f2c1d330d246b3144ac Mon Sep 17 00:00:00 2001
From: Andrey Makarov 
Date: Sat, 15 May 2021 07:53:38 +0300
Subject: [PATCH 0810/2472] docgen: escape special characters in titles
 (#18014)

---
 compiler/docgen.nim | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/compiler/docgen.nim b/compiler/docgen.nim
index 23e6ba10d..e07db9188 100644
--- a/compiler/docgen.nim
+++ b/compiler/docgen.nim
@@ -1217,10 +1217,11 @@ proc genOutFile(d: PDoc, groupedToc = false): string =
   else:
     # Modules get an automatic title for the HTML, but no entry in the index.
     title = canonicalImport(d.conf, AbsoluteFile d.filename)
+  title = esc(d.target, title)
   var subtitle = ""
   if d.meta[metaSubtitle] != "":
     dispA(d.conf, subtitle, "
    $1
    ",
-        "\\\\\\vspace{0.5em}\\large $1", [d.meta[metaSubtitle]])
+        "\\\\\\vspace{0.5em}\\large $1", [esc(d.target, d.meta[metaSubtitle])])
 
   var groupsection = getConfigVar(d.conf, "doc.body_toc_groupsection")
   let bodyname = if d.hasToc and not d.isPureRst and not d.conf.isLatexCmd:
@@ -1247,7 +1248,7 @@ proc genOutFile(d: PDoc, groupedToc = false): string =
         "title", title, "subtitle", subtitle, "tableofcontents", toc,
         "moduledesc", d.modDesc, "date", getDateStr(), "time", getClockStr(),
         "content", content, "author", d.meta[metaAuthor],
-        "version", d.meta[metaVersion], "analytics", d.analytics,
+        "version", esc(d.target, d.meta[metaVersion]), "analytics", d.analytics,
         "deprecationMsg", d.modDeprecationMsg]
   else:
     code = content

From 3824fd3f9a303bc885a6f1f2eb7aa28fb5460cb7 Mon Sep 17 00:00:00 2001
From: Andrey Makarov 
Date: Sat, 15 May 2021 09:12:26 +0300
Subject: [PATCH 0811/2472] RST opt.list to have priority over def.list
 (#17845)

---
 lib/packages/docutils/rst.nim |  4 ++--
 tests/stdlib/trst.nim         | 26 ++++++++++++++++++++++++++
 2 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim
index 181929c09..9cbc4efb7 100644
--- a/lib/packages/docutils/rst.nim
+++ b/lib/packages/docutils/rst.nim
@@ -1867,10 +1867,10 @@ proc whichSection(p: RstParser): RstNodeKind =
     elif match(p, p.idx, "(e) ") or match(p, p.idx, "e) ") or
          match(p, p.idx, "e. "):
       result = rnEnumList
-    elif isDefList(p):
-      result = rnDefList
     elif isOptionList(p):
       result = rnOptionList
+    elif isDefList(p):
+      result = rnDefList
     else:
       result = rnParagraph
   of tkWord, tkOther, tkWhite:
diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim
index 905e5143a..e6c5d9700 100644
--- a/tests/stdlib/trst.nim
+++ b/tests/stdlib/trst.nim
@@ -1,6 +1,8 @@
 discard """
   output: '''
 
+[Suite] RST parsing
+
 [Suite] RST indentation
 
 [Suite] RST include directive
@@ -55,6 +57,30 @@ proc toAst(input: string,
   except EParseError:
     discard
 
+suite "RST parsing":
+  test "option list has priority over definition list":
+    check(dedent"""
+        --defusages
+                      file
+        -o            set
+        """.toAst ==
+      dedent"""
+        rnOptionList
+          rnOptionListItem  order=1
+            rnOptionGroup
+              rnLeaf  '--'
+              rnLeaf  'defusages'
+            rnDescription
+              rnInner
+                rnLeaf  'file'
+          rnOptionListItem  order=2
+            rnOptionGroup
+              rnLeaf  '-'
+              rnLeaf  'o'
+            rnDescription
+              rnLeaf  'set'
+        """)
+
 suite "RST indentation":
   test "nested bullet lists":
     let input = dedent """

From 99788ee50426946d954e7311f567414b39f7b8d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20T=2E=20Jarl=C3=B8v?=
 
Date: Sat, 15 May 2021 21:26:15 +0200
Subject: [PATCH 0812/2472] Escape `%00` / `\0` in `dbQuote` (#18015)
 [backport:1.4]

Fix https://github.com/nim-lang/Nim/issues/17925
---
 lib/impure/db_postgres.nim | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/impure/db_postgres.nim b/lib/impure/db_postgres.nim
index cb9e5414a..4edeac2fc 100644
--- a/lib/impure/db_postgres.nim
+++ b/lib/impure/db_postgres.nim
@@ -110,7 +110,9 @@ proc dbQuote*(s: string): string =
   ## DB quotes the string.
   result = "'"
   for c in items(s):
-    if c == '\'': add(result, "''")
+    case c
+    of '\'': add(result, "''")
+    of '\0': add(result, "\\0")
     else: add(result, c)
   add(result, '\'')
 

From 36145236644085c636c76500a189e3afda40c13f Mon Sep 17 00:00:00 2001
From: Clyybber 
Date: Sun, 16 May 2021 00:15:53 +0200
Subject: [PATCH 0813/2472] Rework DFA traversal (#18016)

* enable using dbg: without a context

* Optimally joining first write/last read analysis

* Add test for #18002

* potLastReads -> potentialLastReads
---
 compiler/injectdestructors.nim | 208 ++++++++++++++++-----------------
 tests/arc/tmovebug.nim         |  34 ++++++
 2 files changed, 133 insertions(+), 109 deletions(-)

diff --git a/compiler/injectdestructors.nim b/compiler/injectdestructors.nim
index 152efd8a1..48a852d26 100644
--- a/compiler/injectdestructors.nim
+++ b/compiler/injectdestructors.nim
@@ -46,6 +46,13 @@ type
     sinkArg
 
 const toDebug {.strdefine.} = ""
+when toDebug.len > 0:
+  var shouldDebug = false
+
+template dbg(body) =
+  when toDebug.len > 0:
+    if shouldDebug:
+      body
 
 proc hasDestructor(c: Con; t: PType): bool {.inline.} =
   result = ast.hasDestructor(t)
@@ -54,11 +61,6 @@ proc hasDestructor(c: Con; t: PType): bool {.inline.} =
     if not result and c.graph.config.selectedGC in {gcArc, gcOrc}:
       assert(not containsGarbageCollectedRef(t))
 
-template dbg(body) =
-  when toDebug.len > 0:
-    if c.owner.name.s == toDebug or toDebug == "always":
-      body
-
 proc getTemp(c: var Con; s: var Scope; typ: PType; info: TLineInfo): PNode =
   let sym = newSym(skTemp, getIdent(c.graph.cache, ":tmpD"), nextSymId c.idgen, c.owner, info)
   sym.typ = typ
@@ -75,106 +77,111 @@ import sets, hashes, tables
 
 proc hash(n: PNode): Hash = hash(cast[pointer](n))
 
-type AliasCache = Table[(PNode, PNode), AliasKind]
-proc aliasesCached(cache: var AliasCache, obj, field: PNode): AliasKind =
+proc aliasesCached(cache: var Table[(PNode, PNode), AliasKind], obj, field: PNode): AliasKind =
   let key = (obj, field)
   if not cache.hasKey(key):
     cache[key] = aliases(obj, field)
   cache[key]
 
-proc collectLastReads(cfg: ControlFlowGraph; cache: var AliasCache, lastReads, potLastReads: var IntSet; pc: var int, until: int) =
-  template aliasesCached(obj, field: PNode): untyped =
+type
+  State = ref object
+    lastReads: IntSet
+    potentialLastReads: IntSet
+    notLastReads: IntSet
+    alreadySeen: HashSet[PNode]
+
+proc preprocessCfg(cfg: var ControlFlowGraph) =
+  for i in 0.. cfg.len:
+      cfg[i].dest = cfg.len - i
+
+proc mergeStates(a: var State, b: sink State) =
+  # Inplace for performance:
+  #   lastReads = a.lastReads + b.lastReads
+  #   potentialLastReads = (a.potentialLastReads + b.potentialLastReads) - (a.notLastReads + b.notLastReads)
+  #   notLastReads = a.notLastReads + b.notLastReads
+  #   alreadySeen = a.alreadySeen + b.alreadySeen
+  # b is never nil
+  if a == nil:
+    a = b
+  else:
+    a.lastReads.incl b.lastReads
+    a.potentialLastReads.incl b.potentialLastReads
+    a.potentialLastReads.excl a.notLastReads
+    a.potentialLastReads.excl b.notLastReads
+    a.notLastReads.incl b.notLastReads
+    a.alreadySeen.incl b.alreadySeen
+
+proc computeLastReadsAndFirstWrites(cfg: ControlFlowGraph) =
+  var cache = initTable[(PNode, PNode), AliasKind]()
+  template aliasesCached(obj, field: PNode): AliasKind =
     aliasesCached(cache, obj, field)
-  while pc < until:
-    case cfg[pc].kind
-    of def:
-      let potLastReadsCopy = potLastReads
-      for r in potLastReadsCopy:
-        if cfg[pc].n.aliasesCached(cfg[r].n) == yes:
-          # the path leads to a redefinition of 's' --> sink 's'.
-          lastReads.incl r
-          potLastReads.excl r
-        elif cfg[r].n.aliasesCached(cfg[pc].n) != no:
-          # only partially writes to 's' --> can't sink 's', so this def reads 's'
-          # or maybe writes to 's' --> can't sink 's'
-          cfg[r].n.comment = '\n' & $pc
-          potLastReads.excl r
 
-      inc pc
-    of use:
-      let potLastReadsCopy = potLastReads
-      for r in potLastReadsCopy:
-        if cfg[pc].n.aliasesCached(cfg[r].n) != no or cfg[r].n.aliasesCached(cfg[pc].n) != no:
-          cfg[r].n.comment = '\n' & $pc
-          potLastReads.excl r
+  var cfg = cfg
+  preprocessCfg(cfg)
 
-      potLastReads.incl pc
+  var states = newSeq[State](cfg.len + 1)
+  states[0] = State()
 
-      inc pc
-    of goto:
-      pc += cfg[pc].dest
-    of fork:
-      var variantA = pc + 1
-      var variantB = pc + cfg[pc].dest
-      var potLastReadsA, potLastReadsB = potLastReads
-      var lastReadsA, lastReadsB: IntSet
-      while variantA != variantB and max(variantA, variantB) < cfg.len and min(variantA, variantB) < until:
-        if variantA < variantB:
-          collectLastReads(cfg, cache, lastReadsA, potLastReadsA, variantA, min(variantB, until))
-        else:
-          collectLastReads(cfg, cache, lastReadsB, potLastReadsB, variantB, min(variantA, until))
+  for pc in 0.. sink 's'.
+            state.lastReads.incl r
+            state.potentialLastReads.excl r
+          elif cfg[r].n.aliasesCached(cfg[pc].n) != no:
+            # only partially writes to 's' --> can't sink 's', so this def reads 's'
+            # or maybe writes to 's' --> can't sink 's'
+            cfg[r].n.comment = '\n' & $pc
+            state.potentialLastReads.excl r
+            state.notLastReads.incl r
 
-      # Add those last reads that were turned into last reads on both branches
-      lastReads.incl lastReadsA * lastReadsB
-      # Add those last reads that were turned into last reads on only one branch,
-      # but where the read operation itself also belongs to only that branch
-      lastReads.incl (lastReadsA + lastReadsB) - potLastReads
+        var alreadySeenThisNode = false
+        for s in state.alreadySeen:
+          if cfg[pc].n.aliasesCached(s) != no or s.aliasesCached(cfg[pc].n) != no:
+            alreadySeenThisNode = true; break
+        if alreadySeenThisNode: cfg[pc].n.flags.excl nfFirstWrite
+        else: cfg[pc].n.flags.incl nfFirstWrite
 
-      let oldPotLastReads = potLastReads
-      potLastReads = initIntSet()
+        state.alreadySeen.incl cfg[pc].n
 
-      potLastReads.incl potLastReadsA + potLastReadsB
+        mergeStates(states[pc + 1], move(states[pc]))
+      of use:
+        var potentialLastReadsCopy = state.potentialLastReads
+        for r in potentialLastReadsCopy:
+          if cfg[pc].n.aliasesCached(cfg[r].n) != no or cfg[r].n.aliasesCached(cfg[pc].n) != no:
+            cfg[r].n.comment = '\n' & $pc
+            state.potentialLastReads.excl r
+            state.notLastReads.incl r
 
-      # Remove potential last reads that were invalidated in a branch,
-      # but don't remove those which were turned into last reads on that branch
-      potLastReads.excl ((oldPotLastReads - potLastReadsA) - lastReadsA)
-      potLastReads.excl ((oldPotLastReads - potLastReadsB) - lastReadsB)
+        state.potentialLastReads.incl pc
 
-      pc = min(variantA, variantB)
+        state.alreadySeen.incl cfg[pc].n
 
-proc collectFirstWrites(cfg: ControlFlowGraph; alreadySeen: var HashSet[PNode]; pc: var int, until: int) =
-  while pc < until:
-    case cfg[pc].kind
-    of def:
-      var alreadySeenThisNode = false
-      for s in alreadySeen:
-        if cfg[pc].n.aliases(s) != no or s.aliases(cfg[pc].n) != no:
-          alreadySeenThisNode = true; break
-      if alreadySeenThisNode: cfg[pc].n.flags.excl nfFirstWrite
-      else: cfg[pc].n.flags.incl nfFirstWrite
+        mergeStates(states[pc + 1], move(states[pc]))
+      of goto:
+        mergeStates(states[pc + cfg[pc].dest], move(states[pc]))
+      of fork:
+        var copy = State()
+        copy[] = states[pc][]
+        mergeStates(states[pc + cfg[pc].dest], copy)
+        mergeStates(states[pc + 1], move(states[pc]))
 
-      alreadySeen.incl cfg[pc].n
-
-      inc pc
-    of use:
-      alreadySeen.incl cfg[pc].n
-
-      inc pc
-    of goto:
-      pc += cfg[pc].dest
-    of fork:
-      var variantA = pc + 1
-      var variantB = pc + cfg[pc].dest
-      var alreadySeenA, alreadySeenB = alreadySeen
-      while variantA != variantB and max(variantA, variantB) < cfg.len and min(variantA, variantB) < until:
-        if variantA < variantB:
-          collectFirstWrites(cfg, alreadySeenA, variantA, min(variantB, until))
-        else:
-          collectFirstWrites(cfg, alreadySeenB, variantB, min(variantA, until))
-
-      alreadySeen.incl alreadySeenA + alreadySeenB
-
-      pc = min(variantA, variantB)
+  let lastReads = (states[^1].lastReads + states[^1].potentialLastReads) - states[^1].notLastReads
+  var lastReadTable: Table[PNode, seq[int]]
+  for position, node in cfg:
+    if node.kind == use:
+      lastReadTable.mgetOrPut(node.n, @[]).add position
+  for node, positions in lastReadTable:
+    block checkIfAllPosLastRead:
+      for p in positions:
+        if p notin lastReads: break checkIfAllPosLastRead
+      node.flags.incl nfLastRead
 
 proc isLastRead(n: PNode; c: var Con): bool =
   let m = dfa.skipConvDfa(n)
@@ -1096,6 +1103,8 @@ proc injectDefaultCalls(n: PNode, c: var Con) =
       injectDefaultCalls(n[i], c)
 
 proc injectDestructorCalls*(g: ModuleGraph; idgen: IdGenerator; owner: PSym; n: PNode): PNode =
+  when toDebug.len > 0:
+    shouldDebug = toDebug == owner.name.s or toDebug == "always"
   if sfGeneratedOp in owner.flags or (owner.kind == skIterator and isInlineIterator(owner.typ)):
     return n
   var c = Con(owner: owner, graph: g, g: constructCfg(owner, n), idgen: idgen)
@@ -1107,26 +1116,7 @@ proc injectDestructorCalls*(g: ModuleGraph; idgen: IdGenerator; owner: PSym; n:
   if optCursorInference in g.config.options:
     computeCursors(owner, n, g)
 
-  block:
-    var cache = initTable[(PNode, PNode), AliasKind]()
-    var lastReads, potLastReads: IntSet
-    var pc = 0
-    collectLastReads(c.g, cache, lastReads, potLastReads, pc, c.g.len)
-    lastReads.incl potLastReads
-    var lastReadTable: Table[PNode, seq[int]]
-    for position, node in c.g:
-      if node.kind == use:
-        lastReadTable.mgetOrPut(node.n, @[]).add position
-    for node, positions in lastReadTable:
-      var allPositionsLastRead = true
-      for p in positions:
-        if p notin lastReads: allPositionsLastRead = false; break
-      if allPositionsLastRead:
-        node.flags.incl nfLastRead
-
-    var alreadySeen: HashSet[PNode]
-    pc = 0
-    collectFirstWrites(c.g, alreadySeen, pc, c.g.len)
+  computeLastReadsAndFirstWrites(c.g)
 
   var scope: Scope
   let body = p(n, c, scope, normal)
diff --git a/tests/arc/tmovebug.nim b/tests/arc/tmovebug.nim
index 3ff1c4a0c..7977d330a 100644
--- a/tests/arc/tmovebug.nim
+++ b/tests/arc/tmovebug.nim
@@ -784,3 +784,37 @@ proc main3 =
 
 main3()
 
+# misc
+proc smoltest(x: bool): bool =
+  while true:
+    if true: return x
+
+discard smoltest(true)
+
+# bug #18002
+type
+  TTypeAttachedOp = enum
+    attachedAsgn
+    attachedSink
+    attachedTrace
+
+  PNode = ref object
+    discard
+
+proc genAddrOf(n: PNode) =
+  assert n != nil, "moved?!"
+
+proc atomicClosureOp =
+  let x = PNode()
+
+  genAddrOf:
+    block:
+      x
+
+  case attachedTrace
+  of attachedSink: discard
+  of attachedAsgn: discard
+  of attachedTrace: genAddrOf(x)
+
+atomicClosureOp()
+

From c218f2ba0b8e27110087ea754c11cff123806a94 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Sun, 16 May 2021 10:39:52 +0800
Subject: [PATCH 0814/2472] [std/re]fix terrible and strange interface

---
 lib/impure/re.nim | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/lib/impure/re.nim b/lib/impure/re.nim
index 0c96876b9..95d77033b 100644
--- a/lib/impure/re.nim
+++ b/lib/impure/re.nim
@@ -142,7 +142,7 @@ proc matchOrFind(buf: cstring, pattern: Regex, matches: var openArray[string],
   return rawMatches[1] - rawMatches[0]
 
 proc findBounds*(buf: cstring, pattern: Regex, matches: var openArray[string],
-                 start = 0, bufSize: int): tuple[first, last: int] =
+                 start = 0, bufSize = 0): tuple[first, last: int] =
   ## returns the starting position and end position of `pattern` in `buf`
   ## (where `buf` has length `bufSize` and is not necessarily `'\0'` terminated),
   ## and the captured
@@ -200,7 +200,7 @@ proc findBounds*(s: string, pattern: Regex,
   result = findBounds(cstring(s), pattern, matches, start, s.len)
 
 proc findBounds*(buf: cstring, pattern: Regex,
-                 start = 0, bufSize: int): tuple[first, last: int] =
+                 start = 0, bufSize = 0): tuple[first, last: int] =
   ## returns the `first` and `last` position of `pattern` in `buf`,
   ## where `buf` has length `bufSize` (not necessarily `'\0'` terminated).
   ## If it does not match, `(-1,0)` is returned.
@@ -239,7 +239,7 @@ proc matchLen*(s: string, pattern: Regex, matches: var openArray[string],
   result = matchOrFind(cstring(s), pattern, matches, start.cint, s.len.cint, pcre.ANCHORED)
 
 proc matchLen*(buf: cstring, pattern: Regex, matches: var openArray[string],
-              start = 0, bufSize: int): int {.inline.} =
+              start = 0, bufSize = 0): int {.inline.} =
   ## the same as `match`, but it returns the length of the match,
   ## if there is no match, `-1` is returned. Note that a match length
   ## of zero can happen.
@@ -281,7 +281,7 @@ proc match*(s: string, pattern: Regex, matches: var openArray[string],
   result = matchLen(cstring(s), pattern, matches, start, s.len) != -1
 
 proc match*(buf: cstring, pattern: Regex, matches: var openArray[string],
-           start = 0, bufSize: int): bool {.inline.} =
+           start = 0, bufSize = 0): bool {.inline.} =
   ## returns `true` if `buf[start..
Date: Sun, 16 May 2021 11:10:41 +0800
Subject: [PATCH 0815/2472] Revert "[std/re]fix terrible and strange interface"
 (#18027)

This reverts commit c218f2ba0b8e27110087ea754c11cff123806a94.
---
 lib/impure/re.nim | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/lib/impure/re.nim b/lib/impure/re.nim
index 95d77033b..0c96876b9 100644
--- a/lib/impure/re.nim
+++ b/lib/impure/re.nim
@@ -142,7 +142,7 @@ proc matchOrFind(buf: cstring, pattern: Regex, matches: var openArray[string],
   return rawMatches[1] - rawMatches[0]
 
 proc findBounds*(buf: cstring, pattern: Regex, matches: var openArray[string],
-                 start = 0, bufSize = 0): tuple[first, last: int] =
+                 start = 0, bufSize: int): tuple[first, last: int] =
   ## returns the starting position and end position of `pattern` in `buf`
   ## (where `buf` has length `bufSize` and is not necessarily `'\0'` terminated),
   ## and the captured
@@ -200,7 +200,7 @@ proc findBounds*(s: string, pattern: Regex,
   result = findBounds(cstring(s), pattern, matches, start, s.len)
 
 proc findBounds*(buf: cstring, pattern: Regex,
-                 start = 0, bufSize = 0): tuple[first, last: int] =
+                 start = 0, bufSize: int): tuple[first, last: int] =
   ## returns the `first` and `last` position of `pattern` in `buf`,
   ## where `buf` has length `bufSize` (not necessarily `'\0'` terminated).
   ## If it does not match, `(-1,0)` is returned.
@@ -239,7 +239,7 @@ proc matchLen*(s: string, pattern: Regex, matches: var openArray[string],
   result = matchOrFind(cstring(s), pattern, matches, start.cint, s.len.cint, pcre.ANCHORED)
 
 proc matchLen*(buf: cstring, pattern: Regex, matches: var openArray[string],
-              start = 0, bufSize = 0): int {.inline.} =
+              start = 0, bufSize: int): int {.inline.} =
   ## the same as `match`, but it returns the length of the match,
   ## if there is no match, `-1` is returned. Note that a match length
   ## of zero can happen.
@@ -281,7 +281,7 @@ proc match*(s: string, pattern: Regex, matches: var openArray[string],
   result = matchLen(cstring(s), pattern, matches, start, s.len) != -1
 
 proc match*(buf: cstring, pattern: Regex, matches: var openArray[string],
-           start = 0, bufSize = 0): bool {.inline.} =
+           start = 0, bufSize: int): bool {.inline.} =
   ## returns `true` if `buf[start..
Date: Sun, 16 May 2021 10:03:22 -0700
Subject: [PATCH 0816/2472] more informative error msg for undeclared field
 (`A(badfield: 1)` and `a.badfield = expr`) (#17777)

---
 compiler/semcall.nim                | 15 +++++++++--
 compiler/semobjconstr.nim           |  3 ++-
 tests/errmsgs/tundeclared_field.nim | 40 +++++++++++++++++++++++++++++
 3 files changed, 55 insertions(+), 3 deletions(-)
 create mode 100644 tests/errmsgs/tundeclared_field.nim

diff --git a/compiler/semcall.nim b/compiler/semcall.nim
index 8f29bdf32..facaa89f7 100644
--- a/compiler/semcall.nim
+++ b/compiler/semcall.nim
@@ -411,8 +411,19 @@ proc resolveOverloads(c: PContext, n, orig: PNode,
 
     if overloadsState == csEmpty and result.state == csEmpty:
       if efNoUndeclared notin flags: # for tests/pragmas/tcustom_pragma.nim
-        # xxx adapt/use errorUndeclaredIdentifierHint(c, n, f.ident)
-        localError(c.config, n.info, getMsgDiagnostic(c, flags, n, f))
+        template impl() =
+          # xxx adapt/use errorUndeclaredIdentifierHint(c, n, f.ident)
+          localError(c.config, n.info, getMsgDiagnostic(c, flags, n, f))
+        if n[0].kind == nkIdent and n[0].ident.s == ".=" and n[2].kind == nkIdent:
+          let sym = n[1].typ.sym
+          if sym == nil:
+            impl()
+          else:
+            let field = n[2].ident.s
+            let msg = errUndeclaredField % field & " for type " & getProcHeader(c.config, sym)
+            localError(c.config, orig[2].info, msg)
+        else:
+          impl()
       return
     elif result.state != csMatch:
       if nfExprCall in n.flags:
diff --git a/compiler/semobjconstr.nim b/compiler/semobjconstr.nim
index 6f3cabcbf..16e3c6cee 100644
--- a/compiler/semobjconstr.nim
+++ b/compiler/semobjconstr.nim
@@ -429,7 +429,8 @@ proc semObjConstr(c: PContext, n: PNode, flags: TExprFlags): PNode =
           hasError = true
           break
       # 2) No such field exists in the constructed type
-      localError(c.config, field.info, errUndeclaredFieldX % id.s)
+      let msg = errUndeclaredField % id.s & " for type " & getProcHeader(c.config, t.sym)
+      localError(c.config, field.info, msg)
       hasError = true
       break
 
diff --git a/tests/errmsgs/tundeclared_field.nim b/tests/errmsgs/tundeclared_field.nim
new file mode 100644
index 000000000..2b274ae58
--- /dev/null
+++ b/tests/errmsgs/tundeclared_field.nim
@@ -0,0 +1,40 @@
+discard """
+cmd: '''nim check --hints:off $file'''
+action: reject
+nimout: '''
+tundeclared_field.nim(25, 12) Error: undeclared field: 'bad' for type tundeclared_field.A [type declared in tundeclared_field.nim(22, 8)]
+tundeclared_field.nim(30, 16) Error: undeclared field: 'bad' for type tundeclared_field.A [type declared in tundeclared_field.nim(28, 8)]
+tundeclared_field.nim(36, 4) Error: undeclared field: 'bad' for type tundeclared_field.A [type declared in tundeclared_field.nim(33, 8)]
+tundeclared_field.nim(40, 13) Error: cannot instantiate Foo [type declared in tundeclared_field.nim(39, 8)]
+'''
+"""
+
+
+
+
+
+
+
+
+
+# line 20
+block:
+  type A = object
+    a0: int
+  var a: A
+  discard a.bad
+
+block:
+  type A = object
+    a0: int
+  var a = A(bad: 0)
+
+block:
+  type A = object
+    a0: int
+  var a: A
+  a.bad = 0
+
+block:
+  type Foo[T: SomeInteger] = object
+  var a: Foo[float]

From 63fcb9e5f54fa6d3a65499abbed26a923f0dd337 Mon Sep 17 00:00:00 2001
From: Clyybber 
Date: Sun, 16 May 2021 19:43:52 +0200
Subject: [PATCH 0817/2472] Disable performance hints by default (verbosity 1)
 (#18024)

---
 compiler/lineinfos.nim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim
index 27d7b32e0..47cddb888 100644
--- a/compiler/lineinfos.nim
+++ b/compiler/lineinfos.nim
@@ -199,7 +199,7 @@ proc computeNotesVerbosity(): array[0..3, TNoteKinds] =
   result[2] = result[3] - {hintStackTrace, warnUninit, hintExtendedContext, hintDeclaredLoc}
   result[1] = result[2] - {warnProveField, warnProveIndex,
     warnGcUnsafe, hintPath, hintDependency, hintCodeBegin, hintCodeEnd,
-    hintSource, hintGlobalVar, hintGCStats, hintMsgOrigin}
+    hintSource, hintGlobalVar, hintGCStats, hintMsgOrigin, hintPerformance}
   result[0] = result[1] - {hintSuccessX, hintBuildMode, hintSuccess, hintConf,
     hintProcessing, hintPattern, hintExecuting, hintLinking, hintCC}
 

From d83b25db1ebe7025d95dbce3978219948f5c49d3 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Sun, 16 May 2021 14:54:10 -0700
Subject: [PATCH 0818/2472] fix #18007: std/json now serializes nan,inf,-inf as
 strings instead of invalid json (#18026)

* fix #18007: std/json now serializes nan,inf,-inf as raw strings instead of invalid json

* fix roundtrip

* fix tests

* fix changelog

* simplify

* add runnableExamples

* fix typo [skip ci]
---
 changelog.md                |   2 +
 lib/pure/json.nim           | 121 +++++++++++++++++++++---------------
 lib/std/jsonutils.nim       |   3 +
 tests/stdlib/tjson.nim      |  23 +++++++
 tests/stdlib/tjsonutils.nim |  21 ++++++-
 5 files changed, 119 insertions(+), 51 deletions(-)

diff --git a/changelog.md b/changelog.md
index 98a7a8c44..369b11917 100644
--- a/changelog.md
+++ b/changelog.md
@@ -70,6 +70,8 @@
 - `jsonutils` now serializes/deserializes holey enums as regular enums (via `ord`) instead of as strings.
   Use `-d:nimLegacyJsonutilsHoleyEnum` for a transition period.
 
+- `json` and `jsonutils` now serialize NaN, Inf, -Inf as strings, so that
+  `%[NaN, -Inf]` is the string `["nan","-inf"]` instead of `[nan,-inf]` which was invalid json.
 
 ## Standard library additions and changes
 - Added support for parenthesized expressions in `strformat`
diff --git a/lib/pure/json.nim b/lib/pure/json.nim
index 96ee61fd5..3a9fa5898 100644
--- a/lib/pure/json.nim
+++ b/lib/pure/json.nim
@@ -333,7 +333,16 @@ proc `%`*(n: BiggestInt): JsonNode =
 
 proc `%`*(n: float): JsonNode =
   ## Generic constructor for JSON data. Creates a new `JFloat JsonNode`.
-  result = JsonNode(kind: JFloat, fnum: n)
+  runnableExamples:
+    assert $(%[NaN, Inf, -Inf, 0.0, -0.0, 1.0, 1e-2]) == """["nan","inf","-inf",0.0,-0.0,1.0,0.01]"""
+    assert (%NaN).kind == JString
+    assert (%0.0).kind == JFloat
+  # for those special cases, we could also have used `newJRawNumber` but then
+  # it would've been inconsisten with the case of `parseJson` vs `%` for representing them.
+  if n != n: newJString("nan")
+  elif n == Inf: newJString("inf")
+  elif n == -Inf: newJString("-inf")
+  else: JsonNode(kind: JFloat, fnum: n)
 
 proc `%`*(b: bool): JsonNode =
   ## Generic constructor for JSON data. Creates a new `JBool JsonNode`.
@@ -662,6 +671,47 @@ proc escapeJson*(s: string): string =
   result = newStringOfCap(s.len + s.len shr 3)
   escapeJson(s, result)
 
+proc toUgly*(result: var string, node: JsonNode) =
+  ## Converts `node` to its JSON Representation, without
+  ## regard for human readability. Meant to improve `$` string
+  ## conversion performance.
+  ##
+  ## JSON representation is stored in the passed `result`
+  ##
+  ## This provides higher efficiency than the `pretty` procedure as it
+  ## does **not** attempt to format the resulting JSON to make it human readable.
+  var comma = false
+  case node.kind:
+  of JArray:
+    result.add "["
+    for child in node.elems:
+      if comma: result.add ","
+      else: comma = true
+      result.toUgly child
+    result.add "]"
+  of JObject:
+    result.add "{"
+    for key, value in pairs(node.fields):
+      if comma: result.add ","
+      else: comma = true
+      key.escapeJson(result)
+      result.add ":"
+      result.toUgly value
+    result.add "}"
+  of JString:
+    if node.isUnquoted:
+      result.add node.str
+    else:
+      escapeJson(node.str, result)
+  of JInt:
+    result.addInt(node.num)
+  of JFloat:
+    result.addFloat(node.fnum)
+  of JBool:
+    result.add(if node.bval: "true" else: "false")
+  of JNull:
+    result.add "null"
+
 proc toPretty(result: var string, node: JsonNode, indent = 2, ml = true,
               lstArr = false, currIndent = 0) =
   case node.kind
@@ -689,10 +739,7 @@ proc toPretty(result: var string, node: JsonNode, indent = 2, ml = true,
       result.add("{}")
   of JString:
     if lstArr: result.indent(currIndent)
-    if node.isUnquoted:
-      result.add node.str
-    else:
-      escapeJson(node.str, result)
+    toUgly(result, node)
   of JInt:
     if lstArr: result.indent(currIndent)
     result.addInt(node.num)
@@ -743,47 +790,6 @@ proc pretty*(node: JsonNode, indent = 2): string =
   result = ""
   toPretty(result, node, indent)
 
-proc toUgly*(result: var string, node: JsonNode) =
-  ## Converts `node` to its JSON Representation, without
-  ## regard for human readability. Meant to improve `$` string
-  ## conversion performance.
-  ##
-  ## JSON representation is stored in the passed `result`
-  ##
-  ## This provides higher efficiency than the `pretty` procedure as it
-  ## does **not** attempt to format the resulting JSON to make it human readable.
-  var comma = false
-  case node.kind:
-  of JArray:
-    result.add "["
-    for child in node.elems:
-      if comma: result.add ","
-      else: comma = true
-      result.toUgly child
-    result.add "]"
-  of JObject:
-    result.add "{"
-    for key, value in pairs(node.fields):
-      if comma: result.add ","
-      else: comma = true
-      key.escapeJson(result)
-      result.add ":"
-      result.toUgly value
-    result.add "}"
-  of JString:
-    if node.isUnquoted:
-      result.add node.str
-    else:
-      node.str.escapeJson(result)
-  of JInt:
-    result.addInt(node.num)
-  of JFloat:
-    result.addFloat(node.fnum)
-  of JBool:
-    result.add(if node.bval: "true" else: "false")
-  of JNull:
-    result.add "null"
-
 proc `$`*(node: JsonNode): string =
   ## Converts `node` to its JSON Representation on one line.
   result = newStringOfCap(node.len shl 1)
@@ -1087,11 +1093,26 @@ when defined(nimFixedForwardGeneric):
       dst = cast[T](jsonNode.num)
 
   proc initFromJson[T: SomeFloat](dst: var T; jsonNode: JsonNode; jsonPath: var string) =
-    verifyJsonKind(jsonNode, {JInt, JFloat}, jsonPath)
-    if jsonNode.kind == JFloat:
-      dst = T(jsonNode.fnum)
+    if jsonNode.kind == JString:
+      case jsonNode.str
+      of "nan":
+        let b = NaN
+        dst = T(b)
+        # dst = NaN # would fail some tests because range conversions would cause CT error
+        # in some cases; but this is not a hot-spot inside this branch and backend can optimize this.
+      of "inf":
+        let b = Inf
+        dst = T(b)
+      of "-inf":
+        let b = -Inf
+        dst = T(b)
+      else: raise newException(JsonKindError, "expected 'nan|inf|-inf', got " & jsonNode.str)
     else:
-      dst = T(jsonNode.num)
+      verifyJsonKind(jsonNode, {JInt, JFloat}, jsonPath)
+      if jsonNode.kind == JFloat:
+        dst = T(jsonNode.fnum)
+      else:
+        dst = T(jsonNode.num)
 
   proc initFromJson[T: enum](dst: var T; jsonNode: JsonNode; jsonPath: var string) =
     verifyJsonKind(jsonNode, {JString}, jsonPath)
diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim
index 99f5ce9c6..1f49f60ed 100644
--- a/lib/std/jsonutils.nim
+++ b/lib/std/jsonutils.nim
@@ -12,6 +12,9 @@ runnableExamples:
   let a = (1.5'f32, (b: "b2", a: "a2"), 'x', @[Foo(t: true, z1: -3), nil], [{"name": "John"}.newStringTable])
   let j = a.toJson
   assert j.jsonTo(typeof(a)).toJson == j
+  assert $[NaN, Inf, -Inf, 0.0, -0.0, 1.0, 1e-2].toJson == """["nan","inf","-inf",0.0,-0.0,1.0,0.01]"""
+  assert 0.0.toJson.kind == JFloat
+  assert Inf.toJson.kind == JString
 
 import json, strutils, tables, sets, strtabs, options
 
diff --git a/tests/stdlib/tjson.nim b/tests/stdlib/tjson.nim
index e538baf4f..e757e6c7e 100644
--- a/tests/stdlib/tjson.nim
+++ b/tests/stdlib/tjson.nim
@@ -8,14 +8,27 @@ Note: Macro tests are in tests/stdlib/tjsonmacro.nim
 ]#
 
 import std/[json,parsejson,strutils]
+from std/math import isNaN
 when not defined(js):
   import std/streams
 
 proc testRoundtrip[T](t: T, expected: string) =
+  # checks that `T => json => T2 => json2` is such that json2 = json
   let j = %t
   doAssert $j == expected, $j
   doAssert %(j.to(T)) == j
 
+proc testRoundtripVal[T](t: T, expected: string) =
+  # similar to testRoundtrip, but also checks that the `T => json => T2` is such that `T2 == T`
+  # note that this isn't always possible, e.g. for pointer-like types or nans
+  let j = %t
+  doAssert $j == expected, $j
+  let j2 = ($j).parseJson
+  doAssert $j2 == expected, $(j2, t)
+  let t2 = j2.to(T)
+  doAssert t2 == t
+  doAssert $(%* t2) == expected # sanity check, because -0.0 = 0.0 but their json representation differs
+
 let testJson = parseJson"""{ "a": [1, 2, 3, 4], "b": "asd", "c": "\ud83c\udf83", "d": "\u00E6"}"""
 # nil passthrough
 doAssert(testJson{"doesnt_exist"}{"anything"}.isNil)
@@ -301,6 +314,16 @@ block: # bug #17383
     testRoundtrip(int64.high): "9223372036854775807"
     testRoundtrip(uint64.high): "18446744073709551615"
 
+block: # bug #18007
+  testRoundtrip([NaN, Inf, -Inf, 0.0, -0.0, 1.0]): """["nan","inf","-inf",0.0,-0.0,1.0]"""
+  # pending https://github.com/nim-lang/Nim/issues/18025 use:
+  # testRoundtrip([float32(NaN), Inf, -Inf, 0.0, -0.0, 1.0])
+  let inf = float32(Inf)
+  testRoundtrip([float32(NaN), inf, -inf, 0.0, -0.0, 1.0]): """["nan","inf","-inf",0.0,-0.0,1.0]"""
+  when not defined(js): # because of Infinity vs inf
+    testRoundtripVal([inf, -inf, 0.0, -0.0, 1.0]): """["inf","-inf",0.0,-0.0,1.0]"""
+  let a = parseJson($(%NaN)).to(float)
+  doAssert a.isNaN
 
 block:
   let a = "18446744073709551615"
diff --git a/tests/stdlib/tjsonutils.nim b/tests/stdlib/tjsonutils.nim
index 18d0aa325..c826a79b0 100644
--- a/tests/stdlib/tjsonutils.nim
+++ b/tests/stdlib/tjsonutils.nim
@@ -4,15 +4,28 @@ discard """
 
 import std/jsonutils
 import std/json
+from std/math import isNaN
 
 proc testRoundtrip[T](t: T, expected: string) =
+  # checks that `T => json => T2 => json2` is such that json2 = json
   let j = t.toJson
-  doAssert $j == expected, $j
+  doAssert $j == expected, "\n" & $j & "\n" & expected
   doAssert j.jsonTo(T).toJson == j
   var t2: T
   t2.fromJson(j)
   doAssert t2.toJson == j
 
+proc testRoundtripVal[T](t: T, expected: string) =
+  # similar to testRoundtrip, but also checks that the `T => json => T2` is such that `T2 == T`
+  # note that this isn't always possible, e.g. for pointer-like types.
+  let j = t.toJson
+  let j2 = $j
+  doAssert j2 == expected, j2
+  let j3 = j2.parseJson
+  let t2 = j3.jsonTo(T)
+  doAssert t2 == t
+  doAssert $t2.toJson == j2 # still needed, because -0.0 = 0.0 but their json representation differs
+
 import tables, sets, algorithm, sequtils, options, strtabs
 from strutils import contains
 
@@ -91,6 +104,12 @@ template fn() =
       else:
         testRoundtrip(a): "[9223372036854775807,18446744073709551615]"
 
+  block: # bug #18007
+    testRoundtrip((NaN, Inf, -Inf, 0.0, -0.0, 1.0)): """["nan","inf","-inf",0.0,-0.0,1.0]"""
+    testRoundtrip((float32(NaN), Inf, -Inf, 0.0, -0.0, 1.0)): """["nan","inf","-inf",0.0,-0.0,1.0]"""
+    testRoundtripVal((Inf, -Inf, 0.0, -0.0, 1.0)): """["inf","-inf",0.0,-0.0,1.0]"""
+    doAssert ($NaN.toJson).parseJson.jsonTo(float).isNaN
+
   block: # case object
     type Foo = object
       x0: float

From 2096490b5974cfb11d0f5d37ff6d3a963ef43713 Mon Sep 17 00:00:00 2001
From: Andrey Makarov 
Date: Mon, 17 May 2021 00:55:14 +0300
Subject: [PATCH 0819/2472] follow-up #17930 - inline syntax highlighting
 (#18013)

* follow-up #17930 - inline syntax highlighting

* make closure->nimcall
---
 doc/gc.rst                    | 35 ++++++++++++++++++++++-------------
 lib/packages/docutils/rst.nim | 25 ++++++++++++++-----------
 tests/stdlib/trst.nim         | 21 +++++++++++++++++++++
 3 files changed, 57 insertions(+), 24 deletions(-)

diff --git a/doc/gc.rst b/doc/gc.rst
index 4455afcbe..5de02f73d 100644
--- a/doc/gc.rst
+++ b/doc/gc.rst
@@ -1,9 +1,10 @@
-.. default-role:: code
-
 =======================
 Nim's Memory Management
 =======================
 
+.. default-role:: code
+.. include:: rstcommon.rst
+
 :Author: Andreas Rumpf
 :Version: |nimversion|
 
@@ -29,24 +30,28 @@ and how the memory management strategies other than garbage collectors work.
 Multi-paradigm Memory Management Strategies
 ===========================================
 
+.. default-role:: option
+
 To choose the memory management strategy use the `--gc:` switch.
 
-- `--gc:refc`. This is the default GC. It's a
+--gc:refc    This is the default GC. It's a
   deferred reference counting based garbage collector
   with a simple Mark&Sweep backup GC in order to collect cycles. Heaps are thread-local.
-- `--gc:markAndSweep`. Simple Mark-And-Sweep based garbage collector. Heaps are thread-local.
-- `--gc:boehm`. Boehm based garbage collector, it offers a shared heap.
-- `--gc:go`. Go's garbage collector, useful for interoperability with Go. Offers a shared heap.
-- `--gc:arc`. Plain reference counting with
+--gc:markAndSweep  Simple Mark-And-Sweep based garbage collector.
+  Heaps are thread-local.
+--gc:boehm    Boehm based garbage collector, it offers a shared heap.
+--gc:go    Go's garbage collector, useful for interoperability with Go.
+  Offers a shared heap.
+--gc:arc    Plain reference counting with
   `move semantic optimizations `_, offers a shared heap.
   It offers deterministic performance for `hard realtime`:idx: systems. Reference cycles
   cause memory leaks, beware.
 
-- `--gc:orc`. Same as `--gc:arc` but adds a cycle collector based on "trial deletion".
+--gc:orc    Same as `--gc:arc` but adds a cycle collector based on "trial deletion".
   Unfortunately, that makes its performance profile hard to reason about so it is less
   useful for hard real-time systems.
 
-- `--gc:none`. No memory management strategy nor a garbage collector. Allocated memory is
+--gc:none    No memory management strategy nor a garbage collector. Allocated memory is
   simply never freed. You should use `--gc:arc` instead.
 
 
@@ -62,6 +67,9 @@ Go                 Shared   Cycle Collector   Yes            `--gc:go`
 None               Manual   Manual            Manual         `--gc:none`
 ================== ======== ================= ============== ===================
 
+.. default-role:: code
+.. include:: rstcommon.rst
+
 JavaScript's garbage collector is used for the `JavaScript and NodeJS
 `_ compilation targets.
 The `NimScript `_ target uses the memory management strategy built into
@@ -82,7 +90,7 @@ Soft real-time support
 ----------------------
 
 To enable real-time support, the symbol `useRealtimeGC`:idx: needs to be
-defined via `--define:useRealtimeGC` (you can put this into your config
+defined via `--define:useRealtimeGC`:option: (you can put this into your config
 file as well).
 With this switch the garbage collector supports the following operations:
 
@@ -132,7 +140,7 @@ Time measurement with garbage collectors
 ----------------------------------------
 
 The garbage collectors' way of measuring time uses
-(see `lib/system/timers.nim` for the implementation):
+(see ``lib/system/timers.nim`` for the implementation):
 
 1) `QueryPerformanceCounter` and `QueryPerformanceFrequency` on Windows.
 2) `mach_absolute_time` on Mac OS X.
@@ -170,7 +178,7 @@ Other useful procs from `system `_ you can use to keep track of mem
 * `GC_getStatistics()` Garbage collector statistics as a human-readable string.
 
 These numbers are usually only for the running thread, not for the whole heap,
-with the exception of `--gc:boehm` and `--gc:go`.
+with the exception of `--gc:boehm`:option: and `--gc:go`:option:.
 
 In addition to `GC_ref` and `GC_unref` you can avoid the garbage collector by manually
 allocating memory with procs like `alloc`, `alloc0`, `allocShared`, `allocShared0` or `allocCStringArray`.
@@ -184,7 +192,8 @@ Heap dump
 
 The heap dump feature is still in its infancy, but it already proved
 useful for us, so it might be useful for you. To get a heap dump, compile
-with `-d:nimTypeNames` and call `dumpNumberOfInstances` at a strategic place in your program.
+with `-d:nimTypeNames`:option: and call `dumpNumberOfInstances`
+at a strategic place in your program.
 This produces a list of the used types in your program and for every type
 the total amount of object instances for this type as well as the total
 amount of bytes these instances take up.
diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim
index 9cbc4efb7..da04e9b54 100644
--- a/lib/packages/docutils/rst.nim
+++ b/lib/packages/docutils/rst.nim
@@ -1151,6 +1151,18 @@ proc toOtherRole(n: PRstNode, kind: RstNodeKind, roleName: string): PRstNode =
 proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode =
   var newKind = n.kind
   var newSons = n.sons
+
+  proc finalizeInterpreted(node: PRstNode, newKind: RstNodeKind,
+                           newSons: seq[PRstNode], roleName: string):
+                          PRstNode {.nimcall.} =
+    # fixes interpreted text (`x` or `y`:role:) to proper internal AST format
+    if newKind in {rnUnknownRole, rnCodeFragment}:
+      result = node.toOtherRole(newKind, roleName)
+    elif newKind == rnInlineCode:
+      result = node.toInlineCode(language=roleName)
+    else:
+      result = newRstNode(newKind, newSons)
+
   if isInlineMarkupEnd(p, "_", exact=true) or
       isInlineMarkupEnd(p, "__", exact=true):
     inc p.idx
@@ -1175,19 +1187,10 @@ proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode =
     # a role:
     let (roleName, lastIdx) = getRefname(p, p.idx+1)
     newKind = whichRole(p, roleName)
-    if newKind in {rnUnknownRole, rnCodeFragment}:
-      result = n.toOtherRole(newKind, roleName)
-    elif newKind == rnInlineCode:
-      result = n.toInlineCode(language=roleName)
-    else:
-      result = newRstNode(newKind, newSons)
+    result = n.finalizeInterpreted(newKind, newSons, roleName)
     p.idx = lastIdx + 2
   else:
-    if p.s.currRoleKind == rnInlineCode:
-      result = n.toInlineCode(language=p.s.currRole)
-    else:
-      newKind = p.s.currRoleKind
-      result = newRstNode(newKind, newSons)
+    result = n.finalizeInterpreted(p.s.currRoleKind, newSons, p.s.currRole)
 
 proc matchVerbatim(p: RstParser, start: int, expr: string): int =
   result = start
diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim
index e6c5d9700..71f5a858b 100644
--- a/tests/stdlib/trst.nim
+++ b/tests/stdlib/trst.nim
@@ -371,6 +371,27 @@ suite "RST inline markup":
         rnLeaf  '```'
       """)
 
+  test "interpreted text parsing: code fragments":
+    check(dedent"""
+        .. default-role:: option
+
+        `--gc:refc`""".toAst ==
+      dedent"""
+        rnInner
+          rnDefaultRole
+            rnDirArg
+              rnLeaf  'option'
+            [nil]
+            [nil]
+          rnParagraph
+            rnCodeFragment
+              rnInner
+                rnLeaf  '--'
+                rnLeaf  'gc'
+                rnLeaf  ':'
+                rnLeaf  'refc'
+              rnLeaf  'option'
+        """)
 
   test """interpreted text can be ended with \` """:
     let output = (".. default-role:: literal\n" & """`\``""").toAst

From e904c6d87ccbb453f6f1d649b2c715bd3cebc815 Mon Sep 17 00:00:00 2001
From: Joey 
Date: Mon, 17 May 2021 00:04:39 -0600
Subject: [PATCH 0820/2472] Add `checked` to dom (#18033)

This allows the ability to set a checkbox as checked programmatically. It's different from `setAttribute` because once an input has been clicked on by the user, `setAttribute` no longer works programmatically.
---
 lib/js/dom.nim | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/js/dom.nim b/lib/js/dom.nim
index 303cc178f..16fc96b68 100644
--- a/lib/js/dom.nim
+++ b/lib/js/dom.nim
@@ -1379,6 +1379,9 @@ proc `class=`*(n: Node; v: cstring) {.importcpp: "#.className = #", nodecl.}
 proc value*(n: Node): cstring {.importcpp: "#.value", nodecl.}
 proc `value=`*(n: Node; v: cstring) {.importcpp: "#.value = #", nodecl.}
 
+proc checked*(n: Node): bool {.importcpp: "#.checked", nodecl.}
+proc `checked=`*(n: Node; v: bool) {.importcpp: "#.checked = #", nodecl.}
+
 proc `disabled=`*(n: Node; v: bool) {.importcpp: "#.disabled = #", nodecl.}
 
 when defined(nodejs):

From 8be5344b3bb74d62cff5aa6a18f4223680612ad5 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Mon, 17 May 2021 01:28:52 -0700
Subject: [PATCH 0821/2472] ./koch --nonexistant now fails (#18036)

---
 koch.nim | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/koch.nim b/koch.nim
index 15355ec36..3d7276aee 100644
--- a/koch.nim
+++ b/koch.nim
@@ -634,9 +634,9 @@ proc valgrind(cmd: string) =
   let supp = getAppDir() / "tools" / "nimgrind.supp"
   exec("valgrind --suppressions=" & supp & valcmd)
 
-proc showHelp() =
+proc showHelp(success: bool) =
   quit(HelpText % [VersionAsString & spaces(44-len(VersionAsString)),
-                   CompileDate, CompileTime], QuitSuccess)
+                   CompileDate, CompileTime], if success: QuitSuccess else: QuitFailure)
 
 proc branchDone() =
   let thisBranch = execProcess("git symbolic-ref --short HEAD").strip()
@@ -656,6 +656,7 @@ when isMainModule:
     case op.kind
     of cmdLongOption, cmdShortOption:
       case normalize(op.key)
+      of "help", "h": showHelp(success = true)
       of "latest": latest = true
       of "stable": latest = false
       of "nim": nimExe = op.val.absolutePath # absolute so still works with changeDir
@@ -663,7 +664,7 @@ when isMainModule:
         localDocsOnly = true
         if op.val.len > 0:
           localDocsOut = op.val.absolutePath
-      else: showHelp()
+      else: showHelp(success = false)
     of cmdArgument:
       case normalize(op.key)
       of "boot": boot(op.cmdLineRest)
@@ -705,6 +706,6 @@ when isMainModule:
         exec("nimble install -y fusion@$#" % suffix)
       of "ic": icTest(op.cmdLineRest)
       of "branchdone": branchDone()
-      else: showHelp()
+      else: showHelp(success = false)
       break
     of cmdEnd: break

From fac5bae7b7d87aeec48c7252029c2852ee157ac9 Mon Sep 17 00:00:00 2001
From: Clyybber 
Date: Mon, 17 May 2021 13:48:10 +0200
Subject: [PATCH 0822/2472] Fix running testament c nimble-packages without
 batch arg (#18023)

* Fix running testament c nimble-packages without batch arg

* Fix
---
 testament/categories.nim | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/testament/categories.nim b/testament/categories.nim
index 1253a2b8c..1f0bd6acf 100644
--- a/testament/categories.nim
+++ b/testament/categories.nim
@@ -424,11 +424,12 @@ proc listPackages(packageFilter: string): seq[NimblePackage] =
     # at least should be a regex; a substring match makes no sense.
     result = pkgs.filterIt(packageFilter in it.name)
   else:
-    let pkgs1 = pkgs.filterIt(it.allowFailure)
-    let pkgs2 = pkgs.filterIt(not it.allowFailure)
     if testamentData0.batchArg == "allowed_failures":
-      result = pkgs1
+      result = pkgs.filterIt(it.allowFailure)
+    elif testamentData0.testamentNumBatch == 0:
+      result = pkgs
     else:
+      let pkgs2 = pkgs.filterIt(not it.allowFailure)
       for i in 0..
Date: Tue, 18 May 2021 09:16:07 +0300
Subject: [PATCH 0823/2472] add sink and lent annotations for xmltree and
 streams (#18037)

---
 lib/pure/streams.nim |  2 +-
 lib/pure/xmltree.nim | 30 +++++++++++++++---------------
 2 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim
index 3cc5ea038..7dc81148f 100644
--- a/lib/pure/streams.nim
+++ b/lib/pure/streams.nim
@@ -1252,7 +1252,7 @@ else: # after 1.3 or JS not defined
     var s = StringStream(s)
     s.data = ""
 
-  proc newStringStream*(s: string = ""): owned StringStream =
+  proc newStringStream*(s: sink string = ""): owned StringStream =
     ## Creates a new stream from the string `s`.
     ##
     ## See also:
diff --git a/lib/pure/xmltree.nim b/lib/pure/xmltree.nim
index af7e4d508..fdb2c3fc5 100644
--- a/lib/pure/xmltree.nim
+++ b/lib/pure/xmltree.nim
@@ -70,7 +70,7 @@ proc newXmlNode(kind: XmlNodeKind): XmlNode =
   ## Creates a new ``XmlNode``.
   result = XmlNode(k: kind)
 
-proc newElement*(tag: string): XmlNode =
+proc newElement*(tag: sink string): XmlNode =
   ## Creates a new ``XmlNode`` of kind ``xnElement`` with the given `tag`.
   ##
   ## See also:
@@ -89,7 +89,7 @@ proc newElement*(tag: string): XmlNode =
   result.s = @[]
   # init attributes lazily to save memory
 
-proc newText*(text: string): XmlNode =
+proc newText*(text: sink string): XmlNode =
   ## Creates a new ``XmlNode`` of kind ``xnText`` with the text `text`.
   runnableExamples:
     var b = newText("my text")
@@ -99,13 +99,13 @@ proc newText*(text: string): XmlNode =
   result = newXmlNode(xnText)
   result.fText = text
 
-proc newVerbatimText*(text: string): XmlNode {.since: (1, 3).} =
+proc newVerbatimText*(text: sink string): XmlNode {.since: (1, 3).} =
   ## Creates a new ``XmlNode`` of kind ``xnVerbatimText`` with the text `text`.
   ## **Since**: Version 1.3.
   result = newXmlNode(xnVerbatimText)
   result.fText = text
 
-proc newComment*(comment: string): XmlNode =
+proc newComment*(comment: sink string): XmlNode =
   ## Creates a new ``XmlNode`` of kind ``xnComment`` with the text `comment`.
   runnableExamples:
     var c = newComment("my comment")
@@ -115,7 +115,7 @@ proc newComment*(comment: string): XmlNode =
   result = newXmlNode(xnComment)
   result.fText = comment
 
-proc newCData*(cdata: string): XmlNode =
+proc newCData*(cdata: sink string): XmlNode =
   ## Creates a new ``XmlNode`` of kind ``xnCData`` with the text `cdata`.
   runnableExamples:
     var d = newCData("my cdata")
@@ -135,7 +135,7 @@ proc newEntity*(entity: string): XmlNode =
   result = newXmlNode(xnEntity)
   result.fText = entity
 
-proc newXmlTree*(tag: string, children: openArray[XmlNode],
+proc newXmlTree*(tag: sink string, children: openArray[XmlNode],
                  attributes: XmlAttributes = nil): XmlNode =
   ## Creates a new XML tree with `tag`, `children` and `attributes`.
   ##
@@ -151,7 +151,7 @@ proc newXmlTree*(tag: string, children: openArray[XmlNode],
     h.add newEntity("some entity")
     let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes
     let k = newXmlTree("treeTag", [g, h], att)
-  
+
     doAssert $k == """
   some text
   &some entity;
@@ -163,7 +163,7 @@ proc newXmlTree*(tag: string, children: openArray[XmlNode],
   for i in 0..children.len-1: result.s[i] = children[i]
   result.fAttr = attributes
 
-proc text*(n: XmlNode): string {.inline.} =
+proc text*(n: XmlNode): lent string {.inline.} =
   ## Gets the associated text with the node `n`.
   ##
   ## `n` can be a CDATA, Text, comment, or entity node.
@@ -181,7 +181,7 @@ proc text*(n: XmlNode): string {.inline.} =
   assert n.k in {xnText, xnComment, xnCData, xnEntity}
   result = n.fText
 
-proc `text=`*(n: XmlNode, text: string){.inline.} =
+proc `text=`*(n: XmlNode, text: sink string) {.inline.} =
   ## Sets the associated text with the node `n`.
   ##
   ## `n` can be a CDATA, Text, comment, or entity node.
@@ -199,7 +199,7 @@ proc `text=`*(n: XmlNode, text: string){.inline.} =
   assert n.k in {xnText, xnComment, xnCData, xnEntity}
   n.fText = text
 
-proc tag*(n: XmlNode): string {.inline.} =
+proc tag*(n: XmlNode): lent string {.inline.} =
   ## Gets the tag name of `n`.
   ##
   ## `n` has to be an ``xnElement`` node.
@@ -220,7 +220,7 @@ proc tag*(n: XmlNode): string {.inline.} =
   assert n.k == xnElement
   result = n.fTag
 
-proc `tag=`*(n: XmlNode, tag: string) {.inline.} =
+proc `tag=`*(n: XmlNode, tag: sink string) {.inline.} =
   ## Sets the tag name of `n`.
   ##
   ## `n` has to be an ``xnElement`` node.
@@ -389,13 +389,13 @@ proc clear*(n: var XmlNode) =
     var g = newElement("myTag")
     g.add newText("some text")
     g.add newComment("this is comment")
-  
+
     var h = newElement("secondTag")
     h.add newEntity("some entity")
-  
+
     let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes
     var k = newXmlTree("treeTag", [g, h], att)
-  
+
     doAssert $k == """
   some text
   &some entity;
@@ -447,7 +447,7 @@ proc toXmlAttributes*(keyValuePairs: varargs[tuple[key,
     let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes
     var j = newElement("myTag")
     j.attrs = att
-  
+
     doAssert $j == """"""
 
   newStringTable(keyValuePairs)

From 7f077a76fea389ce4f55e18857c499fadb3958fa Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Tue, 18 May 2021 06:10:19 -0700
Subject: [PATCH 0824/2472] jsonutils: add customization for toJson via
 `ToJsonOptions`; generalize symbolName; add symbolRank (#18029)

* jsonutils: add customization for toJson via `ToJsonOptions`

* add enumutils.symbolRank

* lookup table implementation for HoleyEnum

* cleanup

* changelog

* fixup

* Update lib/std/jsonutils.nim

Co-authored-by: Andreas Rumpf 
---
 changelog.md                |  5 ++-
 lib/std/enumutils.nim       | 65 +++++++++++++++++++++++++++++++++++--
 lib/std/jsonutils.nim       | 43 +++++++++++++++++-------
 tests/stdlib/tjsonutils.nim | 13 ++++++++
 4 files changed, 112 insertions(+), 14 deletions(-)

diff --git a/changelog.md b/changelog.md
index 369b11917..97e749337 100644
--- a/changelog.md
+++ b/changelog.md
@@ -118,9 +118,11 @@
 - `json.%`,`json.to`, `jsonutils.formJson`,`jsonutils.toJson` now work with `uint|uint64`
   instead of raising (as in 1.4) or giving wrong results (as in 1.2).
 
+- `jsonutils` now handles `cstring` (including as Table key), and `set`.
+
 - added `jsonutils.jsonTo` overload with `opt = Joptions()` param.
 
-- `jsonutils` now handles `cstring` (including as Table key), and `set`.
+- `jsonutils.toJson` now supports customization via `ToJsonOptions`.
 
 - Added an overload for the `collect` macro that inferes the container type based
   on the syntax of the last expression. Works with std seqs, tables and sets.
@@ -138,6 +140,7 @@
 - Added `std/enumutils` module. Added `genEnumCaseStmt` macro that generates case statement to parse string to enum.
   Added `items` for enums with holes.
   Added `symbolName` to return the enum symbol name ignoring the human readable name.
+  Added `symbolRank` to return the index in which an enum member is listed in an enum.
 
 - Added `typetraits.HoleyEnum` for enums with holes, `OrdinalEnum` for enums without holes.
 
diff --git a/lib/std/enumutils.nim b/lib/std/enumutils.nim
index 6195ae07d..09cf24f51 100644
--- a/lib/std/enumutils.nim
+++ b/lib/std/enumutils.nim
@@ -86,8 +86,67 @@ iterator items*[T: HoleyEnum](E: typedesc[T]): T =
     assert B[float].toSeq == [B[float].b0, B[float].b1]
   for a in enumFullRange(E): yield a
 
-func symbolName*[T: OrdinalEnum](a: T): string =
+func span(T: typedesc[HoleyEnum]): int =
+  (T.high.ord - T.low.ord) + 1
+
+const invalidSlot = uint8.high
+
+proc genLookup[T: typedesc[HoleyEnum]](_: T): auto =
+  const n = span(T)
+  var ret: array[n, uint8]
+  var i = 0
+  assert n <= invalidSlot.int
+  for ai in mitems(ret): ai = invalidSlot
+  for ai in items(T):
+    ret[ai.ord - T.low.ord] = uint8(i)
+    inc(i)
+  return ret
+
+func symbolRankImpl[T](a: T): int {.inline.} =
+  const n = T.span
+  const thres = 255 # must be <= `invalidSlot`, but this should be tuned.
+  when n <= thres:
+    const lookup = genLookup(T)
+    let lookup2 {.global.} = lookup # xxx improve pending https://github.com/timotheecour/Nim/issues/553
+    #[
+    This could be optimized using a hash adapted to `T` (possible since it's known at CT)
+    to get better key distribution before indexing into the lookup table table.
+    ]#
+    {.noSideEffect.}: # because it's immutable
+      let ret = lookup2[ord(a) - T.low.ord]
+    if ret != invalidSlot: return ret.int
+  else:
+    var i = 0
+    # we could also generate a case statement as optimization
+    for ai in items(T):
+      if ai == a: return i
+      inc(i)
+  raise newException(IndexDefect, $ord(a) & " invalid for " & $T)
+
+template symbolRank*[T: enum](a: T): int =
+  ## Returns the index in which `a` is listed in `T`.
+  ##
+  ## The cost for a `HoleyEnum` is implementation defined, currently optimized
+  ## for small enums, otherwise is `O(T.enumLen)`.
+  runnableExamples:
+    type
+      A = enum a0 = -3, a1 = 10, a2, a3 = (20, "f3Alt") # HoleyEnum
+      B = enum b0, b1, b2 # OrdinalEnum
+      C = enum c0 = 10, c1, c2 # OrdinalEnum
+    assert a2.symbolRank == 2
+    assert b2.symbolRank == 2
+    assert c2.symbolRank == 2
+    assert c2.ord == 12
+    assert a2.ord == 11
+    var invalid = 7.A
+    doAssertRaises(IndexDefect): discard invalid.symbolRank
+  when T is Ordinal: ord(a) - T.low.ord.static
+  else: symbolRankImpl(a)
+
+func symbolName*[T: enum](a: T): string =
   ## Returns the symbol name of an enum.
+  ##
+  ## This uses `symbolRank`.
   runnableExamples:
     type B = enum
       b0 = (10, "kb0")
@@ -97,5 +156,7 @@ func symbolName*[T: OrdinalEnum](a: T): string =
     assert b.symbolName == "b0"
     assert $b == "kb0"
     static: assert B.high.symbolName == "b2"
+    type C = enum c0 = -3, c1 = 4, c2 = 20 # HoleyEnum
+    assert c1.symbolName == "c1"
   const names = enumNames(T)
-  names[a.ord - T.low.ord]
+  names[a.symbolRank]
diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim
index 1f49f60ed..1e222e3a2 100644
--- a/lib/std/jsonutils.nim
+++ b/lib/std/jsonutils.nim
@@ -31,9 +31,11 @@ add a way to customize serialization, for e.g.:
 ]#
 
 import macros
+from enumutils import symbolName
+from typetraits import OrdinalEnum
 
 type
-  Joptions* = object
+  Joptions* = object # xxx rename FromJsonOptions
     ## Options controlling the behavior of `fromJson`.
     allowExtraKeys*: bool
       ## If `true` Nim's object to which the JSON is parsed is not required to
@@ -42,6 +44,17 @@ type
       ## If `true` Nim's object to which JSON is parsed is allowed to have
       ## fields without corresponding JSON keys.
     # in future work: a key rename could be added
+  EnumMode* = enum
+    joptEnumOrd
+    joptEnumSymbol
+    joptEnumString
+  ToJsonOptions* = object
+    enumMode*: EnumMode
+    # xxx charMode
+
+proc initToJsonOptions*(): ToJsonOptions =
+  ## initializes `ToJsonOptions` with sane options.
+  ToJsonOptions(enumMode: joptEnumOrd)
 
 proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".}
 proc distinctBase(T: typedesc): typedesc {.magic: "TypeTrait".}
@@ -261,33 +274,41 @@ proc jsonTo*(b: JsonNode, T: typedesc, opt = Joptions()): T =
   ## reverse of `toJson`
   fromJson(result, b, opt)
 
-proc toJson*[T](a: T): JsonNode =
+proc toJson*[T](a: T, opt = initToJsonOptions()): JsonNode =
   ## serializes `a` to json; uses `toJsonHook(a: T)` if it's in scope to
   ## customize serialization, see strtabs.toJsonHook for an example.
   when compiles(toJsonHook(a)): result = toJsonHook(a)
   elif T is object | tuple:
     when T is object or isNamedTuple(T):
       result = newJObject()
-      for k, v in a.fieldPairs: result[k] = toJson(v)
+      for k, v in a.fieldPairs: result[k] = toJson(v, opt)
     else:
       result = newJArray()
-      for v in a.fields: result.add toJson(v)
+      for v in a.fields: result.add toJson(v, opt)
   elif T is ref | ptr:
     if system.`==`(a, nil): result = newJNull()
-    else: result = toJson(a[])
+    else: result = toJson(a[], opt)
   elif T is array | seq | set:
     result = newJArray()
-    for ai in a: result.add toJson(ai)
-  elif T is pointer: result = toJson(cast[int](a))
+    for ai in a: result.add toJson(ai, opt)
+  elif T is pointer: result = toJson(cast[int](a), opt)
     # edge case: `a == nil` could've also led to `newJNull()`, but this results
     # in simpler code for `toJson` and `fromJson`.
-  elif T is distinct: result = toJson(a.distinctBase)
+  elif T is distinct: result = toJson(a.distinctBase, opt)
   elif T is bool: result = %(a)
   elif T is SomeInteger: result = %a
-  elif T is Ordinal: result = %(a.ord)
   elif T is enum:
-    when defined(nimLegacyJsonutilsHoleyEnum): result = %a
-    else: result = %(a.ord)
+    case opt.enumMode
+    of joptEnumOrd:
+      when T is Ordinal or not defined(nimLegacyJsonutilsHoleyEnum): %(a.ord)
+      else: toJson($a, opt)
+    of joptEnumSymbol:
+      when T is OrdinalEnum:
+        toJson(symbolName(a), opt)
+      else:
+        toJson($a, opt)
+    of joptEnumString: toJson($a, opt)
+  elif T is Ordinal: result = %(a.ord)
   elif T is cstring: (if a == nil: result = newJNull() else: result = % $a)
   else: result = %a
 
diff --git a/tests/stdlib/tjsonutils.nim b/tests/stdlib/tjsonutils.nim
index c826a79b0..a55b0ca1d 100644
--- a/tests/stdlib/tjsonutils.nim
+++ b/tests/stdlib/tjsonutils.nim
@@ -35,6 +35,13 @@ type Foo = ref object
 proc `==`(a, b: Foo): bool =
   a.id == b.id
 
+type MyEnum = enum me0, me1 = "me1Alt", me2, me3, me4
+
+proc `$`(a: MyEnum): string =
+  # putting this here pending https://github.com/nim-lang/Nim/issues/13747
+  if a == me2: "me2Modif"
+  else: system.`$`(a)
+
 template fn() = 
   block: # toJson, jsonTo
     type Foo = distinct float
@@ -83,6 +90,12 @@ template fn() =
     doAssert b2.ord == 1 # explains the `1`
     testRoundtrip(a): """[1,2,3]"""
 
+  block: # ToJsonOptions
+    let a = (me1, me2)
+    doAssert $a.toJson() == "[1,2]"
+    doAssert $a.toJson(ToJsonOptions(enumMode: joptEnumSymbol)) == """["me1","me2"]"""
+    doAssert $a.toJson(ToJsonOptions(enumMode: joptEnumString)) == """["me1Alt","me2Modif"]"""
+
   block: # set
     type Foo = enum f1, f2, f3, f4, f5
     type Goo = enum g1 = 10, g2 = 15, g3 = 17, g4

From 31143c68fc7438406df39207b30594e775b4f97e Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Tue, 18 May 2021 10:48:13 -0700
Subject: [PATCH 0825/2472] disable pkg/fidget refs
 https://github.com/treeform/fidget/issues/155 (#18043)

---
 testament/important_packages.nim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/testament/important_packages.nim b/testament/important_packages.nim
index 1a61d3bbf..b84155eee 100644
--- a/testament/important_packages.nim
+++ b/testament/important_packages.nim
@@ -61,7 +61,7 @@ pkg "delaunay"
 pkg "docopt"
 pkg "easygl", "nim c -o:egl -r src/easygl.nim", "https://github.com/jackmott/easygl"
 pkg "elvis"
-pkg "fidget"
+pkg "fidget", allowFailure = true # pending https://github.com/treeform/fidget/issues/155
 pkg "fragments", "nim c -r fragments/dsl.nim"
 pkg "fusion"
 pkg "gara"

From 53935b8b27643215b677cc152f563a7fb7cb84fb Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Tue, 18 May 2021 21:45:37 +0200
Subject: [PATCH 0826/2472] ARC: fixes memory leaks with newSeq used in a loop
 [backport:1.4] (#18040)

* ARC: fixes memory leaks with newSeq used in a loop [backport:1.4]
* Update tests/arc/tnewseq_legacy.nim
---
 compiler/ccgexprs.nim        |  7 ++++++-
 lib/system/seqs_v2.nim       |  4 ++++
 tests/arc/tnewseq_legacy.nim | 13 +++++++++++++
 3 files changed, 23 insertions(+), 1 deletion(-)
 create mode 100644 tests/arc/tnewseq_legacy.nim

diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim
index b5ad24cc1..7cb403a94 100644
--- a/compiler/ccgexprs.nim
+++ b/compiler/ccgexprs.nim
@@ -2319,7 +2319,12 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
       gcUsage(p.config, e)
     else:
       genNewFinalize(p, e)
-  of mNewSeq: genNewSeq(p, e)
+  of mNewSeq:
+    if optSeqDestructors in p.config.globalOptions:
+      e[1] = makeAddr(e[1], p.module.idgen)
+      genCall(p, e, d)
+    else:
+      genNewSeq(p, e)
   of mNewSeqOfCap: genNewSeqOfCap(p, e, d)
   of mSizeOf:
     let t = e[1].typ.skipTypes({tyTypeDesc})
diff --git a/lib/system/seqs_v2.nim b/lib/system/seqs_v2.nim
index b7f24ecd5..375eef340 100644
--- a/lib/system/seqs_v2.nim
+++ b/lib/system/seqs_v2.nim
@@ -124,3 +124,7 @@ proc setLen[T](s: var seq[T], newlen: Natural) =
       if xu.p == nil or xu.p.cap < newlen:
         xu.p = cast[typeof(xu.p)](prepareSeqAdd(oldLen, xu.p, newlen - oldLen, sizeof(T), alignof(T)))
       xu.len = newlen
+
+proc newSeq[T](s: var seq[T], len: Natural) =
+  shrink(s, 0)
+  setLen(s, len)
diff --git a/tests/arc/tnewseq_legacy.nim b/tests/arc/tnewseq_legacy.nim
new file mode 100644
index 000000000..4730d2c2b
--- /dev/null
+++ b/tests/arc/tnewseq_legacy.nim
@@ -0,0 +1,13 @@
+discard """
+  output: "(allocCount: 201, deallocCount: 201)"
+  cmd: "nim c --gc:orc -d:nimAllocStats $file"
+"""
+
+proc main(prefix: string) =
+  var c: seq[string]
+  for i in 0..<100:
+    newSeq(c, 100)
+    c[i] = prefix & $i
+
+main("abc")
+echo getAllocStats()

From 558644725ddcb72ae4e0b34ad5cfb1d38e2c5385 Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Wed, 19 May 2021 16:00:14 +0200
Subject: [PATCH 0827/2472] fixes #17943 (#18045)

* fixes #17943

* IC: test 'nim check --ic:on' for the full Nim compiler

* Update testament/testament.nim

Co-authored-by: Clyybber 

Co-authored-by: Clyybber 
---
 compiler/ic/ic.nim       |  3 +++
 testament/categories.nim | 33 +++++++++++++++++++--------------
 testament/testament.nim  |  4 ++--
 3 files changed, 24 insertions(+), 16 deletions(-)

diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim
index 6530cb6c2..bd3be27f0 100644
--- a/compiler/ic/ic.nim
+++ b/compiler/ic/ic.nim
@@ -507,6 +507,9 @@ proc toPackedNodeIgnoreProcDefs(n: PNode, encoder: var PackedEncoder; m: var Pac
   of nkStmtList, nkStmtListExpr:
     for it in n:
       toPackedNodeIgnoreProcDefs(it, encoder, m)
+  of nkImportStmt, nkImportExceptStmt, nkExportStmt, nkExportExceptStmt,
+     nkFromStmt, nkIncludeStmt:
+    discard "nothing to do"
   else:
     toPackedNode(n, m.topLevel, encoder, m)
 
diff --git a/testament/categories.nim b/testament/categories.nim
index 1f0bd6acf..28ede1ba0 100644
--- a/testament/categories.nim
+++ b/testament/categories.nim
@@ -502,7 +502,7 @@ proc testNimblePackages(r: var TResults; cat: Category; packageFilter: string) =
 proc icTests(r: var TResults; testsDir: string, cat: Category, options: string;
              isNavigatorTest: bool) =
   const
-    tooltests = ["compiler/nim.nim", "tools/nimgrep.nim"]
+    tooltests = ["compiler/nim.nim"]
     writeOnly = " --incremental:writeonly "
     readOnly = " --incremental:readonly "
     incrementalOn = " --incremental:on -d:nimIcIntegrityChecks "
@@ -518,6 +518,24 @@ proc icTests(r: var TResults; testsDir: string, cat: Category, options: string;
     test.spec.targets = {getTestSpecTarget()}
     testSpecWithNimcache(r, test, nimcache)
 
+  template checkTest() =
+    var test = makeRawTest(file, options, cat)
+    test.spec.cmd = compilerPrefix & " check --hint[Conf]:off --warnings:off --ic:on $options " & file
+    testSpecWithNimcache(r, test, nimcache)
+
+  if not isNavigatorTest:
+    for file in tooltests:
+      let nimcache = nimcacheDir(file, options, getTestSpecTarget())
+      removeDir(nimcache)
+
+      let oldPassed = r.passed
+      checkTest()
+
+      if r.passed == oldPassed+1:
+        checkTest()
+        if r.passed == oldPassed+2:
+          checkTest()
+
   const tempExt = "_temp.nim"
   for it in walkDirRec(testsDir):
     if isTestFile(it) and not it.endsWith(tempExt):
@@ -532,19 +550,6 @@ proc icTests(r: var TResults; testsDir: string, cat: Category, options: string;
         editedTest(if isNavigatorTest: navTestConfig else: incrementalOn)
         if r.passed != oldPassed+1: break
 
-  if not isNavigatorTest and false:
-    for file in tooltests:
-      let nimcache = nimcacheDir(file, options, getTestSpecTarget())
-      removeDir(nimcache)
-
-      let oldPassed = r.passed
-      test writeOnly
-
-      if r.passed == oldPassed+1:
-        test readOnly
-        if r.passed == oldPassed+2:
-          test readOnly
-
 # ----------------------------------------------------------------------------
 
 const AdditionalCategories = ["debugger", "examples", "lib", "ic", "navigator"]
diff --git a/testament/testament.nim b/testament/testament.nim
index 076123882..a8dc0557c 100644
--- a/testament/testament.nim
+++ b/testament/testament.nim
@@ -500,7 +500,7 @@ proc equalModuloLastNewline(a, b: string): bool =
 proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec,
                     target: TTarget, nimcache: string, extraOptions = "") =
   test.startTime = epochTime()
-  template callNimCompilerImpl(): untyped = 
+  template callNimCompilerImpl(): untyped =
     # xxx this used to also pass: `--stdout --hint:Path:off`, but was done inconsistently
     # with other branches
     callNimCompiler(expected.getCmd, test.name, test.options, nimcache, target, extraOptions)
@@ -649,9 +649,9 @@ proc makeRawTest(test, options: string, cat: Category): TTest {.used.} =
   result.name = test
   result.options = options
   result.spec = initSpec(addFileExt(test, ".nim"))
-  result.startTime = epochTime()
   result.spec.action = actionCompile
   result.spec.targets = {getTestSpecTarget()}
+  result.startTime = epochTime()
 
 # TODO: fix these files
 const disabledFilesDefault = @[

From 7052503ca8d360f11fabf476f7450dc3f794a602 Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Wed, 19 May 2021 19:18:16 +0200
Subject: [PATCH 0828/2472] make strformat part of the prelude (#18046)

---
 changelog.md         | 3 +++
 lib/pure/prelude.nim | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/changelog.md b/changelog.md
index 97e749337..f9c48e0bf 100644
--- a/changelog.md
+++ b/changelog.md
@@ -73,6 +73,9 @@
 - `json` and `jsonutils` now serialize NaN, Inf, -Inf as strings, so that
   `%[NaN, -Inf]` is the string `["nan","-inf"]` instead of `[nan,-inf]` which was invalid json.
 
+- `strformat` is now part of `include std/prelude`.
+
+
 ## Standard library additions and changes
 - Added support for parenthesized expressions in `strformat`
 
diff --git a/lib/pure/prelude.nim b/lib/pure/prelude.nim
index f1728b5f7..3c98d9d57 100644
--- a/lib/pure/prelude.nim
+++ b/lib/pure/prelude.nim
@@ -25,4 +25,4 @@ when defined(nimdoc) and isMainModule:
   # xxx `nim doc -b:js -d:nodejs --doccmd:-d:nodejs lib/pure/prelude.nim` fails for some reason
   # specific to `nim doc`, but the code otherwise works with nodejs.
 
-import std/[os, strutils, times, parseutils, hashes, tables, sets, sequtils, parseopt]
+import std/[os, strutils, times, parseutils, hashes, tables, sets, sequtils, parseopt, strformat]

From a1c82c39af812b54cd3dd1e472c9088457fb7dc5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= 
Date: Wed, 19 May 2021 19:19:11 +0200
Subject: [PATCH 0829/2472] asyncdispatch+stackTraceOverride: fix premature
 collection (#18039) [backport:1.2]

Copying StackTraceEntry instances when nimStackTraceOverride is defined
breaks the link between a cstring field that's supposed to point at
another string field in the same object.

Sometimes, the original object is garbage collected, that memory region
reused for storing other strings, so when the StackTraceEntry copy tries
to use its cstring pointer to construct a traceback message, it accesses
unrelated strings.

This only happens for async tracebacks and this patch prevents that by
making sure we only use the string fields when nimStackTraceOverride is
defined.

Async tracebacks also beautified slightly by getting rid of an extra line
that was supposed to be commented out, along with the corresponding debugging output.

There's also a micro-optimisation to avoid concatenating two strings just
to get their combined length.
---
 changelog.md                     |  1 +
 lib/pure/asyncfutures.nim        | 46 ++++++++++++++++++++++----------
 lib/system/exceptions.nim        |  2 +-
 tests/async/tasync_traceback.nim |  4 +--
 4 files changed, 36 insertions(+), 17 deletions(-)

diff --git a/changelog.md b/changelog.md
index f9c48e0bf..992cd89c0 100644
--- a/changelog.md
+++ b/changelog.md
@@ -316,6 +316,7 @@
 
 - Added `copyWithin` [for `seq` and `array` for JavaScript targets](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/copyWithin).
 
+- Fixed premature garbage collection in asyncdispatch, when a stack trace override is in place.
 
 ## Language changes
 
diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim
index 0178ee4d6..af3fbb3ec 100644
--- a/lib/pure/asyncfutures.nim
+++ b/lib/pure/asyncfutures.nim
@@ -278,19 +278,33 @@ proc `callback=`*[T](future: Future[T],
   ## If future has already completed then `cb` will be called immediately.
   future.callback = proc () = cb(future)
 
+template getFilenameProcname(entry: StackTraceEntry): (string, string) =
+  when compiles(entry.filenameStr) and compiles(entry.procnameStr):
+    # We can't rely on "entry.filename" and "entry.procname" still being valid
+    # cstring pointers, because the "string.data" buffers they pointed to might
+    # be already garbage collected (this entry being a non-shallow copy,
+    # "entry.filename" no longer points to "entry.filenameStr.data", but to the
+    # buffer of the original object).
+    (entry.filenameStr, entry.procnameStr)
+  else:
+    ($entry.filename, $entry.procname)
+
 proc getHint(entry: StackTraceEntry): string =
   ## We try to provide some hints about stack trace entries that the user
   ## may not be familiar with, in particular calls inside the stdlib.
+
+  let (filename, procname) = getFilenameProcname(entry)
+
   result = ""
-  if entry.procname == cstring"processPendingCallbacks":
-    if cmpIgnoreStyle(entry.filename, "asyncdispatch.nim") == 0:
+  if procname == "processPendingCallbacks":
+    if cmpIgnoreStyle(filename, "asyncdispatch.nim") == 0:
       return "Executes pending callbacks"
-  elif entry.procname == cstring"poll":
-    if cmpIgnoreStyle(entry.filename, "asyncdispatch.nim") == 0:
+  elif procname == "poll":
+    if cmpIgnoreStyle(filename, "asyncdispatch.nim") == 0:
       return "Processes asynchronous completion events"
 
-  if entry.procname.endsWith(NimAsyncContinueSuffix):
-    if cmpIgnoreStyle(entry.filename, "asyncmacro.nim") == 0:
+  if procname.endsWith(NimAsyncContinueSuffix):
+    if cmpIgnoreStyle(filename, "asyncmacro.nim") == 0:
       return "Resumes an async procedure"
 
 proc `$`*(stackTraceEntries: seq[StackTraceEntry]): string =
@@ -303,16 +317,20 @@ proc `$`*(stackTraceEntries: seq[StackTraceEntry]): string =
   # Find longest filename & line number combo for alignment purposes.
   var longestLeft = 0
   for entry in entries:
-    if entry.procname.isNil: continue
+    let (filename, procname) = getFilenameProcname(entry)
 
-    let left = $entry.filename & $entry.line
-    if left.len > longestLeft:
-      longestLeft = left.len
+    if procname == "": continue
+
+    let leftLen = filename.len + len($entry.line)
+    if leftLen > longestLeft:
+      longestLeft = leftLen
 
   var indent = 2
   # Format the entries.
   for entry in entries:
-    if entry.procname.isNil:
+    let (filename, procname) = getFilenameProcname(entry)
+
+    if procname == "":
       if entry.line == reraisedFromBegin:
         result.add(spaces(indent) & "#[\n")
         indent.inc(2)
@@ -321,11 +339,11 @@ proc `$`*(stackTraceEntries: seq[StackTraceEntry]): string =
         result.add(spaces(indent) & "]#\n")
       continue
 
-    let left = "$#($#)" % [$entry.filename, $entry.line]
+    let left = "$#($#)" % [filename, $entry.line]
     result.add((spaces(indent) & "$#$# $#\n") % [
       left,
       spaces(longestLeft - left.len + 2),
-      $entry.procname
+      procname
     ])
     let hint = getHint(entry)
     if hint.len > 0:
@@ -349,9 +367,9 @@ proc injectStacktrace[T](future: Future[T]) =
     newMsg.add($entries)
 
     newMsg.add("Exception message: " & exceptionMsg & "\n")
-    newMsg.add("Exception type:")
 
     # # For debugging purposes
+    # newMsg.add("Exception type:")
     # for entry in getStackTraceEntries(future.error):
     #   newMsg.add "\n" & $entry
     future.error.msg = newMsg
diff --git a/lib/system/exceptions.nim b/lib/system/exceptions.nim
index b5f4fc325..5dcd77bd0 100644
--- a/lib/system/exceptions.nim
+++ b/lib/system/exceptions.nim
@@ -29,7 +29,7 @@ type
       programCounter*: uint ## Program counter - will be used to get the rest of the info,
                             ## when `$` is called on this type. We can't use
                             ## "cuintptr_t" in here.
-      procnameStr*, filenameStr*: string ## GC-ed objects holding the cstrings in "procname" and "filename"
+      procnameStr*, filenameStr*: string ## GC-ed alternatives to "procname" and "filename"
 
   Exception* {.compilerproc, magic: "Exception".} = object of RootObj ## \
     ## Base exception class.
diff --git a/tests/async/tasync_traceback.nim b/tests/async/tasync_traceback.nim
index 9f787929b..cd16b2257 100644
--- a/tests/async/tasync_traceback.nim
+++ b/tests/async/tasync_traceback.nim
@@ -86,7 +86,7 @@ Async traceback:
     asyncfutures\.nim\(\d+?\)\s+?read
   \]#
 Exception message: b failure
-Exception type:
+
 
 bar failure
 Async traceback:
@@ -114,7 +114,7 @@ Async traceback:
     asyncfutures\.nim\(\d+?\)\s+?read
   \]#
 Exception message: bar failure
-Exception type:
+
 """
 
 # TODO: is asyncmacro good enough location for fooIter traceback/debugging? just put the callsite info for all?

From df429fa28772e077faa30dd6e3a701abf48c7669 Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Thu, 20 May 2021 11:52:46 +0200
Subject: [PATCH 0830/2472] config system: special case -d:release and
 -d:danger [backport:1.4] (#18051)

---
 changelog.md              |  4 ++++
 compiler/commands.nim     | 19 +++++++++++++++----
 compiler/main.nim         |  3 ++-
 config/nim.cfg            |  2 ++
 tests/newconfig/tfoo.nim  |  4 ++--
 tests/newconfig/tfoo.nims |  2 ++
 6 files changed, 27 insertions(+), 7 deletions(-)

diff --git a/changelog.md b/changelog.md
index 992cd89c0..4c30d67a7 100644
--- a/changelog.md
+++ b/changelog.md
@@ -75,6 +75,10 @@
 
 - `strformat` is now part of `include std/prelude`.
 
+- The configuration subsystem now allows for `-d:release` and `-d:danger` to work as expected.
+  The downside is that these defines now have custom logic that doesn't apply for
+  other defines.
+
 
 ## Standard library additions and changes
 - Added support for parenthesized expressions in `strformat`
diff --git a/compiler/commands.nim b/compiler/commands.nim
index b9c8e9eb3..605e2930c 100644
--- a/compiler/commands.nim
+++ b/compiler/commands.nim
@@ -482,6 +482,19 @@ proc setCommandEarly*(conf: ConfigRef, command: string) =
   else:
     conf.foreignPackageNotes = foreignPackageNotesDefault
 
+proc specialDefine(conf: ConfigRef, key: string) =
+  # Keep this syncronized with the default config/nim.cfg!
+  if cmpIgnoreStyle(key, "nimQuirky") == 0:
+    conf.exc = excQuirky
+  elif cmpIgnoreStyle(key, "release") == 0 or cmpIgnoreStyle(key, "danger") == 0:
+    conf.options.excl {optStackTrace, optLineTrace, optLineDir, optOptimizeSize}
+    conf.globalOptions.excl {optExcessiveStackTrace, optCDebug}
+    conf.options.incl optOptimizeSpeed
+  if cmpIgnoreStyle(key, "danger") == 0 or cmpIgnoreStyle(key, "quick") == 0:
+    conf.options.excl {optObjCheck, optFieldCheck, optRangeCheck, optBoundsCheck,
+      optOverflowCheck, optAssert, optStackTrace, optLineTrace, optLineDir}
+    conf.globalOptions.excl {optCDebug}
+
 proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
                     conf: ConfigRef) =
   var
@@ -548,12 +561,10 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     expectArg(conf, switch, arg, pass, info)
     if {':', '='} in arg:
       splitSwitch(conf, arg, key, val, pass, info)
-      if cmpIgnoreStyle(key, "nimQuirky") == 0:
-        conf.exc = excQuirky
+      specialDefine(conf, key)
       defineSymbol(conf.symbols, key, val)
     else:
-      if cmpIgnoreStyle(arg, "nimQuirky") == 0:
-        conf.exc = excQuirky
+      specialDefine(conf, arg)
       defineSymbol(conf.symbols, arg)
   of "undef", "u":
     expectArg(conf, switch, arg, pass, info)
diff --git a/compiler/main.nim b/compiler/main.nim
index e2b5f4b67..71e549ab8 100644
--- a/compiler/main.nim
+++ b/compiler/main.nim
@@ -417,7 +417,8 @@ proc mainCommand*(graph: ModuleGraph) =
       "project", project,
       "output", output,
       ])
-    rawMessage(conf, hintBuildMode, build)
+    if conf.cmd in cmdBackends:
+      rawMessage(conf, hintBuildMode, build)
 
   when PrintRopeCacheStats:
     echo "rope cache stats: "
diff --git a/config/nim.cfg b/config/nim.cfg
index d3ab75825..3b964d124 100644
--- a/config/nim.cfg
+++ b/config/nim.cfg
@@ -50,6 +50,7 @@ path="$lib/pure"
 @end
 nimblepath="$home/.nimble/pkgs/"
 
+# Syncronize with compiler/commands.specialDefine
 @if danger or quick:
   obj_checks:off
   field_checks:off
@@ -63,6 +64,7 @@ nimblepath="$home/.nimble/pkgs/"
   line_dir:off
 @end
 
+# Syncronize with compiler/commands.specialDefine
 @if release or danger:
   stacktrace:off
   excessiveStackTrace:off
diff --git a/tests/newconfig/tfoo.nim b/tests/newconfig/tfoo.nim
index 6654202c5..0c6ded470 100644
--- a/tests/newconfig/tfoo.nim
+++ b/tests/newconfig/tfoo.nim
@@ -1,6 +1,6 @@
 discard """
   cmd: "nim default --hint:cc:off --hint:cc $file"
-  output: '''hello world! 0.5'''
+  output: '''hello world! 0.5 true'''
   nimout: '''[NimScript] exec: gcc -v'''
 """
 
@@ -10,4 +10,4 @@ when not defined(definedefine):
 import math, mfriends
 
 discard gen[int]()
-echo "hello world! ", ln 2.0
+echo "hello world! ", ln 2.0, " ", compileOption("opt", "speed")
diff --git a/tests/newconfig/tfoo.nims b/tests/newconfig/tfoo.nims
index 6f0048afb..0cb950227 100644
--- a/tests/newconfig/tfoo.nims
+++ b/tests/newconfig/tfoo.nims
@@ -3,6 +3,8 @@ mode = ScriptMode.Whatif
 
 exec "gcc -v"
 
+--define:release
+
 --forceBuild
 --path: "../friends"
 

From 6a5973882bb0c4134a0e454ea4ae69dc54815f27 Mon Sep 17 00:00:00 2001
From: Juan Carlos 
Date: Thu, 20 May 2021 10:48:46 -0300
Subject: [PATCH 0831/2472] jsgen improve spacing (#18048)

* Fix inconsistent spacings in generated JS of jsgen
---
 compiler/jsgen.nim   | 60 ++++++++++++++++++++++----------------------
 compiler/jstypes.nim |  4 +--
 2 files changed, 32 insertions(+), 32 deletions(-)

diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim
index 53ce45e96..3b183b9d6 100644
--- a/compiler/jsgen.nim
+++ b/compiler/jsgen.nim
@@ -322,7 +322,7 @@ proc isSimpleExpr(p: PProc; n: PNode): bool =
 
 proc getTemp(p: PProc, defineInLocals: bool = true): Rope =
   inc(p.unique)
-  result = "Tmp$1" % [rope(p.unique)]
+  result = "Temporary$1" % [rope(p.unique)]
   if defineInLocals:
     p.locals.add(p.indentLine("var $1;$n" % [result]))
 
@@ -489,12 +489,12 @@ proc maybeMakeTempAssignable(p: PProc, n: PNode; x: TCompRes): tuple[a, tmp: Rop
       if optBoundsCheck in p.options:
         useMagic(p, "chckIndx")
         if first == 0: # save a couple chars
-          index.res = "chckIndx($1, 0, ($2).length-1)" % [index.res, tmp1]
+          index.res = "chckIndx($1, 0, ($2).length - 1)" % [index.res, tmp1]
         else:
-          index.res = "chckIndx($1, $2, ($3).length+($2)-1)-($2)" % [
+          index.res = "chckIndx($1, $2, ($3).length + ($2) - 1) - ($2)" % [
             index.res, rope(first), tmp1]
       elif first != 0:
-        index.res = "($1)-($2)" % [index.res, rope(first)]
+        index.res = "($1) - ($2)" % [index.res, rope(first)]
       else:
         discard # index.res = index.res
       let (n1, tmp2) = maybeMakeTemp(p, n[1], index)
@@ -607,12 +607,12 @@ proc arithAux(p: PProc, n: PNode, r: var TCompRes, op: TMagic) =
     if n[1].typ.size <= 4:
       applyFormat("($1 << $2)", "($1 << $2)")
     else:
-      applyFormat("($1 * Math.pow(2,$2))", "($1 * Math.pow(2,$2))")
+      applyFormat("($1 * Math.pow(2, $2))", "($1 * Math.pow(2, $2))")
   of mAshrI:
     if n[1].typ.size <= 4:
       applyFormat("($1 >> $2)", "($1 >> $2)")
     else:
-      applyFormat("Math.floor($1 / Math.pow(2,$2))", "Math.floor($1 / Math.pow(2,$2))")
+      applyFormat("Math.floor($1 / Math.pow(2, $2))", "Math.floor($1 / Math.pow(2, $2))")
   of mBitandI: applyFormat("($1 & $2)", "($1 & $2)")
   of mBitorI: applyFormat("($1 | $2)", "($1 | $2)")
   of mBitxorI: applyFormat("($1 ^ $2)", "($1 ^ $2)")
@@ -656,8 +656,8 @@ proc arithAux(p: PProc, n: PNode, r: var TCompRes, op: TMagic) =
   of mUnaryMinusF64: applyFormat("-($1)", "-($1)")
   of mCharToStr: applyFormat("nimCharToStr($1)", "nimCharToStr($1)")
   of mBoolToStr: applyFormat("nimBoolToStr($1)", "nimBoolToStr($1)")
-  of mIntToStr: applyFormat("cstrToNimstr(($1)+\"\")", "cstrToNimstr(($1)+\"\")")
-  of mInt64ToStr: applyFormat("cstrToNimstr(($1)+\"\")", "cstrToNimstr(($1)+\"\")")
+  of mIntToStr: applyFormat("cstrToNimstr(($1) + \"\")", "cstrToNimstr(($1) + \"\")")
+  of mInt64ToStr: applyFormat("cstrToNimstr(($1) + \"\")", "cstrToNimstr(($1) + \"\")")
   of mFloatToStr:
     useMagic(p, "nimFloatToString")
     applyFormat "cstrToNimstr(nimFloatToString($1))"
@@ -704,7 +704,7 @@ proc hasFrameInfo(p: PProc): bool =
       ((p.prc == nil) or not (sfPure in p.prc.flags))
 
 proc lineDir(config: ConfigRef, info: TLineInfo, line: int): Rope =
-  ropes.`%`("// line $2 \"$1\"$n",
+  ropes.`%`("/* line $2 \"$1\" */$n",
          [rope(toFullPath(config, info)), rope(line)])
 
 proc genLineDir(p: PProc, n: PNode) =
@@ -725,9 +725,9 @@ proc genWhileStmt(p: PProc, n: PNode) =
   p.blocks[^1].id = -p.unique
   p.blocks[^1].isLoop = true
   let labl = p.unique.rope
-  lineF(p, "L$1: while (true) {$n", [labl])
+  lineF(p, "Label$1: while (true) {$n", [labl])
   p.nested: gen(p, n[0], cond)
-  lineF(p, "if (!$1) break L$2;$n",
+  lineF(p, "if (!$1) break Label$2;$n",
        [cond.res, labl])
   p.nested: genStmt(p, n[1])
   lineF(p, "}$n", [labl])
@@ -750,8 +750,8 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) =
   #  try {
   #    stmts;
   #    --excHandler;
-  #  } catch (EXC) {
-  #    var prevJSError = lastJSError; lastJSError = EXC;
+  #  } catch (EXCEPTION) {
+  #    var prevJSError = lastJSError; lastJSError = EXCEPTION;
   #    framePtr = tmpFramePtr;
   #    --excHandler;
   #    if (e.typ && e.typ == NTI433 || e.typ == NTI2321) {
@@ -785,8 +785,8 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) =
   moveInto(p, a, r)
   var generalCatchBranchExists = false
   if catchBranchesExist:
-    p.body.addf("--excHandler;$n} catch (EXC) {$n var prevJSError = lastJSError;$n" &
-        " lastJSError = EXC;$n --excHandler;$n", [])
+    p.body.addf("--excHandler;$n} catch (EXCEPTION) {$n var prevJSError = lastJSError;$n" &
+        " lastJSError = EXCEPTION;$n --excHandler;$n", [])
     line(p, "framePtr = $1;$n" % [tmpFramePtr])
   while i < n.len and n[i].kind == nkExceptBranch:
     if n[i].len == 1:
@@ -928,12 +928,12 @@ proc genBlock(p: PProc, n: PNode, r: var TCompRes) =
     sym.loc.k = locOther
     sym.position = idx+1
   let labl = p.unique
-  lineF(p, "L$1: do {$n", [labl.rope])
+  lineF(p, "Label$1: do {$n", [labl.rope])
   setLen(p.blocks, idx + 1)
   p.blocks[idx].id = - p.unique # negative because it isn't used yet
   gen(p, n[1], r)
   setLen(p.blocks, idx)
-  lineF(p, "} while(false);$n", [labl.rope])
+  lineF(p, "} while (false);$n", [labl.rope])
 
 proc genBreakStmt(p: PProc, n: PNode) =
   var idx: int
@@ -951,7 +951,7 @@ proc genBreakStmt(p: PProc, n: PNode) =
     if idx < 0 or not p.blocks[idx].isLoop:
       internalError(p.config, n.info, "no loop to break")
   p.blocks[idx].id = abs(p.blocks[idx].id) # label is used
-  lineF(p, "break L$1;$n", [rope(p.blocks[idx].id)])
+  lineF(p, "break Label$1;$n", [rope(p.blocks[idx].id)])
 
 proc genAsmOrEmitStmt(p: PProc, n: PNode) =
   genLineDir(p, n)
@@ -1248,12 +1248,12 @@ proc genArrayAddr(p: PProc, n: PNode, r: var TCompRes) =
   if optBoundsCheck in p.options:
     useMagic(p, "chckIndx")
     if first == 0: # save a couple chars
-      r.res = "chckIndx($1, 0, ($2).length-1)" % [b.res, tmp]
+      r.res = "chckIndx($1, 0, ($2).length - 1)" % [b.res, tmp]
     else:
-      r.res = "chckIndx($1, $2, ($3).length+($2)-1)-($2)" % [
+      r.res = "chckIndx($1, $2, ($3).length + ($2) - 1) - ($2)" % [
         b.res, rope(first), tmp]
   elif first != 0:
-    r.res = "($1)-($2)" % [b.res, rope(first)]
+    r.res = "($1) - ($2)" % [b.res, rope(first)]
   else:
     r.res = b.res
   r.kind = resExpr
@@ -1876,13 +1876,13 @@ proc genNewSeq(p: PProc, n: PNode) =
   gen(p, n[1], x)
   gen(p, n[2], y)
   let t = skipTypes(n[1].typ, abstractVar)[0]
-  lineF(p, "$1 = new Array($2); for (var i=0;i<$2;++i) {$1[i]=$3;}", [
+  lineF(p, "$1 = new Array($2); for (var i = 0 ; i < $2 ; ++i) { $1[i] = $3; }", [
     x.rdLoc, y.rdLoc, createVar(p, t, false)])
 
 proc genOrd(p: PProc, n: PNode, r: var TCompRes) =
   case skipTypes(n[1].typ, abstractVar + abstractRange).kind
   of tyEnum, tyInt..tyUInt64, tyChar: gen(p, n[1], r)
-  of tyBool: unaryExpr(p, n, r, "", "($1 ? 1:0)")
+  of tyBool: unaryExpr(p, n, r, "", "($1 ? 1 : 0)")
   else: internalError(p.config, n.info, "genOrd")
 
 proc genConStrStr(p: PProc, n: PNode, r: var TCompRes) =
@@ -2122,7 +2122,7 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) =
     let t = skipTypes(n[1].typ, abstractVar)[0]
     let (a, tmp) = maybeMakeTemp(p, n[1], x)
     let (b, tmp2) = maybeMakeTemp(p, n[2], y)
-    r.res = """if ($1.length < $2) { for (var i=$4.length;i<$5;++i) $4.push($3); }
+    r.res = """if ($1.length < $2) { for (var i = $4.length ; i < $5 ; ++i) $4.push($3); }
                else { $4.length = $5; }""" % [a, b, createVar(p, t, false), tmp, tmp2]
     r.kind = resExpr
   of mCard: unaryExpr(p, n, r, "SetCard", "SetCard($1)")
@@ -2159,7 +2159,7 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) =
     gen(p, n[1], x)
     gen(p, n[2], y)
     gen(p, n[3], z)
-    r.res = "($1.slice($2, $3+1))" % [x.rdLoc, y.rdLoc, z.rdLoc]
+    r.res = "($1.slice($2, $3 + 1))" % [x.rdLoc, y.rdLoc, z.rdLoc]
     r.kind = resExpr
   of mMove:
     genMove(p, n, r)
@@ -2281,7 +2281,7 @@ proc genConv(p: PProc, n: PNode, r: var TCompRes) =
     r.res = "(!!($1))" % [r.res]
     r.kind = resExpr
   elif toInt:
-    r.res = "(($1)|0)" % [r.res]
+    r.res = "(($1) | 0)" % [r.res]
   else:
     # TODO: What types must we handle here?
     discard
@@ -2337,7 +2337,7 @@ proc genReturnStmt(p: PProc, n: PNode) =
 
 proc frameCreate(p: PProc; procname, filename: Rope): Rope =
   const frameFmt =
-    "var F={procname:$1,prev:framePtr,filename:$2,line:0};$n"
+    "var F = {procname: $1, prev: framePtr, filename: $2, line: 0};$n"
 
   result = p.indentLine(frameFmt % [procname, filename])
   result.add p.indentLine(ropes.`%`("framePtr = F;$n", []))
@@ -2431,10 +2431,10 @@ proc genProc(oldProc: PProc, prc: PSym): Rope =
       # references will end up calling the reloaded code.
       var thunkName = name
       name = name & "IMLP"
-      result.add("function $#() { return $#.apply(this, arguments); }$n" %
+      result.add("\Lfunction $#() { return $#.apply(this, arguments); }$n" %
                  [thunkName, name])
 
-    def = "function $#($#) {$n$#$#$#$#$#" %
+    def = "\Lfunction $#($#) {$n$#$#$#$#$#" %
             [ name,
               header,
               optionalLine(p.globals),
@@ -2482,7 +2482,7 @@ proc genCast(p: PProc, n: PNode, r: var TCompRes) =
     elif fromUint:
       if src.size == 4 and dest.size == 4:
         # XXX prevent multi evaluations
-        r.res = "($1|0)" % [r.res]
+        r.res = "($1 | 0)" % [r.res]
       else:
         let trimmer = unsignedTrimmer(dest.size)
         let minuend = case dest.size
diff --git a/compiler/jstypes.nim b/compiler/jstypes.nim
index 81f03dc12..5b684b60c 100644
--- a/compiler/jstypes.nim
+++ b/compiler/jstypes.nim
@@ -135,14 +135,14 @@ proc genTypeInfo(p: PProc, typ: PType): Rope =
     prepend(p.g.typeInfo, s)
   of tyVar, tyLent, tyRef, tyPtr, tySequence, tyRange, tySet:
     var s =
-      "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" %
+      "var $1 = {size: 0, kind: $2, base: null, node: null, finalizer: null};$n" %
               [result, rope(ord(t.kind))]
     prepend(p.g.typeInfo, s)
     p.g.typeInfo.addf("$1.base = $2;$n",
          [result, genTypeInfo(p, t.lastSon)])
   of tyArray:
     var s =
-      "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" %
+      "var $1 = {size: 0, kind: $2, base: null, node: null, finalizer: null};$n" %
               [result, rope(ord(t.kind))]
     prepend(p.g.typeInfo, s)
     p.g.typeInfo.addf("$1.base = $2;$n",

From 9f7e2e30573a377770fa630f12a47bac09751282 Mon Sep 17 00:00:00 2001
From: Andrey Makarov 
Date: Fri, 21 May 2021 07:54:20 +0300
Subject: [PATCH 0832/2472] docs: make inline markup more compatible with
 Markdown (#18053)

fixes https://github.com/timotheecour/Nim/issues/739
---
 doc/regexprs.txt              |   2 +-
 lib/impure/db_sqlite.nim      |   2 +-
 lib/packages/docutils/rst.nim | 147 ++++++++++++++++++++++------------
 lib/posix/posix_utils.nim     |   2 +-
 tests/stdlib/trst.nim         |  65 +++++++++++++--
 tests/stdlib/trstgen.nim      |  12 +--
 6 files changed, 168 insertions(+), 62 deletions(-)

diff --git a/doc/regexprs.txt b/doc/regexprs.txt
index b7370d858..9ec08b810 100644
--- a/doc/regexprs.txt
+++ b/doc/regexprs.txt
@@ -146,7 +146,7 @@ character          meaning
 After ``\x``, from zero to two hexadecimal digits are read (letters can be in
 upper or lower case). In UTF-8 mode, any number of hexadecimal digits may
 appear between ``\x{`` and ``}``, but the value of the character code must be
-less than 2**31 (that is, the maximum hexadecimal value is 7FFFFFFF). If
+less than 2^31 (that is, the maximum hexadecimal value is 7FFFFFFF). If
 characters other than hexadecimal digits appear between ``\x{`` and ``}``, or
 if there is no terminating ``}``, this form of escape is not recognized.
 Instead, the initial ``\x`` will be interpreted as a basic hexadecimal escape,
diff --git a/lib/impure/db_sqlite.nim b/lib/impure/db_sqlite.nim
index 832407960..7bd807a12 100644
--- a/lib/impure/db_sqlite.nim
+++ b/lib/impure/db_sqlite.nim
@@ -152,7 +152,7 @@
 ## Instead, a `seq[string]` is returned for each row.
 ##
 ## The reasoning is as follows:
-## 1. it's close to what many DBs offer natively (char**)
+## 1. it's close to what many DBs offer natively (`char**`:c:)
 ## 2. it hides the number of types that the DB supports
 ##    (int? int64? decimal up to 10 places? geo coords?)
 ## 3. it's convenient when all you do is to forward the data to somewhere else (echo, log, put the data into a new query)
diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim
index da04e9b54..dae692fb7 100644
--- a/lib/packages/docutils/rst.nim
+++ b/lib/packages/docutils/rst.nim
@@ -130,6 +130,32 @@
 ## .. warning:: Using Nim-specific features can cause other RST implementations
 ##   to fail on your document.
 ##
+## Idiosyncrasies
+## --------------
+##
+## Currently we do **not** aim at 100% Markdown or RST compatibility in inline
+## markup recognition rules because that would provide very little user value.
+## This parser has 2 modes for inline markup:
+##
+## 1) Markdown-like mode which is enabled by `roPreferMarkdown` option
+##    (turned **on** by default).
+##
+##    .. Note:: RST features like directives are still turned **on**
+##
+## 2) Compatibility mode which is RST rules.
+##
+## .. Note:: in both modes the parser interpretes text between single
+##    backticks (code) identically:
+##      backslash does not escape; the only exception: ``\`` folowed by `
+##      does escape so that we can always input a single backtick ` in
+##      inline code. However that makes impossible to input code with
+##      ``\`` at the end in *single* backticks, one must use *double*
+##      backticks::
+##
+##        `\`   -- WRONG
+##        ``\`` -- GOOD
+##        So single backticks can always be input: `\`` will turn to ` code
+##
 ## Limitations
 ## -----------
 ##
@@ -994,8 +1020,22 @@ proc expect(p: var RstParser, tok: string) =
   if currentTok(p).symbol == tok: inc p.idx
   else: rstMessage(p, meExpected, tok)
 
-proc isInlineMarkupEnd(p: RstParser, markup: string, exact: bool): bool =
+proc inlineMarkdownEnd(p: RstParser): bool =
+  result = prevTok(p).kind notin {tkIndent, tkWhite}
+  ## (For a special case of ` we don't allow spaces surrounding it
+  ## unlike original Markdown because this behavior confusing/useless)
+
+proc inlineRstEnd(p: RstParser): bool =
   # rst rules: https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#inline-markup-recognition-rules
+  # Rule 2:
+  result = prevTok(p).kind notin {tkIndent, tkWhite}
+  if not result: return
+  # Rule 7:
+  result = nextTok(p).kind in {tkIndent, tkWhite, tkEof} or
+      nextTok(p).symbol[0] in
+      {'\'', '\"', ')', ']', '}', '>', '-', '/', '\\', ':', '.', ',', ';', '!', '?', '_'}
+
+proc isInlineMarkupEnd(p: RstParser, markup: string, exact: bool): bool =
   if exact:
     result = currentTok(p).symbol == markup
   else:
@@ -1004,55 +1044,58 @@ proc isInlineMarkupEnd(p: RstParser, markup: string, exact: bool): bool =
       # check that escaping may have splitted `` to 2 tokens ` and `
       result = currentTok(p).symbol == "`" and prevTok(p).symbol == "`"
   if not result: return
-  # Rule 2:
-  result = prevTok(p).kind notin {tkIndent, tkWhite}
-  if not result: return
-  # Rule 7:
-  result = nextTok(p).kind in {tkIndent, tkWhite, tkEof} or
-      (roPreferMarkdown in p.s.options and
-        markup in ["``", "`"] and
-        nextTok(p).kind in {tkIndent, tkWhite, tkWord, tkEof}) or
-      nextTok(p).symbol[0] in
-      {'\'', '\"', ')', ']', '}', '>', '-', '/', '\\', ':', '.', ',', ';', '!', '?', '_'}
-  if not result: return
-  # Rule 4:
-  if p.idx > 0:
-    # see bug #17260; for now `\` must be written ``\``, likewise with sequences
-    # ending in an un-escaped `\`; `\\` is legal but not `\\\` for example;
-    # for this reason we can't use `["``", "`"]` here.
-    if markup != "``" and prevTok(p).symbol == "\\":
-      result = false
+  # surroundings check
+  if markup in ["_", "__"]:
+    result = inlineRstEnd(p)
+  else:
+    if roPreferMarkdown in p.s.options: result = inlineMarkdownEnd(p)
+    else: result = inlineRstEnd(p)
 
-proc isInlineMarkupStart(p: RstParser, markup: string): bool =
-  # rst rules: https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#inline-markup-recognition-rules
-  var d: char
-  if markup != "_`":
-    result = currentTok(p).symbol == markup
-  else:  # _` is a 2 token case
-    result = currentTok(p).symbol == "_" and nextTok(p).symbol == "`"
+proc rstRuleSurround(p: RstParser): bool =
+  result = true
+  # Rules 4 & 5:
+  if p.idx > 0:
+    var d: char
+    var c = prevTok(p).symbol[0]
+    case c
+    of '\'', '\"': d = c
+    of '(': d = ')'
+    of '[': d = ']'
+    of '{': d = '}'
+    of '<': d = '>'
+    else: d = '\0'
+    if d != '\0': result = nextTok(p).symbol[0] != d
+
+proc inlineMarkdownStart(p: RstParser): bool =
+  result = nextTok(p).kind notin {tkIndent, tkWhite, tkEof}
   if not result: return
-  # Rule 6:
+  # this rst rule is really nice, let us use it in Markdown mode too.
+  result = rstRuleSurround(p)
+
+proc inlineRstStart(p: RstParser): bool =
+  ## rst rules: https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#inline-markup-recognition-rules
+  # Rule 6
   result = p.idx == 0 or prevTok(p).kind in {tkIndent, tkWhite} or
-      (markup in ["``", "`"] and prevTok(p).kind in {tkIndent, tkWhite, tkWord}) or
       prevTok(p).symbol[0] in {'\'', '\"', '(', '[', '{', '<', '-', '/', ':', '_'}
   if not result: return
   # Rule 1:
   result = nextTok(p).kind notin {tkIndent, tkWhite, tkEof}
   if not result: return
-  # Rules 4 & 5:
-  if p.idx > 0:
-    if prevTok(p).symbol == "\\":
-      result = false
-    else:
-      var c = prevTok(p).symbol[0]
-      case c
-      of '\'', '\"': d = c
-      of '(': d = ')'
-      of '[': d = ']'
-      of '{': d = '}'
-      of '<': d = '>'
-      else: d = '\0'
-      if d != '\0': result = nextTok(p).symbol[0] != d
+  result = rstRuleSurround(p)
+
+proc isInlineMarkupStart(p: RstParser, markup: string): bool =
+  if markup != "_`":
+    result = currentTok(p).symbol == markup
+  else:  # _` is a 2 token case
+    result = currentTok(p).symbol == "_" and nextTok(p).symbol == "`"
+  if not result: return
+  # surroundings check
+  if markup in ["_", "__", "[", "|"]:
+    # Note: we require space/punctuation even before [markdown link](...)
+    result = inlineRstStart(p)
+  else:
+    if roPreferMarkdown in p.s.options: result = inlineMarkdownStart(p)
+    else: result = inlineRstStart(p)
 
 proc match(p: RstParser, start: int, expr: string): bool =
   # regular expressions are:
@@ -1263,10 +1306,7 @@ proc parseWordOrRef(p: var RstParser, father: PRstNode) =
 
 proc parseBackslash(p: var RstParser, father: PRstNode) =
   assert(currentTok(p).kind == tkPunct)
-  if currentTok(p).symbol == "\\\\":
-    father.add newLeaf("\\")
-    inc p.idx
-  elif currentTok(p).symbol == "\\":
+  if currentTok(p).symbol == "\\":
     # XXX: Unicode?
     inc p.idx
     if currentTok(p).kind != tkWhite: father.add(newLeaf(p))
@@ -1297,11 +1337,20 @@ proc parseUntil(p: var RstParser, father: PRstNode, postfix: string,
         break
       else:
         if postfix == "`":
-          if prevTok(p).symbol == "\\" and currentTok(p).symbol == "`":
-            father.sons[^1] = newLeaf(p) # instead, we should use lookahead
+          if currentTok(p).symbol == "\\":
+            if nextTok(p).symbol == "\\":
+              father.add newLeaf("\\")
+              father.add newLeaf("\\")
+              inc p.idx, 2
+            elif nextTok(p).symbol == "`":  # escape `
+              father.add newLeaf("`")
+              inc p.idx, 2
+            else:
+              father.add newLeaf("\\")
+              inc p.idx
           else:
             father.add(newLeaf(p))
-          inc p.idx
+            inc p.idx
         else:
           if interpretBackslash:
             parseBackslash(p, father)
diff --git a/lib/posix/posix_utils.nim b/lib/posix/posix_utils.nim
index aeec73a45..c2d5aab56 100644
--- a/lib/posix/posix_utils.nim
+++ b/lib/posix/posix_utils.nim
@@ -7,7 +7,7 @@
 #
 
 ## A set of helpers for the POSIX module.
-## Raw interfaces are in the other posix*.nim files.
+## Raw interfaces are in the other ``posix*.nim`` files.
 
 # Where possible, contribute OS-independent procs in `os `_ instead.
 
diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim
index 71f5a858b..ec34edc91 100644
--- a/tests/stdlib/trst.nim
+++ b/tests/stdlib/trst.nim
@@ -23,7 +23,7 @@ import std/private/miscdollars
 import os
 
 proc toAst(input: string,
-            rstOptions: RstParseOptions = {roSupportMarkdown, roNimFile},
+            rstOptions: RstParseOptions = {roPreferMarkdown, roSupportMarkdown, roNimFile},
             error: ref string = nil,
             warnings: ref seq[string] = nil): string =
   ## If `error` is nil then no errors should be generated.
@@ -36,10 +36,11 @@ proc toAst(input: string,
     toLocation(message, filename, line, col + ColRstOffset)
     message.add " $1: $2" % [$mc, a]
     if mc == mcError:
-      doAssert error != nil, "unexpected RST error '" & message & "'"
+      if error == nil:
+        raise newException(EParseError, "[unexpected error] " & message)
       error[] = message
       # we check only first error because subsequent ones may be meaningless
-      raise newException(EParseError, message)
+      raise newException(EParseError, "")
     else:
       doAssert warnings != nil, "unexpected RST warning '" & message & "'"
       warnings[].add message
@@ -54,8 +55,9 @@ proc toAst(input: string,
     var rst = rstParse(input, filen, line=LineRstInit, column=ColRstInit,
                        dummyHasToc, rstOptions, myFindFile, testMsgHandler)
     result = renderRstToStr(rst)
-  except EParseError:
-    discard
+  except EParseError as e:
+    if e.msg != "":
+      result = e.msg
 
 suite "RST parsing":
   test "option list has priority over definition list":
@@ -326,6 +328,28 @@ suite "RST escaping":
       """)
 
 suite "RST inline markup":
+  test "* and ** surrounded by spaces are not inline markup":
+    check("a * b * c ** d ** e".toAst == dedent"""
+      rnInner
+        rnLeaf  'a'
+        rnLeaf  ' '
+        rnLeaf  '*'
+        rnLeaf  ' '
+        rnLeaf  'b'
+        rnLeaf  ' '
+        rnLeaf  '*'
+        rnLeaf  ' '
+        rnLeaf  'c'
+        rnLeaf  ' '
+        rnLeaf  '**'
+        rnLeaf  ' '
+        rnLeaf  'd'
+        rnLeaf  ' '
+        rnLeaf  '**'
+        rnLeaf  ' '
+        rnLeaf  'e'
+      """)
+
   test "end-string has repeating symbols":
     check("*emphasis content****".toAst == dedent"""
       rnEmphasis
@@ -420,6 +444,37 @@ suite "RST inline markup":
           rnLeaf  'proc `+`'
       """)
 
+    check("""`\\`""".toAst ==
+      dedent"""
+        rnInlineCode
+          rnDirArg
+            rnLeaf  'nim'
+          [nil]
+          rnLiteralBlock
+            rnLeaf  '\\'
+        """)
+
+  test "Markdown-style code/backtick":
+    # no whitespace is required before `
+    check("`try`...`except`".toAst ==
+      dedent"""
+        rnInner
+          rnInlineCode
+            rnDirArg
+              rnLeaf  'nim'
+            [nil]
+            rnLiteralBlock
+              rnLeaf  'try'
+          rnLeaf  '...'
+          rnInlineCode
+            rnDirArg
+              rnLeaf  'nim'
+            [nil]
+            rnLiteralBlock
+              rnLeaf  'except'
+        """)
+
+
   test """inline literals can contain \ anywhere""":
     check("""``\``""".toAst == dedent"""
       rnInlineLiteral
diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim
index 667fec780..864728686 100644
--- a/tests/stdlib/trstgen.nim
+++ b/tests/stdlib/trstgen.nim
@@ -10,7 +10,7 @@ import unittest, strutils, strtabs
 import std/private/miscdollars
 
 proc toHtml(input: string,
-            rstOptions: RstParseOptions = {roSupportMarkdown, roNimFile},
+            rstOptions: RstParseOptions = {roPreferMarkdown, roSupportMarkdown, roNimFile},
             error: ref string = nil,
             warnings: ref seq[string] = nil): string =
   ## If `error` is nil then no errors should be generated.
@@ -23,18 +23,20 @@ proc toHtml(input: string,
     toLocation(message, filename, line, col + ColRstOffset)
     message.add " $1: $2" % [$mc, a]
     if mc == mcError:
-      doAssert error != nil, "unexpected RST error '" & message & "'"
+      if error == nil:
+        raise newException(EParseError, "[unexpected error] " & message)
       error[] = message
       # we check only first error because subsequent ones may be meaningless
-      raise newException(EParseError, message)
+      raise newException(EParseError, "")
     else:
       doAssert warnings != nil, "unexpected RST warning '" & message & "'"
       warnings[].add message
   try:
     result = rstToHtml(input, rstOptions, defaultConfig(),
                        msgHandler=testMsgHandler)
-  except EParseError:
-    discard
+  except EParseError as e:
+    if e.msg != "":
+      result = e.msg
 
 # inline code tags (for parsing originated from highlite.nim)
 proc id(str: string): string = """"""  & str & ""

From e12597589f2a8cdaabe09e8b2b92ff0fbe3727ef Mon Sep 17 00:00:00 2001
From: Juan Carlos 
Date: Sat, 22 May 2021 08:12:30 -0300
Subject: [PATCH 0833/2472] Error message minor (#18021)

* Update compiler/semtypes.nim

Co-authored-by: Timothee Cour 
Co-authored-by: flywind 
Co-authored-by: Andreas Rumpf 
---
 compiler/lineinfos.nim | 2 +-
 compiler/semtypes.nim  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim
index 47cddb888..bf81aed02 100644
--- a/compiler/lineinfos.nim
+++ b/compiler/lineinfos.nim
@@ -110,7 +110,7 @@ const
     warnFieldXNotSupported: "field '$1' not supported",
     warnRstStyle: "RST style: $1",
     warnCommentXIgnored: "comment '$1' ignored",
-    warnTypelessParam: "'$1' has no type. Typeless parameters are deprecated; only allowed for 'template'",
+    warnTypelessParam: "", # deadcode
     warnUseBase: "use {.base.} for base methods; baseless methods are deprecated",
     warnWriteToForeignHeap: "write to foreign heap",
     warnUnsafeCode: "unsafe code: '$1'",
diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim
index 97e37f8b7..84556ca1a 100644
--- a/compiler/semtypes.nim
+++ b/compiler/semtypes.nim
@@ -1297,7 +1297,7 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode,
         let param = strTableGet(c.signatures, arg.name)
         if param != nil: typ = param.typ
         else:
-          localError(c.config, a.info, "typeless parameters are obsolete")
+          localError(c.config, a.info, "parameter '$1' requires a type" % param.name.s)
           typ = errorType(c)
       let lifted = liftParamType(c, kind, genericParams, typ,
                                  arg.name.s, arg.info)

From 1636c05d138a7a52c5b6cb3cafe6ab84f59f086b Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Sun, 23 May 2021 01:02:04 -0700
Subject: [PATCH 0834/2472] close #5540 generic object with generic field
 evaluated too early (#18062)

---
 tests/misc/t5540.nim | 45 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)
 create mode 100644 tests/misc/t5540.nim

diff --git a/tests/misc/t5540.nim b/tests/misc/t5540.nim
new file mode 100644
index 000000000..6a19e70e1
--- /dev/null
+++ b/tests/misc/t5540.nim
@@ -0,0 +1,45 @@
+# bug #5540; works in 1.2.0
+# fails in 1.0 (Error: cannot generate VM code for)
+# fails in 0.18.0 (Error: type mismatch: got )
+
+block:
+  type
+    Fruit = object
+    Yellow = object
+      a: int
+  template getColor(x: typedesc[Fruit]): typedesc = Yellow
+  type
+    Banana[T] = object
+      b: T
+      a: getColor(Fruit)
+    Apple[T] = object
+      a: T
+      b: getColor(T)
+  block:
+    var x: Banana[int]
+    doAssert x.b == 0
+    doAssert x.a is Yellow
+  block:
+    var x: Apple[Fruit]
+    doAssert x.b is Yellow
+
+block:
+  type
+    Fruit = object
+    Yellow = object
+      a: int
+    
+  template getColor(x: typedesc[Fruit]): typedesc = Yellow
+
+  type
+    Banana[T] = object
+      b: T
+      a: getColor(Fruit)
+
+    Apple[T] = object
+      a: T
+      b: getColor(T)
+      
+  var x: Banana[int]
+  x.b = 13
+  x.a.a = 17

From 61630c6aee56acf688c48c00f5a6711a59e79616 Mon Sep 17 00:00:00 2001
From: Euan 
Date: Sun, 23 May 2021 17:23:20 +0100
Subject: [PATCH 0835/2472] Use gmake for NetBSD as well as FreeBSD/OpenBSD
 (#18064)

---
 ci/funs.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ci/funs.sh b/ci/funs.sh
index cad005246..0e4f0eb3b 100644
--- a/ci/funs.sh
+++ b/ci/funs.sh
@@ -55,6 +55,8 @@ _nimBuildCsourcesIfNeeded(){
     makeX=gmake
   elif [ "$unamestr" = 'OpenBSD' ]; then
     makeX=gmake
+  elif [ "$unamestr" = 'NetBSD' ]; then
+    makeX=gmake
   else
     makeX=make
   fi

From 1421a3bf265e45cea7c26dcaf3be148ea02db5f0 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Sun, 23 May 2021 12:28:26 -0700
Subject: [PATCH 0836/2472] pkg/pixie: use latest (#18069)

---
 testament/important_packages.nim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/testament/important_packages.nim b/testament/important_packages.nim
index b84155eee..975dd63ad 100644
--- a/testament/important_packages.nim
+++ b/testament/important_packages.nim
@@ -118,7 +118,7 @@ pkg "optionsutils"
 pkg "ormin", "nim c -o:orminn ormin.nim"
 pkg "parsetoml"
 pkg "patty"
-pkg "pixie", useHead = false
+pkg "pixie"
 pkg "plotly", "nim c examples/all.nim"
 pkg "pnm"
 pkg "polypbren"

From d217888e5679aff063668930bd00892c4f5cb2b3 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Sun, 23 May 2021 14:06:14 -0700
Subject: [PATCH 0837/2472] close #18009 parseJson JInt vs JFloat; preserve
 -0.0 as JFloat to distinguish from 0.0 (#18067)

---
 lib/pure/json.nim           |  2 +-
 tests/stdlib/tjsonutils.nim | 17 ++++++++++++++++-
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/lib/pure/json.nim b/lib/pure/json.nim
index 3a9fa5898..d5bbb86b8 100644
--- a/lib/pure/json.nim
+++ b/lib/pure/json.nim
@@ -938,7 +938,7 @@ when defined(js):
     of "[object Array]": return JArray
     of "[object Object]": return JObject
     of "[object Number]":
-      if isInteger(x):
+      if isInteger(x) and 1.0 / cast[float](x) != -Inf: # preserve -0.0 as float
         if isSafeInteger(x):
           return JInt
         else:
diff --git a/tests/stdlib/tjsonutils.nim b/tests/stdlib/tjsonutils.nim
index a55b0ca1d..62486b896 100644
--- a/tests/stdlib/tjsonutils.nim
+++ b/tests/stdlib/tjsonutils.nim
@@ -4,7 +4,8 @@ discard """
 
 import std/jsonutils
 import std/json
-from std/math import isNaN
+from std/math import isNaN, signbit
+from stdtest/testutils import whenRuntimeJs
 
 proc testRoundtrip[T](t: T, expected: string) =
   # checks that `T => json => T2 => json2` is such that json2 = json
@@ -123,6 +124,20 @@ template fn() =
     testRoundtripVal((Inf, -Inf, 0.0, -0.0, 1.0)): """["inf","-inf",0.0,-0.0,1.0]"""
     doAssert ($NaN.toJson).parseJson.jsonTo(float).isNaN
 
+  block: # bug #18009; unfixable unless we change parseJson (which would have overhead),
+         # but at least we can guarantee that the distinction between 0.0 and -0.0 is preserved.
+    let a = (0, 0.0, -0.0, 0.5, 1, 1.0)
+    testRoundtripVal(a): "[0,0.0,-0.0,0.5,1,1.0]"
+    let a2 = $($a.toJson).parseJson
+    whenRuntimeJs:
+      doAssert a2 == "[0,0,-0.0,0.5,1,1]"
+    do:
+      doAssert a2 == "[0,0.0,-0.0,0.5,1,1.0]"
+    let b = a2.parseJson.jsonTo(type(a))
+    doAssert not b[1].signbit
+    doAssert b[2].signbit
+    doAssert not b[3].signbit
+
   block: # case object
     type Foo = object
       x0: float

From 478f717377c4cd60cfce112b8b21d58031b118b4 Mon Sep 17 00:00:00 2001
From: xioren <40043405+xioren@users.noreply.github.com>
Date: Mon, 24 May 2021 20:56:31 -0700
Subject: [PATCH 0838/2472] Move async example to asynchronous version of proc
 (#18078)

* improve runnableExamples in std/httpclient

* Add synchronous example.

* Update lib/pure/httpclient.nim

Co-authored-by: Timothee Cour 
---
 lib/pure/httpclient.nim | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim
index 86708dc40..0dbf8a045 100644
--- a/lib/pure/httpclient.nim
+++ b/lib/pure/httpclient.nim
@@ -574,15 +574,11 @@ proc newHttpClient*(userAgent = defUserAgent, maxRedirects = 5,
   ##
   ## `headers` specifies the HTTP Headers.
   runnableExamples:
-    import std/[asyncdispatch, httpclient, strutils]
+    import std/strutils
 
-    proc asyncProc(): Future[string] {.async.} =
-      var client = newAsyncHttpClient()
-      return await client.getContent("http://example.com")
-
-    let exampleHtml = waitFor asyncProc()
+    let exampleHtml = newHttpClient().getContent("http://example.com")
     assert "Example Domain" in exampleHtml
-    assert not ("Pizza" in exampleHtml)
+    assert "Pizza" notin exampleHtml
 
   new result
   result.headers = headers
@@ -616,6 +612,17 @@ proc newAsyncHttpClient*(userAgent = defUserAgent, maxRedirects = 5,
   ## connections.
   ##
   ## `headers` specifies the HTTP Headers.
+  runnableExamples:
+    import std/[asyncdispatch, strutils]
+
+    proc asyncProc(): Future[string] {.async.} =
+      let client = newAsyncHttpClient()
+      result = await client.getContent("http://example.com")
+
+    let exampleHtml = waitFor asyncProc()
+    assert "Example Domain" in exampleHtml
+    assert "Pizza" notin exampleHtml
+  
   new result
   result.headers = headers
   result.userAgent = userAgent

From b59dc3b255d778cefdbef2c0c3ff55d43892c11c Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Wed, 26 May 2021 00:41:50 -0700
Subject: [PATCH 0839/2472] remove some custom logic in testament around flags,
 testExec (#18090)

* remove some custom logic in testament around flags, testExec

* remove testExec, custom logic around flags from testament

* fixup
---
 testament/categories.nim   | 23 -----------------------
 testament/testament.nim    | 16 ----------------
 tests/flags/tgenscript.nim |  6 ------
 tests/misc/trunner.nim     | 33 ++++++++++++++++++++++++++++++---
 4 files changed, 30 insertions(+), 48 deletions(-)
 delete mode 100644 tests/flags/tgenscript.nim

diff --git a/testament/categories.nim b/testament/categories.nim
index 28ede1ba0..510e90c53 100644
--- a/testament/categories.nim
+++ b/testament/categories.nim
@@ -23,7 +23,6 @@ const
     "debugger",
     "dll",
     "examples",
-    "flags",
     "gc",
     "io",
     "js",
@@ -48,26 +47,6 @@ proc isTestFile*(file: string): bool =
   let (_, name, ext) = splitFile(file)
   result = ext == ".nim" and name.startsWith("t")
 
-# --------------------- flags tests -------------------------------------------
-
-proc flagTests(r: var TResults, cat: Category, options: string) =
-  # --genscript
-  const filename = testsDir/"flags"/"tgenscript"
-  const genopts = " --genscript"
-  let nimcache = nimcacheDir(filename, genopts, targetC)
-  testSpec r, makeTest(filename, genopts, cat)
-
-  when defined(windows):
-    testExec r, makeTest(filename, " cmd /c cd " & nimcache &
-                         " && compile_tgenscript.bat", cat)
-
-  elif defined(posix):
-    testExec r, makeTest(filename, " sh -c \"cd " & nimcache &
-                         " && sh compile_tgenscript.sh\"", cat)
-
-  # Run
-  testExec r, makeTest(filename, " " & nimcache / "tgenscript", cat)
-
 # --------------------- DLL generation tests ----------------------------------
 
 proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) =
@@ -705,8 +684,6 @@ proc processCategory(r: var TResults, cat: Category,
         jsTests(r, cat, options)
     of "dll":
       dllTests(r, cat, options)
-    of "flags":
-      flagTests(r, cat, options)
     of "gc":
       gcTests(r, cat, options)
     of "longgc":
diff --git a/testament/testament.nim b/testament/testament.nim
index a8dc0557c..2edefdb01 100644
--- a/testament/testament.nim
+++ b/testament/testament.nim
@@ -621,22 +621,6 @@ proc testC(r: var TResults, test: TTest, action: TTestAction) =
     if exitCode != 0: given.err = reExitcodesDiffer
   if given.err == reSuccess: inc(r.passed)
 
-proc testExec(r: var TResults, test: TTest) =
-  # runs executable or script, just goes by exit code
-  if not checkDisabled(r, test): return
-
-  inc(r.total)
-  let (outp, errC) = execCmdEx(test.options.strip())
-  var given: TSpec
-  if errC == 0:
-    given.err = reSuccess
-  else:
-    given.err = reExitcodesDiffer
-    given.msg = outp
-
-  if given.err == reSuccess: inc(r.passed)
-  r.addResult(test, targetC, "", given.msg, given.err)
-
 proc makeTest(test, options: string, cat: Category): TTest =
   result.cat = cat
   result.name = test
diff --git a/tests/flags/tgenscript.nim b/tests/flags/tgenscript.nim
deleted file mode 100644
index d58395a40..000000000
--- a/tests/flags/tgenscript.nim
+++ /dev/null
@@ -1,6 +0,0 @@
-discard """
-  targets: "c"
-  action: compile
-"""
-
-echo "--genscript"
diff --git a/tests/misc/trunner.nim b/tests/misc/trunner.nim
index 1b679d92c..3297b3a24 100644
--- a/tests/misc/trunner.nim
+++ b/tests/misc/trunner.nim
@@ -33,16 +33,23 @@ const
 
 proc runNimCmd(file, options = "", rtarg = ""): auto =
   let fileabs = testsDir / file.unixToNativePath
-  doAssert fileabs.fileExists, fileabs
+  # doAssert fileabs.fileExists, fileabs # disabled because this allows passing `nim r --eval:code fakefile`
   let cmd = fmt"{nim} {mode} {options} --hints:off {fileabs} {rtarg}"
   result = execCmdEx(cmd)
-  when false:  echo result[0] & "\n" & result[1] # for debugging
+  when false: # for debugging
+    echo cmd
+    echo result[0] & "\n" & $result[1]
 
 proc runNimCmdChk(file, options = "", rtarg = ""): string =
   let (ret, status) = runNimCmd(file, options, rtarg = rtarg)
   doAssert status == 0, $(file, options) & "\n" & ret
   ret
 
+proc genShellCmd(filename: string): string =
+  let filename = filename.quoteShell
+  when defined(windows): "cmd /c " & filename # or "cmd /c " ?
+  else: "sh " & filename
+
 when defined(nimTrunnerFfi):
   block: # mevalffi
     when defined(openbsd):
@@ -71,7 +78,9 @@ foo:0.03:asdf:103:105
 ret=[s1:foobar s2:foobar age:25 pi:3.14]
 """, output
 
-else: # don't run twice the same test
+elif not defined(nimTestsTrunnerDebugging):
+  # don't run twice the same test with `nimTrunnerFfi`
+  # use `-d:nimTestsTrunnerDebugging` for debugging convenience when you want to just run 1 test
   import std/strutils
   import std/json
   template check2(msg) = doAssert msg in output, output
@@ -330,3 +339,21 @@ running: v2
     doAssert "D20210428T161003" in j["defined_symbols"].to(seq[string])
     doAssert j["version"].to(string) == NimVersion
     doAssert j["nimExe"].to(string) == getCurrentCompilerExe()
+
+  block: # genscript
+    const nimcache2 = buildDir / "D20210524T212851"
+    removeDir(nimcache2)
+    let input = "tgenscript_fakefile" # no need for a real file, --eval is good enough
+    let output = runNimCmdChk(input, fmt"""--genscript --nimcache:{nimcache2.quoteShell} --eval:"echo(12345)" """)
+    doAssert output.len == 0, output
+    let ext = when defined(windows): ".bat" else: ".sh"
+    let filename = fmt"compile_{input}{ext}" # synchronize with `generateScript`
+    doAssert fileExists(nimcache2/filename), nimcache2/filename
+    let (outp, status) = execCmdEx(genShellCmd(filename), options = {poStdErrToStdOut}, workingDir = nimcache2)
+    doAssert status == 0, outp
+    let (outp2, status2) = execCmdEx(nimcache2 / input, options = {poStdErrToStdOut})
+    doAssert outp2 == "12345\n", outp2
+    doAssert status2 == 0
+
+else:
+  discard # only during debugging, tests added here will run with `-d:nimTestsTrunnerDebugging` enabled

From c495628255330a559e3398663cc8872cca18b04d Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Wed, 26 May 2021 00:43:30 -0700
Subject: [PATCH 0840/2472] refactor common code in CI pipelines (#18035)

* refactor CI with nimInternalInstallDepsWindows

* refactor CI with nimInternalBuildKochAndRunCI

* fixup
---
 .builds/freebsd.yml               |  6 +-----
 .builds/openbsd_0.yml             |  6 +-----
 .builds/openbsd_1.yml             |  6 +-----
 .github/workflows/ci_docs.yml     |  8 +++-----
 .github/workflows/ci_packages.yml | 22 ++++++----------------
 azure-pipelines.yml               | 16 ++++------------
 ci/funs.sh                        | 17 +++++++++++++++++
 tools/ci_generate.nim             |  6 +-----
 8 files changed, 34 insertions(+), 53 deletions(-)

diff --git a/.builds/freebsd.yml b/.builds/freebsd.yml
index 8f336771a..15d09dda0 100644
--- a/.builds/freebsd.yml
+++ b/.builds/freebsd.yml
@@ -22,15 +22,11 @@ tasks:
     set -e
     cd Nim
     . ci/funs.sh && nimBuildCsourcesIfNeeded
-    $nim_csources c --skipUserCfg --skipParentCfg koch
     echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
 - test: |
     set -e
     cd Nim
-    if ! ./koch runCI; then
-      nim r tools/ci_testresults.nim
-      exit 1
-    fi
+    . ci/funs.sh && nimInternalBuildKochAndRunCI
 triggers:
 - action: email
   condition: failure
diff --git a/.builds/openbsd_0.yml b/.builds/openbsd_0.yml
index c19cd63c6..c3b2fd43e 100644
--- a/.builds/openbsd_0.yml
+++ b/.builds/openbsd_0.yml
@@ -22,15 +22,11 @@ tasks:
     set -e
     cd Nim
     . ci/funs.sh && nimBuildCsourcesIfNeeded
-    $nim_csources c --skipUserCfg --skipParentCfg koch
     echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
 - test: |
     set -e
     cd Nim
-    if ! ./koch runCI; then
-      nim r tools/ci_testresults.nim
-      exit 1
-    fi
+    . ci/funs.sh && nimInternalBuildKochAndRunCI
 triggers:
 - action: email
   condition: failure
diff --git a/.builds/openbsd_1.yml b/.builds/openbsd_1.yml
index 06f5f2ae5..e98ec4640 100644
--- a/.builds/openbsd_1.yml
+++ b/.builds/openbsd_1.yml
@@ -22,15 +22,11 @@ tasks:
     set -e
     cd Nim
     . ci/funs.sh && nimBuildCsourcesIfNeeded
-    $nim_csources c --skipUserCfg --skipParentCfg koch
     echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
 - test: |
     set -e
     cd Nim
-    if ! ./koch runCI; then
-      nim r tools/ci_testresults.nim
-      exit 1
-    fi
+    . ci/funs.sh && nimInternalBuildKochAndRunCI
 triggers:
 - action: email
   condition: failure
diff --git a/.github/workflows/ci_docs.yml b/.github/workflows/ci_docs.yml
index 364c05051..ddef94fbe 100644
--- a/.github/workflows/ci_docs.yml
+++ b/.github/workflows/ci_docs.yml
@@ -59,11 +59,9 @@ jobs:
         if: runner.os == 'Windows'
         shell: bash
         run: |
-          mkdir dist
-          curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
-          curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
-          7z x dist/mingw64.7z -odist
-          7z x dist/dlls.zip -obin
+          set -e
+          . ci/funs.sh
+          nimInternalInstallDepsWindows
           echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
 
       - name: 'Add build binaries to PATH'
diff --git a/.github/workflows/ci_packages.yml b/.github/workflows/ci_packages.yml
index d033aba10..aeb29e147 100644
--- a/.github/workflows/ci_packages.yml
+++ b/.github/workflows/ci_packages.yml
@@ -40,12 +40,10 @@ jobs:
         if: runner.os == 'Windows'
         shell: bash
         run: |
-          mkdir dist
-          curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
-          curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
-          7z x dist/mingw64.7z -odist
-          7z x dist/dlls.zip -obin
-          echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
+          set -e
+          . ci/funs.sh
+          nimInternalInstallDepsWindows
+          echo_run echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
 
       - name: 'Add build binaries to PATH'
         shell: bash
@@ -59,14 +57,6 @@ jobs:
         shell: bash
         run: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc ucpu='${{ matrix.cpu }}'
 
-      - name: 'Build koch'
+      - name: 'koch, Run CI'
         shell: bash
-        run: nim c koch
-      - name: 'Run CI'
-        shell: bash
-        run: ./koch runCI
-
-      - name: 'Show failed tests'
-        if: failure()
-        shell: bash
-        run: nim c -r tools/ci_testresults.nim
+        run: . ci/funs.sh && nimInternalBuildKochAndRunCI
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 112fb0a62..f0c3b2d0a 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -133,11 +133,7 @@ jobs:
     - bash: |
         set -e
         . ci/funs.sh
-        echo_run mkdir dist
-        echo_run curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
-        echo_run curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
-        echo_run 7z x dist/mingw64.7z -odist
-        echo_run 7z x dist/dlls.zip -obin
+        nimInternalInstallDepsWindows
         echo_run echo '##vso[task.prependpath]$(System.DefaultWorkingDirectory)/dist/mingw64/bin'
 
       displayName: 'Install dependencies (Windows)'
@@ -163,13 +159,9 @@ jobs:
     #   condition: and(succeeded(), eq(variables['skipci'], 'false'))
     #   displayName: 'Restore built csourcesAny'
 
-    - bash: nim c koch
+    - bash: . ci/funs.sh && nimInternalBuildKochAndRunCI
+      # would could also show on failure: echo '##vso[task.complete result=Failed]'
       condition: and(succeeded(), eq(variables['skipci'], 'false'))
-      displayName: 'Build koch'
-
-      # set result to omit the "bash exited with error code '1'" message
-    - bash: ./koch runCI || echo '##vso[task.complete result=Failed]'
-      condition: and(succeeded(), eq(variables['skipci'], 'false'))
-      displayName: 'Run CI'
+      displayName: 'koch, Run CI'
       env:
         SYSTEM_ACCESSTOKEN: $(System.AccessToken)
diff --git a/ci/funs.sh b/ci/funs.sh
index 0e4f0eb3b..9772535be 100644
--- a/ci/funs.sh
+++ b/ci/funs.sh
@@ -31,6 +31,23 @@ nimIsCiSkip(){
   fi
 }
 
+nimInternalInstallDepsWindows(){
+  echo_run mkdir dist
+  echo_run curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
+  echo_run curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
+  echo_run 7z x dist/mingw64.7z -odist
+  echo_run 7z x dist/dlls.zip -obin
+}
+
+nimInternalBuildKochAndRunCI(){
+  echo_run nim c koch
+  if ! echo_run ./koch runCI; then
+    echo_run echo "runCI failed"
+    echo_run nim r -tools/ci_testresults.nim
+    return 1
+  fi
+}
+
 nimDefineVars(){
   . config/build_config.txt
   nim_csources=bin/nim_csources_$nim_csourcesHash
diff --git a/tools/ci_generate.nim b/tools/ci_generate.nim
index f08cd5409..52b84f0d8 100644
--- a/tools/ci_generate.nim
+++ b/tools/ci_generate.nim
@@ -29,15 +29,11 @@ tasks:
     set -e
     cd Nim
     . ci/funs.sh && nimBuildCsourcesIfNeeded
-    $nim_csources c --skipUserCfg --skipParentCfg koch
     echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
 - test: |
     set -e
     cd Nim
-    if ! ./koch runCI; then
-      nim r tools/ci_testresults.nim
-      exit 1
-    fi
+    . ci/funs.sh && nimInternalBuildKochAndRunCI
 triggers:
 - action: email
   condition: failure

From 8df55d0ad735277683631bb8e5966ee75545de2b Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Wed, 26 May 2021 00:44:00 -0700
Subject: [PATCH 0841/2472] close #3482 no more cgen error with typed templates
 (#18094)

---
 tests/misc/t3482.nim | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 tests/misc/t3482.nim

diff --git a/tests/misc/t3482.nim b/tests/misc/t3482.nim
new file mode 100644
index 000000000..33b3b8f40
--- /dev/null
+++ b/tests/misc/t3482.nim
@@ -0,0 +1,15 @@
+discard """
+  action: reject
+  nimout: "t3482.nim(13, 8) Error: undeclared identifier: 'output'"
+"""
+# bug #3482 (correct behavior since 1.4.0, cgen error in 1.2.0)
+template foo*(body: typed) =
+  if true:
+    body
+
+proc test =
+  foo:
+    var output = ""
+  echo output.len
+
+test()

From 0b2bbcaa23f91b06bc06dd2cf35d4d26a6d24aa8 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Wed, 26 May 2021 00:46:53 -0700
Subject: [PATCH 0842/2472] fix #18077 testament now parses `cmd` properly
 (#18086)

---
 testament/categories.nim |  3 ++-
 testament/testament.nim  | 53 ++++++++++++++++++++++++++++------------
 tests/misc/t18077.nim    | 21 ++++++++++++++++
 3 files changed, 61 insertions(+), 16 deletions(-)
 create mode 100644 tests/misc/t18077.nim

diff --git a/testament/categories.nim b/testament/categories.nim
index 510e90c53..8cf90666c 100644
--- a/testament/categories.nim
+++ b/testament/categories.nim
@@ -87,8 +87,9 @@ proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) =
     var hcri = makeTest("tests/dll/nimhcr_integration.nim",
                                    options & " --forceBuild --hotCodeReloading:on" & rpath, cat)
     let nimcache = nimcacheDir(hcri.name, hcri.options, getTestSpecTarget())
-    hcri.args = prepareTestArgs(hcri.spec.getCmd, hcri.name,
+    let cmd = prepareTestCmd(hcri.spec.getCmd, hcri.name,
                                 hcri.options, nimcache, getTestSpecTarget())
+    hcri.testArgs = cmd.parseCmdLine
     testSpec r, hcri
 
 proc dllTests(r: var TResults, cat: Category, options: string) =
diff --git a/testament/testament.nim b/testament/testament.nim
index 2edefdb01..1eb925122 100644
--- a/testament/testament.nim
+++ b/testament/testament.nim
@@ -85,7 +85,7 @@ type
     name: string
     cat: Category
     options: string
-    args: seq[string]
+    testArgs: seq[string]
     spec: TSpec
     startTime: float
     debugInfo: string
@@ -154,24 +154,23 @@ proc nimcacheDir(filename, options: string, target: TTarget): string =
   let hashInput = options & $target
   result = "nimcache" / (filename & '_' & hashInput.getMD5)
 
-proc prepareTestArgs(cmdTemplate, filename, options, nimcache: string,
-                     target: TTarget, extraOptions = ""): seq[string] =
+proc prepareTestCmd(cmdTemplate, filename, options, nimcache: string,
+                     target: TTarget, extraOptions = ""): string =
   var options = target.defaultOptions & ' ' & options
-  # improve pending https://github.com/nim-lang/Nim/issues/14343
-  if nimcache.len > 0: options.add ' ' & ("--nimCache:" & nimcache).quoteShell
+  if nimcache.len > 0: options.add(" --nimCache:$#" % nimcache.quoteShell)
   options.add ' ' & extraOptions
-  result = parseCmdLine(cmdTemplate % ["target", targetToCmd[target],
+  # we avoid using `parseCmdLine` which is buggy, refs bug #14343
+  result = cmdTemplate % ["target", targetToCmd[target],
                       "options", options, "file", filename.quoteShell,
-                      "filedir", filename.getFileDir(), "nim", compilerPrefix])
+                      "filedir", filename.getFileDir(), "nim", compilerPrefix]
 
 proc callNimCompiler(cmdTemplate, filename, options, nimcache: string,
                      target: TTarget, extraOptions = ""): TSpec =
-  let c = prepareTestArgs(cmdTemplate, filename, options, nimcache, target,
+  result.cmd = prepareTestCmd(cmdTemplate, filename, options, nimcache, target,
                           extraOptions)
-  result.cmd = quoteShellCommand(c)
-  verboseCmd(c.quoteShellCommand)
-  var p = startProcess(command = c[0], args = c[1 .. ^1],
-                       options = {poStdErrToStdOut, poUsePath})
+  verboseCmd(result.cmd)
+  var p = startProcess(command = result.cmd,
+                       options = {poStdErrToStdOut, poUsePath, poEvalCommand})
   let outp = p.outputStream
   var foundSuccessMsg = false
   var foundErrorMsg = false
@@ -224,9 +223,33 @@ proc callNimCompiler(cmdTemplate, filename, options, nimcache: string,
 
 proc callCCompiler(cmdTemplate, filename, options: string,
                   target: TTarget): TSpec =
-  let c = prepareTestArgs(cmdTemplate, filename, options, nimcache = "", target)
+  let cmd = prepareTestCmd(cmdTemplate, filename, options, nimcache = "", target)
+  doAssert false
+  #[
+  this code hasn't been run in a while, and should be removed which simplifies code
+  there are better ways to do this anyways (e.g. running c code from a nim file)
+  
+  the only place where this is called is:
+  `testC r, makeTest("tests/realtimeGC/cmain", cOptions, cat), actionRun`
+  which isn't run unless you call:
+  XDG_CONFIG_HOME= nim r --lib:lib --stacktrace:on testament/testament.nim r longgc
+
+  and this fails since at least nim 1.0 with:
+  testament/testament.nim(851) testament
+  testament/testament.nim(822) main
+  testament/categories.nim(713) processCategory
+  testament/categories.nim(189) longGCTests
+  testament/testament.nim(644) makeTest
+  testament/specs.nim(251) parseSpec
+  testament/specs.nim(184) extractSpec
+  lib/system/io.nim(861) readFile
+  Error: unhandled exception: cannot open: tests/realtimeGC/cmain.nim [IOError]
+
+  Also, `c[5 .. ^1]` is too magical.
+  ]#
+  let c = cmd.parseCmdLine
   var p = startProcess(command = "gcc", args = c[5 .. ^1],
-                       options = {poStdErrToStdOut, poUsePath})
+                       options = {poStdErrToStdOut, poUsePath, poEvalCommand})
   let outp = p.outputStream
   var x = newStringOfCap(120)
   result.nimout = ""
@@ -525,7 +548,7 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec,
                       reExeNotFound)
         else:
           var exeCmd: string
-          var args = test.args
+          var args = test.testArgs
           if isJsTarget:
             exeCmd = nodejs
             # see D20210217T215950
diff --git a/tests/misc/t18077.nim b/tests/misc/t18077.nim
new file mode 100644
index 000000000..6cd05d575
--- /dev/null
+++ b/tests/misc/t18077.nim
@@ -0,0 +1,21 @@
+discard """
+  cmd: '''nim doc -d:nimTestsT18077b:4 --doccmd:"-d:nimTestsT18077 -d:nimTestsT18077b:3 --hints:off" $file'''
+  action: compile
+"""
+
+# bug #18077
+
+const nimTestsT18077b {.intdefine.} = 1
+
+static:
+  when defined(nimdoc):
+    doAssert nimTestsT18077b == 4
+    doAssert not defined(nimTestsT18077)
+  else:
+    doAssert defined(nimTestsT18077)
+    doAssert nimTestsT18077b == 3
+
+runnableExamples:
+  const nimTestsT18077b {.intdefine.} = 2
+  doAssert nimTestsT18077b == 3
+  doAssert defined(nimTestsT18077)

From 3b1aabdcffafbbf37ad02f013f754f67dfe2461c Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Wed, 26 May 2021 00:51:48 -0700
Subject: [PATCH 0843/2472] change `--hint[X] => --hint:X` in nim repo (avoids
 shell quoting issues) (#18085)

---
 compiler/nim.cfg                          |  4 ++--
 compiler/nimfix/nimfix.nim.cfg            |  2 +-
 config/nim.cfg                            |  4 ++--
 doc/advopt.txt                            | 10 +++++-----
 doc/filters.rst                           |  2 +-
 doc/nimc.rst                              |  8 ++++----
 drnim/nim.cfg                             |  2 +-
 nimsuggest/nimsuggest.nim.cfg             |  2 +-
 testament/categories.nim                  |  4 ++--
 tests/effects/toutparam.nim               |  2 +-
 tests/osproc/treadlines.nim               |  2 +-
 tests/pragmas/t8741.nim                   |  2 +-
 tests/range/tcompiletime_range_checks.nim |  2 +-
 tests/tools/tunused_imports.nim           |  2 +-
 tools/nimweb.nim                          |  2 +-
 15 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/compiler/nim.cfg b/compiler/nim.cfg
index 40e93d523..9ecd00b0b 100644
--- a/compiler/nim.cfg
+++ b/compiler/nim.cfg
@@ -1,6 +1,6 @@
 # Special configuration file for the Nim project
 
-hint[XDeclaredButNotUsed]:off
+hint:XDeclaredButNotUsed:off
 
 define:booting
 define:nimcore
@@ -22,5 +22,5 @@ define:useStdoutAsStdmsg
 #gc:markAndSweep
 
 @if nimHasWarningObservableStores:
-  warning[ObservableStores]: off
+  warning:ObservableStores: off
 @end
diff --git a/compiler/nimfix/nimfix.nim.cfg b/compiler/nimfix/nimfix.nim.cfg
index 0d9dbfa4b..27ebf10b9 100644
--- a/compiler/nimfix/nimfix.nim.cfg
+++ b/compiler/nimfix/nimfix.nim.cfg
@@ -1,7 +1,7 @@
 # Special configuration file for the Nim project
 # gc:markAndSweep
 
-hint[XDeclaredButNotUsed]:off
+hint:XDeclaredButNotUsed:off
 path:"$projectPath/.."
 
 path:"$lib/packages/docutils"
diff --git a/config/nim.cfg b/config/nim.cfg
index 3b964d124..c0a844e03 100644
--- a/config/nim.cfg
+++ b/config/nim.cfg
@@ -13,8 +13,8 @@ cc = gcc
 # additional options always passed to the compiler:
 --parallel_build: "0" # 0 to auto-detect number of processors
 
-hint[LineTooLong]=off
-#hint[XDeclaredButNotUsed]=off
+hint:LineTooLong:off
+#hint:XDeclaredButNotUsed:off
 
 # Examples of how to setup a cross-compiler:
 
diff --git a/doc/advopt.txt b/doc/advopt.txt
index e50fb243b..ea3370880 100644
--- a/doc/advopt.txt
+++ b/doc/advopt.txt
@@ -47,12 +47,12 @@ Advanced options:
                             an implementation defined set of suggestions.
   -w:on|off|list, --warnings:on|off|list
                             turn all warnings on|off or list all available
-  --warning[X]:on|off       turn specific warning X on|off
+  --warning:X:on|off        turn specific warning X on|off.
+                            `warning:X` means `warning:X:on`, as with similar flags.
   --hints:on|off|list       turn all hints on|off or list all available
-  --hint[X]:on|off          turn specific hint X on|off
-  --warningAsError[X]:on|off
-                            turn specific warning X into an error on|off
-  --hintAsError[X]:on|off   turn specific hint X into an error on|off
+  --hint:X:on|off           turn specific hint X on|off.
+  --warningAsError:X:on|off turn specific warning X into an error on|off
+  --hintAsError:X:on|off    turn specific hint X into an error on|off
   --styleCheck:off|hint|error
                             produce hints or errors for Nim identifiers that
                             do not adhere to Nim's official style guide
diff --git a/doc/filters.rst b/doc/filters.rst
index c6d436194..e237744cb 100644
--- a/doc/filters.rst
+++ b/doc/filters.rst
@@ -25,7 +25,7 @@ just like an ordinary procedure call with named or positional arguments. The
 available parameters depend on the invoked filter. Before version 0.12.0 of
 the language `#!` was used instead of `#?`.
 
-**Hint:** With `--hint[codeBegin]:on`:option: or `--verbosity:2`:option:
+**Hint:** With `--hint:codeBegin:on`:option: or `--verbosity:2`:option:
 (or higher) while compiling or `nim check`:cmd:, Nim lists the processed code after
 each filter application.
 
diff --git a/doc/nimc.rst b/doc/nimc.rst
index 47f7243cc..e467a33af 100644
--- a/doc/nimc.rst
+++ b/doc/nimc.rst
@@ -53,8 +53,8 @@ Advanced command-line switches are:
 List of warnings
 ----------------
 
-Each warning can be activated individually with `--warning[NAME]:on|off`:option: or
-in a `push` pragma.
+Each warning can be activated individually with `--warning:NAME:on|off`:option: or
+in a `push` pragma with `{.warning[NAME]:on|off.}`.
 
 ==========================       ============================================
 Name                             Description
@@ -77,8 +77,8 @@ User                             Some user-defined warning.
 List of hints
 -------------
 
-Each hint can be activated individually with `--hint[NAME]:on|off`:option: or in a
-`push` pragma.
+Each hint can be activated individually with `--hint:NAME:on|off`:option: or in a
+`push` pragma with `{.hint[NAME]:on|off.}`.
 
 ==========================       ============================================
 Name                             Description
diff --git a/drnim/nim.cfg b/drnim/nim.cfg
index 58c1725d9..57dca974c 100644
--- a/drnim/nim.cfg
+++ b/drnim/nim.cfg
@@ -1,6 +1,6 @@
 # Special configuration file for the Nim project
 
-hint[XDeclaredButNotUsed]:off
+hint:XDeclaredButNotUsed:off
 
 define:booting
 define:nimcore
diff --git a/nimsuggest/nimsuggest.nim.cfg b/nimsuggest/nimsuggest.nim.cfg
index 394449740..6e5ee8c71 100644
--- a/nimsuggest/nimsuggest.nim.cfg
+++ b/nimsuggest/nimsuggest.nim.cfg
@@ -2,7 +2,7 @@
 
 gc:markAndSweep
 
-hint[XDeclaredButNotUsed]:off
+hint:XDeclaredButNotUsed:off
 
 path:"$lib/packages/docutils"
 
diff --git a/testament/categories.nim b/testament/categories.nim
index 8cf90666c..0a3595ff4 100644
--- a/testament/categories.nim
+++ b/testament/categories.nim
@@ -486,7 +486,7 @@ proc icTests(r: var TResults; testsDir: string, cat: Category, options: string;
     writeOnly = " --incremental:writeonly "
     readOnly = " --incremental:readonly "
     incrementalOn = " --incremental:on -d:nimIcIntegrityChecks "
-    navTestConfig = " --ic:on -d:nimIcNavigatorTests --hint[Conf]:off --warnings:off "
+    navTestConfig = " --ic:on -d:nimIcNavigatorTests --hint:Conf:off --warnings:off "
 
   template test(x: untyped) =
     testSpecWithNimcache(r, makeRawTest(file, x & options, cat), nimcache)
@@ -500,7 +500,7 @@ proc icTests(r: var TResults; testsDir: string, cat: Category, options: string;
 
   template checkTest() =
     var test = makeRawTest(file, options, cat)
-    test.spec.cmd = compilerPrefix & " check --hint[Conf]:off --warnings:off --ic:on $options " & file
+    test.spec.cmd = compilerPrefix & " check --hint:Conf:off --warnings:off --ic:on $options " & file
     testSpecWithNimcache(r, test, nimcache)
 
   if not isNavigatorTest:
diff --git a/tests/effects/toutparam.nim b/tests/effects/toutparam.nim
index 00572aa6b..1126aa77e 100644
--- a/tests/effects/toutparam.nim
+++ b/tests/effects/toutparam.nim
@@ -1,5 +1,5 @@
 discard """
-  cmd: '''nim c --warningAsError[Uninit]:on --skipCfg --skipParentCfg $file'''
+  cmd: '''nim c --warningAsError:Uninit:on --skipCfg --skipParentCfg $file'''
   errormsg: "use explicit initialization of 'x' for clarity [Uninit]"
   line: 24
   disabled: "true"
diff --git a/tests/osproc/treadlines.nim b/tests/osproc/treadlines.nim
index bcde19d7f..200a7c299 100644
--- a/tests/osproc/treadlines.nim
+++ b/tests/osproc/treadlines.nim
@@ -13,7 +13,7 @@ var ps: seq[Process] # compile & run 2 progs in parallel
 const nim = getCurrentCompilerExe()
 for prog in ["a", "b"]:
   ps.add startProcess(nim, "",
-                      ["r", "--hint[Conf]=off", "--hint[Processing]=off", prog],
+                      ["r", "--hint:Conf:off", "--hint:Processing:off", prog],
                       options = {poUsePath, poDaemon, poStdErrToStdOut})
 
 for p in ps:
diff --git a/tests/pragmas/t8741.nim b/tests/pragmas/t8741.nim
index 61a449c01..221c732b0 100644
--- a/tests/pragmas/t8741.nim
+++ b/tests/pragmas/t8741.nim
@@ -1,5 +1,5 @@
 discard """
-  cmd: "nim check --hint[processing]:off $file"
+  cmd: "nim check --hint:processing:off $file"
   errormsg: "3 is not two"
   nimout: '''t8741.nim(13, 9) Error: cannot attach a custom pragma to 'a'
 t8741.nim(29, 15) template/generic instantiation of `onlyTwo` from here
diff --git a/tests/range/tcompiletime_range_checks.nim b/tests/range/tcompiletime_range_checks.nim
index 37095e0b7..29e2c48a8 100644
--- a/tests/range/tcompiletime_range_checks.nim
+++ b/tests/range/tcompiletime_range_checks.nim
@@ -1,5 +1,5 @@
 discard """
-  cmd: "nim check --hint[Processing]:off --hint[Conf]:off $file"
+  cmd: "nim check --hint:Processing:off --hint:Conf:off $file"
   errormsg: "18446744073709551615 can't be converted to int8"
   nimout: '''tcompiletime_range_checks.nim(36, 21) Error: 2147483648 can't be converted to int32
 tcompiletime_range_checks.nim(37, 23) Error: -1 can't be converted to uint64
diff --git a/tests/tools/tunused_imports.nim b/tests/tools/tunused_imports.nim
index bce906634..31d6cf7d7 100644
--- a/tests/tools/tunused_imports.nim
+++ b/tests/tools/tunused_imports.nim
@@ -1,5 +1,5 @@
 discard """
-  cmd: '''nim c --hint[Processing]:off $file'''
+  cmd: '''nim c --hint:Processing:off $file'''
   nimout: '''
 tunused_imports.nim(11, 10) Warning: BEGIN [User]
 tunused_imports.nim(36, 10) Warning: END [User]
diff --git a/tools/nimweb.nim b/tools/nimweb.nim
index f71b5f3be..ccc80dfcf 100644
--- a/tools/nimweb.nim
+++ b/tools/nimweb.nim
@@ -53,7 +53,7 @@ proc initConfigData(c: var TConfigData) =
   c.webdoc = @[]
   c.pdf = @[]
   c.infile = ""
-  c.nimArgs = "--hint[Conf]:off --hint[Path]:off --hint[Processing]:off -d:boot "
+  c.nimArgs = "--hint:Conf:off --hint:Path:off --hint:Processing:off -d:boot "
   c.gitURL = "https://github.com/nim-lang/Nim"
   c.docHTMLOutput = "doc/html"
   c.webUploadOutput = "web/upload"

From 1e0165186bb8539cfd8aca1a7af8d01dc278bd46 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Wed, 26 May 2021 11:49:01 -0700
Subject: [PATCH 0844/2472] testament: remove deadcode related to realtimeGC,
 testC, callCCompiler (#18087)

---
 testament/categories.nim    | 22 +++---------
 testament/testament.nim     | 63 ---------------------------------
 tests/readme.md             |  3 +-
 tests/realtimeGC/cmain.c    | 69 -------------------------------------
 tests/realtimeGC/readme.txt | 10 ------
 tests/realtimeGC/tmain.nim  | 10 +++++-
 6 files changed, 15 insertions(+), 162 deletions(-)
 delete mode 100644 tests/realtimeGC/cmain.c
 delete mode 100644 tests/realtimeGC/readme.txt

diff --git a/testament/categories.nim b/testament/categories.nim
index 0a3595ff4..7f05797ff 100644
--- a/testament/categories.nim
+++ b/testament/categories.nim
@@ -28,7 +28,6 @@ const
     "js",
     "ic",
     "lib",
-    "longgc",
     "manyloc",
     "nimble-packages",
     "niminaction",
@@ -39,7 +38,6 @@ const
     "coroutines",
     "osproc",
     "shouldfail",
-    "dir with space",
     "destructor"
   ]
 
@@ -156,19 +154,6 @@ proc gcTests(r: var TResults, cat: Category, options: string) =
   test "cyclecollector"
   testWithoutBoehm "trace_globals"
 
-proc longGCTests(r: var TResults, cat: Category, options: string) =
-  when defined(windows):
-    let cOptions = "-ldl -DWIN"
-  else:
-    let cOptions = "-ldl"
-
-  var c = initResults()
-  # According to ioTests, this should compile the file
-  testSpec c, makeTest("tests/realtimeGC/shared", options, cat)
-  #        ^- why is this not appended to r? Should this be discarded?
-  testC r, makeTest("tests/realtimeGC/cmain", cOptions, cat), actionRun
-  testSpec r, makeTest("tests/realtimeGC/nmain", options & "--threads: on", cat)
-
 # ------------------------- threading tests -----------------------------------
 
 proc threadTests(r: var TResults, cat: Category, options: string) =
@@ -186,6 +171,11 @@ proc ioTests(r: var TResults, cat: Category, options: string) =
   # dummy compile result:
   var c = initResults()
   testSpec c, makeTest("tests/system/helpers/readall_echo", options, cat)
+  #        ^- why is this not appended to r? Should this be discarded?
+  # EDIT: this should be replaced by something like in D20210524T180826,
+  # likewise in similar instances where `testSpec c` is used, or more generally
+  # when a test depends on another test, as it makes tests non-independent,
+  # creating complications for batching and megatest logic.
   testSpec r, makeTest("tests/system/tio", options, cat)
 
 # ------------------------- async tests ---------------------------------------
@@ -687,8 +677,6 @@ proc processCategory(r: var TResults, cat: Category,
       dllTests(r, cat, options)
     of "gc":
       gcTests(r, cat, options)
-    of "longgc":
-      longGCTests(r, cat, options)
     of "debugger":
       debuggerTests(r, cat, options)
     of "manyloc":
diff --git a/testament/testament.nim b/testament/testament.nim
index 1eb925122..789c987c0 100644
--- a/testament/testament.nim
+++ b/testament/testament.nim
@@ -221,53 +221,6 @@ proc callNimCompiler(cmdTemplate, filename, options, nimcache: string,
     result.msg = matches[0]
   trimUnitSep result.msg
 
-proc callCCompiler(cmdTemplate, filename, options: string,
-                  target: TTarget): TSpec =
-  let cmd = prepareTestCmd(cmdTemplate, filename, options, nimcache = "", target)
-  doAssert false
-  #[
-  this code hasn't been run in a while, and should be removed which simplifies code
-  there are better ways to do this anyways (e.g. running c code from a nim file)
-  
-  the only place where this is called is:
-  `testC r, makeTest("tests/realtimeGC/cmain", cOptions, cat), actionRun`
-  which isn't run unless you call:
-  XDG_CONFIG_HOME= nim r --lib:lib --stacktrace:on testament/testament.nim r longgc
-
-  and this fails since at least nim 1.0 with:
-  testament/testament.nim(851) testament
-  testament/testament.nim(822) main
-  testament/categories.nim(713) processCategory
-  testament/categories.nim(189) longGCTests
-  testament/testament.nim(644) makeTest
-  testament/specs.nim(251) parseSpec
-  testament/specs.nim(184) extractSpec
-  lib/system/io.nim(861) readFile
-  Error: unhandled exception: cannot open: tests/realtimeGC/cmain.nim [IOError]
-
-  Also, `c[5 .. ^1]` is too magical.
-  ]#
-  let c = cmd.parseCmdLine
-  var p = startProcess(command = "gcc", args = c[5 .. ^1],
-                       options = {poStdErrToStdOut, poUsePath, poEvalCommand})
-  let outp = p.outputStream
-  var x = newStringOfCap(120)
-  result.nimout = ""
-  result.msg = ""
-  result.file = ""
-  result.output = ""
-  result.line = -1
-  while true:
-    if outp.readLine(x):
-      result.nimout.add(x & '\n')
-    elif not running(p):
-      break
-  close(p)
-  if p.peekExitCode == 0:
-    result.err = reSuccess
-  else:
-    result.err = reNimcCrash
-
 proc initResults: TResults =
   result.total = 0
   result.passed = 0
@@ -628,22 +581,6 @@ proc testSpecWithNimcache(r: var TResults, test: TTest; nimcache: string) {.used
     var testClone = test
     testSpecHelper(r, testClone, test.spec, target, nimcache)
 
-proc testC(r: var TResults, test: TTest, action: TTestAction) =
-  # runs C code. Doesn't support any specs, just goes by exit code.
-  if not checkDisabled(r, test): return
-
-  let tname = test.name.addFileExt(".c")
-  inc(r.total)
-  maybeStyledEcho "Processing ", fgCyan, extractFilename(tname)
-  var given = callCCompiler(getCmd(TSpec()), test.name & ".c", test.options, targetC)
-  if given.err != reSuccess:
-    r.addResult(test, targetC, "", given.msg, given.err)
-  elif action == actionRun:
-    let exeFile = changeFileExt(test.name, ExeExt)
-    var (_, exitCode) = execCmdEx(exeFile, options = {poStdErrToStdOut, poUsePath})
-    if exitCode != 0: given.err = reExitcodesDiffer
-  if given.err == reSuccess: inc(r.passed)
-
 proc makeTest(test, options: string, cat: Category): TTest =
   result.cat = cat
   result.name = test
diff --git a/tests/readme.md b/tests/readme.md
index 69be92051..dca7ec152 100644
--- a/tests/readme.md
+++ b/tests/readme.md
@@ -55,5 +55,4 @@ testing, which is slower).
 The folder ``dll`` contains simple DLL tests.
 
 The folder ``realtimeGC`` contains a test for validating that the realtime GC
-can run properly without linking against the nimrtl.dll/so. It includes a C
-client and platform specific build files for manual compilation.
+can run properly without linking against the nimrtl.dll/so.
diff --git a/tests/realtimeGC/cmain.c b/tests/realtimeGC/cmain.c
deleted file mode 100644
index 4bc5bd026..000000000
--- a/tests/realtimeGC/cmain.c
+++ /dev/null
@@ -1,69 +0,0 @@
-/* xxx consider removing, this seems redundant with tmain.nim */
-
-#ifdef WIN
-#include 
-#else
-#include 
-#include  /* for sleep(3) */
-#endif
-#include 
-#include 
-#include 
-#include 
-#include 
-
-#define RUNTIME (15*60)
-
-
-typedef void (*pFunc)(void);
-
-int main(int argc, char* argv[])
-{
-    int i;
-    void* hndl;
-    pFunc status;
-    pFunc count;
-    pFunc checkOccupiedMem;
-
-#ifdef WIN
-    hndl = (void*) LoadLibrary((char const*)"./tests/realtimeGC/shared.dll");
-    status = (pFunc)GetProcAddress((HMODULE) hndl, (char const*)"status");
-    count = (pFunc)GetProcAddress((HMODULE) hndl, (char const*)"count");
-    checkOccupiedMem = (pFunc)GetProcAddress((HMODULE) hndl, (char const*)"checkOccupiedMem");
-#else /* OSX || NIX xxx: OSX assumes dylib*/
-    hndl = (void*) dlopen((char const*)"./tests/realtimeGC/libshared.so", RTLD_LAZY);
-    status = (pFunc) dlsym(hndl, (char const*)"status");
-    count = (pFunc) dlsym(hndl, (char const*)"count");
-    checkOccupiedMem = (pFunc) dlsym(hndl, (char const*)"checkOccupiedMem");
-#endif
-
-    assert(hndl);
-    assert(status);
-    assert(count);
-    assert(checkOccupiedMem);
-
-    time_t startTime = time((time_t*)0);
-    time_t runTime = (time_t)(RUNTIME);
-    time_t accumTime = 0;
-    while (accumTime < runTime) {
-        for (i = 0; i < 10; i++)
-            count();
-        /* printf("1. sleeping...\n"); */
-        sleep(1);
-        for (i = 0; i < 10; i++)
-            status();
-        /* printf("2. sleeping...\n"); */
-        sleep(1);
-        checkOccupiedMem();
-        accumTime = time((time_t*)0) - startTime;
-        /* printf("--- Minutes left to run: %d\n", (int)(runTime-accumTime)/60); */
-    }
-    printf("Cleaning up the shared object pointer...\n");
-#ifdef WIN
-    FreeLibrary((HMODULE)hndl);
-#else /* OSX || NIX */
-    dlclose(hndl);
-#endif
-    printf("Done\n");
-    return 0;
-}
diff --git a/tests/realtimeGC/readme.txt b/tests/realtimeGC/readme.txt
deleted file mode 100644
index c5837ee14..000000000
--- a/tests/realtimeGC/readme.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-Test the realtime GC without linking nimrtl.dll/so.
-
-To build by hand and run the test for 35 minutes:
-    $ nim r --threads:on -d:runtimeSecs:2100 tests/realtimeGC/nmain.nim
-
-```
-xxx do we still need tests/realtimeGC/cmain.c?
-if so, tests/realtimeGC/cmain.c needs to updated and factorized with nmain.nim to avoid duplication (even if it's a C file)
-```
-    $ gcc -o tests/realtimeGC/cmain tests/realtimeGC/cmain.c -ldl
\ No newline at end of file
diff --git a/tests/realtimeGC/tmain.nim b/tests/realtimeGC/tmain.nim
index f1515a549..ca0177e3d 100644
--- a/tests/realtimeGC/tmain.nim
+++ b/tests/realtimeGC/tmain.nim
@@ -9,6 +9,13 @@ these dont' seem needed --debuginfo
 nor these from the previous main.nim.cfg: --app:console
 ]#
 
+#[
+Test the realtime GC without linking nimrtl.dll/so.
+
+To build by hand and run the test for 35 minutes:
+`nim r --threads:on -d:runtimeSecs:2100 tests/realtimeGC/tmain.nim`
+]#
+
 import times, os, strformat, strutils
 from stdtest/specialpaths import buildDir
 # import threadpool
@@ -16,9 +23,10 @@ from stdtest/specialpaths import buildDir
 const runtimeSecs {.intdefine.} = 5
 
 const file = "shared.nim"
-const dllname = buildDir / (DynlibFormat % file)
+const dllname = buildDir / (DynlibFormat % "shared_D20210524T180506")
 
 static:
+  # D20210524T180826:here we compile the dependency on the fly
   let nim = getCurrentCompilerExe()
   let (output, exitCode) = gorgeEx(fmt"{nim} c -o:{dllname} --debuginfo --app:lib --threads:on -d:release -d:useRealtimeGC {file}")
   doAssert exitCode == 0, output

From d62a298a2834b9a35aac4d922a810a772cd3bc1d Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Thu, 27 May 2021 09:04:10 -0700
Subject: [PATCH 0845/2472] fix typo in ci/funs.sh (#18115)

---
 ci/funs.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/funs.sh b/ci/funs.sh
index 9772535be..4a8887658 100644
--- a/ci/funs.sh
+++ b/ci/funs.sh
@@ -43,7 +43,7 @@ nimInternalBuildKochAndRunCI(){
   echo_run nim c koch
   if ! echo_run ./koch runCI; then
     echo_run echo "runCI failed"
-    echo_run nim r -tools/ci_testresults.nim
+    echo_run nim r tools/ci_testresults.nim
     return 1
   fi
 }

From 9eae6b4fe312035daadd8dcb0ca488a3e5897c11 Mon Sep 17 00:00:00 2001
From: vabresto <77133146+vabresto@users.noreply.github.com>
Date: Fri, 28 May 2021 18:17:37 -0400
Subject: [PATCH 0846/2472] fix #17456 flaky test
 tasynchttpserver_transferencoding (#18052)

---
 .../tasynchttpserver_transferencoding.nim     | 55 +++++++++++--------
 1 file changed, 32 insertions(+), 23 deletions(-)

diff --git a/tests/stdlib/tasynchttpserver_transferencoding.nim b/tests/stdlib/tasynchttpserver_transferencoding.nim
index bfa0e2f69..fd79ce3a0 100644
--- a/tests/stdlib/tasynchttpserver_transferencoding.nim
+++ b/tests/stdlib/tasynchttpserver_transferencoding.nim
@@ -1,3 +1,7 @@
+discard """
+  matrix: "--gc:arc --threads:on; --gc:arc --threads:on -d:danger; --threads:on"
+"""
+
 import httpclient, asynchttpserver, asyncdispatch, asyncfutures
 import net
 
@@ -10,33 +14,38 @@ Transfer-Encoding:chunked
 
 """
 
-template genTest(input, expected) =
-  var sanity = false
-  proc handler(request: Request) {.async.} =
-      doAssert(request.body == expected)
-      doAssert(request.headers.hasKey("Transfer-Encoding"))
-      doAssert(not request.headers.hasKey("Content-Length"))
-      sanity = true
-      await request.respond(Http200, "Good")
+template genTest(input, expected: string) =
+  proc handler(request: Request, future: Future[bool]) {.async, gcsafe.} =
+    doAssert(request.body == expected)
+    doAssert(request.headers.hasKey("Transfer-Encoding"))
+    doAssert(not request.headers.hasKey("Content-Length"))
+    future.complete(true)
+    await request.respond(Http200, "Good")
 
-  proc runSleepLoop(server: AsyncHttpServer) {.async.} = 
+  proc sendData(data: string, port: Port) {.async.} =
+    var socket = newSocket()
+    defer: socket.close()
+
+    socket.connect("127.0.0.1", port)
+    socket.send(data)
+
+  proc runTest(): Future[bool] {.async.} =
+    var handlerFuture = newFuture[bool]("runTest")
+    let data = postBegin & input
+    let server = newAsyncHttpServer()
     server.listen(Port(0))
-    proc wrapper() = 
-      waitFor server.acceptRequest(handler)
-    asyncdispatch.callSoon wrapper
 
-  let server = newAsyncHttpServer()
-  waitFor runSleepLoop(server)
-  let data = postBegin & input
-  var socket = newSocket()
-  socket.connect("127.0.0.1", server.getPort)
-  socket.send(data)
-  waitFor sleepAsync(10)
-  socket.close()
-  server.close()
+    proc wrapper(request: Request): Future[void] {.gcsafe, closure.} =
+      handler(request, handlerFuture)
+    
+    asyncCheck sendData(data, server.getPort)
+    asyncCheck server.acceptRequest(wrapper)
+    doAssert await handlerFuture
+    
+    server.close()
+    return true
 
-  # Verify we ran the handler and its asserts
-  doAssert(sanity)
+  doAssert waitFor runTest()
 
 block:
   const expected = "hello=world"

From c792154b030c1bfec9ed0020ddaf985d5f034ca3 Mon Sep 17 00:00:00 2001
From: quantimnot <54247259+quantimnot@users.noreply.github.com>
Date: Sat, 29 May 2021 15:39:46 -0400
Subject: [PATCH 0847/2472] Minor doc fixes; follow up to 17258 and 17259
 (#18123)

Co-authored-by: quantimnot 
---
 doc/manual.rst                           | 6 +++---
 doc/manual_experimental_strictnotnil.rst | 2 +-
 doc/tut1.rst                             | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/doc/manual.rst b/doc/manual.rst
index 2b85f5c34..6a0dec120 100644
--- a/doc/manual.rst
+++ b/doc/manual.rst
@@ -363,7 +363,7 @@ contain the following `escape sequences`:idx:\ :
   ``\\``                   `backslash`:idx:
   ``\"``                   `quotation mark`:idx:
   ``\'``                   `apostrophe`:idx:
-  ``\\`` '0'..'9'+         `character with decimal value d`:idx:;
+  ``\`` '0'..'9'+         `character with decimal value d`:idx:;
                            all decimal digits directly
                            following are used for the character
   ``\a``                   `alert`:idx:
@@ -473,7 +473,7 @@ literals:
   ``\\``                   `backslash`:idx:
   ``\"``                   `quotation mark`:idx:
   ``\'``                   `apostrophe`:idx:
-  ``\\`` '0'..'9'+         `character with decimal value d`:idx:;
+  ``\`` '0'..'9'+          `character with decimal value d`:idx:;
                            all decimal digits directly
                            following are used for the character
   ``\a``                   `alert`:idx:
@@ -7825,7 +7825,7 @@ Threads
 
 To enable thread support the `--threads:on`:option: command-line switch needs to
 be used. The system_ module then contains several threading primitives.
-See the `threads `_ and `channels `_ modules
+See the `threads `_ and `channels `_ modules
 for the low-level thread API. There are also high-level parallelism constructs
 available. See `spawn `_ for
 further details.
diff --git a/doc/manual_experimental_strictnotnil.rst b/doc/manual_experimental_strictnotnil.rst
index c7c045683..cb3bfeca9 100644
--- a/doc/manual_experimental_strictnotnil.rst
+++ b/doc/manual_experimental_strictnotnil.rst
@@ -143,7 +143,7 @@ Merging usually merges maps and alias sets: nilabilities are merged like this:
     else:
       MaybeNil
 
-Special handling is for `.isNil` and ` == nil`, also for `not`, `and` and `or`.
+Special handling is for `.isNil` and `== nil`, also for `not`, `and` and `or`.
 
 `not` reverses the nilability, `and` is similar to "forking" : the right expression is checked in the layer resulting from the left one and `or` is similar to "merging": the right and left expression should be both checked in the original layer.
 
diff --git a/doc/tut1.rst b/doc/tut1.rst
index 2f19ae173..210ea324e 100644
--- a/doc/tut1.rst
+++ b/doc/tut1.rst
@@ -114,7 +114,7 @@ String and character literals
 -----------------------------
 
 String literals are enclosed in double-quotes; character literals in single
-quotes. Special characters are escaped with ``\\``: ``\n`` means newline, ``\t``
+quotes. Special characters are escaped with ``\``: ``\n`` means newline, ``\t``
 means tabulator, etc. There are also *raw* string literals:
 
 .. code-block:: Nim

From cfe19247e856950f7facaf7adbc50bca8dec3990 Mon Sep 17 00:00:00 2001
From: vabresto <77133146+vabresto@users.noreply.github.com>
Date: Sat, 29 May 2021 15:44:39 -0400
Subject: [PATCH 0848/2472] Disable tasynchttpserver_transferencoding on
 FreeBSD (#18122)

---
 tests/stdlib/tasynchttpserver_transferencoding.nim | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/stdlib/tasynchttpserver_transferencoding.nim b/tests/stdlib/tasynchttpserver_transferencoding.nim
index fd79ce3a0..cdd30d04b 100644
--- a/tests/stdlib/tasynchttpserver_transferencoding.nim
+++ b/tests/stdlib/tasynchttpserver_transferencoding.nim
@@ -1,5 +1,6 @@
 discard """
   matrix: "--gc:arc --threads:on; --gc:arc --threads:on -d:danger; --threads:on"
+  disabled: "freebsd"
 """
 
 import httpclient, asynchttpserver, asyncdispatch, asyncfutures

From 4a7f2c386cde91b784106905f6e70d3954d8fae9 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Sun, 30 May 2021 13:37:21 -0700
Subject: [PATCH 0849/2472] close #16569: deprecated `reversed(a, start, last)`
 overload, use toOpenArray instead (#18047)

* close #16569: deprecated reversed overload, use toOpenArray instead

* [skip ci] change wording in changelog per review

* fixup
---
 changelog.md           |  2 ++
 lib/pure/algorithm.nim | 39 ++++++++++-----------------------------
 2 files changed, 12 insertions(+), 29 deletions(-)

diff --git a/changelog.md b/changelog.md
index 4c30d67a7..809728b16 100644
--- a/changelog.md
+++ b/changelog.md
@@ -75,6 +75,8 @@
 
 - `strformat` is now part of `include std/prelude`.
 
+- Deprecated `proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T]` in `std/algorithm`.
+
 - The configuration subsystem now allows for `-d:release` and `-d:danger` to work as expected.
   The downside is that these defines now have custom logic that doesn't apply for
   other defines.
diff --git a/lib/pure/algorithm.nim b/lib/pure/algorithm.nim
index f4b3f46be..c96f599e8 100644
--- a/lib/pure/algorithm.nim
+++ b/lib/pure/algorithm.nim
@@ -131,40 +131,21 @@ proc reverse*[T](a: var openArray[T]) =
   # the max is needed, since a.high is -1 if a is empty
   reverse(a, 0, max(0, a.high))
 
-proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T] =
-  ## Returns the reverse of the slice `a[first..last]`.
-  ##
-  ## If an invalid range is passed, it raises `IndexDefect`.
+proc reversed*[T](a: openArray[T]): seq[T] {.inline.} =
+  ## Returns the elements of `a` in reverse order.
   ##
   ## **See also:**
-  ## * `reverse proc<#reverse,openArray[T],Natural,Natural>`_ reverse a slice
   ## * `reverse proc<#reverse,openArray[T]>`_
   runnableExamples:
-    let
-      a = [1, 2, 3, 4, 5, 6]
-      b = a.reversed(1, 3)
-    assert b == @[4, 3, 2]
-  assert last >= first - 1
-  var i = last - first
-  var x = first.int
-  result = newSeq[T](i + 1)
-  while i >= 0:
-    result[i] = a[x]
-    dec(i)
-    inc(x)
+    assert [10, 11, 12].reversed == @[12, 11, 10]
+    assert seq[string].default.reversed == @[]
+  let n = a.len
+  result.setLen(n)
+  for i in 0..`_ reverse a slice
-  ## * `reverse proc<#reverse,openArray[T]>`_
-  runnableExamples:
-    let
-      a = [1, 2, 3, 4, 5, 6]
-      b = reversed(a)
-    assert b == @[6, 5, 4, 3, 2, 1]
-  reversed(a, 0, a.high)
+proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T]
+  {.inline, deprecated: "use: `reversed(toOpenArray(a, first, last))`".} =
+  reversed(toOpenArray(a, first, last))
 
 proc binarySearch*[T, K](a: openArray[T], key: K,
                          cmp: proc (x: T, y: K): int {.closure.}): int =

From a6bd6c7ed803dcef41b49343b5cd60607b984ca8 Mon Sep 17 00:00:00 2001
From: Yuriy Glukhov 
Date: Sun, 30 May 2021 23:38:33 +0300
Subject: [PATCH 0850/2472] Fixes #17849 (#18055) [backport:1.2]

* Fixes #17849
* Update compiler/closureiters.nim

Co-authored-by: Andreas Rumpf 
---
 compiler/closureiters.nim  |  6 ++++++
 tests/iter/tyieldintry.nim | 16 +++++++++++++++-
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/compiler/closureiters.nim b/compiler/closureiters.nim
index e474e5ba2..adc5a4873 100644
--- a/compiler/closureiters.nim
+++ b/compiler/closureiters.nim
@@ -606,6 +606,12 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
             internalError(ctx.g.config, "lowerStmtListExpr(nkCaseStmt): " & $branch.kind)
         result.add(n)
         result.add(ctx.newEnvVarAccess(tmp))
+      elif n[0].kind == nkStmtListExpr:
+        result = newNodeI(nkStmtList, n.info)
+        let (st, ex) = exprToStmtList(n[0])
+        result.add(st)
+        n[0] = ex
+        result.add(n)
 
   of nkCallKinds, nkChckRange, nkChckRangeF, nkChckRange64:
     var ns = false
diff --git a/tests/iter/tyieldintry.nim b/tests/iter/tyieldintry.nim
index 35df55c24..3d5260800 100644
--- a/tests/iter/tyieldintry.nim
+++ b/tests/iter/tyieldintry.nim
@@ -483,5 +483,19 @@ block: # nnkChckRange
 
   test(it, 1, 2, 3)
 
-echo "ok"
+block: #17849 - yield in case subject
+  template yieldInCase: int =
+    yield 2
+    3
 
+  iterator it(): int {.closure.} =
+    yield 1
+    case yieldInCase()
+    of 1: checkpoint(11)
+    of 3: checkpoint(13)
+    else: checkpoint(14)
+    yield 5
+
+  test(it, 1, 2, 13, 5)
+
+echo "ok"

From 4274e6e4d700ef72275344afed44d0b658729761 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Sun, 30 May 2021 13:39:17 -0700
Subject: [PATCH 0851/2472] upgrade ubuntu 16.04 (not supported starting dec
 2021) => 18.04; revive Linux_i386 (#18107)

---
 .gitlab-ci.yml      |  4 +++-
 azure-pipelines.yml | 14 +++++++-------
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ad9e565a6..31908d223 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,4 +1,6 @@
-image: ubuntu:16.04
+# xxx unused, out of date
+
+image: ubuntu:18.04
 
 stages:
   - pre-build
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index f0c3b2d0a..a70a86fae 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -19,14 +19,14 @@ jobs:
   strategy:
     matrix:
       Linux_amd64:
-        vmImage: 'ubuntu-16.04'
+        vmImage: 'ubuntu-18.04'
         CPU: amd64
-      # pending bug #17325 (broken again)
-      # Linux_i386:
-      #   # bug #17325: fails on 'ubuntu-16.04' because it now errors with:
-      #   # g++-multilib : Depends: gcc-multilib (>= 4:5.3.1-1ubuntu1) but it is not going to be installed
-      #   vmImage: 'ubuntu-18.04'
-      #   CPU: i386
+      # regularly breaks, refs bug #17325
+      Linux_i386:
+        # on 'ubuntu-16.04' (not supported anymore anyways) it errored with:
+        # g++-multilib : Depends: gcc-multilib (>= 4:5.3.1-1ubuntu1) but it is not going to be installed
+        vmImage: 'ubuntu-18.04'
+        CPU: i386
       OSX_amd64:
         vmImage: 'macOS-10.15'
         CPU: amd64

From e2ab08603db6499063b3ff0f16776b4ed20c36f0 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Sun, 30 May 2021 13:40:42 -0700
Subject: [PATCH 0852/2472] fix warnings/hints in nimdoc/tester.nim (#18083)

* fix warnings/hints in nimdoc/tester.nim

* improve err msg for nimdoc/tester.nim and change flag from fixup to nimTestsNimdocFixup

* address comment: put back quit instead of doAssert
---
 nimdoc/tester.nim                            | 12 +++++++-----
 nimdoc/testproject/expected/testproject.html |  8 ++++----
 nimdoc/testproject/testproject.nim           |  9 +++++----
 3 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/nimdoc/tester.nim b/nimdoc/tester.nim
index 9daa0bb51..1e438b97b 100644
--- a/nimdoc/tester.nim
+++ b/nimdoc/tester.nim
@@ -1,10 +1,12 @@
 # Small program that runs the test cases for 'nim doc'.
 # To run this, cd to the git repo root, and run "nim r nimdoc/tester.nim".
-# to change expected results (after carefully verifying everything), use -d:fixup
+# to change expected results (after carefully verifying everything), use -d:nimTestsNimdocFixup
 
 import strutils, os
 from std/private/gitutils import diffFiles
 
+const fixup = defined(nimTestsNimdocFixup)
+
 var
   failures = 0
 
@@ -62,7 +64,7 @@ let
                                      "$1/$2.nim" % [test1Dir, test1PrjName]],
                               buildIndex: @["--out:$1/$2/theindex.html" % [test1Dir, test1DocsDir],
                                             "$1/$2" % [test1Dir, test1DocsDir]])
-testNimDoc(test1Dir, test1DocsDir, test1Switches, defined(fixup))
+testNimDoc(test1Dir, test1DocsDir, test1Switches, fixup)
 
 # Test "nim doc --out:.. --index:on .."
 let
@@ -75,7 +77,7 @@ let
                                      "$1/$2.nim" % [test2Dir, test2PrjName]],
                               buildIndex: @["--out:$1/$2/theindex.html" % [test2Dir, test2DocsDir],
                                             "$1/$2" % [test2Dir, test2DocsDir]])
-testNimDoc(test2Dir, test2DocsDir, test2Switches, defined(fixup))
+testNimDoc(test2Dir, test2DocsDir, test2Switches, fixup)
 
-# Check for failures
-if failures > 0: quit($failures & " failures occurred.")
+if failures > 0:
+  quit "$# failures occurred; see note in tester.nim regarding -d:nimTestsNimdocFixup" %  $failures
diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html
index f23f6bb42..d2fe38021 100644
--- a/nimdoc/testproject/expected/testproject.html
+++ b/nimdoc/testproject/expected/testproject.html
@@ -790,7 +790,7 @@ the c printf. etc.
 
    
 
 
-
    Example:
    
+
    Example: cmd: --hint:XDeclaredButNotUsed:off
    
 
    ## mullitline string litterals are tricky as their indentation can span
 ## below that of the runnableExamples
 let s1a = """
@@ -865,21 +865,21 @@ There is no block quote after blank lines at the beginning.
 
    Methods
    
 
    
 
-
    method method1(self: Moo) {....raises: [], tags: [].}
    
+
    method method1(self: Moo) {.base, ...raises: [], tags: [].}
    
 
    
 
 foo1
 
 
    
 
-
    method method2(self: Moo): int {....raises: [], tags: [].}
    
+
    method method2(self: Moo): int {.base, ...raises: [], tags: [].}
    
 
    
 
 foo2
 
 
    
 
-
    method method3(self: Moo): int {....raises: [], tags: [].}
    
+
    method method3(self: Moo): int {.base, ...raises: [], tags: [].}
    
 
    
 
 foo3
diff --git a/nimdoc/testproject/testproject.nim b/nimdoc/testproject/testproject.nim
index d435efa66..b88b41098 100644
--- a/nimdoc/testproject/testproject.nim
+++ b/nimdoc/testproject/testproject.nim
@@ -131,6 +131,7 @@ when true:
       # BUG: this currently this won't be run since not exported
       # but probably should
       doAssert false
+  if false: bazNonExported() # silence XDeclaredButNotUsed
 
   proc z17*() =
     # BUG: a comment before 1st doc comment currently doesn't prevent
@@ -212,7 +213,7 @@ when true: # tests RST inside comments
 
 when true: # multiline string litterals
   proc tripleStrLitTest*() =
-    runnableExamples:
+    runnableExamples("--hint:XDeclaredButNotUsed:off"):
       ## mullitline string litterals are tricky as their indentation can span
       ## below that of the runnableExamples
       let s1a = """
@@ -252,12 +253,12 @@ at indent 0
 
 when true: # methods; issue #14691
   type Moo = object
-  method method1*(self: Moo) =
+  method method1*(self: Moo) {.base.} =
     ## foo1
-  method method2*(self: Moo): int =
+  method method2*(self: Moo): int {.base.} =
     ## foo2
     result = 1
-  method method3*(self: Moo): int =
+  method method3*(self: Moo): int {.base.} =
     ## foo3
     1
 

From 50e98e6efaca05b60b33db0c878f04951a43b3b8 Mon Sep 17 00:00:00 2001
From: Miran 
Date: Sun, 30 May 2021 23:55:51 +0200
Subject: [PATCH 0853/2472] hashes for refs should be an opt-in feature
 (#18098)

---
 changelog.md                  | 17 +++++++++++++----
 lib/pure/hashes.nim           |  2 +-
 tests/collections/ttables.nim |  1 +
 tests/stdlib/thashes.nim      |  1 +
 4 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/changelog.md b/changelog.md
index 809728b16..279ba38dd 100644
--- a/changelog.md
+++ b/changelog.md
@@ -50,8 +50,9 @@
 - On POSIX systems, we now ignore `SIGPIPE` signals, use `-d:nimLegacySigpipeHandler`
   for previous behavior.
 
-- `hashes.hash` now supports `object` and `ref` (can be overloaded in user code).
-  For a transition period, use `-d:nimLegacyNoHashRef` to avoid defining `hash(ref)`.
+- `hashes.hash` can now support `object` and `ref` (can be overloaded in user code),
+  if `-d:nimEnableHashRef` is used.
+
 - `hashes.hash(proc|ptr|ref|pointer)` now calls `hash(int)` and honors `-d:nimIntHash1`,
   `hashes.hash(closure)` has also been improved.
 
@@ -82,7 +83,9 @@
   other defines.
 
 
+
 ## Standard library additions and changes
+
 - Added support for parenthesized expressions in `strformat`
 
 - Fixed buffer overflow bugs in `net`
@@ -99,7 +102,6 @@
   the OpenSSL DLLs (e.g. libssl-1_1-x64.dll, libcrypto-1_1-x64.dll) you
   now also need to ship `cacert.pem` with your `.exe` file.
 
-
 - Make `{.requiresInit.}` pragma to work for `distinct` types.
 
 - Added a macros `enumLen` for returning the number of items in an enum to the
@@ -324,6 +326,8 @@
 
 - Fixed premature garbage collection in asyncdispatch, when a stack trace override is in place.
 
+
+
 ## Language changes
 
 - `nimscript` now handles `except Exception as e`.
@@ -368,6 +372,8 @@
 
 - `typeof(voidStmt)` now works and returns `void`.
 
+
+
 ## Compiler changes
 
 - Added `--declaredlocs` to show symbol declaration location in messages.
@@ -394,7 +400,8 @@
 - TLS: OSX now uses native TLS (`--tlsEmulation:off`), TLS now works with importcpp non-POD types,
   such types must use `.cppNonPod` and `--tlsEmulation:off`should be used.
 
-- Now array literals(JS backend) uses JS typed arrays when the corresponding js typed array exists, for example `[byte(1), 2, 3]` generates `new Uint8Array([1, 2, 3])`.
+- Now array literals(JS backend) uses JS typed arrays when the corresponding js typed array exists,
+  for example `[byte(1), 2, 3]` generates `new Uint8Array([1, 2, 3])`.
 
 - docgen: rst files can now use single backticks instead of double backticks and correctly render
   in both rst2html (as before) as well as common tools rendering rst directly (e.g. github), by
@@ -426,6 +433,8 @@
   the official Nim style guide. To be enabled, this has to be combined either
   with `--styleCheck:error` or `--styleCheck:hint`.
 
+
+
 ## Tool changes
 
 - The rst parser now supports markdown table syntax.
diff --git a/lib/pure/hashes.nim b/lib/pure/hashes.nim
index f61fbdf2b..46b4fc948 100644
--- a/lib/pure/hashes.nim
+++ b/lib/pure/hashes.nim
@@ -236,7 +236,7 @@ proc hash*[T](x: ptr[T]): Hash {.inline.} =
     assert cast[pointer](a[0].addr).hash == a[0].addr.hash
   hash(cast[pointer](x))
 
-when not defined(nimLegacyNoHashRef):
+when defined(nimEnableHashRef):
   proc hash*[T](x: ref[T]): Hash {.inline.} =
     ## Efficient `hash` overload.
     runnableExamples:
diff --git a/tests/collections/ttables.nim b/tests/collections/ttables.nim
index 638f4241b..751509062 100644
--- a/tests/collections/ttables.nim
+++ b/tests/collections/ttables.nim
@@ -8,6 +8,7 @@ And we get here
 '''
 joinable: false
 targets: "c cpp js"
+matrix: "-d:nimEnableHashRef"
 """
 
 # xxx wrap in a template to test in VM, see https://github.com/timotheecour/Nim/issues/534#issuecomment-769565033
diff --git a/tests/stdlib/thashes.nim b/tests/stdlib/thashes.nim
index 46576ef12..1e9b02d0c 100644
--- a/tests/stdlib/thashes.nim
+++ b/tests/stdlib/thashes.nim
@@ -1,5 +1,6 @@
 discard """
   targets: "c cpp js"
+  matrix: "-d:nimEnableHashRef"
 """
 
 import std/hashes

From 98ea61f09b4bfdc13c911b1ff3fb404dfc0d1338 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Sun, 30 May 2021 14:57:25 -0700
Subject: [PATCH 0854/2472] dont silence hints in system.nim (#18034)

---
 lib/system.nim | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/system.nim b/lib/system.nim
index be30665dd..942d29fe8 100644
--- a/lib/system.nim
+++ b/lib/system.nim
@@ -112,7 +112,7 @@ proc compileOption*(option, arg: string): bool {.
       discard "compiled with optimization for size and uses Boehm's GC"
 
 {.push warning[GcMem]: off, warning[Uninit]: off.}
-{.push hints: off.}
+# {.push hints: off.}
 
 proc `or`*(a, b: typedesc): typedesc {.magic: "TypeTrait", noSideEffect.}
   ## Constructs an `or` meta class.
@@ -2471,7 +2471,7 @@ proc quit*(errormsg: string, errorcode = QuitFailure) {.noreturn.} =
   quit(errorcode)
 
 {.pop.} # checks: off
-{.pop.} # hints: off
+# {.pop.} # hints: off
 
 proc `/`*(x, y: int): float {.inline, noSideEffect.} =
   ## Division of integers that results in a float.

From 18b477431138d944b90e7a6e6e0412496634b518 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Mon, 31 May 2021 01:51:20 -0700
Subject: [PATCH 0855/2472] document macros.unpackVarargs (#18106)

* deprecate macros.unpackVarargs

* un-deprecate unpackVarargs and add docs+runnableExamples

* update examples + tests with varargs[typed]
---
 lib/core/macros.nim              | 15 ++++++++++
 tests/destructor/tatomicptrs.nim |  1 +
 tests/stdlib/tmacros.nim         | 50 ++++++++++++++++++++++++++++++++
 3 files changed, 66 insertions(+)

diff --git a/lib/core/macros.nim b/lib/core/macros.nim
index 79c3ba28b..89f90a23b 100644
--- a/lib/core/macros.nim
+++ b/lib/core/macros.nim
@@ -1682,6 +1682,21 @@ macro getCustomPragmaVal*(n: typed, cp: typed): untyped =
     error(n.repr & " doesn't have a pragma named " & cp.repr, n)
 
 macro unpackVarargs*(callee: untyped; args: varargs[untyped]): untyped =
+  ## Calls `callee` with `args` unpacked as individual arguments.
+  ## This is useful in 2 cases:
+  ## * when forwarding `varargs[T]` for some typed `T`
+  ## * when forwarding `varargs[untyped]` when `args` can potentially be empty,
+  ##   due to a compiler limitation
+  runnableExamples:
+    template call1(fun: typed; args: varargs[untyped]): untyped =
+      unpackVarargs(fun, args)
+      # when varargsLen(args) > 0: fun(args) else: fun() # this would also work
+    template call2(fun: typed; args: varargs[typed]): untyped =
+      unpackVarargs(fun, args)
+    proc fn1(a = 0, b = 1) = discard (a, b)
+    call1(fn1, 10, 11)
+    call1(fn1) # `args` is empty in this case
+    if false: call2(echo, 10, 11) # would print 1011
   result = newCall(callee)
   for i in 0 ..< args.len:
     result.add args[i]
diff --git a/tests/destructor/tatomicptrs.nim b/tests/destructor/tatomicptrs.nim
index 7313afbf5..36f0cab8a 100644
--- a/tests/destructor/tatomicptrs.nim
+++ b/tests/destructor/tatomicptrs.nim
@@ -72,6 +72,7 @@ template `.=`*[T](s: SharedPtr[T]; field, value: untyped) =
 from macros import unpackVarargs
 
 template `.()`*[T](s: SharedPtr[T]; field: untyped, args: varargs[untyped]): untyped =
+  # xxx this isn't used, the test should be improved
   unpackVarargs(s.x.field, args)
 
 
diff --git a/tests/stdlib/tmacros.nim b/tests/stdlib/tmacros.nim
index effe1032f..52ea1e898 100644
--- a/tests/stdlib/tmacros.nim
+++ b/tests/stdlib/tmacros.nim
@@ -10,3 +10,53 @@ block: # hasArgOfName
 
 block: # bug #17454
   proc f(v: NimNode): string {.raises: [].} = $v
+
+block: # unpackVarargs
+  block:
+    proc bar1(a: varargs[int]): string =
+      for ai in a: result.add " " & $ai
+    proc bar2(a: varargs[int]) =
+      let s1 = bar1(a)
+      let s2 = unpackVarargs(bar1, a) # `unpackVarargs` makes no difference here
+      doAssert s1 == s2
+    bar2(1, 2, 3)
+    bar2(1)
+    bar2()
+
+  block:
+    template call1(fun: typed; args: varargs[untyped]): untyped =
+      unpackVarargs(fun, args)
+    template call2(fun: typed; args: varargs[untyped]): untyped =
+      # fun(args) # works except for last case with empty `args`, pending bug #9996
+      when varargsLen(args) > 0: fun(args)
+      else: fun()
+
+    proc fn1(a = 0, b = 1) = discard (a, b)
+
+    call1(fn1)
+    call1(fn1, 10)
+    call1(fn1, 10, 11)
+
+    call2(fn1)
+    call2(fn1, 10)
+    call2(fn1, 10, 11)
+
+  block:
+    template call1(fun: typed; args: varargs[typed]): untyped =
+      unpackVarargs(fun, args)
+    template call2(fun: typed; args: varargs[typed]): untyped =
+      # xxx this would give a confusing error message:
+      # required type for a: varargs[typed] [varargs] but expression '[10]' is of type: varargs[typed] [varargs]
+      when varargsLen(args) > 0: fun(args)
+      else: fun()
+    macro toString(a: varargs[typed, `$`]): string =
+      var msg = genSym(nskVar, "msg")
+      result = newStmtList()
+      result.add quote do:
+        var `msg` = ""
+      for ai in a:
+        result.add quote do: `msg`.add $`ai`
+      result.add quote do: `msg`
+    doAssert call1(toString) == ""
+    doAssert call1(toString, 10) == "10"
+    doAssert call1(toString, 10, 11) == "1011"

From 064fe18de61c2aae5c215077ba1dd0352affa271 Mon Sep 17 00:00:00 2001
From: n5m <72841454+n5m@users.noreply.github.com>
Date: Mon, 31 May 2021 09:20:15 +0000
Subject: [PATCH 0856/2472] improve fast returns of find and rfind (#18127)

---
 lib/pure/strutils.nim      |  4 +++-
 tests/stdlib/tstrutils.nim | 28 ++++++++++++++++++++++++++++
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim
index 5bbd850cc..3fb67ead1 100644
--- a/lib/pure/strutils.nim
+++ b/lib/pure/strutils.nim
@@ -1890,7 +1890,7 @@ func find*(s, sub: string, start: Natural = 0, last = 0): int {.rtl,
   ## See also:
   ## * `rfind func<#rfind,string,string,Natural>`_
   ## * `replace func<#replace,string,string,string>`_
-  if sub.len > s.len: return -1
+  if sub.len > s.len - start: return -1
   if sub.len == 1: return find(s, sub[0], start, last)
 
   template useSkipTable {.dirty.} =
@@ -1967,6 +1967,8 @@ func rfind*(s, sub: string, start: Natural = 0, last = -1): int {.rtl,
   ## * `find func<#find,string,string,Natural,int>`_
   if sub.len == 0:
     return -1
+  if sub.len > s.len - start:
+    return -1
   let last = if last == -1: s.high else: last
   result = 0
   for i in countdown(last - sub.len + 1, start):
diff --git a/tests/stdlib/tstrutils.nim b/tests/stdlib/tstrutils.nim
index 771dddcaf..637661ba3 100644
--- a/tests/stdlib/tstrutils.nim
+++ b/tests/stdlib/tstrutils.nim
@@ -243,6 +243,34 @@ template main() =
     doAssert "/1/2/3".rfind('/', last=1) == 0
     doAssert "/1/2/3".rfind('0') == -1
 
+    block:
+      const haystack: string = "ABCABABABABCAB"
+      doAssert haystack.len == 14
+      doAssert haystack.rfind("ABC") == 9
+      doAssert haystack.rfind("ABC", last=13) == 9
+      doAssert haystack.rfind("ABC", last=12) == 9
+      doAssert haystack.rfind("ABC", last=11) == 9
+      doAssert haystack.rfind("ABC", last=10) == 0
+
+      doAssert haystack.rfind("ABC", start=0) == 9
+      doAssert haystack.rfind("ABC", start=1) == 9
+      doAssert haystack.rfind("ABC", start=2) == 9
+      doAssert haystack.rfind("ABC", start=9) == 9
+      doAssert haystack.rfind("ABC", start=10) == -1
+      doAssert haystack.rfind("ABC", start=11) == -1
+      doAssert haystack.rfind("ABC", start=12) == -1
+
+      doAssert haystack.rfind("ABC", start=0, last=13) == 9
+      doAssert haystack.rfind("ABC", start=0, last=12) == 9
+      doAssert haystack.rfind("ABC", start=0, last=11) == 9
+      doAssert haystack.rfind("ABC", start=0, last=10) == 0
+      doAssert haystack.rfind("ABC", start=1, last=10) == -1
+
+    doAssert "".rfind("/") == -1
+    doAssert "/".rfind("/") == 0
+    doAssert "/".rfind("//") == -1
+    doAssert "///".rfind("//", start=3) == -1
+
   block: # trimZeros
     var x = "1200"
     x.trimZeros()

From b7ad29e692ffd9d12fdf4d42cd8ef2a63b9340d1 Mon Sep 17 00:00:00 2001
From: Saem Ghani 
Date: Mon, 31 May 2021 04:27:44 -0700
Subject: [PATCH 0857/2472] fix #18113 (#18124)

---
 compiler/semtempl.nim     |  8 ++++++--
 tests/proc/t17157.nim     |  1 +
 tests/template/t18113.nim | 14 ++++++++++++++
 3 files changed, 21 insertions(+), 2 deletions(-)
 create mode 100644 tests/template/t18113.nim

diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim
index 40502daf4..181cc01db 100644
--- a/compiler/semtempl.nim
+++ b/compiler/semtempl.nim
@@ -610,8 +610,6 @@ proc semTemplateDef(c: PContext, n: PNode): PNode =
        s.owner.name.s == "vm" and s.name.s == "stackTrace":
       incl(s.flags, sfCallsite)
 
-  s.ast = n
-
   styleCheckDef(c.config, s)
   onDef(n[namePos].info, s)
   # check parameter list:
@@ -664,6 +662,12 @@ proc semTemplateDef(c: PContext, n: PNode): PNode =
   semIdeForTemplateOrGeneric(c, n[bodyPos], ctx.cursorInBody)
   closeScope(c)
   popOwner(c)
+
+  # set the symbol AST after pragmas, at least. This stops pragma that have
+  # been pushed (implicit) to be explicitly added to the template definition
+  # and misapplied to the body. see #18113
+  s.ast = n
+
   if sfCustomPragma in s.flags:
     if n[bodyPos].kind != nkEmpty:
       localError(c.config, n[bodyPos].info, errImplOfXNotAllowed % s.name.s)
diff --git a/tests/proc/t17157.nim b/tests/proc/t17157.nim
index 020e93fce..2927eeee8 100644
--- a/tests/proc/t17157.nim
+++ b/tests/proc/t17157.nim
@@ -1,5 +1,6 @@
 discard """
   errormsg: "'untyped' is only allowed in templates and macros or magic procs"
+  disabled: true
 """
 
 template something(op: proc (v: untyped): void): void =
diff --git a/tests/template/t18113.nim b/tests/template/t18113.nim
new file mode 100644
index 000000000..a84b3fd0e
--- /dev/null
+++ b/tests/template/t18113.nim
@@ -0,0 +1,14 @@
+# ensure template pragma handling doesn't eagerly attempt to add an implicit
+# 'pushed' pragma to the evaluation of any intermediate AST prior to
+# substitution.
+ 
+# bug #18113
+
+import sequtils
+
+{.push raises: [Defect].}
+
+var a = toSeq([1, 2, 3, 5, 10]).filterIt(it > 5)
+
+doAssert a.len == 1
+doAssert a[0] == 10

From a36efb59b5a74db6c5bbe6c0997c0221d0f55491 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Mon, 31 May 2021 04:39:19 -0700
Subject: [PATCH 0858/2472] fix #16256: nimout:  should give error
 (vacuously true); improve a few tests (#18089)

* fix #16256: nimout:  should give error (vacuously true); improve some tests

* renamed:    tests/stdlib/t9710.nim -> tests/misc/t9710.nim

* improve tests

* fix non-DRY tests

* improve $nim_prs_D/tests/stdlib/t9091.nim

* renamed:    tests/stdlib/t9091.nim -> tests/misc/t9091.nim

* fixup

* address comment: doAssert => result.parseErrors
---
 testament/specs.nim              |  5 ++
 tests/{stdlib => misc}/t9091.nim | 15 +++---
 tests/misc/t9710.nim             |  6 +++
 tests/stdlib/t9710.nim           | 11 ----
 tests/stdlib/tgetprotobyname.nim | 10 ----
 tests/stdlib/tnativesockets.nim  | 46 ++++++++---------
 tests/stdlib/tvarints.nim        | 86 +++++++++++++-------------------
 7 files changed, 73 insertions(+), 106 deletions(-)
 rename tests/{stdlib => misc}/t9091.nim (70%)
 create mode 100644 tests/misc/t9710.nim
 delete mode 100644 tests/stdlib/t9710.nim

diff --git a/testament/specs.nim b/testament/specs.nim
index a3271aeaa..6fd0ab22d 100644
--- a/testament/specs.nim
+++ b/testament/specs.nim
@@ -253,6 +253,7 @@ proc parseSpec*(filename: string): TSpec =
   var p: CfgParser
   open(p, ss, filename, 1)
   var flags: HashSet[string]
+  var nimoutFound = false
   while true:
     var e = next(p)
     case e.kind
@@ -309,6 +310,7 @@ proc parseSpec*(filename: string): TSpec =
         result.action = actionReject
       of "nimout":
         result.nimout = e.value
+        nimoutFound = true
       of "nimoutfull":
         result.nimoutFull = parseCfgBool(e.value)
       of "batchable":
@@ -401,6 +403,9 @@ proc parseSpec*(filename: string): TSpec =
   if skips.anyIt(it in result.file):
     result.err = reDisabled
 
+  if nimoutFound and result.nimout.len == 0 and not result.nimoutFull:
+    result.parseErrors.addLine "empty `nimout` is vacuously true, use `nimoutFull:true` if intentional"
+
   result.inCurrentBatch = isCurrentBatch(testamentData0, filename) or result.unbatchable
   if not result.inCurrentBatch:
     result.err = reDisabled
diff --git a/tests/stdlib/t9091.nim b/tests/misc/t9091.nim
similarity index 70%
rename from tests/stdlib/t9091.nim
rename to tests/misc/t9091.nim
index 8419479a7..6e7a98ca5 100644
--- a/tests/stdlib/t9091.nim
+++ b/tests/misc/t9091.nim
@@ -1,10 +1,5 @@
-discard """
-  targets:  "c"
-  output:   "test AObj"
-  action:   "compile"
-  exitcode: 0
-  timeout:  60.0
-"""
+# bug #9091
+
 import streams
 
 block:
@@ -18,6 +13,8 @@ block:
   let mi = new Mine
 
   str.write(mi)
+  str.setPosition 0
+  doAssert str.readAll == "sure"
 
 block:
   type
@@ -27,10 +24,10 @@ block:
   proc foo(a: int): string = ""
 
   proc test(args: varargs[string, foo]) =
-    echo "varargs"
+    doAssert false
 
   proc test(a: AObj) =
-    echo "test AObj"
+    discard
 
   let x = AObj()
   test(x)
diff --git a/tests/misc/t9710.nim b/tests/misc/t9710.nim
new file mode 100644
index 000000000..c65cb7bf4
--- /dev/null
+++ b/tests/misc/t9710.nim
@@ -0,0 +1,6 @@
+discard """
+  matrix: "--debugger:native"
+"""
+# bug #9710
+for i in 1 || 200:
+  discard i
diff --git a/tests/stdlib/t9710.nim b/tests/stdlib/t9710.nim
deleted file mode 100644
index f3ed860df..000000000
--- a/tests/stdlib/t9710.nim
+++ /dev/null
@@ -1,11 +0,0 @@
-discard """
-  cmd:      "nim c -r --debugger:native --panics:on $options $file"
-  targets:  "c"
-  nimout:   ""
-  action:   "run"
-  exitcode: 0
-  timeout:  60.0
-"""
-
-for i in 1 || 200:
-  discard i
diff --git a/tests/stdlib/tgetprotobyname.nim b/tests/stdlib/tgetprotobyname.nim
index b4df05102..0b60d059f 100644
--- a/tests/stdlib/tgetprotobyname.nim
+++ b/tests/stdlib/tgetprotobyname.nim
@@ -1,15 +1,5 @@
-discard """
-  cmd:      "nim c -r --styleCheck:hint --panics:on $options $file"
-  targets:  "c"
-  nimout:   ""
-  action:   "run"
-  exitcode: 0
-  timeout:  60.0
-"""
-
 import nativesockets
 
-
 when not defined(netbsd):
   # Ref: https://github.com/nim-lang/Nim/issues/15452 - NetBSD doesn't define an `ip` protocol
   doAssert getProtoByName("ip") == 0
diff --git a/tests/stdlib/tnativesockets.nim b/tests/stdlib/tnativesockets.nim
index b0cfd09cf..6a1a00881 100644
--- a/tests/stdlib/tnativesockets.nim
+++ b/tests/stdlib/tnativesockets.nim
@@ -1,29 +1,25 @@
-discard """
-  cmd:      "nim c -r --styleCheck:hint --panics:on $options $file"
-  targets:  "c"
-  nimout:   ""
-  action:   "run"
-  exitcode: 0
-  timeout:  60.0
-"""
-
-import nativesockets
+import std/nativesockets
+import stdtest/testutils
 
+block:
+  let hostname = getHostname()
+  doAssert hostname.len > 0
 
 when defined(windows):
-  doAssert toInt(IPPROTO_IP) == 0.cint
-  doAssert toInt(IPPROTO_ICMP) == 1.cint
-  doAssert toInt(IPPROTO_TCP) == 6.cint
-  doAssert toInt(IPPROTO_UDP) == 17.cint
-  doAssert toInt(IPPROTO_IPV6) == 41.cint
-  doAssert toInt(IPPROTO_ICMPV6) == 58.cint
-  doAssert toInt(IPPROTO_RAW) == 20.cint
+  assertAll:
+    toInt(IPPROTO_IP) == 0
+    toInt(IPPROTO_ICMP) == 1
+    toInt(IPPROTO_TCP) == 6
+    toInt(IPPROTO_UDP) == 17
+    toInt(IPPROTO_IPV6) == 41
+    toInt(IPPROTO_ICMPV6) == 58
+    toInt(IPPROTO_RAW) == 20
 
-  # no changes to enum value
-  doAssert ord(IPPROTO_TCP) == 6
-  doAssert ord(IPPROTO_UDP) == 17
-  doAssert ord(IPPROTO_IP) == 18
-  doAssert ord(IPPROTO_IPV6) == 19
-  doAssert ord(IPPROTO_RAW) == 20
-  doAssert ord(IPPROTO_ICMP) == 21
-  doAssert ord(IPPROTO_ICMPV6) == 22
+    # no changes to enum value
+    ord(IPPROTO_TCP) == 6
+    ord(IPPROTO_UDP) == 17
+    ord(IPPROTO_IP) == 18
+    ord(IPPROTO_IPV6) == 19
+    ord(IPPROTO_RAW) == 20
+    ord(IPPROTO_ICMP) == 21
+    ord(IPPROTO_ICMPV6) == 22
diff --git a/tests/stdlib/tvarints.nim b/tests/stdlib/tvarints.nim
index dcdb756ce..772121f9f 100644
--- a/tests/stdlib/tvarints.nim
+++ b/tests/stdlib/tvarints.nim
@@ -1,15 +1,6 @@
-discard """
-  cmd:      "nim c -r --styleCheck:hint --panics:on $options $file"
-  matrix:   "-d:danger; -d:release"
-  targets:  "c cpp"
-  nimout:   ""
-  action:   "run"
-  exitcode: 0
-  timeout:  60.0
-"""
-
 import std/varints
 
+# xxx doesn't work with js: tvarints.nim(18, 14) `wrLen == rdLen`  [AssertionDefect]
 
 block:
   var dest: array[50, byte]
@@ -39,46 +30,39 @@ block:
 block:
   var hugeIntArray: array[50, byte]
   var readedInt: uint64
-  doAssert writeVu64(hugeIntArray, 0.uint64) == readVu64(hugeIntArray, readedInt)
-  doAssert readedInt == 0.uint64
-  doAssert writeVu64(hugeIntArray, uint64.high) == readVu64(hugeIntArray, readedInt)
-  doAssert readedInt == uint64.high
-  doAssert writeVu64(hugeIntArray, uint64(int64.high)) == readVu64(hugeIntArray, readedInt)
-  doAssert readedInt == uint64(int64.high)
-  doAssert writeVu64(hugeIntArray, uint64(int32.high)) == readVu64(hugeIntArray, readedInt)
-  doAssert readedInt == uint64(int32.high)
-  doAssert writeVu64(hugeIntArray, uint64(int16.high)) == readVu64(hugeIntArray, readedInt)
-  doAssert readedInt == uint64(int16.high)
-  doAssert writeVu64(hugeIntArray, uint64(int8.high)) == readVu64(hugeIntArray, readedInt)
-  doAssert readedInt == uint64(int8.high)
-  doAssert writeVu64(hugeIntArray, cast[uint64](0.0)) == readVu64(hugeIntArray, readedInt)
-  doAssert readedInt == cast[uint64](0.0)
-  doAssert writeVu64(hugeIntArray, cast[uint64](-0.0)) == readVu64(hugeIntArray, readedInt)
-  doAssert readedInt == cast[uint64](-0.0)
-  doAssert writeVu64(hugeIntArray, cast[uint64](0.1)) == readVu64(hugeIntArray, readedInt)
-  doAssert readedInt == cast[uint64](0.1)
-  doAssert writeVu64(hugeIntArray, cast[uint64](0.9555555555555555555555501)) == readVu64(hugeIntArray, readedInt)
-  doAssert readedInt == cast[uint64](0.9555555555555555555555501)
-  doAssert writeVu64(hugeIntArray, cast[uint64](+Inf)) == readVu64(hugeIntArray, readedInt)
-  doAssert readedInt == cast[uint64](+Inf)
-  doAssert writeVu64(hugeIntArray, cast[uint64](NegInf)) == readVu64(hugeIntArray, readedInt)
-  doAssert readedInt == cast[uint64](NegInf)
-  doAssert writeVu64(hugeIntArray, cast[uint64](Nan)) == readVu64(hugeIntArray, readedInt)
-  doAssert readedInt == cast[uint64](Nan)
-  doAssert writeVu64(hugeIntArray, cast[uint64](3.1415926535897932384626433)) == readVu64(hugeIntArray, readedInt)
-  doAssert readedInt == cast[uint64](3.1415926535897932384626433)
-  doAssert writeVu64(hugeIntArray, cast[uint64](2.71828182845904523536028747)) == readVu64(hugeIntArray, readedInt)
-  doAssert readedInt == cast[uint64](2.71828182845904523536028747)
+
+  template chk(a) =
+    let b = cast[uint64](a)
+    doAssert writeVu64(hugeIntArray, b) == readVu64(hugeIntArray, readedInt)
+    doAssert readedInt == b
+
+  chk 0
+  chk uint64.high
+  chk int64.high
+  chk int32.high
+  chk int16.high
+  chk int16.high
+  chk int8.high
+  chk 0.0
+  chk -0.0
+  chk 0.1
+  chk Inf
+  chk NegInf
+  chk Nan
+  chk 3.1415926535897932384626433
 
 block:
-  doAssert encodeZigzag(decodeZigzag(0.uint64)) == 0.uint64
-  doAssert encodeZigzag(decodeZigzag(uint64(uint32.high))) == uint64(uint32.high)
-  doAssert encodeZigzag(decodeZigzag(uint64(int32.high))) == uint64(int32.high)
-  doAssert encodeZigzag(decodeZigzag(uint64(int16.high))) == uint64(int16.high)
-  doAssert encodeZigzag(decodeZigzag(uint64(int8.high))) == uint64(int8.high)
-  doAssert encodeZigzag(decodeZigzag(cast[uint64](0.0))) == cast[uint64](0.0)
-  doAssert encodeZigzag(decodeZigzag(cast[uint64](0.1))) == cast[uint64](0.1)
-  doAssert encodeZigzag(decodeZigzag(cast[uint64](0.9555555555555555555555501))) == cast[uint64](0.9555555555555555555555501)
-  doAssert encodeZigzag(decodeZigzag(cast[uint64](+Inf))) == cast[uint64](+Inf)
-  doAssert encodeZigzag(decodeZigzag(cast[uint64](3.1415926535897932384626433))) == cast[uint64](3.1415926535897932384626433)
-  doAssert encodeZigzag(decodeZigzag(cast[uint64](2.71828182845904523536028747))) == cast[uint64](2.71828182845904523536028747)
+  template chk(a) =
+    let b = cast[uint64](a)
+    doAssert encodeZigzag(decodeZigzag(b)) == b
+  chk 0
+  chk uint32.high
+  chk int32.high
+  chk int16.high
+  chk int8.high
+  chk 0.0
+  chk 0.1
+  chk 0.9555555555555555555555501
+  chk Inf
+  chk 3.1415926535897932384626433
+  chk 2.71828182845904523536028747

From 60cbdbf37ac66624f78edd5b3d575c80c6603679 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Mon, 31 May 2021 13:14:50 -0700
Subject: [PATCH 0859/2472] close #18092 document elif in case statements
 (#18105)

* close #18092 [skip ci] document elif in case statements

* fixup

* clarify spec; mention special rule for string in case statements

* address comments
---
 doc/manual.rst | 30 ++++++++++++++++++------------
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/doc/manual.rst b/doc/manual.rst
index 6a0dec120..ff34b62ca 100644
--- a/doc/manual.rst
+++ b/doc/manual.rst
@@ -2981,11 +2981,13 @@ Example:
 
 .. code-block:: nim
 
-  case readline(stdin)
+  let line = readline(stdin)
+  case line
   of "delete-everything", "restart-computer":
     echo "permission denied"
   of "go-for-a-walk":     echo "please yourself"
-  else:                   echo "unknown command"
+  elif line.len == 0:     echo "empty" # optional, must come after `of` branches
+  else:                   echo "unknown command" # ditto
 
   # indentation of the branches is also allowed; and so is an optional colon
   # after the selecting expression:
@@ -2996,19 +2998,23 @@ Example:
     else:                   echo "unknown command"
 
 
-The `case` statement is similar to the if statement, but it represents
+The `case` statement is similar to the `if` statement, but it represents
 a multi-branch selection. The expression after the keyword `case` is
 evaluated and if its value is in a *slicelist* the corresponding statements
 (after the `of` keyword) are executed. If the value is not in any
-given *slicelist* the `else` part is executed. If there is no `else`
-part and not all possible values that `expr` can hold occur in a
-*slicelist*, a static error occurs. This holds only for expressions of
-ordinal types. "All possible values" of `expr` are determined by `expr`'s
-type. To suppress the static error an `else` part with an
-empty `discard` statement should be used.
+given *slicelist*, trailing `elif` and `else` parts are executed using same
+semantics as for `if` statement, and `elif` is handled just like `else: if`.
+If there are no `else` or `elif` parts and not
+all possible values that `expr` can hold occur in a *slicelist*, a static error occurs.
+This holds only for expressions of ordinal types.
+"All possible values" of `expr` are determined by `expr`'s type.
+To suppress the static error an `else: discard` should be used.
 
 For non-ordinal types, it is not possible to list every possible value and so
 these always require an `else` part.
+An exception to this rule is for the `string` type, which currently doesn't
+require a trailing `else` or `elif` branch; it's unspecified whether this will
+keep working in future versions.
 
 Because case statements are checked for exhaustiveness during semantic analysis,
 the value in every `of` branch must be a constant expression.
@@ -3054,15 +3060,15 @@ won't work:
   var foo = Foo(x: @[])
   foo.get_x().add("asd")
 
-This can be fixed by explicitly using `return`:
+This can be fixed by explicitly using `result` or `return`:
 
 .. code-block:: nim
   proc get_x(x: Foo): var seq[string] =
     case true
     of true:
-      return x.x
+      result = x.x
     else:
-      return x.x
+      result = x.x
 
 
 When statement

From 9559350e34a62614d1f7de0945095a4c4f4f5ece Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Mon, 31 May 2021 13:16:33 -0700
Subject: [PATCH 0860/2472] add `os.getCacheDir` (#18126)

* add `os.getCacheDir`

* fixup

* address comments
---
 changelog.md         |  3 ++-
 lib/pure/os.nim      | 44 ++++++++++++++++++++++++++++++++++++++------
 tests/stdlib/tos.nim |  3 +++
 3 files changed, 43 insertions(+), 7 deletions(-)

diff --git a/changelog.md b/changelog.md
index 279ba38dd..d404b1965 100644
--- a/changelog.md
+++ b/changelog.md
@@ -175,10 +175,11 @@
 - `--gc:orc` is now 10% faster than previously for common workloads. If
   you have trouble with its changed behavior, compile with `-d:nimOldOrc`.
 
-
 - `os.FileInfo` (returned by `getFileInfo`) now contains `blockSize`,
   determining preferred I/O block size for this file object.
 
+- Added `os.getCacheDir()` to return platform specific cache directory.
+
 - Added a simpler to use `io.readChars` overload.
 
 - Added `**` to jsffi.
diff --git a/lib/pure/os.nim b/lib/pure/os.nim
index 287fbe125..c48d0d84f 100644
--- a/lib/pure/os.nim
+++ b/lib/pure/os.nim
@@ -922,9 +922,7 @@ proc getConfigDir*(): string {.rtl, extern: "nos$1",
   ## See also:
   ## * `getHomeDir proc <#getHomeDir>`_
   ## * `getTempDir proc <#getTempDir>`_
-  ## * `expandTilde proc <#expandTilde,string>`_
-  ## * `getCurrentDir proc <#getCurrentDir>`_
-  ## * `setCurrentDir proc <#setCurrentDir,string>`_
+  ## * `getCacheDir proc <#getCacheDir>`_
   runnableExamples:
     from std/strutils import endsWith
     # See `getHomeDir` for behavior regarding trailing DirSep.
@@ -935,6 +933,42 @@ proc getConfigDir*(): string {.rtl, extern: "nos$1",
     result = getEnv("XDG_CONFIG_HOME", getEnv("HOME") / ".config")
   result.normalizePathEnd(trailingSep = defined(nimLegacyHomeDir))
 
+proc getCacheDir*(): string =
+  ## Returns the cache directory of the current user for applications.
+  ## 
+  ## This makes use of the following environment variables:
+  ##
+  ## * On Windows: `getEnv("LOCALAPPDATA")`
+  ##
+  ## * On macOS: `getEnv("XDG_CACHE_HOME", getEnv("HOME") / "Library/Caches")`
+  ##
+  ## * On other platforms: `getEnv("XDG_CACHE_HOME", getEnv("HOME") / ".cache")`
+  ##
+  ## **See also:**
+  ## * `getHomeDir proc <#getHomeDir>`_
+  ## * `getTempDir proc <#getTempDir>`_
+  ## * `getConfigDir proc <#getConfigDir>`_
+  # follows https://crates.io/crates/platform-dirs
+  when defined(windows):
+    result = getEnv("LOCALAPPDATA")
+  elif defined(osx):
+    result = getEnv("XDG_CACHE_HOME", getEnv("HOME") / "Library/Caches")
+  else:
+    result = getEnv("XDG_CACHE_HOME", getEnv("HOME") / ".cache")
+  result.normalizePathEnd(false)
+
+proc getCacheDir*(app: string): string =
+  ## Returns the cache directory for an application `app`.
+  ##
+  ## * On windows, this uses: `getCacheDir() / app / "cache"`
+  ##
+  ## * On other platforms, this uses: `getCacheDir() / app`
+  when defined(windows):
+    getCacheDir() / app / "cache"
+  else:
+    getCacheDir() / app
+
+
 when defined(windows):
   type DWORD = uint32
 
@@ -972,9 +1006,7 @@ proc getTempDir*(): string {.rtl, extern: "nos$1",
   ## See also:
   ## * `getHomeDir proc <#getHomeDir>`_
   ## * `getConfigDir proc <#getConfigDir>`_
-  ## * `expandTilde proc <#expandTilde,string>`_
-  ## * `getCurrentDir proc <#getCurrentDir>`_
-  ## * `setCurrentDir proc <#setCurrentDir,string>`_
+  ## * `getCacheDir proc <#getCacheDir>`_
   runnableExamples:
     from std/strutils import endsWith
     # See `getHomeDir` for behavior regarding trailing DirSep.
diff --git a/tests/stdlib/tos.nim b/tests/stdlib/tos.nim
index 1fcee5a2e..b9769d05c 100644
--- a/tests/stdlib/tos.nim
+++ b/tests/stdlib/tos.nim
@@ -556,6 +556,9 @@ block getTempDir:
       else:
         doAssert getTempDir() == "/tmp"
 
+block: # getCacheDir
+  doAssert getCacheDir().dirExists
+
 block osenv:
   block delEnv:
     const dummyEnvVar = "DUMMY_ENV_VAR" # This env var wouldn't be likely to exist to begin with

From 369a7d12467831295c12f5f51aa8c69ec3b3596a Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Mon, 31 May 2021 13:17:52 -0700
Subject: [PATCH 0861/2472] jsonutils.toJson now serializes JsonNode as is by
 default (#18097)

* jsonutils.toJson now serializes JsonNode as is (without deep copy nor treating it as a regular ref object)

* JsonNodeMode
---
 changelog.md                |  4 +++-
 lib/std/jsonutils.nim       | 20 ++++++++++++++++----
 tests/stdlib/tjsonutils.nim | 22 ++++++++++++++++++++++
 3 files changed, 41 insertions(+), 5 deletions(-)

diff --git a/changelog.md b/changelog.md
index d404b1965..9806a6011 100644
--- a/changelog.md
+++ b/changelog.md
@@ -69,7 +69,9 @@
   previous behavior for a transition time, see PR #17467.
 
 - `jsonutils` now serializes/deserializes holey enums as regular enums (via `ord`) instead of as strings.
-  Use `-d:nimLegacyJsonutilsHoleyEnum` for a transition period.
+  Use `-d:nimLegacyJsonutilsHoleyEnum` for a transition period. `toJson` now serializes `JsonNode`
+  as is via reference (without a deep copy) instead of treating `JsonNode` as a regular ref object,
+  this can be customized via `jsonNodeMode`.
 
 - `json` and `jsonutils` now serialize NaN, Inf, -Inf as strings, so that
   `%[NaN, -Inf]` is the string `["nan","-inf"]` instead of `[nan,-inf]` which was invalid json.
diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim
index 1e222e3a2..e33ae4401 100644
--- a/lib/std/jsonutils.nim
+++ b/lib/std/jsonutils.nim
@@ -48,13 +48,18 @@ type
     joptEnumOrd
     joptEnumSymbol
     joptEnumString
+  JsonNodeMode* = enum ## controls `toJson` for JsonNode types
+    joptJsonNodeAsRef ## returns the ref as is
+    joptJsonNodeAsCopy ## returns a deep copy of the JsonNode
+    joptJsonNodeAsObject ## treats JsonNode as a regular ref object
   ToJsonOptions* = object
     enumMode*: EnumMode
-    # xxx charMode
+    jsonNodeMode*: JsonNodeMode
+    # xxx charMode, etc
 
 proc initToJsonOptions*(): ToJsonOptions =
   ## initializes `ToJsonOptions` with sane options.
-  ToJsonOptions(enumMode: joptEnumOrd)
+  ToJsonOptions(enumMode: joptEnumOrd, jsonNodeMode: joptJsonNodeAsRef)
 
 proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".}
 proc distinctBase(T: typedesc): typedesc {.magic: "TypeTrait".}
@@ -286,8 +291,15 @@ proc toJson*[T](a: T, opt = initToJsonOptions()): JsonNode =
       result = newJArray()
       for v in a.fields: result.add toJson(v, opt)
   elif T is ref | ptr:
-    if system.`==`(a, nil): result = newJNull()
-    else: result = toJson(a[], opt)
+    template impl =
+      if system.`==`(a, nil): result = newJNull()
+      else: result = toJson(a[], opt)
+    when T is JsonNode:
+      case opt.jsonNodeMode
+      of joptJsonNodeAsRef: result = a
+      of joptJsonNodeAsCopy: result = copy(a)
+      of joptJsonNodeAsObject: impl()
+    else: impl()
   elif T is array | seq | set:
     result = newJArray()
     for ai in a: result.add toJson(ai, opt)
diff --git a/tests/stdlib/tjsonutils.nim b/tests/stdlib/tjsonutils.nim
index 62486b896..88c05facf 100644
--- a/tests/stdlib/tjsonutils.nim
+++ b/tests/stdlib/tjsonutils.nim
@@ -91,6 +91,28 @@ template fn() =
     doAssert b2.ord == 1 # explains the `1`
     testRoundtrip(a): """[1,2,3]"""
 
+  block: # JsonNode
+    let a = ((1, 2.5, "abc").toJson, (3, 4.5, "foo"))
+    testRoundtripVal(a): """[[1,2.5,"abc"],[3,4.5,"foo"]]"""
+
+    block:
+      template toInt(a): untyped = cast[int](a)
+
+      let a = 3.toJson
+      let b = (a, a)
+
+      let c1 = b.toJson
+      doAssert c1[0].toInt == a.toInt
+      doAssert c1[1].toInt == a.toInt
+
+      let c2 = b.toJson(ToJsonOptions(jsonNodeMode: joptJsonNodeAsCopy))
+      doAssert c2[0].toInt != a.toInt
+      doAssert c2[1].toInt != c2[0].toInt
+      doAssert c2[1] == c2[0]
+
+      let c3 = b.toJson(ToJsonOptions(jsonNodeMode: joptJsonNodeAsObject))
+      doAssert $c3 == """[{"isUnquoted":false,"kind":2,"num":3},{"isUnquoted":false,"kind":2,"num":3}]"""
+
   block: # ToJsonOptions
     let a = (me1, me2)
     doAssert $a.toJson() == "[1,2]"

From 06d50bfd4cac6611c5fc775c8130cad87e4dcee1 Mon Sep 17 00:00:00 2001
From: Dylan Modesitt 
Date: Mon, 31 May 2021 16:51:32 -0400
Subject: [PATCH 0862/2472] Fixes #5034 illformed AST from getImpl with proc
 returning value (#17976)

* Fixes 5034

* address comments
---
 compiler/semstmts.nim          | 25 +++++++++++++++++++++----
 tests/macros/tmacrogetimpl.nim | 31 +++++++++++++++++++++++++++++++
 2 files changed, 52 insertions(+), 4 deletions(-)
 create mode 100644 tests/macros/tmacrogetimpl.nim

diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim
index b4630294b..085146f7f 100644
--- a/compiler/semstmts.nim
+++ b/compiler/semstmts.nim
@@ -1439,16 +1439,33 @@ proc semBorrow(c: PContext, n: PNode, s: PSym) =
   else:
     localError(c.config, n.info, errNoSymbolToBorrowFromFound)
 
+proc swapResult(n: PNode, sRes: PSym, dNode: PNode) =
+  ## Swap nodes that are (skResult) symbols to d(estination)Node.
+  for i in 0.. resultPos and n[resultPos] != nil:
-      if n[resultPos].sym.kind != skResult or n[resultPos].sym.owner != getCurrOwner(c):
+      if n[resultPos].sym.kind != skResult:
         localError(c.config, n.info, "incorrect result proc symbol")
+      if n[resultPos].sym.owner != getCurrOwner(c):
+        # re-write result with new ownership, and re-write the proc accordingly
+        let sResSym = n[resultPos].sym
+        genResSym(s)
+        n[resultPos] = newSymNode(s)
+        swapResult(n, sResSym, n[resultPos])
       c.p.resultSym = n[resultPos].sym
     else:
-      var s = newSym(skResult, getIdent(c.cache, "result"), nextSymId c.idgen, getCurrOwner(c), n.info)
-      s.typ = t
-      incl(s.flags, sfUsed)
+      genResSym(s)
       c.p.resultSym = s
       n.add newSymNode(c.p.resultSym)
     addParamOrResult(c, c.p.resultSym, owner)
diff --git a/tests/macros/tmacrogetimpl.nim b/tests/macros/tmacrogetimpl.nim
new file mode 100644
index 000000000..1d996ff29
--- /dev/null
+++ b/tests/macros/tmacrogetimpl.nim
@@ -0,0 +1,31 @@
+import macros
+
+# bug #5034
+
+macro copyImpl(srsProc: typed, toSym: untyped) =
+  result = copyNimTree(getImplTransformed(srsProc))
+  result[0] = ident $toSym.toStrLit()
+
+proc foo1(x: float, one: bool = true): float =
+  if one:
+    return 1'f
+  result = x
+
+proc bar1(what: string): string =
+  ## be a little more adversarial with `skResult`
+  proc buzz: string =
+    result = "lightyear"
+  if what == "buzz":
+    result = "buzz " & buzz()
+  else:
+    result = what
+  return result
+
+copyImpl(foo1, foo2)
+doAssert foo1(1'f) == 1.0
+doAssert foo2(10.0, false) == 10.0
+doAssert foo2(10.0) == 1.0
+
+copyImpl(bar1, bar2)
+doAssert bar1("buzz") == "buzz lightyear"
+doAssert bar1("macros") == "macros"

From c0e8199acc9f49caa36bae34d103600d112b66f3 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Tue, 1 Jun 2021 05:28:22 +0800
Subject: [PATCH 0863/2472] [std/re] fix findBounds and find procs (#18028)

* [std/re] make interface consistent

* tiny

* revert
---
 lib/impure/re.nim | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/impure/re.nim b/lib/impure/re.nim
index 0c96876b9..40860cc88 100644
--- a/lib/impure/re.nim
+++ b/lib/impure/re.nim
@@ -171,7 +171,7 @@ proc findBounds*(s: string, pattern: Regex, matches: var openArray[string],
 
 proc findBounds*(buf: cstring, pattern: Regex,
                  matches: var openArray[tuple[first, last: int]],
-                 start = 0, bufSize = 0): tuple[first, last: int] =
+                 start = 0, bufSize: int): tuple[first, last: int] =
   ## returns the starting position and end position of `pattern` in `buf`
   ## (where `buf` has length `bufSize` and is not necessarily `'\0'` terminated),
   ## and the captured substrings in the array `matches`.
@@ -290,7 +290,7 @@ proc match*(buf: cstring, pattern: Regex, matches: var openArray[string],
   result = matchLen(buf, pattern, matches, start, bufSize) != -1
 
 proc find*(buf: cstring, pattern: Regex, matches: var openArray[string],
-           start = 0, bufSize = 0): int =
+           start = 0, bufSize: int): int =
   ## returns the starting position of `pattern` in `buf` and the captured
   ## substrings in the array `matches`. If it does not match, nothing
   ## is written into `matches` and `-1` is returned.

From c2e3dc0ed16a854981769b426e9e327663447df7 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Wed, 2 Jun 2021 00:16:25 +0800
Subject: [PATCH 0864/2472] close #18129 Add setCurrentException for JS backend
 (#18145)

* [std/re] make interface consistent

* tiny

* revert

* close #18129  add setCurrentException

* changelog entry
---
 changelog.md                       | 2 +-
 lib/system/jssys.nim               | 3 +++
 tests/exception/tsetexceptions.nim | 8 ++++++++
 3 files changed, 12 insertions(+), 1 deletion(-)
 create mode 100644 tests/exception/tsetexceptions.nim

diff --git a/changelog.md b/changelog.md
index 9806a6011..900b23ad8 100644
--- a/changelog.md
+++ b/changelog.md
@@ -329,7 +329,7 @@
 
 - Fixed premature garbage collection in asyncdispatch, when a stack trace override is in place.
 
-
+- Added setCurrentException for JS backend.
 
 ## Language changes
 
diff --git a/lib/system/jssys.nim b/lib/system/jssys.nim
index ef06437e5..371cb7962 100644
--- a/lib/system/jssys.nim
+++ b/lib/system/jssys.nim
@@ -69,6 +69,9 @@ proc getCurrentExceptionMsg*(): string =
         return $msg
   return ""
 
+proc setCurrentException*(exc: ref Exception) =
+  lastJSError = cast[PJSError](exc)
+
 proc auxWriteStackTrace(f: PCallFrame): string =
   type
     TempFrame = tuple[procname: cstring, line: int, filename: cstring]
diff --git a/tests/exception/tsetexceptions.nim b/tests/exception/tsetexceptions.nim
new file mode 100644
index 000000000..557fc1898
--- /dev/null
+++ b/tests/exception/tsetexceptions.nim
@@ -0,0 +1,8 @@
+discard """
+  targets: "c cpp js"
+"""
+
+let ex = newException(CatchableError, "test")
+setCurrentException(ex)
+doAssert getCurrentException().msg == ex.msg
+doAssert getCurrentExceptionMsg() == ex.msg

From ba3ec7b04954a9a60adb6e1a64405f7644f799ee Mon Sep 17 00:00:00 2001
From: Andrey Makarov 
Date: Tue, 1 Jun 2021 21:47:23 +0300
Subject: [PATCH 0865/2472] docs: Latex generation improvements (#18141)

* docs: improve Latex generation

* make it work on Windows + fix ] escaping

* minor fixes with escapes and style
---
 compiler/docgen.nim              |   2 +-
 config/nimdoc.tex.cfg            | 202 ++++++++++++++++++++++---------
 lib/packages/docutils/rstast.nim |  18 ++-
 lib/packages/docutils/rstgen.nim | 140 +++++++++++----------
 tests/stdlib/trstgen.nim         |   4 +-
 tools/kochdocs.nim               |  14 +--
 6 files changed, 252 insertions(+), 128 deletions(-)

diff --git a/compiler/docgen.nim b/compiler/docgen.nim
index e07db9188..18c568ba0 100644
--- a/compiler/docgen.nim
+++ b/compiler/docgen.nim
@@ -549,7 +549,7 @@ proc getAllRunnableExamplesImpl(d: PDoc; n: PNode, dest: var string,
         var msg = "Example:"
         if rdoccmd.len > 0: msg.add " cmd: " & rdoccmd
         dispA(d.conf, dest, "\n
    $1
    \n",
-            "\n\\textbf{$1}\n", [msg])
+            "\n\n\\textbf{$1}\n", [msg])
         inc d.listingCounter
         let id = $d.listingCounter
         dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim", ""])
diff --git a/config/nimdoc.tex.cfg b/config/nimdoc.tex.cfg
index b19b25678..75382ce26 100644
--- a/config/nimdoc.tex.cfg
+++ b/config/nimdoc.tex.cfg
@@ -8,7 +8,8 @@ split.item.toc = "20"
 # after this number of characters
 
 doc.section = """
-\rsthA{$sectionTitle}\label{$sectionID}
+\rsthA[$sectionTitle]{$sectionTitle}\label{$sectionID}
+
 $content
 """
 
@@ -17,12 +18,16 @@ doc.section.toc = ""
 
 doc.item = """
 
+\vspace{1em}
 \phantomsection\addcontentsline{toc}{subsection}{$uniqueName}
 
-\begin{rstpre}
+\begin{rstdocitem}
 $header
-\end{rstpre}
+\end{rstdocitem}
+
+\begin{addmargin}[0.05\linewidth]{0pt}
 $desc
+\end{addmargin}
 """
 
 doc.item.toc = ""
@@ -42,46 +47,118 @@ $content
 
 # $1 - number of listing in document, $2 - language (e.g. langNim), $3 - anchor
 doc.listing_start = "\\begin{rstpre}\n"
-doc.listing_end = "\n\\end{rstpre}\n"
+doc.listing_end = "\n\\end{rstpre}\n\n"
 
 doc.file = """
 % This file was generated by Nim.
 % Generated: $date $time UTC
-\documentclass[a4paper]{article}
-\usepackage[left=2cm,right=3cm,top=3cm,bottom=3cm]{geometry}
-\usepackage[utf8]{inputenc}
-\usepackage[T1]{fontenc}
+%
+% Compile it by:   xelatex    (up to 3 times to get labels generated)
+%                  -------
+%
+\documentclass[a4paper,11pt]{article}
+\usepackage[a4paper,xetex,left=3cm,right=3cm,top=1.5cm,bottom=2cm]{geometry}
+
+%   for 2-sided printing with larger inner "binding" margin
+%\usepackage[a4paper,xetex,twoside,left=4cm,right=2cm,top=1.5cm,bottom=2cm]{geometry}
+%   for e-readers with 1.77:1 aspect ratio (e.g. 1920x1080)
+%\usepackage[xetex,paperheight=27.6cm,paperwidth=15.5cm,left=3mm,right=3mm,top=3mm,bottom=3mm]{geometry}
+%   for e-readers with 1.45:1 aspect ratio (e.g. 1200x825)
+%\usepackage[xetex,paperheight=22.5cm,paperwidth=15.5cm,left=3mm,right=3mm,top=3mm,bottom=3mm]{geometry}
+%   for e-readers with 1.33:1 aspect ratio (e.g. 1872x1404)
+%\usepackage[xetex,paperheight=20.7cm,paperwidth=15.5cm,left=3mm,right=3mm,top=3mm,bottom=3mm]{geometry}
+
+\usepackage{fontspec}
+% logic to select default font with some fall-back fonts.
+\IfFontExistsTF{Times New Roman}{%
+  \setmainfont{Times New Roman}  % the default font
+  \typeout{========================================= nim: using Times New Roman}
+}{
+  \IfFontExistsTF{FreeSerif}{%
+    \setmainfont{FreeSerif}  % fallback #1 - official GNU font, resembles Times
+    \typeout{========================================= nim: using FreeSerif}
+  }{
+    \IfFontExistsTF{DejaVuSerif}{%
+      \setmainfont{DejaVuSerif}  % fallback #2 - very widespread free font
+      \typeout{========================================= nim: using DejaVuSerif}
+    }{
+      \typeout{!!!!!!!!!!!!!!!!!!! Fonts not found !!!!!!!!!!!!!!!!!!!!!!!}
+    }
+  }
+}
+
+% default monospace font for code:
+\usepackage{GoMono}
+\usepackage{relsize}
+% make this monospace font 2 steps smaller to hold 80-character line
+\newcommand{\rstverbblockfont}{\smaller[2]}
+\newcommand{\rstverbinlinefont}{\smaller}
+
+\usepackage{parskip}  % paragraphs delimited by vertical space, no indent
 \usepackage{graphicx}
-\usepackage{lmodern}
-\usepackage{fancyvrb, courier}
-\usepackage{tabularx}
-\usepackage{hyperref}
-\usepackage{enumitem}  % for option list, enumList, and rstfootnote
+
+\usepackage{dingbat} % for \carriagereturn, etc
+\usepackage{fvextra}  % for code blocks (works better than original fancyvrb)
+\fvset{
+  breaklines,
+  breakafter={=}:|\_\{\}[](){,}.;+-*/'",
+  breaksymbolleft=\color{red}{\ensuremath{\hookrightarrow}}, 
+  breaksymbolright=\color{red}{\small\carriagereturn}
+}
+\fvinlineset{%
+   breaklines,
+   breakafter={=}:|\_\{\}[](){,}.;+-*/'",
+     % that does not work at all when we underline inline code by ulem :-(
+   commandchars=\\\{\}
+}
+
+\usepackage{scrextend}  % for the `addmargin` environment
 
 \usepackage{xcolor}
-\usepackage[tikz]{mdframed}
-\usetikzlibrary{shadows}
-\mdfsetup{%
-linewidth=3,
-topline=false,
-rightline=false,
-bottomline=false}
+\usepackage[urlbordercolor=blue,linkbordercolor=cyan,
+            pdfborderstyle={/S/U/W 1}]{hyperref}
+\usepackage{enumitem}  % for option list, enumList, and rstfootnote
 
-\begin{document}
-\title{$title $version $subtitle}
-\author{$author}
+\usepackage[most]{tcolorbox}  % boxes around admonitions, code blocks, doc.item
 
-\tolerance 1414 
-\hbadness 1414 
-\emergencystretch 1.5em 
-\hfuzz 0.3pt 
-\widowpenalty=10000 
-\vfuzz \hfuzz 
-\raggedbottom 
+\newtcolorbox{rstadmonition}[1][]{blanker, breakable,
+     left=3mm, right=3mm, top=1mm, bottom=1mm,
+     before upper=\indent, parbox=false, #1}
+
+\definecolor{rstframecolor}{rgb}{0.85, 0.8, 0.6}
+
+\newtcolorbox{rstprebox}[1][]{blanker, breakable,
+     left=3mm, right=3mm, top=1mm, bottom=1mm,
+     borderline ={0.1em}{0pt}{rstframecolor},
+     before upper=\indent, parbox=false, #1}
+
+\newenvironment{rstpre}{%
+\VerbatimEnvironment\begingroup\begin{rstprebox}%
+\begin{Verbatim}[fontsize=\rstverbblockfont , commandchars=\\\{\}]}%
+{\end{Verbatim}\end{rstprebox}\endgroup}
+
+\newtcolorbox{rstdocitembox}[1][]{blanker, breakable,
+     left=3mm, right=3mm, top=1mm, bottom=1mm,
+     borderline ={1pt}{0pt}{cyan},
+     before upper=\indent, parbox=false, #1}
+
+% Inline code formatting: grey underline,
+% use \Verb from fvextras e.g. to display -- correctly as double -
+\usepackage[normalem]{ulem}
+\newcommand\rstuline{\bgroup\markoverwith{\textcolor{rstframecolor}{\rule[-0.8ex]{2pt}{1.0pt}}}\ULon}
+
+\newcommand{\rstcode}[1]{%
+{\rstverbinlinefont\Verb{\rstuline{#1}}}%
+}
+
+\newcommand{\rstcodeitem}[1]{\Verb{#1}}
+
+\newenvironment{rstdocitem}{%
+\VerbatimEnvironment\begingroup\begin{rstdocitembox}%
+\begin{Verbatim}[fontsize=\rstverbblockfont , commandchars=\\\{\}]}%
+{\end{Verbatim}\end{rstdocitembox}\endgroup}
 
-\maketitle
 
-\newenvironment{rstpre}{\VerbatimEnvironment\begingroup\begin{Verbatim}[fontsize=\footnotesize , commandchars=\\\{\}]}{\end{Verbatim}\endgroup}
 \newenvironment{rstfootnote}{\begin{description}[labelindent=1em,leftmargin=1em,labelwidth=2.6em]}{\end{description}}
 \ifdim\linewidth<30em
   \def\rstoptleftmargin{0.4\linewidth}
@@ -93,37 +170,41 @@ bottomline=false}
 \newenvironment{rstoptlist}{%
 \begin{description}[font=\sffamily\bfseries,style=nextline,leftmargin=\rstoptleftmargin,labelwidth=\rstoptlabelwidth]}{\end{description}}
 
-% to pack tabularx into a new environment, special syntax is needed :-(
-\newenvironment{rsttab}[1]{\tabularx{\linewidth}{#1}}{\endtabularx}
+\usepackage{tabulary}  % tables with adjustable cell width and no overflow
+% make tabulary prevent overflows (https://tex.stackexchange.com/a/195088)
+\tymin=60pt
+\tymax=\maxdimen
+% to pack tabulary into a new environment, special syntax is needed :-(
+\newenvironment{rsttab}[1]{\tabulary{\linewidth}{#1}}{\endtabulary}
 
 \newcommand{\rstsub}[1]{\raisebox{-0.5ex}{\scriptsize{#1}}}
 \newcommand{\rstsup}[1]{\raisebox{0.5ex}{\scriptsize{#1}}}
 
-\newcommand{\rsthA}[1]{\section{#1}}
-\newcommand{\rsthB}[1]{\subsection{#1}}
-\newcommand{\rsthC}[1]{\subsubsection{#1}}
-\newcommand{\rsthD}[1]{\paragraph{#1}}
-\newcommand{\rsthE}[1]{\paragraph{#1}}
+\newcommand{\rsthA}[2][]{\section[#1]{#2}}
+\newcommand{\rsthB}[2][]{\subsection[#1]{#2}}
+\newcommand{\rsthC}[2][]{\subsubsection[#1]{#2}}
+\newcommand{\rsthD}[2][]{\paragraph[#1]{#2}}
+\newcommand{\rsthE}[2][]{\paragraph[#1]{#2}}
 
-\newcommand{\rstovA}[1]{\section*{#1}}
-\newcommand{\rstovB}[1]{\subsection*{#1}}
-\newcommand{\rstovC}[1]{\subsubsection*{#1}}
-\newcommand{\rstovD}[1]{\paragraph*{#1}}
-\newcommand{\rstovE}[1]{\paragraph*{#1}}
+\newcommand{\rstovA}[2][]{\section*[#1]{#2}}
+\newcommand{\rstovB}[2][]{\subsection*[#1]{#2}}
+\newcommand{\rstovC}[2][]{\subsubsection*[#1]{#2}}
+\newcommand{\rstovD}[2][]{\paragraph*[#1]{#2}}
+\newcommand{\rstovE}[2][]{\paragraph*[#1]{#2}}
 
 % Syntax highlighting:
-\newcommand{\spanDecNumber}[1]{#1}
-\newcommand{\spanBinNumber}[1]{#1}
-\newcommand{\spanHexNumber}[1]{#1}
-\newcommand{\spanOctNumber}[1]{#1}
-\newcommand{\spanFloatNumber}[1]{#1}
+\newcommand{\spanDecNumber}[1]{\textbf{\textcolor{darkgray}{#1}}}
+\newcommand{\spanBinNumber}[1]{\textbf{\textcolor{darkgray}{#1}}}
+\newcommand{\spanHexNumber}[1]{\textbf{\textcolor{darkgray}{#1}}}
+\newcommand{\spanOctNumber}[1]{\textbf{\textcolor{darkgray}{#1}}}
+\newcommand{\spanFloatNumber}[1]{\textbf{\textcolor{darkgray}{#1}}}
 \newcommand{\spanIdentifier}[1]{#1}
 \newcommand{\spanKeyword}[1]{\textbf{#1}}
-\newcommand{\spanStringLit}[1]{#1}
-\newcommand{\spanLongStringLit}[1]{#1}
+\newcommand{\spanStringLit}[1]{\textbf{\textcolor{darkgray}{#1}}}
+\newcommand{\spanLongStringLit}[1]{\textbf{\textcolor{darkgray}{#1}}}
 \newcommand{\spanCharLit}[1]{#1}
 \newcommand{\spanEscapeSequence}[1]{#1}
-\newcommand{\spanOperator}[1]{#1}
+\newcommand{\spanOperator}[1]{\textbf{#1}}
 \newcommand{\spanPunctuation}[1]{#1}
 \newcommand{\spanComment}[1]{\emph{#1}}
 \newcommand{\spanLongComment}[1]{\emph{#1}}
@@ -132,7 +213,7 @@ bottomline=false}
 \newcommand{\spanTagEnd}[1]{#1}
 \newcommand{\spanKey}[1]{#1}
 \newcommand{\spanValue}[1]{#1}
-\newcommand{\spanRawData}[1]{#1}
+\newcommand{\spanRawData}[1]{\textbf{\textcolor{darkgray}{#1}}}
 \newcommand{\spanAssembler}[1]{#1}
 \newcommand{\spanPreprocessor}[1]{#1}
 \newcommand{\spanDirective}[1]{#1}
@@ -142,11 +223,20 @@ bottomline=false}
 \newcommand{\spanLabel}[1]{#1}
 \newcommand{\spanReference}[1]{#1}
 \newcommand{\spanOther}[1]{#1}
-\newcommand{\spantok}[1]{\frame{#1}}
+\newcommand{\spantok}[1]{\fbox{#1}}
 \newcommand{\spanPrompt}[1]{\textcolor{red}{\textbf{#1}}}
-\newcommand{\spanProgramOutput}[1]{\textcolor{gray}{\textbf{#1}}}
+\newcommand{\spanProgramOutput}[1]{\textcolor{darkgray}{\textbf{#1}}}
 \newcommand{\spanprogram}[1]{\textbf{\underline{#1}}}
-\newcommand{\spanoption}[1]{\textbf{#1}}
+\newcommand{\spanoption}[1]{\textbf{\textcolor{darkgray}{#1}}}
+
+\begin{document}
+\title{$title $version $subtitle}
+\author{$author}
+
+% Never allow text overflow to margin:
+\setlength\emergencystretch{\hsize}\hbadness=10000
+
+\maketitle
 
 $content
 \end{document}
diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim
index 81e3ba6d9..2489ce40c 100644
--- a/lib/packages/docutils/rstast.nim
+++ b/lib/packages/docutils/rstast.nim
@@ -349,8 +349,24 @@ proc renderRstToJson*(node: PRstNode): string =
   ##   }
   renderRstToJsonNode(node).pretty
 
+proc renderRstToText*(node: PRstNode): string =
+  ## minimal text representation of markup node
+  const code = {rnCodeFragment, rnInterpretedText, rnInlineLiteral, rnInlineCode}
+  if node == nil:
+    return ""
+  case node.kind
+  of rnLeaf, rnSmiley:
+    result.add node.text
+  else:
+    if node.kind in code: result.add "`"
+    for i in 0 ..< node.sons.len:
+      if node.kind in {rnInlineCode, rnCodeBlock} and i == 0:
+        continue  # omit language specifier
+      result.add renderRstToText(node.sons[i])
+    if node.kind in code: result.add "`"
+
 proc renderRstToStr*(node: PRstNode, indent=0): string =
-  ## Writes the parsed RST `node` into a compact string
+  ## Writes the parsed RST `node` into an AST tree with compact string
   ## representation in the format (one line per every sub-node):
   ## ``indent - kind - [text|level|order|adType] - anchor (if non-zero)``
   ## (suitable for debugging of RST parsing).
diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim
index 5ab6f28ee..e3f889283 100644
--- a/lib/packages/docutils/rstgen.nim
+++ b/lib/packages/docutils/rstgen.nim
@@ -60,6 +60,10 @@ type
   MetaEnum* = enum
     metaNone, metaTitle, metaSubtitle, metaAuthor, metaVersion
 
+  EscapeMode = enum  # in Latex text inside options [] and URLs is
+                     # escaped slightly differently than in normal text
+    emText, emOption, emUrl  # emText is currently used for code also
+
   RstGenerator* = object of RootObj
     target*: OutputTarget
     config*: StringTableRef
@@ -84,6 +88,7 @@ type
     id*: int               ## A counter useful for generating IDs.
     onTestSnippet*: proc (d: var RstGenerator; filename, cmd: string; status: int;
                           content: string)
+    escMode*: EscapeMode
 
   PDoc = var RstGenerator ## Alias to type less.
 
@@ -161,6 +166,7 @@ proc initRstGenerator*(g: var RstGenerator, target: OutputTarget,
   g.findFile = findFile
   g.currentSection = ""
   g.id = 0
+  g.escMode = emText
   let fileParts = filename.splitFile
   if fileParts.ext == ".nim":
     g.currentSection = "Module " & fileParts.name
@@ -189,28 +195,34 @@ proc addHtmlChar(dest: var string, c: char) =
   of '\"': add(dest, """)
   else: add(dest, c)
 
-proc addRtfChar(dest: var string, c: char) =
+proc addTexChar(dest: var string, c: char, escMode: EscapeMode) =
+  ## Escapes 10 special Latex characters and sometimes ` and [, ].
+  ## TODO: @ is always a normal symbol (besides the header), am I wrong?
+  ## All escapes that need to work in text and code blocks (`emText` mode)
+  ## should start from \ (to be compatible with fancyvrb/fvextra).
   case c
-  of '{': add(dest, "\\{")
-  of '}': add(dest, "\\}")
-  of '\\': add(dest, "\\\\")
-  else: add(dest, c)
-
-proc addTexChar(dest: var string, c: char) =
-  # Escapes 10 special Latex characters. Note that [, ], and ` are not
-  # considered as such. TODO: neither is @, am I wrong?
-  case c
-  of '_', '{', '}', '$', '&', '#', '%': add(dest, "\\" & c)
+  of '_', '$', '&', '#', '%': add(dest, "\\" & c)
   # \~ and \^ have a special meaning unless they are followed by {}
   of '~', '^': add(dest, "\\" & c & "{}")
+  # Latex loves to substitute ` to opening quote, even in texttt mode!
+  of '`': add(dest, "\\textasciigrave{}")
   # add {} to avoid gobbling up space by \textbackslash
   of '\\': add(dest, "\\textbackslash{}")
+  # Using { and } in URL in Latex: https://tex.stackexchange.com/a/469175
+  of '{':
+    add(dest, if escMode == emUrl: "\\%7B" else: "\\{")
+  of '}':
+    add(dest, if escMode == emUrl: "\\%7D" else: "\\}")
+  of ']':
+    # escape ] inside an optional argument in e.g. \section[static[T]]{..
+    add(dest, if escMode == emOption: "\\text{]}" else: "]")
   else: add(dest, c)
 
-proc escChar*(target: OutputTarget, dest: var string, c: char) {.inline.} =
+proc escChar*(target: OutputTarget, dest: var string,
+              c: char, escMode: EscapeMode) {.inline.} =
   case target
   of outHtml:  addHtmlChar(dest, c)
-  of outLatex: addTexChar(dest, c)
+  of outLatex: addTexChar(dest, c, escMode)
 
 proc addSplitter(target: OutputTarget; dest: var string) {.inline.} =
   case target
@@ -229,7 +241,7 @@ proc nextSplitPoint*(s: string, start: int): int =
     inc(result)
   dec(result)                 # last valid index
 
-proc esc*(target: OutputTarget, s: string, splitAfter = -1): string =
+proc esc*(target: OutputTarget, s: string, splitAfter = -1, escMode = emText): string =
   ## Escapes the HTML.
   result = ""
   if splitAfter >= 0:
@@ -240,11 +252,11 @@ proc esc*(target: OutputTarget, s: string, splitAfter = -1): string =
       #if (splitter != " ") or (partLen + k - j + 1 > splitAfter):
       partLen = 0
       addSplitter(target, result)
-      for i in countup(j, k): escChar(target, result, s[i])
+      for i in countup(j, k): escChar(target, result, s[i], escMode)
       inc(partLen, k - j + 1)
       j = k + 1
   else:
-    for i in countup(0, len(s) - 1): escChar(target, result, s[i])
+    for i in countup(0, len(s) - 1): escChar(target, result, s[i], escMode)
 
 
 proc disp(target: OutputTarget, xml, tex: string): string =
@@ -760,6 +772,8 @@ proc renderHeadline(d: PDoc, n: PRstNode, result: var string) =
       sectionPrefix = rstnodeToRefname(n2) & "-"
       break
   var refname = sectionPrefix & rstnodeToRefname(n)
+  var tocName = esc(d.target, renderRstToText(n), escMode = emOption)
+    # for Latex: simple text without commands that may break TOC/hyperref
   if d.hasToc:
     var length = len(d.tocPart)
     setLen(d.tocPart, length + 1)
@@ -768,13 +782,14 @@ proc renderHeadline(d: PDoc, n: PRstNode, result: var string) =
     d.tocPart[length].header = tmp
 
     dispA(d.target, result, "\n$3", "\\rsth$4{$3}$2\n",
-      [$n.level, refname.idS, tmp, $chr(n.level - 1 + ord('A')), refname])
+      "$2 href=\"#$5\">$3", "\\rsth$4[$6]{$3}$2\n",
+      [$n.level, refname.idS, tmp,
+       $chr(n.level - 1 + ord('A')), refname, tocName])
   else:
     dispA(d.target, result, "\n$3",
-                            "\\rsth$4{$3}$2\n", [
+                            "\\rsth$4[$5]{$3}$2\n", [
         $n.level, refname.idS, tmp,
-        $chr(n.level - 1 + ord('A'))])
+        $chr(n.level - 1 + ord('A')), tocName])
 
   # Generate index entry using spaces to indicate TOC level for the output HTML.
   assert n.level >= 0
@@ -802,9 +817,10 @@ proc renderOverline(d: PDoc, n: PRstNode, result: var string) =
     var tmp = ""
     for i in countup(0, len(n) - 1): renderRstToOut(d, n.sons[i], tmp)
     d.currentSection = tmp
+    var tocName = esc(d.target, renderRstToText(n), escMode=emOption)
     dispA(d.target, result, "
    $3
    ",
-                   "\\rstov$4{$3}$2\n", [$n.level,
-        rstnodeToRefname(n).idS, tmp, $chr(n.level - 1 + ord('A'))])
+                   "\\rstov$4[$5]{$3}$2\n", [$n.level,
+        rstnodeToRefname(n).idS, tmp, $chr(n.level - 1 + ord('A')), tocName])
 
 
 proc renderTocEntry(d: PDoc, e: TocEntry, result: var string) =
@@ -870,7 +886,7 @@ proc renderImage(d: PDoc, n: PRstNode, result: var string) =
     htmlOut = ""
 
   # support for `:target:` links for images:
-  var target = esc(d.target, getFieldValue(n, "target").strip())
+  var target = esc(d.target, getFieldValue(n, "target").strip(), escMode=emUrl)
   if target.len > 0:
     # `htmlOut` needs to be of the following format for link to work for images:
     # 
@@ -1031,16 +1047,16 @@ proc renderCode(d: PDoc, n: PRstNode, result: var string) =
       blockEnd = "    "
   of outLatex:
     if n.kind == rnCodeBlock:
-      blockStart = "\n\n\\begin{rstpre}" & n.anchor.idS & "\n"
-      blockEnd = "\n\\end{rstpre}\n"
+      blockStart = "\n\n" & n.anchor.idS & "\\begin{rstpre}\n"
+      blockEnd = "\n\\end{rstpre}\n\n"
     else:  # rnInlineCode
-      blockStart = "\\texttt{"
+      blockStart = "\\rstcode{"
       blockEnd = "}"
   dispA(d.target, result, blockStart, blockStart, [])
   if params.lang == langNone:
     if len(params.langStr) > 0:
       d.msgHandler(d.filename, 1, 0, mwUnsupportedLanguage, params.langStr)
-    for letter in m.text: escChar(d.target, result, letter)
+    for letter in m.text: escChar(d.target, result, letter, emText)
   else:
     renderCodeLang(result, params.lang, m.text, d.target)
   dispA(d.target, result, blockEnd, blockEnd)
@@ -1055,9 +1071,8 @@ proc renderContainer(d: PDoc, n: PRstNode, result: var string) =
     dispA(d.target, result, "
    $2
    ", "$2", [arg, tmp])
 
 proc texColumns(n: PRstNode): string =
-  result = ""
   let nColumns = if n.sons.len > 0: len(n.sons[0]) else: 1
-  for i in countup(1, nColumns): add(result, "|X")
+  result = "L".repeat(nColumns)
 
 proc renderField(d: PDoc, n: PRstNode, result: var string) =
   var b = false
@@ -1144,19 +1159,38 @@ proc renderAdmonition(d: PDoc, n: PRstNode, result: var string) =
   renderAux(d, n,
       htmlHead & "" & txt &
         ":\n" & "$1\n",
-      "\n\n\\begin{mdframed}[linecolor=" & texColor & "]$2\n" &
+      "\n\n\\begin{rstadmonition}[borderline west={0.2em}{0pt}{" &
+        texColor & "}]$2\n" &
         "{" & texSz & "\\color{" & texColor & "}{\\textbf{" & txt & ":}}} " &
-        "$1\n\\end{mdframed}\n",
+        "$1\n\\end{rstadmonition}\n",
       result)
 
+proc renderHyperlink(d: PDoc, text, link: PRstNode, result: var string, external: bool) =
+  var linkStr = ""
+  block:
+    let mode = d.escMode
+    d.escMode = emUrl
+    renderRstToOut(d, link, linkStr)
+    d.escMode = mode
+  var textStr = ""
+  renderRstToOut(d, text, textStr)
+  if external:
+    dispA(d.target, result,
+      "$1",
+      "\\href{$2}{$1}", [textStr, linkStr])
+  else:
+    dispA(d.target, result,
+      "$1",
+      "\\hyperlink{$2}{$1} (p.~\\pageref{$2})", [textStr, linkStr])
+
 proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) =
   if n == nil: return
   case n.kind
   of rnInner: renderAux(d, n, result)
   of rnHeadline, rnMarkdownHeadline: renderHeadline(d, n, result)
   of rnOverline: renderOverline(d, n, result)
-  of rnTransition: renderAux(d, n, "\n", "\\hrule$2\n", result)
-  of rnParagraph: renderAux(d, n, "$1
    \n", "$2\n$1\n\n", result)
+  of rnTransition: renderAux(d, n, "\n", "\n\n\\vspace{0.6em}\\hrule$2\n", result)
+  of rnParagraph: renderAux(d, n, "$1
    \n", "\n\n$2\n$1\n\n", result)
   of rnBulletList:
     renderAux(d, n, "$1\n",
                     "\\begin{itemize}\n$2\n$1\\end{itemize}\n", result)
@@ -1202,7 +1236,7 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) =
     renderAux(d, n,
         "
    " &
         "$1
    ",
-        "\\item[$1]", result)
+        "\\item[\\rstcodeitem{\\spanoption{$1}}]", result)
   of rnDescription:
     renderAux(d, n, "
    $1
    ",
         " $1\n", result)
@@ -1210,7 +1244,7 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) =
     doAssert false, "renderRstToOut"
   of rnLiteralBlock:
     renderAux(d, n, "$1
    \n",
-                    "\\begin{rstpre}\n$2\n$1\n\\end{rstpre}\n", result)
+                    "\n\n\\begin{rstpre}\n$2\n$1\n\\end{rstpre}\n\n", result)
   of rnQuotedLiteralBlock:
     doAssert false, "renderRstToOut"
   of rnLineBlock:
@@ -1237,8 +1271,8 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) =
   of rnTable, rnGridTable, rnMarkdownTable:
     renderAux(d, n,
       "$1",
-      "\\begin{table}\n$2\n\\begin{rsttab}{" &
-        texColumns(n) & "|}\n\\hline\n$1\\end{rsttab}\\end{table}", result)
+      "\n$2\n\\begin{rsttab}{" &
+        texColumns(n) & "}\n\\hline\n$1\\end{rsttab}", result)
   of rnTableRow:
     if len(n) >= 1:
       if d.target == outLatex:
@@ -1275,21 +1309,13 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) =
       "\\item[\\textsuperscript{[$3]}]$2 $1\n",
       [body, n.anchor.idS, mark, n.anchor])
   of rnRef:
-    var tmp = ""
-    renderAux(d, n, tmp)
-    dispA(d.target, result,
-      "$1",
-      "$1\\ref{$2}", [tmp, rstnodeToRefname(n)])
+    renderHyperlink(d, text=n.sons[0], link=n.sons[0], result, external=false)
   of rnStandaloneHyperlink:
-    renderAux(d, n,
-      "$1",
-      "\\href{$1}{$1}", result)
+    renderHyperlink(d, text=n.sons[0], link=n.sons[0], result, external=true)
   of rnInternalRef:
-    var tmp = ""
-    renderAux(d, n.sons[0], tmp)
-    dispA(d.target, result,
-      "$1",
-      "\\hyperlink{$2}{$1} (p.~\\pageref{$2})", [tmp, n.sons[1].text])
+    renderHyperlink(d, text=n.sons[0], link=n.sons[1], result, external=false)
+  of rnHyperlink:
+    renderHyperlink(d, text=n.sons[0], link=n.sons[1], result, external=true)
   of rnFootnoteRef:
     var tmp = "["
     renderAux(d, n.sons[0], tmp)
@@ -1299,14 +1325,6 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) =
           "$1",
       "\\textsuperscript{\\hyperlink{$2}{\\textbf{$1}}}",
       [tmp, n.sons[1].text])
-  of rnHyperlink:
-    var tmp0 = ""
-    var tmp1 = ""
-    renderRstToOut(d, n.sons[0], tmp0)
-    renderRstToOut(d, n.sons[1], tmp1)
-    dispA(d.target, result,
-      "$1",
-      "\\href{$2}{$1}", [tmp0, tmp1])
   of rnDirArg, rnRaw: renderAux(d, n, result)
   of rnRawHtml:
     if d.target != outLatex and not lastSon(n).isNil:
@@ -1334,7 +1352,7 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) =
       dispA(d.target, result,
             "" &
               "$1",
-            "\\texttt{\\span$2{$1}}", [tmp0, class])
+            "\\rstcode{\\span$2{$1}}", [tmp0, class])
     else:  # rnUnknownRole, not necessarily code/monospace font
       dispA(d.target, result, "$1", "\\span$2{$1}",
             [tmp0, class])
@@ -1351,7 +1369,7 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) =
   of rnInlineLiteral, rnInterpretedText:
     renderAux(d, n,
       "$1",
-      "\\texttt{$1}", result)
+      "\\rstcode{$1}", result)
   of rnInlineTarget:
     var tmp = ""
     renderAux(d, n, tmp)
@@ -1360,7 +1378,7 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) =
       "\\label{$2}\\hypertarget{$2}{$1}",
       [tmp, rstnodeToRefname(n)])
   of rnSmiley: renderSmiley(d, n, result)
-  of rnLeaf: result.add(esc(d.target, n.text))
+  of rnLeaf: result.add(esc(d.target, n.text, escMode=d.escMode))
   of rnContents: d.hasToc = true
   of rnDefaultRole: discard
   of rnTitle:
diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim
index 864728686..34b172935 100644
--- a/tests/stdlib/trstgen.nim
+++ b/tests/stdlib/trstgen.nim
@@ -259,7 +259,7 @@ A2     A3
 A4     A5
 ====   === """
     let output1 = rstToLatex(input1, {})
-    doAssert "{|X|X|}" in output1  # 2 columns
+    doAssert "{LL}" in output1  # 2 columns
     doAssert count(output1, "\\\\") == 4  # 4 rows
     for cell in ["H0", "H1", "A0", "A1", "A2", "A3", "A4", "A5"]:
       doAssert cell in output1
@@ -274,7 +274,7 @@ A0     A1   X
        Ax   Y
 ====   ===  = """
     let output2 = rstToLatex(input2, {})
-    doAssert "{|X|X|X|}" in output2  # 3 columns
+    doAssert "{LLL}" in output2  # 3 columns
     doAssert count(output2, "\\\\") == 2  # 2 rows
     for cell in ["H0", "H1", "H", "A0", "A1", "X", "Ax", "Y"]:
       doAssert cell in output2
diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim
index 621dc643f..f25564fad 100644
--- a/tools/kochdocs.nim
+++ b/tools/kochdocs.nim
@@ -284,22 +284,22 @@ proc buildDoc(nimArgs, destPath: string) =
 
 proc nim2pdf(src: string, dst: string, nimArgs: string) =
   # xxx expose as a `nim` command or in some other reusable way.
-  let outDir = "build" / "pdflatextmp" # xxx factor pending https://github.com/timotheecour/Nim/issues/616
+  let outDir = "build" / "xelatextmp" # xxx factor pending https://github.com/timotheecour/Nim/issues/616
   # note: this will generate temporary files in gitignored `outDir`: aux toc log out tex
   exec("$# rst2tex $# --outdir:$# $#" % [findNim().quoteShell(), nimArgs, outDir.quoteShell, src.quoteShell])
   let texFile = outDir / src.lastPathPart.changeFileExt("tex")
-  for i in 0..<2: # call LaTeX twice to get cross references right:
-    let pdflatexLog = outDir / "pdflatex.log"
+  for i in 0..<3: # call LaTeX three times to get cross references right:
+    let xelatexLog = outDir / "xelatex.log"
     # `>` should work on windows, if not, we can use `execCmdEx`
-    let cmd = "pdflatex -interaction=nonstopmode -output-directory=$# $# > $#" % [outDir.quoteShell, texFile.quoteShell, pdflatexLog.quoteShell]
-    exec(cmd) # on error, user can inspect `pdflatexLog`
+    let cmd = "xelatex -interaction=nonstopmode -output-directory=$# $# > $#" % [outDir.quoteShell, texFile.quoteShell, xelatexLog.quoteShell]
+    exec(cmd) # on error, user can inspect `xelatexLog`
   moveFile(texFile.changeFileExt("pdf"), dst)
 
 proc buildPdfDoc*(nimArgs, destPath: string) =
   var pdfList: seq[string]
   createDir(destPath)
-  if os.execShellCmd("pdflatex -version") != 0:
-    doAssert false, "pdflatex not found" # or, raise an exception
+  if os.execShellCmd("xelatex -version") != 0:
+    doAssert false, "xelatex not found" # or, raise an exception
   else:
     for src in items(rstPdfList):
       let dst = destPath / src.lastPathPart.changeFileExt("pdf")

From 63db2b19bf0787db89ce89c8861e976d44e33cbd Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Tue, 1 Jun 2021 22:29:53 +0200
Subject: [PATCH 0866/2472] use dragonbox algorithm; alternative to #18008
 (#18139)

* use dragonbox algorithm; alternative to #18008
* removed unsafe code
---
 changelog.md               |    4 +
 compiler/condsyms.nim      |    2 +
 lib/system/dragonbox.nim   | 1347 ++++++++++++++++++++++++++++++++++++
 lib/system/formatfloat.nim |   97 +--
 tests/float/tfloat6.nim    |    8 +-
 5 files changed, 1408 insertions(+), 50 deletions(-)
 create mode 100644 lib/system/dragonbox.nim

diff --git a/changelog.md b/changelog.md
index 900b23ad8..a23aedbdf 100644
--- a/changelog.md
+++ b/changelog.md
@@ -76,6 +76,10 @@
 - `json` and `jsonutils` now serialize NaN, Inf, -Inf as strings, so that
   `%[NaN, -Inf]` is the string `["nan","-inf"]` instead of `[nan,-inf]` which was invalid json.
 
+- `system.addFloat` now uses the "Dragonbox" algorithm, which ensures correct roundtrips of floating point
+  numbers, that the minimum length representation of a floating point number is used and correct rounding.
+  Use `-d:nimLegacyAddFloat` for a transition period.
+
 - `strformat` is now part of `include std/prelude`.
 
 - Deprecated `proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T]` in `std/algorithm`.
diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim
index 6a49584c8..ff0c603cb 100644
--- a/compiler/condsyms.nim
+++ b/compiler/condsyms.nim
@@ -134,3 +134,5 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimHasUnifiedTuple")
   defineSymbol("nimHasIterable")
   defineSymbol("nimHasTypeofVoid")
+
+  defineSymbol("nimHasDragonBox")
diff --git a/lib/system/dragonbox.nim b/lib/system/dragonbox.nim
new file mode 100644
index 000000000..db608c630
--- /dev/null
+++ b/lib/system/dragonbox.nim
@@ -0,0 +1,1347 @@
+##  Copyright 2020 Junekey Jeon
+##  Copyright 2020 Alexander Bolz
+##
+##  Distributed under the Boost Software License, Version 1.0.
+##   (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt)
+
+##  char* output_end = Dtoa(buffer, value);
+##
+##  Converts the given double-precision number into decimal form and stores the result in the given
+##  buffer.
+##
+##  The buffer must be large enough, i.e. >= DtoaMinBufferLength.
+##  The output format is similar to printf("%g").
+##  The output is _not_ null-terminted.
+##
+##  The output is optimal, i.e. the output string
+##   1. rounds back to the input number when read in (using round-to-nearest-even)
+##   2. is as short as possible,
+##   3. is as close to the input number as possible.
+##
+##  Note:
+##  This function may temporarily write up to DtoaMinBufferLength characters into the buffer.
+
+const
+  dtoaMinBufferLength*: cint = 64
+
+## --------------------------------------------------------------------------------------------------
+##  This file contains an implementation of Junekey Jeon's Dragonbox algorithm.
+##
+##  It is a simplified version of the reference implementation found here:
+##  https://github.com/jk-jeon/dragonbox
+##
+##  The reference implementation also works with single-precision floating-point numbers and
+##  has options to configure the rounding mode.
+## --------------------------------------------------------------------------------------------------
+
+template dragonbox_Assert*(x: untyped): untyped =
+  assert(x)
+
+## ==================================================================================================
+##
+## ==================================================================================================
+
+type
+  ValueType* = float
+  BitsType* = uint64
+
+type
+  Double* = object
+    bits*: BitsType
+
+const                         ##  = p   (includes the hidden bit)
+  significandSize*: int32 = 53
+
+const ##   static constexpr int32_t   MaxExponent     = 1024 - 1 - (SignificandSize - 1);
+     ##   static constexpr int32_t   MinExponent     = std::numeric_limits::min_exponent - 1 - (SignificandSize - 1);
+  exponentBias*: int32 = 1024 - 1 + (significandSize - 1)
+
+const
+  maxIeeeExponent*: BitsType = BitsType(2 * 1024 - 1)
+
+const                         ##  = 2^(p-1)
+  hiddenBit*: BitsType = BitsType(1) shl (significandSize - 1)
+
+const                         ##  = 2^(p-1) - 1
+  significandMask*: BitsType = hiddenBit - 1
+
+const
+  exponentMask*: BitsType = maxIeeeExponent shl (significandSize - 1)
+
+const
+  signMask*: BitsType = not (not BitsType(0) shr 1)
+
+proc constructDouble*(bits: BitsType): Double {.constructor.} =
+  result.bits = bits
+
+proc constructDouble*(value: ValueType): Double {.constructor.} =
+  result.bits = cast[typeof(result.bits)](value)
+
+proc physicalSignificand*(this: Double): BitsType {.noSideEffect.} =
+  return this.bits and significandMask
+
+proc physicalExponent*(this: Double): BitsType {.noSideEffect.} =
+  return (this.bits and exponentMask) shr (significandSize - 1)
+
+proc isFinite*(this: Double): bool {.noSideEffect.} =
+  return (this.bits and exponentMask) != exponentMask
+
+proc isInf*(this: Double): bool {.noSideEffect.} =
+  return (this.bits and exponentMask) == exponentMask and
+      (this.bits and significandMask) == 0
+
+proc isNaN*(this: Double): bool {.noSideEffect.} =
+  return (this.bits and exponentMask) == exponentMask and
+      (this.bits and significandMask) != 0
+
+proc isZero*(this: Double): bool {.noSideEffect.} =
+  return (this.bits and not signMask) == 0
+
+proc signBit*(this: Double): int {.noSideEffect.} =
+  return ord((this.bits and signMask) != 0)
+
+##  namespace
+## ==================================================================================================
+##
+## ==================================================================================================
+##  Returns floor(x / 2^n).
+##
+##  Technically, right-shift of negative integers is implementation defined...
+##  Should easily be optimized into SAR (or equivalent) instruction.
+
+proc floorDivPow2*(x: int32; n: int32): int32 {.inline.} =
+  return x shr n
+
+proc floorLog2Pow10*(e: int32): int32 {.inline.} =
+  dragonbox_Assert(e >= -1233)
+  dragonbox_Assert(e <= 1233)
+  return floorDivPow2(e * 1741647, 19)
+
+proc floorLog10Pow2*(e: int32): int32 {.inline.} =
+  dragonbox_Assert(e >= -1500)
+  dragonbox_Assert(e <= 1500)
+  return floorDivPow2(e * 1262611, 22)
+
+proc floorLog10ThreeQuartersPow2*(e: int32): int32 {.inline.} =
+  dragonbox_Assert(e >= -1500)
+  dragonbox_Assert(e <= 1500)
+  return floorDivPow2(e * 1262611 - 524031, 22)
+
+## ==================================================================================================
+##
+## ==================================================================================================
+
+type
+  uint64x2* {.bycopy.} = object
+    hi*: uint64
+    lo*: uint64
+
+
+proc computePow10*(k: int32): uint64x2 {.inline.} =
+  const
+    kMin: int32 = -292
+  const
+    kMax: int32 = 326
+  const
+    pow10: array[kMax - kMin + 1, uint64x2] = [
+      uint64x2(hi: 0xFF77B1FCBEBCDC4F'u, lo: 0x25E8E89C13BB0F7B'u),
+      uint64x2(hi: 0x9FAACF3DF73609B1'u, lo: 0x77B191618C54E9AD'u),
+      uint64x2(hi: 0xC795830D75038C1D'u, lo: 0xD59DF5B9EF6A2418'u),
+      uint64x2(hi: 0xF97AE3D0D2446F25'u, lo: 0x4B0573286B44AD1E'u),
+      uint64x2(hi: 0x9BECCE62836AC577'u, lo: 0x4EE367F9430AEC33'u),
+      uint64x2(hi: 0xC2E801FB244576D5'u, lo: 0x229C41F793CDA740'u),
+      uint64x2(hi: 0xF3A20279ED56D48A'u, lo: 0x6B43527578C11110'u),
+      uint64x2(hi: 0x9845418C345644D6'u, lo: 0x830A13896B78AAAA'u),
+      uint64x2(hi: 0xBE5691EF416BD60C'u, lo: 0x23CC986BC656D554'u),
+      uint64x2(hi: 0xEDEC366B11C6CB8F'u, lo: 0x2CBFBE86B7EC8AA9'u),
+      uint64x2(hi: 0x94B3A202EB1C3F39'u, lo: 0x7BF7D71432F3D6AA'u),
+      uint64x2(hi: 0xB9E08A83A5E34F07'u, lo: 0xDAF5CCD93FB0CC54'u),
+      uint64x2(hi: 0xE858AD248F5C22C9'u, lo: 0xD1B3400F8F9CFF69'u),
+      uint64x2(hi: 0x91376C36D99995BE'u, lo: 0x23100809B9C21FA2'u),
+      uint64x2(hi: 0xB58547448FFFFB2D'u, lo: 0xABD40A0C2832A78B'u),
+      uint64x2(hi: 0xE2E69915B3FFF9F9'u, lo: 0x16C90C8F323F516D'u),
+      uint64x2(hi: 0x8DD01FAD907FFC3B'u, lo: 0xAE3DA7D97F6792E4'u),
+      uint64x2(hi: 0xB1442798F49FFB4A'u, lo: 0x99CD11CFDF41779D'u),
+      uint64x2(hi: 0xDD95317F31C7FA1D'u, lo: 0x40405643D711D584'u),
+      uint64x2(hi: 0x8A7D3EEF7F1CFC52'u, lo: 0x482835EA666B2573'u),
+      uint64x2(hi: 0xAD1C8EAB5EE43B66'u, lo: 0xDA3243650005EED0'u),
+      uint64x2(hi: 0xD863B256369D4A40'u, lo: 0x90BED43E40076A83'u),
+      uint64x2(hi: 0x873E4F75E2224E68'u, lo: 0x5A7744A6E804A292'u),
+      uint64x2(hi: 0xA90DE3535AAAE202'u, lo: 0x711515D0A205CB37'u),
+      uint64x2(hi: 0xD3515C2831559A83'u, lo: 0x0D5A5B44CA873E04'u),
+      uint64x2(hi: 0x8412D9991ED58091'u, lo: 0xE858790AFE9486C3'u),
+      uint64x2(hi: 0xA5178FFF668AE0B6'u, lo: 0x626E974DBE39A873'u),
+      uint64x2(hi: 0xCE5D73FF402D98E3'u, lo: 0xFB0A3D212DC81290'u),
+      uint64x2(hi: 0x80FA687F881C7F8E'u, lo: 0x7CE66634BC9D0B9A'u),
+      uint64x2(hi: 0xA139029F6A239F72'u, lo: 0x1C1FFFC1EBC44E81'u),
+      uint64x2(hi: 0xC987434744AC874E'u, lo: 0xA327FFB266B56221'u),
+      uint64x2(hi: 0xFBE9141915D7A922'u, lo: 0x4BF1FF9F0062BAA9'u),
+      uint64x2(hi: 0x9D71AC8FADA6C9B5'u, lo: 0x6F773FC3603DB4AA'u),
+      uint64x2(hi: 0xC4CE17B399107C22'u, lo: 0xCB550FB4384D21D4'u),
+      uint64x2(hi: 0xF6019DA07F549B2B'u, lo: 0x7E2A53A146606A49'u),
+      uint64x2(hi: 0x99C102844F94E0FB'u, lo: 0x2EDA7444CBFC426E'u),
+      uint64x2(hi: 0xC0314325637A1939'u, lo: 0xFA911155FEFB5309'u),
+      uint64x2(hi: 0xF03D93EEBC589F88'u, lo: 0x793555AB7EBA27CB'u),
+      uint64x2(hi: 0x96267C7535B763B5'u, lo: 0x4BC1558B2F3458DF'u),
+      uint64x2(hi: 0xBBB01B9283253CA2'u, lo: 0x9EB1AAEDFB016F17'u),
+      uint64x2(hi: 0xEA9C227723EE8BCB'u, lo: 0x465E15A979C1CADD'u),
+      uint64x2(hi: 0x92A1958A7675175F'u, lo: 0x0BFACD89EC191ECA'u),
+      uint64x2(hi: 0xB749FAED14125D36'u, lo: 0xCEF980EC671F667C'u),
+      uint64x2(hi: 0xE51C79A85916F484'u, lo: 0x82B7E12780E7401B'u),
+      uint64x2(hi: 0x8F31CC0937AE58D2'u, lo: 0xD1B2ECB8B0908811'u),
+      uint64x2(hi: 0xB2FE3F0B8599EF07'u, lo: 0x861FA7E6DCB4AA16'u),
+      uint64x2(hi: 0xDFBDCECE67006AC9'u, lo: 0x67A791E093E1D49B'u),
+      uint64x2(hi: 0x8BD6A141006042BD'u, lo: 0xE0C8BB2C5C6D24E1'u),
+      uint64x2(hi: 0xAECC49914078536D'u, lo: 0x58FAE9F773886E19'u),
+      uint64x2(hi: 0xDA7F5BF590966848'u, lo: 0xAF39A475506A899F'u),
+      uint64x2(hi: 0x888F99797A5E012D'u, lo: 0x6D8406C952429604'u),
+      uint64x2(hi: 0xAAB37FD7D8F58178'u, lo: 0xC8E5087BA6D33B84'u),
+      uint64x2(hi: 0xD5605FCDCF32E1D6'u, lo: 0xFB1E4A9A90880A65'u),
+      uint64x2(hi: 0x855C3BE0A17FCD26'u, lo: 0x5CF2EEA09A550680'u),
+      uint64x2(hi: 0xA6B34AD8C9DFC06F'u, lo: 0xF42FAA48C0EA481F'u),
+      uint64x2(hi: 0xD0601D8EFC57B08B'u, lo: 0xF13B94DAF124DA27'u),
+      uint64x2(hi: 0x823C12795DB6CE57'u, lo: 0x76C53D08D6B70859'u),
+      uint64x2(hi: 0xA2CB1717B52481ED'u, lo: 0x54768C4B0C64CA6F'u),
+      uint64x2(hi: 0xCB7DDCDDA26DA268'u, lo: 0xA9942F5DCF7DFD0A'u),
+      uint64x2(hi: 0xFE5D54150B090B02'u, lo: 0xD3F93B35435D7C4D'u),
+      uint64x2(hi: 0x9EFA548D26E5A6E1'u, lo: 0xC47BC5014A1A6DB0'u),
+      uint64x2(hi: 0xC6B8E9B0709F109A'u, lo: 0x359AB6419CA1091C'u),
+      uint64x2(hi: 0xF867241C8CC6D4C0'u, lo: 0xC30163D203C94B63'u),
+      uint64x2(hi: 0x9B407691D7FC44F8'u, lo: 0x79E0DE63425DCF1E'u),
+      uint64x2(hi: 0xC21094364DFB5636'u, lo: 0x985915FC12F542E5'u),
+      uint64x2(hi: 0xF294B943E17A2BC4'u, lo: 0x3E6F5B7B17B2939E'u),
+      uint64x2(hi: 0x979CF3CA6CEC5B5A'u, lo: 0xA705992CEECF9C43'u),
+      uint64x2(hi: 0xBD8430BD08277231'u, lo: 0x50C6FF782A838354'u),
+      uint64x2(hi: 0xECE53CEC4A314EBD'u, lo: 0xA4F8BF5635246429'u),
+      uint64x2(hi: 0x940F4613AE5ED136'u, lo: 0x871B7795E136BE9A'u),
+      uint64x2(hi: 0xB913179899F68584'u, lo: 0x28E2557B59846E40'u),
+      uint64x2(hi: 0xE757DD7EC07426E5'u, lo: 0x331AEADA2FE589D0'u),
+      uint64x2(hi: 0x9096EA6F3848984F'u, lo: 0x3FF0D2C85DEF7622'u),
+      uint64x2(hi: 0xB4BCA50B065ABE63'u, lo: 0x0FED077A756B53AA'u),
+      uint64x2(hi: 0xE1EBCE4DC7F16DFB'u, lo: 0xD3E8495912C62895'u),
+      uint64x2(hi: 0x8D3360F09CF6E4BD'u, lo: 0x64712DD7ABBBD95D'u),
+      uint64x2(hi: 0xB080392CC4349DEC'u, lo: 0xBD8D794D96AACFB4'u),
+      uint64x2(hi: 0xDCA04777F541C567'u, lo: 0xECF0D7A0FC5583A1'u),
+      uint64x2(hi: 0x89E42CAAF9491B60'u, lo: 0xF41686C49DB57245'u),
+      uint64x2(hi: 0xAC5D37D5B79B6239'u, lo: 0x311C2875C522CED6'u),
+      uint64x2(hi: 0xD77485CB25823AC7'u, lo: 0x7D633293366B828C'u),
+      uint64x2(hi: 0x86A8D39EF77164BC'u, lo: 0xAE5DFF9C02033198'u),
+      uint64x2(hi: 0xA8530886B54DBDEB'u, lo: 0xD9F57F830283FDFD'u),
+      uint64x2(hi: 0xD267CAA862A12D66'u, lo: 0xD072DF63C324FD7C'u),
+      uint64x2(hi: 0x8380DEA93DA4BC60'u, lo: 0x4247CB9E59F71E6E'u),
+      uint64x2(hi: 0xA46116538D0DEB78'u, lo: 0x52D9BE85F074E609'u),
+      uint64x2(hi: 0xCD795BE870516656'u, lo: 0x67902E276C921F8C'u),
+      uint64x2(hi: 0x806BD9714632DFF6'u, lo: 0x00BA1CD8A3DB53B7'u),
+      uint64x2(hi: 0xA086CFCD97BF97F3'u, lo: 0x80E8A40ECCD228A5'u),
+      uint64x2(hi: 0xC8A883C0FDAF7DF0'u, lo: 0x6122CD128006B2CE'u),
+      uint64x2(hi: 0xFAD2A4B13D1B5D6C'u, lo: 0x796B805720085F82'u),
+      uint64x2(hi: 0x9CC3A6EEC6311A63'u, lo: 0xCBE3303674053BB1'u),
+      uint64x2(hi: 0xC3F490AA77BD60FC'u, lo: 0xBEDBFC4411068A9D'u),
+      uint64x2(hi: 0xF4F1B4D515ACB93B'u, lo: 0xEE92FB5515482D45'u),
+      uint64x2(hi: 0x991711052D8BF3C5'u, lo: 0x751BDD152D4D1C4B'u),
+      uint64x2(hi: 0xBF5CD54678EEF0B6'u, lo: 0xD262D45A78A0635E'u),
+      uint64x2(hi: 0xEF340A98172AACE4'u, lo: 0x86FB897116C87C35'u),
+      uint64x2(hi: 0x9580869F0E7AAC0E'u, lo: 0xD45D35E6AE3D4DA1'u),
+      uint64x2(hi: 0xBAE0A846D2195712'u, lo: 0x8974836059CCA10A'u),
+      uint64x2(hi: 0xE998D258869FACD7'u, lo: 0x2BD1A438703FC94C'u),
+      uint64x2(hi: 0x91FF83775423CC06'u, lo: 0x7B6306A34627DDD0'u),
+      uint64x2(hi: 0xB67F6455292CBF08'u, lo: 0x1A3BC84C17B1D543'u),
+      uint64x2(hi: 0xE41F3D6A7377EECA'u, lo: 0x20CABA5F1D9E4A94'u),
+      uint64x2(hi: 0x8E938662882AF53E'u, lo: 0x547EB47B7282EE9D'u),
+      uint64x2(hi: 0xB23867FB2A35B28D'u, lo: 0xE99E619A4F23AA44'u),
+      uint64x2(hi: 0xDEC681F9F4C31F31'u, lo: 0x6405FA00E2EC94D5'u),
+      uint64x2(hi: 0x8B3C113C38F9F37E'u, lo: 0xDE83BC408DD3DD05'u),
+      uint64x2(hi: 0xAE0B158B4738705E'u, lo: 0x9624AB50B148D446'u),
+      uint64x2(hi: 0xD98DDAEE19068C76'u, lo: 0x3BADD624DD9B0958'u),
+      uint64x2(hi: 0x87F8A8D4CFA417C9'u, lo: 0xE54CA5D70A80E5D7'u),
+      uint64x2(hi: 0xA9F6D30A038D1DBC'u, lo: 0x5E9FCF4CCD211F4D'u),
+      uint64x2(hi: 0xD47487CC8470652B'u, lo: 0x7647C32000696720'u),
+      uint64x2(hi: 0x84C8D4DFD2C63F3B'u, lo: 0x29ECD9F40041E074'u),
+      uint64x2(hi: 0xA5FB0A17C777CF09'u, lo: 0xF468107100525891'u),
+      uint64x2(hi: 0xCF79CC9DB955C2CC'u, lo: 0x7182148D4066EEB5'u),
+      uint64x2(hi: 0x81AC1FE293D599BF'u, lo: 0xC6F14CD848405531'u),
+      uint64x2(hi: 0xA21727DB38CB002F'u, lo: 0xB8ADA00E5A506A7D'u),
+      uint64x2(hi: 0xCA9CF1D206FDC03B'u, lo: 0xA6D90811F0E4851D'u),
+      uint64x2(hi: 0xFD442E4688BD304A'u, lo: 0x908F4A166D1DA664'u),
+      uint64x2(hi: 0x9E4A9CEC15763E2E'u, lo: 0x9A598E4E043287FF'u),
+      uint64x2(hi: 0xC5DD44271AD3CDBA'u, lo: 0x40EFF1E1853F29FE'u),
+      uint64x2(hi: 0xF7549530E188C128'u, lo: 0xD12BEE59E68EF47D'u),
+      uint64x2(hi: 0x9A94DD3E8CF578B9'u, lo: 0x82BB74F8301958CF'u),
+      uint64x2(hi: 0xC13A148E3032D6E7'u, lo: 0xE36A52363C1FAF02'u),
+      uint64x2(hi: 0xF18899B1BC3F8CA1'u, lo: 0xDC44E6C3CB279AC2'u),
+      uint64x2(hi: 0x96F5600F15A7B7E5'u, lo: 0x29AB103A5EF8C0BA'u),
+      uint64x2(hi: 0xBCB2B812DB11A5DE'u, lo: 0x7415D448F6B6F0E8'u),
+      uint64x2(hi: 0xEBDF661791D60F56'u, lo: 0x111B495B3464AD22'u),
+      uint64x2(hi: 0x936B9FCEBB25C995'u, lo: 0xCAB10DD900BEEC35'u),
+      uint64x2(hi: 0xB84687C269EF3BFB'u, lo: 0x3D5D514F40EEA743'u),
+      uint64x2(hi: 0xE65829B3046B0AFA'u, lo: 0x0CB4A5A3112A5113'u),
+      uint64x2(hi: 0x8FF71A0FE2C2E6DC'u, lo: 0x47F0E785EABA72AC'u),
+      uint64x2(hi: 0xB3F4E093DB73A093'u, lo: 0x59ED216765690F57'u),
+      uint64x2(hi: 0xE0F218B8D25088B8'u, lo: 0x306869C13EC3532D'u),
+      uint64x2(hi: 0x8C974F7383725573'u, lo: 0x1E414218C73A13FC'u),
+      uint64x2(hi: 0xAFBD2350644EEACF'u, lo: 0xE5D1929EF90898FB'u),
+      uint64x2(hi: 0xDBAC6C247D62A583'u, lo: 0xDF45F746B74ABF3A'u),
+      uint64x2(hi: 0x894BC396CE5DA772'u, lo: 0x6B8BBA8C328EB784'u),
+      uint64x2(hi: 0xAB9EB47C81F5114F'u, lo: 0x066EA92F3F326565'u),
+      uint64x2(hi: 0xD686619BA27255A2'u, lo: 0xC80A537B0EFEFEBE'u),
+      uint64x2(hi: 0x8613FD0145877585'u, lo: 0xBD06742CE95F5F37'u),
+      uint64x2(hi: 0xA798FC4196E952E7'u, lo: 0x2C48113823B73705'u),
+      uint64x2(hi: 0xD17F3B51FCA3A7A0'u, lo: 0xF75A15862CA504C6'u),
+      uint64x2(hi: 0x82EF85133DE648C4'u, lo: 0x9A984D73DBE722FC'u),
+      uint64x2(hi: 0xA3AB66580D5FDAF5'u, lo: 0xC13E60D0D2E0EBBB'u),
+      uint64x2(hi: 0xCC963FEE10B7D1B3'u, lo: 0x318DF905079926A9'u),
+      uint64x2(hi: 0xFFBBCFE994E5C61F'u, lo: 0xFDF17746497F7053'u),
+      uint64x2(hi: 0x9FD561F1FD0F9BD3'u, lo: 0xFEB6EA8BEDEFA634'u),
+      uint64x2(hi: 0xC7CABA6E7C5382C8'u, lo: 0xFE64A52EE96B8FC1'u),
+      uint64x2(hi: 0xF9BD690A1B68637B'u, lo: 0x3DFDCE7AA3C673B1'u),
+      uint64x2(hi: 0x9C1661A651213E2D'u, lo: 0x06BEA10CA65C084F'u),
+      uint64x2(hi: 0xC31BFA0FE5698DB8'u, lo: 0x486E494FCFF30A63'u),
+      uint64x2(hi: 0xF3E2F893DEC3F126'u, lo: 0x5A89DBA3C3EFCCFB'u),
+      uint64x2(hi: 0x986DDB5C6B3A76B7'u, lo: 0xF89629465A75E01D'u),
+      uint64x2(hi: 0xBE89523386091465'u, lo: 0xF6BBB397F1135824'u),
+      uint64x2(hi: 0xEE2BA6C0678B597F'u, lo: 0x746AA07DED582E2D'u),
+      uint64x2(hi: 0x94DB483840B717EF'u, lo: 0xA8C2A44EB4571CDD'u),
+      uint64x2(hi: 0xBA121A4650E4DDEB'u, lo: 0x92F34D62616CE414'u),
+      uint64x2(hi: 0xE896A0D7E51E1566'u, lo: 0x77B020BAF9C81D18'u),
+      uint64x2(hi: 0x915E2486EF32CD60'u, lo: 0x0ACE1474DC1D122F'u),
+      uint64x2(hi: 0xB5B5ADA8AAFF80B8'u, lo: 0x0D819992132456BB'u),
+      uint64x2(hi: 0xE3231912D5BF60E6'u, lo: 0x10E1FFF697ED6C6A'u),
+      uint64x2(hi: 0x8DF5EFABC5979C8F'u, lo: 0xCA8D3FFA1EF463C2'u),
+      uint64x2(hi: 0xB1736B96B6FD83B3'u, lo: 0xBD308FF8A6B17CB3'u),
+      uint64x2(hi: 0xDDD0467C64BCE4A0'u, lo: 0xAC7CB3F6D05DDBDF'u),
+      uint64x2(hi: 0x8AA22C0DBEF60EE4'u, lo: 0x6BCDF07A423AA96C'u),
+      uint64x2(hi: 0xAD4AB7112EB3929D'u, lo: 0x86C16C98D2C953C7'u),
+      uint64x2(hi: 0xD89D64D57A607744'u, lo: 0xE871C7BF077BA8B8'u),
+      uint64x2(hi: 0x87625F056C7C4A8B'u, lo: 0x11471CD764AD4973'u),
+      uint64x2(hi: 0xA93AF6C6C79B5D2D'u, lo: 0xD598E40D3DD89BD0'u),
+      uint64x2(hi: 0xD389B47879823479'u, lo: 0x4AFF1D108D4EC2C4'u),
+      uint64x2(hi: 0x843610CB4BF160CB'u, lo: 0xCEDF722A585139BB'u),
+      uint64x2(hi: 0xA54394FE1EEDB8FE'u, lo: 0xC2974EB4EE658829'u),
+      uint64x2(hi: 0xCE947A3DA6A9273E'u, lo: 0x733D226229FEEA33'u),
+      uint64x2(hi: 0x811CCC668829B887'u, lo: 0x0806357D5A3F5260'u),
+      uint64x2(hi: 0xA163FF802A3426A8'u, lo: 0xCA07C2DCB0CF26F8'u),
+      uint64x2(hi: 0xC9BCFF6034C13052'u, lo: 0xFC89B393DD02F0B6'u),
+      uint64x2(hi: 0xFC2C3F3841F17C67'u, lo: 0xBBAC2078D443ACE3'u),
+      uint64x2(hi: 0x9D9BA7832936EDC0'u, lo: 0xD54B944B84AA4C0E'u),
+      uint64x2(hi: 0xC5029163F384A931'u, lo: 0x0A9E795E65D4DF12'u),
+      uint64x2(hi: 0xF64335BCF065D37D'u, lo: 0x4D4617B5FF4A16D6'u),
+      uint64x2(hi: 0x99EA0196163FA42E'u, lo: 0x504BCED1BF8E4E46'u),
+      uint64x2(hi: 0xC06481FB9BCF8D39'u, lo: 0xE45EC2862F71E1D7'u),
+      uint64x2(hi: 0xF07DA27A82C37088'u, lo: 0x5D767327BB4E5A4D'u),
+      uint64x2(hi: 0x964E858C91BA2655'u, lo: 0x3A6A07F8D510F870'u),
+      uint64x2(hi: 0xBBE226EFB628AFEA'u, lo: 0x890489F70A55368C'u),
+      uint64x2(hi: 0xEADAB0ABA3B2DBE5'u, lo: 0x2B45AC74CCEA842F'u),
+      uint64x2(hi: 0x92C8AE6B464FC96F'u, lo: 0x3B0B8BC90012929E'u),
+      uint64x2(hi: 0xB77ADA0617E3BBCB'u, lo: 0x09CE6EBB40173745'u),
+      uint64x2(hi: 0xE55990879DDCAABD'u, lo: 0xCC420A6A101D0516'u),
+      uint64x2(hi: 0x8F57FA54C2A9EAB6'u, lo: 0x9FA946824A12232E'u),
+      uint64x2(hi: 0xB32DF8E9F3546564'u, lo: 0x47939822DC96ABFA'u),
+      uint64x2(hi: 0xDFF9772470297EBD'u, lo: 0x59787E2B93BC56F8'u),
+      uint64x2(hi: 0x8BFBEA76C619EF36'u, lo: 0x57EB4EDB3C55B65B'u),
+      uint64x2(hi: 0xAEFAE51477A06B03'u, lo: 0xEDE622920B6B23F2'u),
+      uint64x2(hi: 0xDAB99E59958885C4'u, lo: 0xE95FAB368E45ECEE'u),
+      uint64x2(hi: 0x88B402F7FD75539B'u, lo: 0x11DBCB0218EBB415'u),
+      uint64x2(hi: 0xAAE103B5FCD2A881'u, lo: 0xD652BDC29F26A11A'u),
+      uint64x2(hi: 0xD59944A37C0752A2'u, lo: 0x4BE76D3346F04960'u),
+      uint64x2(hi: 0x857FCAE62D8493A5'u, lo: 0x6F70A4400C562DDC'u),
+      uint64x2(hi: 0xA6DFBD9FB8E5B88E'u, lo: 0xCB4CCD500F6BB953'u),
+      uint64x2(hi: 0xD097AD07A71F26B2'u, lo: 0x7E2000A41346A7A8'u),
+      uint64x2(hi: 0x825ECC24C873782F'u, lo: 0x8ED400668C0C28C9'u),
+      uint64x2(hi: 0xA2F67F2DFA90563B'u, lo: 0x728900802F0F32FB'u),
+      uint64x2(hi: 0xCBB41EF979346BCA'u, lo: 0x4F2B40A03AD2FFBA'u),
+      uint64x2(hi: 0xFEA126B7D78186BC'u, lo: 0xE2F610C84987BFA9'u),
+      uint64x2(hi: 0x9F24B832E6B0F436'u, lo: 0x0DD9CA7D2DF4D7CA'u),
+      uint64x2(hi: 0xC6EDE63FA05D3143'u, lo: 0x91503D1C79720DBC'u),
+      uint64x2(hi: 0xF8A95FCF88747D94'u, lo: 0x75A44C6397CE912B'u),
+      uint64x2(hi: 0x9B69DBE1B548CE7C'u, lo: 0xC986AFBE3EE11ABB'u),
+      uint64x2(hi: 0xC24452DA229B021B'u, lo: 0xFBE85BADCE996169'u),
+      uint64x2(hi: 0xF2D56790AB41C2A2'u, lo: 0xFAE27299423FB9C4'u),
+      uint64x2(hi: 0x97C560BA6B0919A5'u, lo: 0xDCCD879FC967D41B'u),
+      uint64x2(hi: 0xBDB6B8E905CB600F'u, lo: 0x5400E987BBC1C921'u),
+      uint64x2(hi: 0xED246723473E3813'u, lo: 0x290123E9AAB23B69'u),
+      uint64x2(hi: 0x9436C0760C86E30B'u, lo: 0xF9A0B6720AAF6522'u),
+      uint64x2(hi: 0xB94470938FA89BCE'u, lo: 0xF808E40E8D5B3E6A'u),
+      uint64x2(hi: 0xE7958CB87392C2C2'u, lo: 0xB60B1D1230B20E05'u),
+      uint64x2(hi: 0x90BD77F3483BB9B9'u, lo: 0xB1C6F22B5E6F48C3'u),
+      uint64x2(hi: 0xB4ECD5F01A4AA828'u, lo: 0x1E38AEB6360B1AF4'u),
+      uint64x2(hi: 0xE2280B6C20DD5232'u, lo: 0x25C6DA63C38DE1B1'u),
+      uint64x2(hi: 0x8D590723948A535F'u, lo: 0x579C487E5A38AD0F'u),
+      uint64x2(hi: 0xB0AF48EC79ACE837'u, lo: 0x2D835A9DF0C6D852'u),
+      uint64x2(hi: 0xDCDB1B2798182244'u, lo: 0xF8E431456CF88E66'u),
+      uint64x2(hi: 0x8A08F0F8BF0F156B'u, lo: 0x1B8E9ECB641B5900'u),
+      uint64x2(hi: 0xAC8B2D36EED2DAC5'u, lo: 0xE272467E3D222F40'u),
+      uint64x2(hi: 0xD7ADF884AA879177'u, lo: 0x5B0ED81DCC6ABB10'u),
+      uint64x2(hi: 0x86CCBB52EA94BAEA'u, lo: 0x98E947129FC2B4EA'u),
+      uint64x2(hi: 0xA87FEA27A539E9A5'u, lo: 0x3F2398D747B36225'u),
+      uint64x2(hi: 0xD29FE4B18E88640E'u, lo: 0x8EEC7F0D19A03AAE'u),
+      uint64x2(hi: 0x83A3EEEEF9153E89'u, lo: 0x1953CF68300424AD'u),
+      uint64x2(hi: 0xA48CEAAAB75A8E2B'u, lo: 0x5FA8C3423C052DD8'u),
+      uint64x2(hi: 0xCDB02555653131B6'u, lo: 0x3792F412CB06794E'u),
+      uint64x2(hi: 0x808E17555F3EBF11'u, lo: 0xE2BBD88BBEE40BD1'u),
+      uint64x2(hi: 0xA0B19D2AB70E6ED6'u, lo: 0x5B6ACEAEAE9D0EC5'u),
+      uint64x2(hi: 0xC8DE047564D20A8B'u, lo: 0xF245825A5A445276'u),
+      uint64x2(hi: 0xFB158592BE068D2E'u, lo: 0xEED6E2F0F0D56713'u),
+      uint64x2(hi: 0x9CED737BB6C4183D'u, lo: 0x55464DD69685606C'u),
+      uint64x2(hi: 0xC428D05AA4751E4C'u, lo: 0xAA97E14C3C26B887'u),
+      uint64x2(hi: 0xF53304714D9265DF'u, lo: 0xD53DD99F4B3066A9'u),
+      uint64x2(hi: 0x993FE2C6D07B7FAB'u, lo: 0xE546A8038EFE402A'u),
+      uint64x2(hi: 0xBF8FDB78849A5F96'u, lo: 0xDE98520472BDD034'u),
+      uint64x2(hi: 0xEF73D256A5C0F77C'u, lo: 0x963E66858F6D4441'u),
+      uint64x2(hi: 0x95A8637627989AAD'u, lo: 0xDDE7001379A44AA9'u),
+      uint64x2(hi: 0xBB127C53B17EC159'u, lo: 0x5560C018580D5D53'u),
+      uint64x2(hi: 0xE9D71B689DDE71AF'u, lo: 0xAAB8F01E6E10B4A7'u),
+      uint64x2(hi: 0x9226712162AB070D'u, lo: 0xCAB3961304CA70E9'u),
+      uint64x2(hi: 0xB6B00D69BB55C8D1'u, lo: 0x3D607B97C5FD0D23'u),
+      uint64x2(hi: 0xE45C10C42A2B3B05'u, lo: 0x8CB89A7DB77C506B'u),
+      uint64x2(hi: 0x8EB98A7A9A5B04E3'u, lo: 0x77F3608E92ADB243'u),
+      uint64x2(hi: 0xB267ED1940F1C61C'u, lo: 0x55F038B237591ED4'u),
+      uint64x2(hi: 0xDF01E85F912E37A3'u, lo: 0x6B6C46DEC52F6689'u),
+      uint64x2(hi: 0x8B61313BBABCE2C6'u, lo: 0x2323AC4B3B3DA016'u),
+      uint64x2(hi: 0xAE397D8AA96C1B77'u, lo: 0xABEC975E0A0D081B'u),
+      uint64x2(hi: 0xD9C7DCED53C72255'u, lo: 0x96E7BD358C904A22'u),
+      uint64x2(hi: 0x881CEA14545C7575'u, lo: 0x7E50D64177DA2E55'u),
+      uint64x2(hi: 0xAA242499697392D2'u, lo: 0xDDE50BD1D5D0B9EA'u),
+      uint64x2(hi: 0xD4AD2DBFC3D07787'u, lo: 0x955E4EC64B44E865'u),
+      uint64x2(hi: 0x84EC3C97DA624AB4'u, lo: 0xBD5AF13BEF0B113F'u),
+      uint64x2(hi: 0xA6274BBDD0FADD61'u, lo: 0xECB1AD8AEACDD58F'u),
+      uint64x2(hi: 0xCFB11EAD453994BA'u, lo: 0x67DE18EDA5814AF3'u),
+      uint64x2(hi: 0x81CEB32C4B43FCF4'u, lo: 0x80EACF948770CED8'u),
+      uint64x2(hi: 0xA2425FF75E14FC31'u, lo: 0xA1258379A94D028E'u),
+      uint64x2(hi: 0xCAD2F7F5359A3B3E'u, lo: 0x096EE45813A04331'u),
+      uint64x2(hi: 0xFD87B5F28300CA0D'u, lo: 0x8BCA9D6E188853FD'u),
+      uint64x2(hi: 0x9E74D1B791E07E48'u, lo: 0x775EA264CF55347E'u),
+      uint64x2(hi: 0xC612062576589DDA'u, lo: 0x95364AFE032A819E'u),
+      uint64x2(hi: 0xF79687AED3EEC551'u, lo: 0x3A83DDBD83F52205'u),
+      uint64x2(hi: 0x9ABE14CD44753B52'u, lo: 0xC4926A9672793543'u),
+      uint64x2(hi: 0xC16D9A0095928A27'u, lo: 0x75B7053C0F178294'u),
+      uint64x2(hi: 0xF1C90080BAF72CB1'u, lo: 0x5324C68B12DD6339'u),
+      uint64x2(hi: 0x971DA05074DA7BEE'u, lo: 0xD3F6FC16EBCA5E04'u),
+      uint64x2(hi: 0xBCE5086492111AEA'u, lo: 0x88F4BB1CA6BCF585'u),
+      uint64x2(hi: 0xEC1E4A7DB69561A5'u, lo: 0x2B31E9E3D06C32E6'u),
+      uint64x2(hi: 0x9392EE8E921D5D07'u, lo: 0x3AFF322E62439FD0'u),
+      uint64x2(hi: 0xB877AA3236A4B449'u, lo: 0x09BEFEB9FAD487C3'u),
+      uint64x2(hi: 0xE69594BEC44DE15B'u, lo: 0x4C2EBE687989A9B4'u),
+      uint64x2(hi: 0x901D7CF73AB0ACD9'u, lo: 0x0F9D37014BF60A11'u),
+      uint64x2(hi: 0xB424DC35095CD80F'u, lo: 0x538484C19EF38C95'u),
+      uint64x2(hi: 0xE12E13424BB40E13'u, lo: 0x2865A5F206B06FBA'u),
+      uint64x2(hi: 0x8CBCCC096F5088CB'u, lo: 0xF93F87B7442E45D4'u),
+      uint64x2(hi: 0xAFEBFF0BCB24AAFE'u, lo: 0xF78F69A51539D749'u),
+      uint64x2(hi: 0xDBE6FECEBDEDD5BE'u, lo: 0xB573440E5A884D1C'u),
+      uint64x2(hi: 0x89705F4136B4A597'u, lo: 0x31680A88F8953031'u),
+      uint64x2(hi: 0xABCC77118461CEFC'u, lo: 0xFDC20D2B36BA7C3E'u),
+      uint64x2(hi: 0xD6BF94D5E57A42BC'u, lo: 0x3D32907604691B4D'u),
+      uint64x2(hi: 0x8637BD05AF6C69B5'u, lo: 0xA63F9A49C2C1B110'u),
+      uint64x2(hi: 0xA7C5AC471B478423'u, lo: 0x0FCF80DC33721D54'u),
+      uint64x2(hi: 0xD1B71758E219652B'u, lo: 0xD3C36113404EA4A9'u),
+      uint64x2(hi: 0x83126E978D4FDF3B'u, lo: 0x645A1CAC083126EA'u),
+      uint64x2(hi: 0xA3D70A3D70A3D70A'u, lo: 0x3D70A3D70A3D70A4'u),
+      uint64x2(hi: 0xCCCCCCCCCCCCCCCC'u, lo: 0xCCCCCCCCCCCCCCCD'u),
+      uint64x2(hi: 0x8000000000000000'u, lo: 0x0000000000000000'u),
+      uint64x2(hi: 0xA000000000000000'u, lo: 0x0000000000000000'u),
+      uint64x2(hi: 0xC800000000000000'u, lo: 0x0000000000000000'u),
+      uint64x2(hi: 0xFA00000000000000'u, lo: 0x0000000000000000'u),
+      uint64x2(hi: 0x9C40000000000000'u, lo: 0x0000000000000000'u),
+      uint64x2(hi: 0xC350000000000000'u, lo: 0x0000000000000000'u),
+      uint64x2(hi: 0xF424000000000000'u, lo: 0x0000000000000000'u),
+      uint64x2(hi: 0x9896800000000000'u, lo: 0x0000000000000000'u),
+      uint64x2(hi: 0xBEBC200000000000'u, lo: 0x0000000000000000'u),
+      uint64x2(hi: 0xEE6B280000000000'u, lo: 0x0000000000000000'u),
+      uint64x2(hi: 0x9502F90000000000'u, lo: 0x0000000000000000'u),
+      uint64x2(hi: 0xBA43B74000000000'u, lo: 0x0000000000000000'u),
+      uint64x2(hi: 0xE8D4A51000000000'u, lo: 0x0000000000000000'u),
+      uint64x2(hi: 0x9184E72A00000000'u, lo: 0x0000000000000000'u),
+      uint64x2(hi: 0xB5E620F480000000'u, lo: 0x0000000000000000'u),
+      uint64x2(hi: 0xE35FA931A0000000'u, lo: 0x0000000000000000'u),
+      uint64x2(hi: 0x8E1BC9BF04000000'u, lo: 0x0000000000000000'u),
+      uint64x2(hi: 0xB1A2BC2EC5000000'u, lo: 0x0000000000000000'u),
+      uint64x2(hi: 0xDE0B6B3A76400000'u, lo: 0x0000000000000000'u),
+      uint64x2(hi: 0x8AC7230489E80000'u, lo: 0x0000000000000000'u),
+      uint64x2(hi: 0xAD78EBC5AC620000'u, lo: 0x0000000000000000'u),
+      uint64x2(hi: 0xD8D726B7177A8000'u, lo: 0x0000000000000000'u),
+      uint64x2(hi: 0x878678326EAC9000'u, lo: 0x0000000000000000'u),
+      uint64x2(hi: 0xA968163F0A57B400'u, lo: 0x0000000000000000'u),
+      uint64x2(hi: 0xD3C21BCECCEDA100'u, lo: 0x0000000000000000'u),
+      uint64x2(hi: 0x84595161401484A0'u, lo: 0x0000000000000000'u),
+      uint64x2(hi: 0xA56FA5B99019A5C8'u, lo: 0x0000000000000000'u),
+      uint64x2(hi: 0xCECB8F27F4200F3A'u, lo: 0x0000000000000000'u),
+      uint64x2(hi: 0x813F3978F8940984'u, lo: 0x4000000000000000'u),
+      uint64x2(hi: 0xA18F07D736B90BE5'u, lo: 0x5000000000000000'u),
+      uint64x2(hi: 0xC9F2C9CD04674EDE'u, lo: 0xA400000000000000'u),
+      uint64x2(hi: 0xFC6F7C4045812296'u, lo: 0x4D00000000000000'u),
+      uint64x2(hi: 0x9DC5ADA82B70B59D'u, lo: 0xF020000000000000'u),
+      uint64x2(hi: 0xC5371912364CE305'u, lo: 0x6C28000000000000'u),
+      uint64x2(hi: 0xF684DF56C3E01BC6'u, lo: 0xC732000000000000'u),
+      uint64x2(hi: 0x9A130B963A6C115C'u, lo: 0x3C7F400000000000'u),
+      uint64x2(hi: 0xC097CE7BC90715B3'u, lo: 0x4B9F100000000000'u),
+      uint64x2(hi: 0xF0BDC21ABB48DB20'u, lo: 0x1E86D40000000000'u),
+      uint64x2(hi: 0x96769950B50D88F4'u, lo: 0x1314448000000000'u),
+      uint64x2(hi: 0xBC143FA4E250EB31'u, lo: 0x17D955A000000000'u),
+      uint64x2(hi: 0xEB194F8E1AE525FD'u, lo: 0x5DCFAB0800000000'u),
+      uint64x2(hi: 0x92EFD1B8D0CF37BE'u, lo: 0x5AA1CAE500000000'u),
+      uint64x2(hi: 0xB7ABC627050305AD'u, lo: 0xF14A3D9E40000000'u),
+      uint64x2(hi: 0xE596B7B0C643C719'u, lo: 0x6D9CCD05D0000000'u),
+      uint64x2(hi: 0x8F7E32CE7BEA5C6F'u, lo: 0xE4820023A2000000'u),
+      uint64x2(hi: 0xB35DBF821AE4F38B'u, lo: 0xDDA2802C8A800000'u),
+      uint64x2(hi: 0xE0352F62A19E306E'u, lo: 0xD50B2037AD200000'u),
+      uint64x2(hi: 0x8C213D9DA502DE45'u, lo: 0x4526F422CC340000'u),
+      uint64x2(hi: 0xAF298D050E4395D6'u, lo: 0x9670B12B7F410000'u),
+      uint64x2(hi: 0xDAF3F04651D47B4C'u, lo: 0x3C0CDD765F114000'u),
+      uint64x2(hi: 0x88D8762BF324CD0F'u, lo: 0xA5880A69FB6AC800'u),
+      uint64x2(hi: 0xAB0E93B6EFEE0053'u, lo: 0x8EEA0D047A457A00'u),
+      uint64x2(hi: 0xD5D238A4ABE98068'u, lo: 0x72A4904598D6D880'u),
+      uint64x2(hi: 0x85A36366EB71F041'u, lo: 0x47A6DA2B7F864750'u),
+      uint64x2(hi: 0xA70C3C40A64E6C51'u, lo: 0x999090B65F67D924'u),
+      uint64x2(hi: 0xD0CF4B50CFE20765'u, lo: 0xFFF4B4E3F741CF6D'u),
+      uint64x2(hi: 0x82818F1281ED449F'u, lo: 0xBFF8F10E7A8921A4'u),
+      uint64x2(hi: 0xA321F2D7226895C7'u, lo: 0xAFF72D52192B6A0D'u),
+      uint64x2(hi: 0xCBEA6F8CEB02BB39'u, lo: 0x9BF4F8A69F764490'u),
+      uint64x2(hi: 0xFEE50B7025C36A08'u, lo: 0x02F236D04753D5B4'u),
+      uint64x2(hi: 0x9F4F2726179A2245'u, lo: 0x01D762422C946590'u),
+      uint64x2(hi: 0xC722F0EF9D80AAD6'u, lo: 0x424D3AD2B7B97EF5'u),
+      uint64x2(hi: 0xF8EBAD2B84E0D58B'u, lo: 0xD2E0898765A7DEB2'u),
+      uint64x2(hi: 0x9B934C3B330C8577'u, lo: 0x63CC55F49F88EB2F'u),
+      uint64x2(hi: 0xC2781F49FFCFA6D5'u, lo: 0x3CBF6B71C76B25FB'u),
+      uint64x2(hi: 0xF316271C7FC3908A'u, lo: 0x8BEF464E3945EF7A'u),
+      uint64x2(hi: 0x97EDD871CFDA3A56'u, lo: 0x97758BF0E3CBB5AC'u),
+      uint64x2(hi: 0xBDE94E8E43D0C8EC'u, lo: 0x3D52EEED1CBEA317'u),
+      uint64x2(hi: 0xED63A231D4C4FB27'u, lo: 0x4CA7AAA863EE4BDD'u),
+      uint64x2(hi: 0x945E455F24FB1CF8'u, lo: 0x8FE8CAA93E74EF6A'u),
+      uint64x2(hi: 0xB975D6B6EE39E436'u, lo: 0xB3E2FD538E122B44'u),
+      uint64x2(hi: 0xE7D34C64A9C85D44'u, lo: 0x60DBBCA87196B616'u),
+      uint64x2(hi: 0x90E40FBEEA1D3A4A'u, lo: 0xBC8955E946FE31CD'u),
+      uint64x2(hi: 0xB51D13AEA4A488DD'u, lo: 0x6BABAB6398BDBE41'u),
+      uint64x2(hi: 0xE264589A4DCDAB14'u, lo: 0xC696963C7EED2DD1'u),
+      uint64x2(hi: 0x8D7EB76070A08AEC'u, lo: 0xFC1E1DE5CF543CA2'u),
+      uint64x2(hi: 0xB0DE65388CC8ADA8'u, lo: 0x3B25A55F43294BCB'u),
+      uint64x2(hi: 0xDD15FE86AFFAD912'u, lo: 0x49EF0EB713F39EBE'u),
+      uint64x2(hi: 0x8A2DBF142DFCC7AB'u, lo: 0x6E3569326C784337'u),
+      uint64x2(hi: 0xACB92ED9397BF996'u, lo: 0x49C2C37F07965404'u),
+      uint64x2(hi: 0xD7E77A8F87DAF7FB'u, lo: 0xDC33745EC97BE906'u),
+      uint64x2(hi: 0x86F0AC99B4E8DAFD'u, lo: 0x69A028BB3DED71A3'u),
+      uint64x2(hi: 0xA8ACD7C0222311BC'u, lo: 0xC40832EA0D68CE0C'u),
+      uint64x2(hi: 0xD2D80DB02AABD62B'u, lo: 0xF50A3FA490C30190'u),
+      uint64x2(hi: 0x83C7088E1AAB65DB'u, lo: 0x792667C6DA79E0FA'u),
+      uint64x2(hi: 0xA4B8CAB1A1563F52'u, lo: 0x577001B891185938'u),
+      uint64x2(hi: 0xCDE6FD5E09ABCF26'u, lo: 0xED4C0226B55E6F86'u),
+      uint64x2(hi: 0x80B05E5AC60B6178'u, lo: 0x544F8158315B05B4'u),
+      uint64x2(hi: 0xA0DC75F1778E39D6'u, lo: 0x696361AE3DB1C721'u),
+      uint64x2(hi: 0xC913936DD571C84C'u, lo: 0x03BC3A19CD1E38E9'u),
+      uint64x2(hi: 0xFB5878494ACE3A5F'u, lo: 0x04AB48A04065C723'u),
+      uint64x2(hi: 0x9D174B2DCEC0E47B'u, lo: 0x62EB0D64283F9C76'u),
+      uint64x2(hi: 0xC45D1DF942711D9A'u, lo: 0x3BA5D0BD324F8394'u),
+      uint64x2(hi: 0xF5746577930D6500'u, lo: 0xCA8F44EC7EE36479'u),
+      uint64x2(hi: 0x9968BF6ABBE85F20'u, lo: 0x7E998B13CF4E1ECB'u),
+      uint64x2(hi: 0xBFC2EF456AE276E8'u, lo: 0x9E3FEDD8C321A67E'u),
+      uint64x2(hi: 0xEFB3AB16C59B14A2'u, lo: 0xC5CFE94EF3EA101E'u),
+      uint64x2(hi: 0x95D04AEE3B80ECE5'u, lo: 0xBBA1F1D158724A12'u),
+      uint64x2(hi: 0xBB445DA9CA61281F'u, lo: 0x2A8A6E45AE8EDC97'u),
+      uint64x2(hi: 0xEA1575143CF97226'u, lo: 0xF52D09D71A3293BD'u),
+      uint64x2(hi: 0x924D692CA61BE758'u, lo: 0x593C2626705F9C56'u),
+      uint64x2(hi: 0xB6E0C377CFA2E12E'u, lo: 0x6F8B2FB00C77836C'u),
+      uint64x2(hi: 0xE498F455C38B997A'u, lo: 0x0B6DFB9C0F956447'u),
+      uint64x2(hi: 0x8EDF98B59A373FEC'u, lo: 0x4724BD4189BD5EAC'u),
+      uint64x2(hi: 0xB2977EE300C50FE7'u, lo: 0x58EDEC91EC2CB657'u),
+      uint64x2(hi: 0xDF3D5E9BC0F653E1'u, lo: 0x2F2967B66737E3ED'u),
+      uint64x2(hi: 0x8B865B215899F46C'u, lo: 0xBD79E0D20082EE74'u),
+      uint64x2(hi: 0xAE67F1E9AEC07187'u, lo: 0xECD8590680A3AA11'u),
+      uint64x2(hi: 0xDA01EE641A708DE9'u, lo: 0xE80E6F4820CC9495'u),
+      uint64x2(hi: 0x884134FE908658B2'u, lo: 0x3109058D147FDCDD'u),
+      uint64x2(hi: 0xAA51823E34A7EEDE'u, lo: 0xBD4B46F0599FD415'u),
+      uint64x2(hi: 0xD4E5E2CDC1D1EA96'u, lo: 0x6C9E18AC7007C91A'u),
+      uint64x2(hi: 0x850FADC09923329E'u, lo: 0x03E2CF6BC604DDB0'u),
+      uint64x2(hi: 0xA6539930BF6BFF45'u, lo: 0x84DB8346B786151C'u),
+      uint64x2(hi: 0xCFE87F7CEF46FF16'u, lo: 0xE612641865679A63'u),
+      uint64x2(hi: 0x81F14FAE158C5F6E'u, lo: 0x4FCB7E8F3F60C07E'u),
+      uint64x2(hi: 0xA26DA3999AEF7749'u, lo: 0xE3BE5E330F38F09D'u),
+      uint64x2(hi: 0xCB090C8001AB551C'u, lo: 0x5CADF5BFD3072CC5'u),
+      uint64x2(hi: 0xFDCB4FA002162A63'u, lo: 0x73D9732FC7C8F7F6'u),
+      uint64x2(hi: 0x9E9F11C4014DDA7E'u, lo: 0x2867E7FDDCDD9AFA'u),
+      uint64x2(hi: 0xC646D63501A1511D'u, lo: 0xB281E1FD541501B8'u),
+      uint64x2(hi: 0xF7D88BC24209A565'u, lo: 0x1F225A7CA91A4226'u),
+      uint64x2(hi: 0x9AE757596946075F'u, lo: 0x3375788DE9B06958'u),
+      uint64x2(hi: 0xC1A12D2FC3978937'u, lo: 0x0052D6B1641C83AE'u),
+      uint64x2(hi: 0xF209787BB47D6B84'u, lo: 0xC0678C5DBD23A49A'u),
+      uint64x2(hi: 0x9745EB4D50CE6332'u, lo: 0xF840B7BA963646E0'u),
+      uint64x2(hi: 0xBD176620A501FBFF'u, lo: 0xB650E5A93BC3D898'u),
+      uint64x2(hi: 0xEC5D3FA8CE427AFF'u, lo: 0xA3E51F138AB4CEBE'u),
+      uint64x2(hi: 0x93BA47C980E98CDF'u, lo: 0xC66F336C36B10137'u),
+      uint64x2(hi: 0xB8A8D9BBE123F017'u, lo: 0xB80B0047445D4184'u),
+      uint64x2(hi: 0xE6D3102AD96CEC1D'u, lo: 0xA60DC059157491E5'u),
+      uint64x2(hi: 0x9043EA1AC7E41392'u, lo: 0x87C89837AD68DB2F'u),
+      uint64x2(hi: 0xB454E4A179DD1877'u, lo: 0x29BABE4598C311FB'u),
+      uint64x2(hi: 0xE16A1DC9D8545E94'u, lo: 0xF4296DD6FEF3D67A'u),
+      uint64x2(hi: 0x8CE2529E2734BB1D'u, lo: 0x1899E4A65F58660C'u),
+      uint64x2(hi: 0xB01AE745B101E9E4'u, lo: 0x5EC05DCFF72E7F8F'u),
+      uint64x2(hi: 0xDC21A1171D42645D'u, lo: 0x76707543F4FA1F73'u),
+      uint64x2(hi: 0x899504AE72497EBA'u, lo: 0x6A06494A791C53A8'u),
+      uint64x2(hi: 0xABFA45DA0EDBDE69'u, lo: 0x0487DB9D17636892'u),
+      uint64x2(hi: 0xD6F8D7509292D603'u, lo: 0x45A9D2845D3C42B6'u),
+      uint64x2(hi: 0x865B86925B9BC5C2'u, lo: 0x0B8A2392BA45A9B2'u),
+      uint64x2(hi: 0xA7F26836F282B732'u, lo: 0x8E6CAC7768D7141E'u),
+      uint64x2(hi: 0xD1EF0244AF2364FF'u, lo: 0x3207D795430CD926'u),
+      uint64x2(hi: 0x8335616AED761F1F'u, lo: 0x7F44E6BD49E807B8'u),
+      uint64x2(hi: 0xA402B9C5A8D3A6E7'u, lo: 0x5F16206C9C6209A6'u),
+      uint64x2(hi: 0xCD036837130890A1'u, lo: 0x36DBA887C37A8C0F'u),
+      uint64x2(hi: 0x802221226BE55A64'u, lo: 0xC2494954DA2C9789'u),
+      uint64x2(hi: 0xA02AA96B06DEB0FD'u, lo: 0xF2DB9BAA10B7BD6C'u),
+      uint64x2(hi: 0xC83553C5C8965D3D'u, lo: 0x6F92829494E5ACC7'u),
+      uint64x2(hi: 0xFA42A8B73ABBF48C'u, lo: 0xCB772339BA1F17F9'u),
+      uint64x2(hi: 0x9C69A97284B578D7'u, lo: 0xFF2A760414536EFB'u),
+      uint64x2(hi: 0xC38413CF25E2D70D'u, lo: 0xFEF5138519684ABA'u),
+      uint64x2(hi: 0xF46518C2EF5B8CD1'u, lo: 0x7EB258665FC25D69'u),
+      uint64x2(hi: 0x98BF2F79D5993802'u, lo: 0xEF2F773FFBD97A61'u),
+      uint64x2(hi: 0xBEEEFB584AFF8603'u, lo: 0xAAFB550FFACFD8FA'u),
+      uint64x2(hi: 0xEEAABA2E5DBF6784'u, lo: 0x95BA2A53F983CF38'u),
+      uint64x2(hi: 0x952AB45CFA97A0B2'u, lo: 0xDD945A747BF26183'u),
+      uint64x2(hi: 0xBA756174393D88DF'u, lo: 0x94F971119AEEF9E4'u),
+      uint64x2(hi: 0xE912B9D1478CEB17'u, lo: 0x7A37CD5601AAB85D'u),
+      uint64x2(hi: 0x91ABB422CCB812EE'u, lo: 0xAC62E055C10AB33A'u),
+      uint64x2(hi: 0xB616A12B7FE617AA'u, lo: 0x577B986B314D6009'u),
+      uint64x2(hi: 0xE39C49765FDF9D94'u, lo: 0xED5A7E85FDA0B80B'u),
+      uint64x2(hi: 0x8E41ADE9FBEBC27D'u, lo: 0x14588F13BE847307'u),
+      uint64x2(hi: 0xB1D219647AE6B31C'u, lo: 0x596EB2D8AE258FC8'u),
+      uint64x2(hi: 0xDE469FBD99A05FE3'u, lo: 0x6FCA5F8ED9AEF3BB'u),
+      uint64x2(hi: 0x8AEC23D680043BEE'u, lo: 0x25DE7BB9480D5854'u),
+      uint64x2(hi: 0xADA72CCC20054AE9'u, lo: 0xAF561AA79A10AE6A'u),
+      uint64x2(hi: 0xD910F7FF28069DA4'u, lo: 0x1B2BA1518094DA04'u),
+      uint64x2(hi: 0x87AA9AFF79042286'u, lo: 0x90FB44D2F05D0842'u),
+      uint64x2(hi: 0xA99541BF57452B28'u, lo: 0x353A1607AC744A53'u),
+      uint64x2(hi: 0xD3FA922F2D1675F2'u, lo: 0x42889B8997915CE8'u),
+      uint64x2(hi: 0x847C9B5D7C2E09B7'u, lo: 0x69956135FEBADA11'u),
+      uint64x2(hi: 0xA59BC234DB398C25'u, lo: 0x43FAB9837E699095'u),
+      uint64x2(hi: 0xCF02B2C21207EF2E'u, lo: 0x94F967E45E03F4BB'u),
+      uint64x2(hi: 0x8161AFB94B44F57D'u, lo: 0x1D1BE0EEBAC278F5'u),
+      uint64x2(hi: 0xA1BA1BA79E1632DC'u, lo: 0x6462D92A69731732'u),
+      uint64x2(hi: 0xCA28A291859BBF93'u, lo: 0x7D7B8F7503CFDCFE'u),
+      uint64x2(hi: 0xFCB2CB35E702AF78'u, lo: 0x5CDA735244C3D43E'u),
+      uint64x2(hi: 0x9DEFBF01B061ADAB'u, lo: 0x3A0888136AFA64A7'u),
+      uint64x2(hi: 0xC56BAEC21C7A1916'u, lo: 0x088AAA1845B8FDD0'u),
+      uint64x2(hi: 0xF6C69A72A3989F5B'u, lo: 0x8AAD549E57273D45'u),
+      uint64x2(hi: 0x9A3C2087A63F6399'u, lo: 0x36AC54E2F678864B'u),
+      uint64x2(hi: 0xC0CB28A98FCF3C7F'u, lo: 0x84576A1BB416A7DD'u),
+      uint64x2(hi: 0xF0FDF2D3F3C30B9F'u, lo: 0x656D44A2A11C51D5'u),
+      uint64x2(hi: 0x969EB7C47859E743'u, lo: 0x9F644AE5A4B1B325'u),
+      uint64x2(hi: 0xBC4665B596706114'u, lo: 0x873D5D9F0DDE1FEE'u),
+      uint64x2(hi: 0xEB57FF22FC0C7959'u, lo: 0xA90CB506D155A7EA'u),
+      uint64x2(hi: 0x9316FF75DD87CBD8'u, lo: 0x09A7F12442D588F2'u),
+      uint64x2(hi: 0xB7DCBF5354E9BECE'u, lo: 0x0C11ED6D538AEB2F'u),
+      uint64x2(hi: 0xE5D3EF282A242E81'u, lo: 0x8F1668C8A86DA5FA'u),
+      uint64x2(hi: 0x8FA475791A569D10'u, lo: 0xF96E017D694487BC'u),
+      uint64x2(hi: 0xB38D92D760EC4455'u, lo: 0x37C981DCC395A9AC'u),
+      uint64x2(hi: 0xE070F78D3927556A'u, lo: 0x85BBE253F47B1417'u),
+      uint64x2(hi: 0x8C469AB843B89562'u, lo: 0x93956D7478CCEC8E'u),
+      uint64x2(hi: 0xAF58416654A6BABB'u, lo: 0x387AC8D1970027B2'u),
+      uint64x2(hi: 0xDB2E51BFE9D0696A'u, lo: 0x06997B05FCC0319E'u),
+      uint64x2(hi: 0x88FCF317F22241E2'u, lo: 0x441FECE3BDF81F03'u),
+      uint64x2(hi: 0xAB3C2FDDEEAAD25A'u, lo: 0xD527E81CAD7626C3'u),
+      uint64x2(hi: 0xD60B3BD56A5586F1'u, lo: 0x8A71E223D8D3B074'u),
+      uint64x2(hi: 0x85C7056562757456'u, lo: 0xF6872D5667844E49'u),
+      uint64x2(hi: 0xA738C6BEBB12D16C'u, lo: 0xB428F8AC016561DB'u),
+      uint64x2(hi: 0xD106F86E69D785C7'u, lo: 0xE13336D701BEBA52'u),
+      uint64x2(hi: 0x82A45B450226B39C'u, lo: 0xECC0024661173473'u),
+      uint64x2(hi: 0xA34D721642B06084'u, lo: 0x27F002D7F95D0190'u),
+      uint64x2(hi: 0xCC20CE9BD35C78A5'u, lo: 0x31EC038DF7B441F4'u),
+      uint64x2(hi: 0xFF290242C83396CE'u, lo: 0x7E67047175A15271'u),
+      uint64x2(hi: 0x9F79A169BD203E41'u, lo: 0x0F0062C6E984D386'u),
+      uint64x2(hi: 0xC75809C42C684DD1'u, lo: 0x52C07B78A3E60868'u),
+      uint64x2(hi: 0xF92E0C3537826145'u, lo: 0xA7709A56CCDF8A82'u),
+      uint64x2(hi: 0x9BBCC7A142B17CCB'u, lo: 0x88A66076400BB691'u),
+      uint64x2(hi: 0xC2ABF989935DDBFE'u, lo: 0x6ACFF893D00EA435'u),
+      uint64x2(hi: 0xF356F7EBF83552FE'u, lo: 0x0583F6B8C4124D43'u),
+      uint64x2(hi: 0x98165AF37B2153DE'u, lo: 0xC3727A337A8B704A'u),
+      uint64x2(hi: 0xBE1BF1B059E9A8D6'u, lo: 0x744F18C0592E4C5C'u),
+      uint64x2(hi: 0xEDA2EE1C7064130C'u, lo: 0x1162DEF06F79DF73'u),
+      uint64x2(hi: 0x9485D4D1C63E8BE7'u, lo: 0x8ADDCB5645AC2BA8'u),
+      uint64x2(hi: 0xB9A74A0637CE2EE1'u, lo: 0x6D953E2BD7173692'u),
+      uint64x2(hi: 0xE8111C87C5C1BA99'u, lo: 0xC8FA8DB6CCDD0437'u),
+      uint64x2(hi: 0x910AB1D4DB9914A0'u, lo: 0x1D9C9892400A22A2'u),
+      uint64x2(hi: 0xB54D5E4A127F59C8'u, lo: 0x2503BEB6D00CAB4B'u),
+      uint64x2(hi: 0xE2A0B5DC971F303A'u, lo: 0x2E44AE64840FD61D'u),
+      uint64x2(hi: 0x8DA471A9DE737E24'u, lo: 0x5CEAECFED289E5D2'u),
+      uint64x2(hi: 0xB10D8E1456105DAD'u, lo: 0x7425A83E872C5F47'u),
+      uint64x2(hi: 0xDD50F1996B947518'u, lo: 0xD12F124E28F77719'u),
+      uint64x2(hi: 0x8A5296FFE33CC92F'u, lo: 0x82BD6B70D99AAA6F'u),
+      uint64x2(hi: 0xACE73CBFDC0BFB7B'u, lo: 0x636CC64D1001550B'u),
+      uint64x2(hi: 0xD8210BEFD30EFA5A'u, lo: 0x3C47F7E05401AA4E'u),
+      uint64x2(hi: 0x8714A775E3E95C78'u, lo: 0x65ACFAEC34810A71'u),
+      uint64x2(hi: 0xA8D9D1535CE3B396'u, lo: 0x7F1839A741A14D0D'u),
+      uint64x2(hi: 0xD31045A8341CA07C'u, lo: 0x1EDE48111209A050'u),
+      uint64x2(hi: 0x83EA2B892091E44D'u, lo: 0x934AED0AAB460432'u),
+      uint64x2(hi: 0xA4E4B66B68B65D60'u, lo: 0xF81DA84D5617853F'u),
+      uint64x2(hi: 0xCE1DE40642E3F4B9'u, lo: 0x36251260AB9D668E'u),
+      uint64x2(hi: 0x80D2AE83E9CE78F3'u, lo: 0xC1D72B7C6B426019'u),
+      uint64x2(hi: 0xA1075A24E4421730'u, lo: 0xB24CF65B8612F81F'u),
+      uint64x2(hi: 0xC94930AE1D529CFC'u, lo: 0xDEE033F26797B627'u),
+      uint64x2(hi: 0xFB9B7CD9A4A7443C'u, lo: 0x169840EF017DA3B1'u),
+      uint64x2(hi: 0x9D412E0806E88AA5'u, lo: 0x8E1F289560EE864E'u),
+      uint64x2(hi: 0xC491798A08A2AD4E'u, lo: 0xF1A6F2BAB92A27E2'u),
+      uint64x2(hi: 0xF5B5D7EC8ACB58A2'u, lo: 0xAE10AF696774B1DB'u),
+      uint64x2(hi: 0x9991A6F3D6BF1765'u, lo: 0xACCA6DA1E0A8EF29'u),
+      uint64x2(hi: 0xBFF610B0CC6EDD3F'u, lo: 0x17FD090A58D32AF3'u),
+      uint64x2(hi: 0xEFF394DCFF8A948E'u, lo: 0xDDFC4B4CEF07F5B0'u),
+      uint64x2(hi: 0x95F83D0A1FB69CD9'u, lo: 0x4ABDAF101564F98E'u),
+      uint64x2(hi: 0xBB764C4CA7A4440F'u, lo: 0x9D6D1AD41ABE37F1'u),
+      uint64x2(hi: 0xEA53DF5FD18D5513'u, lo: 0x84C86189216DC5ED'u),
+      uint64x2(hi: 0x92746B9BE2F8552C'u, lo: 0x32FD3CF5B4E49BB4'u),
+      uint64x2(hi: 0xB7118682DBB66A77'u, lo: 0x3FBC8C33221DC2A1'u),
+      uint64x2(hi: 0xE4D5E82392A40515'u, lo: 0x0FABAF3FEAA5334A'u),
+      uint64x2(hi: 0x8F05B1163BA6832D'u, lo: 0x29CB4D87F2A7400E'u),
+      uint64x2(hi: 0xB2C71D5BCA9023F8'u, lo: 0x743E20E9EF511012'u),
+      uint64x2(hi: 0xDF78E4B2BD342CF6'u, lo: 0x914DA9246B255416'u),
+      uint64x2(hi: 0x8BAB8EEFB6409C1A'u, lo: 0x1AD089B6C2F7548E'u),
+      uint64x2(hi: 0xAE9672ABA3D0C320'u, lo: 0xA184AC2473B529B1'u),
+      uint64x2(hi: 0xDA3C0F568CC4F3E8'u, lo: 0xC9E5D72D90A2741E'u),
+      uint64x2(hi: 0x8865899617FB1871'u, lo: 0x7E2FA67C7A658892'u),
+      uint64x2(hi: 0xAA7EEBFB9DF9DE8D'u, lo: 0xDDBB901B98FEEAB7'u),
+      uint64x2(hi: 0xD51EA6FA85785631'u, lo: 0x552A74227F3EA565'u),
+      uint64x2(hi: 0x8533285C936B35DE'u, lo: 0xD53A88958F87275F'u),
+      uint64x2(hi: 0xA67FF273B8460356'u, lo: 0x8A892ABAF368F137'u),
+      uint64x2(hi: 0xD01FEF10A657842C'u, lo: 0x2D2B7569B0432D85'u),
+      uint64x2(hi: 0x8213F56A67F6B29B'u, lo: 0x9C3B29620E29FC73'u),
+      uint64x2(hi: 0xA298F2C501F45F42'u, lo: 0x8349F3BA91B47B8F'u),
+      uint64x2(hi: 0xCB3F2F7642717713'u, lo: 0x241C70A936219A73'u),
+      uint64x2(hi: 0xFE0EFB53D30DD4D7'u, lo: 0xED238CD383AA0110'u),
+      uint64x2(hi: 0x9EC95D1463E8A506'u, lo: 0xF4363804324A40AA'u),
+      uint64x2(hi: 0xC67BB4597CE2CE48'u, lo: 0xB143C6053EDCD0D5'u),
+      uint64x2(hi: 0xF81AA16FDC1B81DA'u, lo: 0xDD94B7868E94050A'u),
+      uint64x2(hi: 0x9B10A4E5E9913128'u, lo: 0xCA7CF2B4191C8326'u),
+      uint64x2(hi: 0xC1D4CE1F63F57D72'u, lo: 0xFD1C2F611F63A3F0'u),
+      uint64x2(hi: 0xF24A01A73CF2DCCF'u, lo: 0xBC633B39673C8CEC'u),
+      uint64x2(hi: 0x976E41088617CA01'u, lo: 0xD5BE0503E085D813'u),
+      uint64x2(hi: 0xBD49D14AA79DBC82'u, lo: 0x4B2D8644D8A74E18'u),
+      uint64x2(hi: 0xEC9C459D51852BA2'u, lo: 0xDDF8E7D60ED1219E'u),
+      uint64x2(hi: 0x93E1AB8252F33B45'u, lo: 0xCABB90E5C942B503'u),
+      uint64x2(hi: 0xB8DA1662E7B00A17'u, lo: 0x3D6A751F3B936243'u),
+      uint64x2(hi: 0xE7109BFBA19C0C9D'u, lo: 0x0CC512670A783AD4'u),
+      uint64x2(hi: 0x906A617D450187E2'u, lo: 0x27FB2B80668B24C5'u),
+      uint64x2(hi: 0xB484F9DC9641E9DA'u, lo: 0xB1F9F660802DEDF6'u),
+      uint64x2(hi: 0xE1A63853BBD26451'u, lo: 0x5E7873F8A0396973'u),
+      uint64x2(hi: 0x8D07E33455637EB2'u, lo: 0xDB0B487B6423E1E8'u),
+      uint64x2(hi: 0xB049DC016ABC5E5F'u, lo: 0x91CE1A9A3D2CDA62'u),
+      uint64x2(hi: 0xDC5C5301C56B75F7'u, lo: 0x7641A140CC7810FB'u),
+      uint64x2(hi: 0x89B9B3E11B6329BA'u, lo: 0xA9E904C87FCB0A9D'u),
+      uint64x2(hi: 0xAC2820D9623BF429'u, lo: 0x546345FA9FBDCD44'u),
+      uint64x2(hi: 0xD732290FBACAF133'u, lo: 0xA97C177947AD4095'u),
+      uint64x2(hi: 0x867F59A9D4BED6C0'u, lo: 0x49ED8EABCCCC485D'u),
+      uint64x2(hi: 0xA81F301449EE8C70'u, lo: 0x5C68F256BFFF5A74'u),
+      uint64x2(hi: 0xD226FC195C6A2F8C'u, lo: 0x73832EEC6FFF3111'u),
+      uint64x2(hi: 0x83585D8FD9C25DB7'u, lo: 0xC831FD53C5FF7EAB'u),
+      uint64x2(hi: 0xA42E74F3D032F525'u, lo: 0xBA3E7CA8B77F5E55'u),
+      uint64x2(hi: 0xCD3A1230C43FB26F'u, lo: 0x28CE1BD2E55F35EB'u),
+      uint64x2(hi: 0x80444B5E7AA7CF85'u, lo: 0x7980D163CF5B81B3'u),
+      uint64x2(hi: 0xA0555E361951C366'u, lo: 0xD7E105BCC332621F'u),
+      uint64x2(hi: 0xC86AB5C39FA63440'u, lo: 0x8DD9472BF3FEFAA7'u),
+      uint64x2(hi: 0xFA856334878FC150'u, lo: 0xB14F98F6F0FEB951'u),
+      uint64x2(hi: 0x9C935E00D4B9D8D2'u, lo: 0x6ED1BF9A569F33D3'u),
+      uint64x2(hi: 0xC3B8358109E84F07'u, lo: 0x0A862F80EC4700C8'u),
+      uint64x2(hi: 0xF4A642E14C6262C8'u, lo: 0xCD27BB612758C0FA'u),
+      uint64x2(hi: 0x98E7E9CCCFBD7DBD'u, lo: 0x8038D51CB897789C'u),
+      uint64x2(hi: 0xBF21E44003ACDD2C'u, lo: 0xE0470A63E6BD56C3'u),
+      uint64x2(hi: 0xEEEA5D5004981478'u, lo: 0x1858CCFCE06CAC74'u),
+      uint64x2(hi: 0x95527A5202DF0CCB'u, lo: 0x0F37801E0C43EBC8'u),
+      uint64x2(hi: 0xBAA718E68396CFFD'u, lo: 0xD30560258F54E6BA'u),
+      uint64x2(hi: 0xE950DF20247C83FD'u, lo: 0x47C6B82EF32A2069'u),
+      uint64x2(hi: 0x91D28B7416CDD27E'u, lo: 0x4CDC331D57FA5441'u),
+      uint64x2(hi: 0xB6472E511C81471D'u, lo: 0xE0133FE4ADF8E952'u),
+      uint64x2(hi: 0xE3D8F9E563A198E5'u, lo: 0x58180FDDD97723A6'u),
+      uint64x2(hi: 0x8E679C2F5E44FF8F'u, lo: 0x570F09EAA7EA7648'u),
+      uint64x2(hi: 0xB201833B35D63F73'u, lo: 0x2CD2CC6551E513DA'u),
+      uint64x2(hi: 0xDE81E40A034BCF4F'u, lo: 0xF8077F7EA65E58D1'u),
+      uint64x2(hi: 0x8B112E86420F6191'u, lo: 0xFB04AFAF27FAF782'u),
+      uint64x2(hi: 0xADD57A27D29339F6'u, lo: 0x79C5DB9AF1F9B563'u),
+      uint64x2(hi: 0xD94AD8B1C7380874'u, lo: 0x18375281AE7822BC'u),
+      uint64x2(hi: 0x87CEC76F1C830548'u, lo: 0x8F2293910D0B15B5'u),
+      uint64x2(hi: 0xA9C2794AE3A3C69A'u, lo: 0xB2EB3875504DDB22'u),
+      uint64x2(hi: 0xD433179D9C8CB841'u, lo: 0x5FA60692A46151EB'u),
+      uint64x2(hi: 0x849FEEC281D7F328'u, lo: 0xDBC7C41BA6BCD333'u),
+      uint64x2(hi: 0xA5C7EA73224DEFF3'u, lo: 0x12B9B522906C0800'u),
+      uint64x2(hi: 0xCF39E50FEAE16BEF'u, lo: 0xD768226B34870A00'u),
+      uint64x2(hi: 0x81842F29F2CCE375'u, lo: 0xE6A1158300D46640'u),
+      uint64x2(hi: 0xA1E53AF46F801C53'u, lo: 0x60495AE3C1097FD0'u),
+      uint64x2(hi: 0xCA5E89B18B602368'u, lo: 0x385BB19CB14BDFC4'u),
+      uint64x2(hi: 0xFCF62C1DEE382C42'u, lo: 0x46729E03DD9ED7B5'u),
+      uint64x2(hi: 0x9E19DB92B4E31BA9'u, lo: 0x6C07A2C26A8346D1'u),
+      uint64x2(hi: 0xC5A05277621BE293'u, lo: 0xC7098B7305241885'u),
+      uint64x2(hi: 0xF70867153AA2DB38'u, lo: 0xB8CBEE4FC66D1EA7'u)]
+  dragonbox_Assert(k >= kMin)
+  dragonbox_Assert(k <= kMax)
+  return pow10[k - kMin]
+
+##  Returns whether value is divisible by 2^e2
+
+proc multipleOfPow2*(value: uint64; e2: int32): bool {.inline.} =
+  dragonbox_Assert(e2 >= 0)
+  return e2 < 64 and (value and ((uint64(1) shl e2) - 1)) == 0
+
+##  Returns whether value is divisible by 5^e5
+
+proc multipleOfPow5*(value: uint64; e5: int32): bool {.inline.} =
+  type
+    MulCmp {.bycopy.} = object
+      mul: uint64
+      cmp: uint64
+
+  const
+    mod5 = [MulCmp(mul: 0x0000000000000001'u, cmp: 0xFFFFFFFFFFFFFFFF'u),
+      MulCmp(mul: 0xCCCCCCCCCCCCCCCD'u, cmp: 0x3333333333333333'u),
+      MulCmp(mul: 0x8F5C28F5C28F5C29'u, cmp: 0x0A3D70A3D70A3D70'u),
+      MulCmp(mul: 0x1CAC083126E978D5'u, cmp: 0x020C49BA5E353F7C'u),
+      MulCmp(mul: 0xD288CE703AFB7E91'u, cmp: 0x0068DB8BAC710CB2'u),
+      MulCmp(mul: 0x5D4E8FB00BCBE61D'u, cmp: 0x0014F8B588E368F0'u),
+      MulCmp(mul: 0x790FB65668C26139'u, cmp: 0x000431BDE82D7B63'u),
+      MulCmp(mul: 0xE5032477AE8D46A5'u, cmp: 0x0000D6BF94D5E57A'u),
+      MulCmp(mul: 0xC767074B22E90E21'u, cmp: 0x00002AF31DC46118'u),
+      MulCmp(mul: 0x8E47CE423A2E9C6D'u, cmp: 0x0000089705F4136B'u),
+      MulCmp(mul: 0x4FA7F60D3ED61F49'u, cmp: 0x000001B7CDFD9D7B'u),
+      MulCmp(mul: 0x0FEE64690C913975'u, cmp: 0x00000057F5FF85E5'u),
+      MulCmp(mul: 0x3662E0E1CF503EB1'u, cmp: 0x000000119799812D'u),
+      MulCmp(mul: 0xA47A2CF9F6433FBD'u, cmp: 0x0000000384B84D09'u),
+      MulCmp(mul: 0x54186F653140A659'u, cmp: 0x00000000B424DC35'u),
+      MulCmp(mul: 0x7738164770402145'u, cmp: 0x0000000024075F3D'u),
+      MulCmp(mul: 0xE4A4D1417CD9A041'u, cmp: 0x000000000734ACA5'u),
+      MulCmp(mul: 0xC75429D9E5C5200D'u, cmp: 0x000000000170EF54'u),
+      MulCmp(mul: 0xC1773B91FAC10669'u, cmp: 0x000000000049C977'u),
+      MulCmp(mul: 0x26B172506559CE15'u, cmp: 0x00000000000EC1E4'u),
+      MulCmp(mul: 0xD489E3A9ADDEC2D1'u, cmp: 0x000000000002F394'u),
+      MulCmp(mul: 0x90E860BB892C8D5D'u, cmp: 0x000000000000971D'u),
+      MulCmp(mul: 0x502E79BF1B6F4F79'u, cmp: 0x0000000000001E39'u),
+      MulCmp(mul: 0xDCD618596BE30FE5'u, cmp: 0x000000000000060B'u),
+      MulCmp(mul: 0x2C2AD1AB7BFA3661'u, cmp: 0x0000000000000135'u)]
+  dragonbox_Assert(e5 >= 0)
+  dragonbox_Assert(e5 <= 24)
+  let m5: MulCmp = mod5[e5]
+  return value * m5.mul <= m5.cmp
+
+type
+  FloatingDecimal64* {.bycopy.} = object
+    significand*: uint64
+    exponent*: int32
+
+
+proc toDecimal64AsymmetricInterval*(e2: int32): FloatingDecimal64 {.inline.} =
+  ##  NB:
+  ##  accept_lower_endpoint = true
+  ##  accept_upper_endpoint = true
+  const
+    P: int32 = significandSize
+  ##  Compute k and beta
+  let minusK: int32 = floorLog10ThreeQuartersPow2(e2)
+  let betaMinus1: int32 = e2 + floorLog2Pow10(-minusK)
+  ##  Compute xi and zi
+  let pow10: uint64x2 = computePow10(-minusK)
+  let lowerEndpoint: uint64 = (pow10.hi - (pow10.hi shr (P + 1))) shr
+      (64 - P - betaMinus1)
+  let upperEndpoint: uint64 = (pow10.hi + (pow10.hi shr (P + 0))) shr
+      (64 - P - betaMinus1)
+  ##  If we don't accept the left endpoint (but we do!) or
+  ##  if the left endpoint is not an integer, increase it
+  let lowerEndpointIsInteger: bool = (2 <= e2 and e2 <= 3)
+  let xi: uint64 = lowerEndpoint + uint64(not lowerEndpointIsInteger)
+  let zi: uint64 = upperEndpoint
+  ##  Try bigger divisor
+  var q: uint64 = zi div 10
+  if q * 10 >= xi:
+    return FloatingDecimal64(significand: q, exponent: minusK + 1)
+  q = ((pow10.hi shr (64 - (P + 1) - betaMinus1)) + 1) div 2
+  ##  When tie occurs, choose one of them according to the rule
+  if e2 == -77:
+    dec(q, ord(q mod 2 != 0))
+    ##  Round to even.
+  else:
+    inc(q, ord(q < xi))
+  return FloatingDecimal64(significand: q, exponent: minusK)
+
+proc computeDelta*(pow10: uint64x2; betaMinus1: int32): uint32 {.inline.} =
+  dragonbox_Assert(betaMinus1 >= 0)
+  dragonbox_Assert(betaMinus1 <= 63)
+  return cast[uint32](pow10.hi shr (64 - 1 - betaMinus1))
+
+when defined(sizeof_Int128):
+  proc mul128*(x: uint64; y: uint64): uint64x2 {.inline.} =
+    ##  1 mulx
+    type
+      uint128T = uint128
+    let p: uint128T = uint128T(x) * y
+    let hi: uint64 = cast[uint64](p shr 64)
+    let lo: uint64 = cast[uint64](p)
+    return (hi, lo)
+
+elif defined(vcc) and defined(cpu64):
+  proc mul128*(x: uint64; y: uint64): uint64x2 {.inline.} =
+    var hi: uint64 = 0
+    var lo: uint64 = umul128(x, y, addr(hi))
+    return (hi, lo)
+
+else:
+  proc lo32*(x: uint64): uint32 {.inline.} =
+    return cast[uint32](x)
+
+  proc hi32*(x: uint64): uint32 {.inline.} =
+    return cast[uint32](x shr 32)
+
+  proc mul128*(a: uint64; b: uint64): uint64x2 {.inline.} =
+    let b00: uint64 = uint64(lo32(a)) * lo32(b)
+    let b01: uint64 = uint64(lo32(a)) * hi32(b)
+    let b10: uint64 = uint64(hi32(a)) * lo32(b)
+    let b11: uint64 = uint64(hi32(a)) * hi32(b)
+    let mid1: uint64 = b10 + hi32(b00)
+    let mid2: uint64 = b01 + lo32(mid1)
+    let hi: uint64 = b11 + hi32(mid1) + hi32(mid2)
+    let lo: uint64 = lo32(b00) or uint64(lo32(mid2)) shl 32
+    return uint64x2(hi: hi, lo: lo)
+
+##  Returns (x * y) / 2^128
+
+proc mulShift*(x: uint64; y: uint64x2): uint64 {.inline.} =
+  ##  2 mulx
+  var p1: uint64x2 = mul128(x, y.hi)
+  var p0: uint64x2 = mul128(x, y.lo)
+  p1.lo += p0.hi
+  inc(p1.hi, ord(p1.lo < p0.hi))
+  return p1.hi
+
+proc mulParity*(twoF: uint64; pow10: uint64x2; betaMinus1: int32): bool {.inline.} =
+  ##  1 mulx, 1 mul
+  dragonbox_Assert(betaMinus1 >= 1)
+  dragonbox_Assert(betaMinus1 <= 63)
+  let p01: uint64 = twoF * pow10.hi
+  let p10: uint64 = mul128(twoF, pow10.lo).hi
+  let mid: uint64 = p01 + p10
+  return (mid and (uint64(1) shl (64 - betaMinus1))) != 0
+
+proc isIntegralEndpoint*(twoF: uint64; e2: int32; minusK: int32): bool {.inline.} =
+  if e2 < -2:
+    return false
+  if e2 <= 9:
+    return true
+  if e2 <= 86:
+    return multipleOfPow5(twoF, minusK)
+  return false
+
+proc isIntegralMidpoint*(twoF: uint64; e2: int32; minusK: int32): bool {.inline.} =
+  if e2 < -4:
+    return multipleOfPow2(twoF, minusK - e2 + 1)
+  if e2 <= 9:
+    return true
+  if e2 <= 86:
+    return multipleOfPow5(twoF, minusK)
+  return false
+
+proc toDecimal64*(ieeeSignificand: uint64; ieeeExponent: uint64): FloatingDecimal64 {.
+    inline.} =
+  const
+    kappa: int32 = 2
+  const
+    bigDivisor: uint32 = 1000
+  ##  10^(kappa + 1)
+  const
+    smallDivisor: uint32 = 100
+  ##  10^(kappa)
+  ##
+  ##  Step 1:
+  ##  integer promotion & Schubfach multiplier calculation.
+  ##
+  var m2: uint64
+  var e2: int32
+  if ieeeExponent != 0:
+    m2 = hiddenBit or ieeeSignificand
+    e2 = cast[int32](ieeeExponent) - exponentBias
+    if 0 <= -e2 and -e2 < significandSize and multipleOfPow2(m2, -e2):
+      ##  Small integer.
+      return FloatingDecimal64(significand: m2 shr -e2, exponent: 0)
+    if ieeeSignificand == 0 and ieeeExponent > 1:
+      ##  Shorter interval case; proceed like Schubfach.
+      return toDecimal64AsymmetricInterval(e2)
+  else:
+    ##  Subnormal case; interval is always regular.
+    m2 = ieeeSignificand
+    e2 = 1 - exponentBias
+  let isEven: bool = (m2 mod 2 == 0)
+  let acceptLower: bool = isEven
+  let acceptUpper: bool = isEven
+  ##  Compute k and beta.
+  let minusK: int32 = floorLog10Pow2(e2) - kappa
+  let betaMinus1: int32 = e2 + floorLog2Pow10(-minusK)
+  dragonbox_Assert(betaMinus1 >= 6)
+  dragonbox_Assert(betaMinus1 <= 9)
+  let pow10: uint64x2 = computePow10(-minusK)
+  ##  Compute delta
+  ##  10^kappa <= delta < 10^(kappa + 1)
+  ##       100 <= delta < 1000
+  let delta: uint32 = computeDelta(pow10, betaMinus1)
+  dragonbox_Assert(delta >= smallDivisor)
+  dragonbox_Assert(delta < bigDivisor)
+  let twoFl: uint64 = 2 * m2 - 1
+  let twoFc: uint64 = 2 * m2
+  let twoFr: uint64 = 2 * m2 + 1
+  ##  (54 bits)
+  ##  Compute zi
+  ##   (54 + 9 = 63 bits)
+  let zi: uint64 = mulShift(twoFr shl betaMinus1, pow10)
+  ##  2 mulx
+  ##
+  ##  Step 2:
+  ##  Try larger divisor.
+  ##
+  var q: uint64 = zi div bigDivisor
+  ##   uint64_t q = Mul128(zi, 0x83126E978D4FDF3Cu).hi >> 9; // 1 mulx
+  var r: uint32 = cast[uint32](zi) - bigDivisor * cast[uint32](q)
+  ##  r = zi % BigDivisor
+  ##  0 <= r < 1000
+  if r < delta:                  ## likely ~50% ?!
+            ##  (r > deltai)
+    ##  Exclude the right endpoint if necessary
+    if r != 0 or acceptUpper or not isIntegralEndpoint(twoFr, e2, minusK):
+      return FloatingDecimal64(significand: q, exponent: minusK + kappa + 1)
+    dragonbox_Assert(q != 0)
+    dec(q)
+    r = bigDivisor
+  elif r == delta:               ## unlikely
+    ##  Compare fractional parts.
+    ##  Check conditions in the order different from the paper
+    ##  to take advantage of short-circuiting
+    if (acceptLower and isIntegralEndpoint(twoFl, e2, minusK)) or
+        mulParity(twoFl, pow10, betaMinus1):
+      return FloatingDecimal64(significand: q, exponent: minusK + kappa + 1)
+  else:
+    discard
+  ##
+  ##  Step 3:
+  ##  Find the significand with the smaller divisor
+  ##
+  q = q * 10
+  ##  1 hmul
+  ##  0 <= r <= 1000
+  let dist: uint32 = r - (delta div 2) + (smallDivisor div 2)
+  let distQ: uint32 = dist div 100
+  ##  1 mul
+  ##   const uint32_t dist_r = dist % 100;
+  q += distQ
+  ##   if /*likely*/ (dist_r == 0)
+  if dist == distQ * 100:
+    ##       const bool approx_y_parity = ((dist ^ (SmallDivisor / 2)) & 1) != 0;
+    let approxYParity: bool = (dist and 1) != 0
+    ##  Check z^(f) >= epsilon^(f)
+    ##  We have either yi == zi - epsiloni or yi == (zi - epsiloni) - 1,
+    ##  where yi == zi - epsiloni if and only if z^(f) >= epsilon^(f)
+    ##  Since there are only 2 possibilities, we only need to care about the
+    ##  parity. Also, zi and r should have the same parity since the divisor
+    ##  is an even number
+    if mulParity(twoFc, pow10, betaMinus1) != approxYParity:
+      dec(q)
+    elif q mod 2 != 0 and isIntegralMidpoint(twoFc, e2, minusK):
+      dec(q)
+  return FloatingDecimal64(significand: q, exponent: minusK + kappa)
+
+## ==================================================================================================
+##  ToChars
+## ==================================================================================================
+
+proc utoa2Digits*(buf: var openArray[char]; pos: int; digits: uint32) {.inline.} =
+  const
+    digits100: array[200, char] = ['0', '0', '0', '1', '0', '2', '0', '3', '0', '4', '0', '5',
+      '0', '6', '0', '7', '0', '8', '0', '9', '1', '0', '1', '1', '1', '2', '1', '3', '1', '4',
+      '1', '5', '1', '6', '1', '7', '1', '8', '1', '9', '2', '0', '2', '1', '2', '2', '2', '3',
+      '2', '4', '2', '5', '2', '6', '2', '7', '2', '8', '2', '9', '3', '0', '3', '1', '3', '2',
+      '3', '3', '3', '4', '3', '5', '3', '6', '3', '7', '3', '8', '3', '9', '4', '0', '4', '1',
+      '4', '2', '4', '3', '4', '4', '4', '5', '4', '6', '4', '7', '4', '8', '4', '9', '5', '0',
+      '5', '1', '5', '2', '5', '3', '5', '4', '5', '5', '5', '6', '5', '7', '5', '8', '5', '9',
+      '6', '0', '6', '1', '6', '2', '6', '3', '6', '4', '6', '5', '6', '6', '6', '7', '6', '8',
+      '6', '9', '7', '0', '7', '1', '7', '2', '7', '3', '7', '4', '7', '5', '7', '6', '7', '7',
+      '7', '8', '7', '9', '8', '0', '8', '1', '8', '2', '8', '3', '8', '4', '8', '5', '8', '6',
+      '8', '7', '8', '8', '8', '9', '9', '0', '9', '1', '9', '2', '9', '3', '9', '4', '9', '5',
+      '9', '6', '9', '7', '9', '8', '9', '9']
+  dragonbox_Assert(digits <= 99)
+  buf[pos] = digits100[2 * digits]
+  buf[pos+1] = digits100[2 * digits + 1]
+  #copyMem(buf, unsafeAddr(digits100[2 * digits]), 2 * sizeof((char)))
+
+proc trailingZeros2Digits*(digits: uint32): int32 {.inline.} =
+  const
+    trailingZeros100: array[100, int8] = [2'i8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,
+      0, 0, 0, 0, 0, 0]
+  dragonbox_Assert(digits <= 99)
+  return trailingZeros100[digits]
+
+when false:
+  template `+!`(x: cstring; offset: int): cstring = cast[cstring](cast[uint](x) + uint(offset))
+
+  template dec(x: cstring; offset=1) = x = cast[cstring](cast[uint](x) - uint(offset))
+  template inc(x: cstring; offset=1) = x = cast[cstring](cast[uint](x) + uint(offset))
+
+  proc memset(x: cstring; ch: char; L: int) {.importc, nodecl.}
+  proc memmove(a, b: cstring; L: int) {.importc, nodecl.}
+
+proc utoa8DigitsSkipTrailingZeros*(buf: var openArray[char]; pos: int; digits: uint32): int32 {.inline.} =
+  dragonbox_Assert(digits >= 1)
+  dragonbox_Assert(digits <= 99999999'u32)
+  let q: uint32 = digits div 10000
+  let r: uint32 = digits mod 10000
+  let qH: uint32 = q div 100
+  let qL: uint32 = q mod 100
+  utoa2Digits(buf, pos, qH)
+  utoa2Digits(buf, pos + 2, qL)
+  if r == 0:
+    return trailingZeros2Digits(if qL == 0: qH else: qL) + (if qL == 0: 6 else: 4)
+  else:
+    let rH: uint32 = r div 100
+    let rL: uint32 = r mod 100
+    utoa2Digits(buf, pos + 4, rH)
+    utoa2Digits(buf, pos + 6, rL)
+    return trailingZeros2Digits(if rL == 0: rH else: rL) + (if rL == 0: 2 else: 0)
+
+proc printDecimalDigitsBackwards*(buf: var openArray[char]; pos: int; output64: uint64): int32 {.inline.} =
+  var pos = pos
+  var output64 = output64
+  var tz: int32 = 0
+  ##  number of trailing zeros removed.
+  var nd: int32 = 0
+  ##  number of decimal digits processed.
+  ##  At most 17 digits remaining
+  if output64 >= 100000000'u64:
+    let q: uint64 = output64 div 100000000'u64
+    let r: uint32 = cast[uint32](output64 mod 100000000'u64)
+    output64 = q
+    dec(pos, 8)
+    if r != 0:
+      tz = utoa8DigitsSkipTrailingZeros(buf, pos, r)
+      dragonbox_Assert(tz >= 0)
+      dragonbox_Assert(tz <= 7)
+    else:
+      tz = 8
+    nd = 8
+  dragonbox_Assert(output64 <= high(uint32))
+  var output = cast[uint32](output64)
+  if output >= 10000:
+    let q: uint32 = output div 10000
+    let r: uint32 = output mod 10000
+    output = q
+    dec(pos, 4)
+    if r != 0:
+      let rH: uint32 = r div 100
+      let rL: uint32 = r mod 100
+      utoa2Digits(buf, pos, rH)
+      utoa2Digits(buf, pos + 2, rL)
+      if tz == nd:
+        inc(tz, trailingZeros2Digits(if rL == 0: rH else: rL) +
+            (if rL == 0: 2 else: 0))
+    else:
+      if tz == nd:
+        inc(tz, 4)
+      else:
+        for i in 0..3: buf[pos+i] = '0'
+      ##  (actually not required...)
+    inc(nd, 4)
+  if output >= 100:
+    let q: uint32 = output div 100
+    let r: uint32 = output mod 100
+    output = q
+    dec(pos, 2)
+    utoa2Digits(buf, pos, r)
+    if tz == nd:
+      inc(tz, trailingZeros2Digits(r))
+    inc(nd, 2)
+    if output >= 100:
+      let q2: uint32 = output div 100
+      let r2: uint32 = output mod 100
+      output = q2
+      dec(pos, 2)
+      utoa2Digits(buf, pos, r2)
+      if tz == nd:
+        inc(tz, trailingZeros2Digits(r2))
+      inc(nd, 2)
+  dragonbox_Assert(output >= 1)
+  dragonbox_Assert(output <= 99)
+  if output >= 10:
+    let q: uint32 = output
+    dec(pos, 2)
+    utoa2Digits(buf, pos, q)
+    if tz == nd:
+      inc(tz, trailingZeros2Digits(q))
+  else:
+    let q: uint32 = output
+    dragonbox_Assert(q >= 1)
+    dragonbox_Assert(q <= 9)
+    dec(pos)
+    buf[pos] = chr(ord('0') + q)
+  return tz
+
+proc decimalLength*(v: uint64): int32 {.inline.} =
+  dragonbox_Assert(v >= 1)
+  dragonbox_Assert(v <= 99999999999999999'u64)
+  if cast[uint32](v shr 32) != 0:
+    if v >= 10000000000000000'u64:
+      return 17
+    if v >= 1000000000000000'u64:
+      return 16
+    if v >= 100000000000000'u64:
+      return 15
+    if v >= 10000000000000'u64:
+      return 14
+    if v >= 1000000000000'u64:
+      return 13
+    if v >= 100000000000'u64:
+      return 12
+    if v >= 10000000000'u64:
+      return 11
+    return 10
+  let v32: uint32 = cast[uint32](v)
+  if v32 >= 1000000000'u:
+    return 10
+  if v32 >= 100000000'u:
+    return 9
+  if v32 >= 10000000'u:
+    return 8
+  if v32 >= 1000000'u:
+    return 7
+  if v32 >= 100000'u:
+    return 6
+  if v32 >= 10000'u:
+    return 5
+  if v32 >= 1000'u:
+    return 4
+  if v32 >= 100'u:
+    return 3
+  if v32 >= 10'u:
+    return 2
+  return 1
+
+proc formatDigits*(buffer: var openArray[char]; pos: int; digits: uint64; decimalExponent: int32;
+                  forceTrailingDotZero = false): int {.inline.} =
+  const
+    minFixedDecimalPoint: int32 = -6
+  const
+    maxFixedDecimalPoint: int32 = 17
+  var pos = pos
+  assert(minFixedDecimalPoint <= -1, "internal error")
+  assert(maxFixedDecimalPoint >= 17, "internal error")
+  dragonbox_Assert(digits >= 1)
+  dragonbox_Assert(digits <= 99999999999999999'u64)
+  dragonbox_Assert(decimalExponent >= -999)
+  dragonbox_Assert(decimalExponent <= 999)
+  var numDigits: int32 = decimalLength(digits)
+  let decimalPoint: int32 = numDigits + decimalExponent
+  let useFixed: bool = minFixedDecimalPoint <= decimalPoint and
+      decimalPoint <= maxFixedDecimalPoint
+  ## Prepare the buffer.
+  for i in 0..<32: buffer[pos+i] = '0'
+  assert(minFixedDecimalPoint >= -30, "internal error")
+  assert(maxFixedDecimalPoint <= 32, "internal error")
+  var decimalDigitsPosition: int32
+  if useFixed:
+    if decimalPoint <= 0:
+      ##  0.[000]digits
+      decimalDigitsPosition = 2 - decimalPoint
+    else:
+      ##  dig.its
+      ##  digits[000]
+      decimalDigitsPosition = 0
+  else:
+    ##  dE+123 or d.igitsE+123
+    decimalDigitsPosition = 1
+  var digitsEnd = pos + int(decimalDigitsPosition + numDigits)
+  let tz: int32 = printDecimalDigitsBackwards(buffer, digitsEnd, digits)
+  dec(digitsEnd, tz)
+  dec(numDigits, tz)
+  ##   decimal_exponent += tz; // => decimal_point unchanged.
+  if useFixed:
+    if decimalPoint <= 0:
+      ##  0.[000]digits
+      buffer[pos+1] = '.'
+      pos = digitsEnd
+    elif decimalPoint < numDigits:
+      ##  dig.its
+      when true: #defined(vcc) and not defined(clang):
+        ##  VC does not inline the memmove call below. (Even if compiled with /arch:AVX2.)
+        ##  However, memcpy will be inlined.
+        var tmp: array[16, char]
+        for i in 0..<16: tmp[i] = buffer[i+pos+decimalPoint]
+        for i in 0..<16: buffer[i+pos+decimalPoint+1] = tmp[i]
+      else:
+        memmove(buffer +! (decimalPoint + 1), buffer +! decimalPoint, 16)
+      buffer[pos+decimalPoint] = '.'
+      pos = digitsEnd + 1
+    else:
+      ##  digits[000]
+      inc(pos, decimalPoint)
+      if forceTrailingDotZero:
+        buffer[pos] = '.'
+        buffer[pos+1] = '0'
+        inc(pos, 2)
+  else:
+    ##  Copy the first digit one place to the left.
+    buffer[pos] = buffer[pos+1]
+    if numDigits == 1:
+      ##  dE+123
+      inc(pos)
+    else:
+      ##  d.igitsE+123
+      buffer[pos+1] = '.'
+      pos = digitsEnd
+    let scientificExponent: int32 = decimalPoint - 1
+    ##       SF_ASSERT(scientific_exponent != 0);
+    buffer[pos] = 'e'
+    buffer[pos+1] = if scientificExponent < 0: '-' else: '+'
+    inc(pos, 2)
+    let k: uint32 = cast[uint32](if scientificExponent < 0: -scientificExponent else: scientificExponent)
+    if k < 10:
+      buffer[pos] = chr(ord('0') + k)
+      inc(pos)
+    elif k < 100:
+      utoa2Digits(buffer, pos, k)
+      inc(pos, 2)
+    else:
+      let q: uint32 = k div 100
+      let r: uint32 = k mod 100
+      buffer[pos] = chr(ord('0') + q)
+      inc(pos)
+      utoa2Digits(buffer, pos, r)
+      inc(pos, 2)
+  return pos
+
+proc toChars*(buffer: var openArray[char]; v: float; forceTrailingDotZero = false): int {.
+    inline.} =
+  var pos = 0
+  let significand: uint64 = physicalSignificand(constructDouble(v))
+  let exponent: uint64 = physicalExponent(constructDouble(v))
+  if exponent != maxIeeeExponent:
+    ##  Finite
+    buffer[pos] = '-'
+    inc(pos, signBit(constructDouble(v)))
+    if exponent != 0 or significand != 0:
+      ##  != 0
+      let dec = toDecimal64(significand, exponent)
+      return formatDigits(buffer, pos, dec.significand, dec.exponent,
+                         forceTrailingDotZero)
+    else:
+      buffer[pos] = '0'
+      buffer[pos+1] = '.'
+      buffer[pos+2] = '0'
+      buffer[pos+3] = ' '
+      inc(pos, if forceTrailingDotZero: 3 else: 1)
+      return pos
+  if significand == 0:
+    buffer[pos] = '-'
+    inc(pos, signBit(constructDouble(v)))
+    buffer[pos] = 'i'
+    buffer[pos+1] = 'n'
+    buffer[pos+2] = 'f'
+    buffer[pos+3] = ' '
+    return pos + 3
+  else:
+    buffer[pos] = 'n'
+    buffer[pos+1] = 'a'
+    buffer[pos+2] = 'n'
+    buffer[pos+3] = ' '
+    return pos + 3
+
+when false:
+  proc toString*(value: float): string =
+    var buffer: array[dtoaMinBufferLength, char]
+    let last = toChars(addr buffer, value)
+    let L = cast[int](last) - cast[int](addr(buffer))
+    result = newString(L)
+    copyMem(addr result[0], addr buffer[0], L)
+
diff --git a/lib/system/formatfloat.nim b/lib/system/formatfloat.nim
index cb46c8c36..41bc56adc 100644
--- a/lib/system/formatfloat.nim
+++ b/lib/system/formatfloat.nim
@@ -7,53 +7,58 @@
 #    distribution, for details about the copyright.
 #
 
-proc c_sprintf(buf, frmt: cstring): cint {.header: "",
-                                    importc: "sprintf", varargs, noSideEffect.}
+when not defined(nimLegacyAddFloat) and not defined(nimscript) and
+    not defined(js) and defined(nimHasDragonBox):
+  import dragonbox
 
-proc writeToBuffer(buf: var array[65, char]; value: cstring) =
-  var i = 0
-  while value[i] != '\0':
-    buf[i] = value[i]
-    inc i
+  proc writeFloatToBuffer*(buf: var array[65, char]; value: BiggestFloat): int =
+    ## This is the implementation to format floats.
+    ##
+    ## returns the amount of bytes written to `buf` not counting the
+    ## terminating '\0' character.
+    result = toChars(buf, value, forceTrailingDotZero=true)
+    buf[result] = '\0'
 
-proc writeFloatToBuffer*(buf: var array[65, char]; value: BiggestFloat): int =
-  ## This is the implementation to format floats in the Nim
-  ## programming language. The specific format for floating point
-  ## numbers is not specified in the Nim programming language and
-  ## might change slightly in the future, but at least wherever you
-  ## format a float, it should be consistent.
-  ##
-  ## returns the amount of bytes written to `buf` not counting the
-  ## terminating '\0' character.
-  ##
-  ## * `buf` - A buffer to write into. The buffer does not need to be
-  ##           initialized and it will be overridden.
-  ##
-  var n: int = c_sprintf(addr buf, "%.16g", value)
-  var hasDot = false
-  for i in 0..n-1:
-    if buf[i] == ',':
-      buf[i] = '.'
-      hasDot = true
-    elif buf[i] in {'a'..'z', 'A'..'Z', '.'}:
-      hasDot = true
-  if not hasDot:
-    buf[n] = '.'
-    buf[n+1] = '0'
-    buf[n+2] = '\0'
-    result = n + 2
-  else:
-    result = n
-  # On Windows nice numbers like '1.#INF', '-1.#INF' or '1.#NAN' or 'nan(ind)'
-  # of '-1.#IND' are produced.
-  # We want to get rid of these here:
-  if buf[n-1] in {'n', 'N', 'D', 'd', ')'}:
-    writeToBuffer(buf, "nan")
-    result = 3
-  elif buf[n-1] == 'F':
-    if buf[0] == '-':
-      writeToBuffer(buf, "-inf")
-      result = 4
+else:
+  proc c_sprintf(buf, frmt: cstring): cint {.header: "",
+                                      importc: "sprintf", varargs, noSideEffect.}
+
+  proc writeToBuffer(buf: var array[65, char]; value: cstring) =
+    var i = 0
+    while value[i] != '\0':
+      buf[i] = value[i]
+      inc i
+
+  proc writeFloatToBuffer*(buf: var array[65, char]; value: BiggestFloat): int =
+    ## This is the implementation to format floats.
+    ##
+    ## returns the amount of bytes written to `buf` not counting the
+    ## terminating '\0' character.
+    var n: int = c_sprintf(addr buf, "%.16g", value)
+    var hasDot = false
+    for i in 0..n-1:
+      if buf[i] == ',':
+        buf[i] = '.'
+        hasDot = true
+      elif buf[i] in {'a'..'z', 'A'..'Z', '.'}:
+        hasDot = true
+    if not hasDot:
+      buf[n] = '.'
+      buf[n+1] = '0'
+      buf[n+2] = '\0'
+      result = n + 2
     else:
-      writeToBuffer(buf, "inf")
+      result = n
+    # On Windows nice numbers like '1.#INF', '-1.#INF' or '1.#NAN' or 'nan(ind)'
+    # of '-1.#IND' are produced.
+    # We want to get rid of these here:
+    if buf[n-1] in {'n', 'N', 'D', 'd', ')'}:
+      writeToBuffer(buf, "nan")
       result = 3
+    elif buf[n-1] == 'F':
+      if buf[0] == '-':
+        writeToBuffer(buf, "-inf")
+        result = 4
+      else:
+        writeToBuffer(buf, "inf")
+        result = 3
diff --git a/tests/float/tfloat6.nim b/tests/float/tfloat6.nim
index c4cd6e932..c0e6e419f 100644
--- a/tests/float/tfloat6.nim
+++ b/tests/float/tfloat6.nim
@@ -1,10 +1,10 @@
 discard """
   output: '''
-1e-06 : 1e-06
-1e-06 : 1e-06
+0.000001 : 0.000001
+0.000001 : 0.000001
 0.001 : 0.001
-1e-06 : 1e-06
-1e-06 : 1e-06
+0.000001 : 0.000001
+0.000001 : 0.000001
 10.000001 : 10.000001
 100.000001 : 100.000001
 '''

From f10eef29b5d77ad45b00b0d4eb7e87f930c2a0a4 Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Wed, 2 Jun 2021 13:13:23 +0200
Subject: [PATCH 0867/2472] fixes #18059 (#18140)

* fixes #18059
---
 compiler/semexprs.nim   | 28 ++++++++++++++++++++++++++--
 compiler/semstmts.nim   | 20 +++++++++++++++++++-
 tests/metatype/tcps.nim | 35 +++++++++++++++++++++++++++++++++++
 3 files changed, 80 insertions(+), 3 deletions(-)
 create mode 100644 tests/metatype/tcps.nim

diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim
index 8c4f25e12..c839d04d9 100644
--- a/compiler/semexprs.nim
+++ b/compiler/semexprs.nim
@@ -54,7 +54,8 @@ proc semOperand(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
   result = semExpr(c, n, flags + {efOperand})
   if result.typ != nil:
     # XXX tyGenericInst here?
-    if result.typ.kind == tyProc and tfUnresolved in result.typ.flags:
+    if result.typ.kind == tyProc and hasUnresolvedParams(result, {efOperand}):
+      #and tfUnresolved in result.typ.flags:
       localError(c.config, n.info, errProcHasNoConcreteType % n.renderTree)
     if result.typ.kind in {tyVar, tyLent}: result = newDeref(result)
   elif {efWantStmt, efAllowStmt} * flags != {}:
@@ -2649,6 +2650,23 @@ proc shouldBeBracketExpr(n: PNode): bool =
           n[0] = be
           return true
 
+proc asBracketExpr(c: PContext; n: PNode): PNode =
+  proc isGeneric(c: PContext; n: PNode): bool =
+    if n.kind in {nkIdent, nkAccQuoted}:
+      let s = qualifiedLookUp(c, n, {})
+      result = s != nil and isGenericRoutineStrict(s)
+
+  assert n.kind in nkCallKinds
+  if n.len > 1 and isGeneric(c, n[1]):
+    let b = n[0]
+    if b.kind in nkSymChoices:
+      for i in 0..
Date: Wed, 2 Jun 2021 13:14:10 +0200
Subject: [PATCH 0868/2472] refactoring: removed empty withBracketExpr template
 (#18152)

---
 compiler/semgnrc.nim  |  6 ++----
 compiler/semtempl.nim | 14 ++++----------
 2 files changed, 6 insertions(+), 14 deletions(-)

diff --git a/compiler/semgnrc.nim b/compiler/semgnrc.nim
index ebd361db6..4f3c50dae 100644
--- a/compiler/semgnrc.nim
+++ b/compiler/semgnrc.nim
@@ -303,8 +303,7 @@ proc semGenericStmt(c: PContext, n: PNode,
     result = newNodeI(nkCall, n.info)
     result.add newIdentNode(getIdent(c.cache, "[]"), n.info)
     for i in 0..
Date: Wed, 2 Jun 2021 09:02:14 -0700
Subject: [PATCH 0870/2472] fix #16993, #18054, #17835 runnableExamples now
 works with templates and nested templates (#18082)

---
 compiler/evaltempl.nim                        | 12 ++--
 lib/system/assertions.nim                     | 47 ++++++---------
 .../expected/subdir/subdir_b/utils.html       |  4 +-
 nimdoc/testproject/expected/testproject.html  | 11 ++--
 nimdoc/testproject/subdir/subdir_b/utils.nim  | 10 +++-
 tests/js/tstdlib_various.nim                  |  2 +-
 tests/nimdoc/trunnableexamples.nim            | 57 ++++++++++++++++++-
 tests/nimdoc/trunnableexamples2.nim           | 11 ++++
 8 files changed, 108 insertions(+), 46 deletions(-)
 create mode 100644 tests/nimdoc/trunnableexamples2.nim

diff --git a/compiler/evaltempl.nim b/compiler/evaltempl.nim
index a85314ac2..8a6bf9287 100644
--- a/compiler/evaltempl.nim
+++ b/compiler/evaltempl.nim
@@ -83,10 +83,14 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
         not c.isDeclarative:
       c.isDeclarative = true
       isDeclarative = true
-    var res = copyNode(c, templ, actual)
-    for i in 0..`_.
+  runnableExamples: assert 1 == 1
+  runnableExamples("--assertions:off"):
+    assert 1 == 2 # no code generated, no failure here
+  runnableExamples("-d:danger"): assert 1 == 2 # ditto
   assertImpl(cond, msg, astToStr(cond), compileOption("assertions"))
 
 template doAssert*(cond: untyped, msg = "") =
   ## Similar to `assert <#assert.t,untyped,string>`_ but is always turned on regardless of `--assertions`.
+  runnableExamples:
+    doAssert 1 == 1 # generates code even when built with `-d:danger` or `--assertions:off`
   assertImpl(cond, msg, astToStr(cond), true)
 
 template onFailedAssert*(msg, code: untyped): untyped {.dirty.} =
   ## Sets an assertion failure handler that will intercept any assert
   ## statements following `onFailedAssert` in the current scope.
+  runnableExamples:
+    type MyError = object of CatchableError
+      lineinfo: tuple[filename: string, line: int, column: int]
+    # block-wide policy to change the failed assert exception type in order to
+    # include a lineinfo
+    onFailedAssert(msg):
+      raise (ref MyError)(msg: msg, lineinfo: instantiationInfo(-2))
+    doAssertRaises(MyError): doAssert false
   template failedAssertImpl(msgIMPL: string): untyped {.dirty.} =
     let msg = msgIMPL
     code
 
 template doAssertRaises*(exception: typedesc, code: untyped) =
   ## Raises `AssertionDefect` if specified `code` does not raise `exception`.
+  runnableExamples:
+    doAssertRaises(ValueError): raise newException(ValueError, "Hello World")
+    doAssertRaises(CatchableError): raise newException(ValueError, "Hello World")
+    doAssertRaises(AssertionDefect): doAssert false
   var wrong = false
   const begin = "expected raising '" & astToStr(exception) & "', instead"
   const msgEnd = " by: " & astToStr(code)
diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html
index 794066cae..826cad02a 100644
--- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html
+++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html
@@ -206,9 +206,9 @@ constructor.
 
    template fromUtilsGen(): untyped
    
 
    
 
-this should be shown in utils.html
+should be shown in utils.html only
 
    Example:
    
-
    assert 3*2 == 6
    ditto
+
    discard "should be in utils.html only, not in module that calls fromUtilsGen"
    ditto
 
 
    
 
diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html
index d2fe38021..513dd507b 100644
--- a/nimdoc/testproject/expected/testproject.html
+++ b/nimdoc/testproject/expected/testproject.html
@@ -431,10 +431,7 @@ window.addEventListener('DOMContentLoaded', main);
 discard "in top2"
    top2 after
 
    Example:
    
 
    import testproject
-discard "in top3"
    top3 after
-
    Example:
    
-
    import testproject
-assert 3*2 == 6
    
+discard "in top3"
    top3 after

    Imports

    @@ -574,7 +571,8 @@ My someFunc. Stuff in fromUtilsGen is called

    Example:

    -
    discard 1
    +
    discard """should be shown as examples for fromUtils3
+       in module calling fromUtilsGen"""
    @@ -957,6 +955,9 @@ cz18
    ok3 +

    Example:

    +
    discard """should be shown as examples for fromUtils2
+       in module calling fromUtilsGen"""
    diff --git a/nimdoc/testproject/subdir/subdir_b/utils.nim b/nimdoc/testproject/subdir/subdir_b/utils.nim index 128e8e481..4e3aa1f10 100644 --- a/nimdoc/testproject/subdir/subdir_b/utils.nim +++ b/nimdoc/testproject/subdir/subdir_b/utils.nim @@ -50,9 +50,9 @@ template bEnum*(): untyped = discard template fromUtilsGen*(): untyped = - ## this should be shown in utils.html + ## should be shown in utils.html only runnableExamples: - assert 3*2 == 6 + discard "should be in utils.html only, not in module that calls fromUtilsGen" ## ditto iterator fromUtils1*(): int = @@ -64,7 +64,11 @@ template fromUtilsGen*(): untyped = template fromUtils2*() = ## ok3 + runnableExamples: + discard """should be shown as examples for fromUtils2 + in module calling fromUtilsGen""" proc fromUtils3*() = ## came form utils but should be shown where `fromUtilsGen` is called - runnableExamples: discard 1 + runnableExamples: discard """should be shown as examples for fromUtils3 + in module calling fromUtilsGen""" diff --git a/tests/js/tstdlib_various.nim b/tests/js/tstdlib_various.nim index a1bb63d46..4b5ce1de8 100644 --- a/tests/js/tstdlib_various.nim +++ b/tests/js/tstdlib_various.nim @@ -153,7 +153,7 @@ block tsplit2: var errored = false try: discard "hello".split("") - except AssertionError: + except AssertionDefect: errored = true doAssert errored diff --git a/tests/nimdoc/trunnableexamples.nim b/tests/nimdoc/trunnableexamples.nim index bc5ea0e5e..ac7a0e26f 100644 --- a/tests/nimdoc/trunnableexamples.nim +++ b/tests/nimdoc/trunnableexamples.nim @@ -1,6 +1,7 @@ discard """ -cmd: "nim doc --doccmd:-d:testFooExternal --hints:off $file" +cmd: "nim doc --doccmd:--hints:off --hints:off $file" action: "compile" +nimoutFull: true nimout: ''' foo1 foo2 @@ -8,12 +9,23 @@ foo3 foo5 foo6 foo7 +in examplesInTemplate1 +doc in outer +doc in inner1 +doc in inner2 foo8 foo9 ''' joinable: false """ +#[ +pending bug #18077, use instead: +cmd: "nim doc --doccmd:'-d:testFooExternal --hints:off' --hints:off $file" +and merge trunnableexamples2 back here +]# +{.define(testFooExternal).} + proc fun*() = runnableExamples: block: # `defer` only allowed inside a block @@ -109,6 +121,47 @@ when true: # runnableExamples with rdoccmd # passing seq (to run with multiple compilation options) runnableExamples(@["-b:cpp", "-b:js"]): discard +when true: # bug #16993 + template examplesInTemplate1*(cond: untyped) = + ## in examplesInTemplate1 + runnableExamples: + echo "in examplesInTemplate1" + discard + examplesInTemplate1 true + examplesInTemplate1 true + examplesInTemplate1 true + +when true: # bug #18054 + template outer*(body: untyped) = + ## outer template doc string. + runnableExamples: + echo "doc in outer" + ## + template inner1*() = + ## inner1 template doc string. + runnableExamples: + echo "doc in inner1" + ## + + template inner2*() = + ## inner2 template doc string. + runnableExamples: + echo "doc in inner2" + body + outer: + inner1() + inner2() + +when true: # bug #17835 + template anyItFake*(s, pred: untyped): bool = + ## Foo + runnableExamples: discard + true + + proc anyItFakeMain*(n: seq[int]): bool = + result = anyItFake(n, it == 0) + # this was giving: Error: runnableExamples must appear before the first non-comment statement + runnableExamples: block: # bug #17279 when int.sizeof == 8: @@ -118,7 +171,7 @@ runnableExamples: # bug #13491 block: proc fun(): int = doAssert false - doAssertRaises(AssertionError, (discard fun())) + doAssertRaises(AssertionDefect, (discard fun())) block: template foo(body) = discard diff --git a/tests/nimdoc/trunnableexamples2.nim b/tests/nimdoc/trunnableexamples2.nim new file mode 100644 index 000000000..5a437744e --- /dev/null +++ b/tests/nimdoc/trunnableexamples2.nim @@ -0,0 +1,11 @@ +discard """ +cmd: "nim doc --doccmd:-d:testFooExternal --hints:off $file" +action: "compile" +joinable: false +""" + +# pending bug #18077, merge back inside trunnableexamples.nim +when true: # runnableExamples with rdoccmd + runnableExamples "-d:testFoo -d:testBar": + doAssert defined(testFoo) and defined(testBar) + doAssert defined(testFooExternal) From f27f3f65df0bab567cc416cd1332a502833bd12f Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 3 Jun 2021 10:59:06 +0800 Subject: [PATCH 0871/2472] [std/hashcommon]improve docs a bit (#18153) * reduce duplicated deprecated messages --- lib/pure/collections/hashcommon.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pure/collections/hashcommon.nim b/lib/pure/collections/hashcommon.nim index 030446176..a169418ce 100644 --- a/lib/pure/collections/hashcommon.nim +++ b/lib/pure/collections/hashcommon.nim @@ -34,10 +34,10 @@ proc slotsNeeded(count: Natural): int {.inline.} = result = nextPowerOfTwo(count * 3 div 2 + 4) proc rightSize*(count: Natural): int {.inline, deprecated: "Deprecated since 1.4.0".} = - ## **Deprecated since Nim v1.4.0**, it is not needed anymore - ## because picking the correct size is done internally. + ## It is not needed anymore because + ## picking the correct size is done internally. ## - ## Return the value of `initialSize` to support `count` items. + ## Returns the value of `initialSize` to support `count` items. ## ## If more items are expected to be added, simply add that ## expected extra amount to the parameter before calling this. From bbce3d2da919bef2900e4061b9b8a99cb84fdf57 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 3 Jun 2021 12:09:04 +0800 Subject: [PATCH 0872/2472] [std/tables] remove unnecessary `do: ` (#18160) --- lib/pure/collections/tables.nim | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index b71bb0845..d617e06e4 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -594,17 +594,17 @@ template withValue*[A, B](t: var Table[A, B], key: A, value, body: untyped) = let u = User(name: "Hello", uid: 99) t[1] = u - t.withValue(1, value) do: + t.withValue(1, value): # block is executed only if `key` in `t` value.name = "Nim" value.uid = 1314 - t.withValue(2, value) do: + t.withValue(2, value): value.name = "No" value.uid = 521 - doAssert t[1].name == "Nim" - doAssert t[1].uid == 1314 + assert t[1].name == "Nim" + assert t[1].uid == 1314 mixin rawGet var hc: Hash @@ -629,16 +629,22 @@ template withValue*[A, B](t: var Table[A, B], key: A, let u = User(name: "Hello", uid: 99) t[1] = u - t.withValue(1, value) do: + t.withValue(1, value): # block is executed only if `key` in `t` value.name = "Nim" value.uid = 1314 - # do: - # # block is executed when `key` not in `t` - # raise newException(KeyError, "Key not found") - doAssert t[1].name == "Nim" - doAssert t[1].uid == 1314 + t.withValue(521, value): + doAssert false + do: + # block is executed when `key` not in `t` + t[1314] = User(name: "exist", uid: 521) + + assert t[1].name == "Nim" + assert t[1].uid == 1314 + assert t[1314].name == "exist" + assert t[1314].uid == 521 + mixin rawGet var hc: Hash var index = rawGet(t, key, hc) From 06960bb9cb6bb548c8f72882169ae65e14f29e89 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 3 Jun 2021 13:35:24 +0800 Subject: [PATCH 0873/2472] Ref #17697 improve withValue docs (#18154) * Ref #17697 improve withValue docs * address comments --- lib/pure/collections/sharedtables.nim | 81 +++++++++++++++++---------- 1 file changed, 50 insertions(+), 31 deletions(-) diff --git a/lib/pure/collections/sharedtables.nim b/lib/pure/collections/sharedtables.nim index 4ac3befb6..2788ec785 100644 --- a/lib/pure/collections/sharedtables.nim +++ b/lib/pure/collections/sharedtables.nim @@ -60,17 +60,27 @@ template withLock(t, x: untyped) = template withValue*[A, B](t: var SharedTable[A, B], key: A, value, body: untyped) = - ## retrieves the value at `t[key]`. + ## Retrieves the value at `t[key]`. ## `value` can be modified in the scope of the `withValue` call. - ## - ## .. code-block:: nim - ## - ## sharedTable.withValue(key, value) do: - ## # block is executed only if `key` in `t` - ## # value is threadsafe in block - ## value.name = "username" - ## value.uid = 1000 - ## + runnableExamples: + var table: SharedTable[string, string] + init(table) + + table["a"] = "x" + table["b"] = "y" + table["c"] = "z" + + table.withValue("a", value): + assert value[] == "x" + + table.withValue("b", value): + value[] = "modified" + + table.withValue("b", value): + assert value[] == "modified" + + table.withValue("nonexistent", value): + assert false # not called acquire(t.lock) try: var hc: Hash @@ -84,20 +94,29 @@ template withValue*[A, B](t: var SharedTable[A, B], key: A, template withValue*[A, B](t: var SharedTable[A, B], key: A, value, body1, body2: untyped) = - ## retrieves the value at `t[key]`. + ## Retrieves the value at `t[key]`. ## `value` can be modified in the scope of the `withValue` call. - ## - ## .. code-block:: nim - ## - ## sharedTable.withValue(key, value) do: - ## # block is executed only if `key` in `t` - ## # value is threadsafe in block - ## value.name = "username" - ## value.uid = 1000 - ## do: - ## # block is executed when `key` not in `t` - ## raise newException(KeyError, "Key not found") - ## + runnableExamples: + var table: SharedTable[string, string] + init(table) + + table["a"] = "x" + table["b"] = "y" + table["c"] = "z" + + + table.withValue("a", value): + value[] = "m" + + table.withValue("d", value): + discard value + doAssert false + do: # if "d" notin table + table["d"] = "n" + + assert table.mget("a") == "m" + assert table.mget("d") == "n" + acquire(t.lock) try: var hc: Hash @@ -112,7 +131,7 @@ template withValue*[A, B](t: var SharedTable[A, B], key: A, release(t.lock) proc mget*[A, B](t: var SharedTable[A, B], key: A): var B = - ## retrieves the value at `t[key]`. The value can be modified. + ## Retrieves the value at `t[key]`. The value can be modified. ## If `key` is not in `t`, the `KeyError` exception is raised. withLock t: var hc: Hash @@ -126,7 +145,7 @@ proc mget*[A, B](t: var SharedTable[A, B], key: A): var B = raise newException(KeyError, "key not found") proc mgetOrPut*[A, B](t: var SharedTable[A, B], key: A, val: B): var B = - ## retrieves value at `t[key]` or puts `val` if not present, either way + ## Retrieves value at `t[key]` or puts `val` if not present, either way ## returning a value which can be modified. **Note**: This is inherently ## unsafe in the context of multi-threading since it returns a pointer ## to `B`. @@ -134,7 +153,7 @@ proc mgetOrPut*[A, B](t: var SharedTable[A, B], key: A, val: B): var B = mgetOrPutImpl(enlarge) proc hasKeyOrPut*[A, B](t: var SharedTable[A, B], key: A, val: B): bool = - ## returns true if `key` is in the table, otherwise inserts `value`. + ## Returns true if `key` is in the table, otherwise inserts `value`. withLock t: hasKeyOrPutImpl(enlarge) @@ -191,28 +210,28 @@ proc withKey*[A, B](t: var SharedTable[A, B], key: A, st_maybeRehashPutImpl(enlarge) proc `[]=`*[A, B](t: var SharedTable[A, B], key: A, val: B) = - ## puts a (key, value)-pair into `t`. + ## Puts a (key, value)-pair into `t`. withLock t: putImpl(enlarge) proc add*[A, B](t: var SharedTable[A, B], key: A, val: B) = - ## puts a new (key, value)-pair into `t` even if `t[key]` already exists. + ## Puts a new (key, value)-pair into `t` even if `t[key]` already exists. ## This can introduce duplicate keys into the table! withLock t: addImpl(enlarge) proc del*[A, B](t: var SharedTable[A, B], key: A) = - ## deletes `key` from hash table `t`. + ## Deletes `key` from hash table `t`. withLock t: delImpl(tabMakeEmpty, tabCellEmpty, tabCellHash) proc len*[A, B](t: var SharedTable[A, B]): int = - ## number of elements in `t` + ## Number of elements in `t`. withLock t: result = t.counter proc init*[A, B](t: var SharedTable[A, B], initialSize = 32) = - ## creates a new hash table that is empty. + ## Creates a new hash table that is empty. ## ## This proc must be called before any other usage of `t`. let initialSize = slotsNeeded(initialSize) From daaa40973c65c46f8d38dc6033f56fffcd7d8d3f Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 3 Jun 2021 12:09:40 +0200 Subject: [PATCH 0874/2472] added float32 schubfach algorithm; wip (#18155) * added float32 schubfach algorithm; wip * fixes #18418 --- compiler/ccgexprs.nim | 6 +- lib/system/dollars.nim | 4 + lib/system/schubfach.nim | 451 +++++++++++++++++++++++++++++++++++++++ lib/system/strmantle.nim | 13 ++ tests/float/tfloat6.nim | 5 + 5 files changed, 478 insertions(+), 1 deletion(-) create mode 100644 lib/system/schubfach.nim diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 7cb403a94..7ed80145b 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -2300,7 +2300,11 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = of mInt64ToStr: genDollar(p, e, d, "#nimInt64ToStr($1)") of mBoolToStr: genDollar(p, e, d, "#nimBoolToStr($1)") of mCharToStr: genDollar(p, e, d, "#nimCharToStr($1)") - of mFloatToStr: genDollar(p, e, d, "#nimFloatToStr($1)") + of mFloatToStr: + if e[1].typ.skipTypes(abstractInst).kind == tyFloat32: + genDollar(p, e, d, "#nimFloat32ToStr($1)") + else: + genDollar(p, e, d, "#nimFloatToStr($1)") of mCStrToStr: genDollar(p, e, d, "#cstrToNimstr($1)") of mStrToStr, mUnown: expr(p, e[1], d) of mIsolate: genCall(p, e, d) diff --git a/lib/system/dollars.nim b/lib/system/dollars.nim index ce4e8e0ca..5a7a77e56 100644 --- a/lib/system/dollars.nim +++ b/lib/system/dollars.nim @@ -58,6 +58,10 @@ proc `$`*(x: float): string {.magic: "FloatToStr", noSideEffect.} ## The stringify operator for a float argument. Returns `x` ## converted to a decimal string. +proc `$`*(x: float32): string {.magic: "FloatToStr", noSideEffect.} + ## The stringify operator for a float32 argument. Returns `x` + ## converted to a decimal string. + proc `$`*(x: bool): string {.magic: "BoolToStr", noSideEffect.} ## The stringify operator for a boolean argument. Returns `x` ## converted to the string "false" or "true". diff --git a/lib/system/schubfach.nim b/lib/system/schubfach.nim new file mode 100644 index 000000000..c344b7425 --- /dev/null +++ b/lib/system/schubfach.nim @@ -0,0 +1,451 @@ +## Copyright 2020 Alexander Bolz +## +## Distributed under the Boost Software License, Version 1.0. +## (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) + +## -------------------------------------------------------------------------------------------------- +## This file contains an implementation of the Schubfach algorithm as described in +## +## [1] Raffaello Giulietti, "The Schubfach way to render doubles", +## https://drive.google.com/open?id=1luHhyQF9zKlM8yJ1nebU0OgVYhfC6CBN +## -------------------------------------------------------------------------------------------------- + +template sf_Assert(x: untyped): untyped = + assert(x) + +## ================================================================================================== +## +## ================================================================================================== + +type + ValueType = float32 + BitsType = uint32 + Single {.bycopy.} = object + bits: BitsType + +const + significandSize: int32 = 24 + MaxExponent = 128 + exponentBias: int32 = MaxExponent - 1 + (significandSize - 1) + maxIeeeExponent: BitsType = BitsType(2 * MaxExponent - 1) + hiddenBit: BitsType = BitsType(1) shl (significandSize - 1) + significandMask: BitsType = hiddenBit - 1 + exponentMask: BitsType = maxIeeeExponent shl (significandSize - 1) + signMask: BitsType = not (not BitsType(0) shr 1) + +proc constructSingle(bits: BitsType): Single {.constructor.} = + result.bits = bits + +proc constructSingle(value: ValueType): Single {.constructor.} = + result.bits = cast[typeof(result.bits)](value) + +proc physicalSignificand(this: Single): BitsType {.noSideEffect.} = + return this.bits and significandMask + +proc physicalExponent(this: Single): BitsType {.noSideEffect.} = + return (this.bits and exponentMask) shr (significandSize - 1) + +proc isFinite(this: Single): bool {.noSideEffect.} = + return (this.bits and exponentMask) != exponentMask + +proc isInf(this: Single): bool {.noSideEffect.} = + return (this.bits and exponentMask) == exponentMask and + (this.bits and significandMask) == 0 + +proc isNaN(this: Single): bool {.noSideEffect.} = + return (this.bits and exponentMask) == exponentMask and + (this.bits and significandMask) != 0 + +proc isZero(this: Single): bool {.noSideEffect.} = + return (this.bits and not signMask) == 0 + +proc signBit(this: Single): int {.noSideEffect.} = + return int((this.bits and signMask) != 0) + +## ================================================================================================== +## Returns floor(x / 2^n). +## +## Technically, right-shift of negative integers is implementation defined... +## Should easily be optimized into SAR (or equivalent) instruction. + +proc floorDivPow2(x: int32; n: int32): int32 {.inline.} = + return x shr n + +## Returns floor(log_10(2^e)) +## static inline int32_t FloorLog10Pow2(int32_t e) +## { +## SF_ASSERT(e >= -1500); +## SF_ASSERT(e <= 1500); +## return FloorDivPow2(e * 1262611, 22); +## } +## Returns floor(log_10(3/4 2^e)) +## static inline int32_t FloorLog10ThreeQuartersPow2(int32_t e) +## { +## SF_ASSERT(e >= -1500); +## SF_ASSERT(e <= 1500); +## return FloorDivPow2(e * 1262611 - 524031, 22); +## } +## Returns floor(log_2(10^e)) + +proc floorLog2Pow10(e: int32): int32 {.inline.} = + sf_Assert(e >= -1233) + sf_Assert(e <= 1233) + return floorDivPow2(e * 1741647, 19) + +proc computePow10Single(k: int32): uint64 {.inline.} = + ## There are unique beta and r such that 10^k = beta 2^r and + ## 2^63 <= beta < 2^64, namely r = floor(log_2 10^k) - 63 and + ## beta = 2^-r 10^k. + ## Let g = ceil(beta), so (g-1) 2^r < 10^k <= g 2^r, with the latter + ## value being a pretty good overestimate for 10^k. + ## NB: Since for all the required exponents k, we have g < 2^64, + ## all constants can be stored in 128-bit integers. + const + kMin: int32 = -31 + kMax: int32 = 45 + g: array[kMax - kMin + 1, uint64] = [0x81CEB32C4B43FCF5'u64, 0xA2425FF75E14FC32'u64, + 0xCAD2F7F5359A3B3F'u64, 0xFD87B5F28300CA0E'u64, 0x9E74D1B791E07E49'u64, + 0xC612062576589DDB'u64, 0xF79687AED3EEC552'u64, 0x9ABE14CD44753B53'u64, + 0xC16D9A0095928A28'u64, 0xF1C90080BAF72CB2'u64, 0x971DA05074DA7BEF'u64, + 0xBCE5086492111AEB'u64, 0xEC1E4A7DB69561A6'u64, 0x9392EE8E921D5D08'u64, + 0xB877AA3236A4B44A'u64, 0xE69594BEC44DE15C'u64, 0x901D7CF73AB0ACDA'u64, + 0xB424DC35095CD810'u64, 0xE12E13424BB40E14'u64, 0x8CBCCC096F5088CC'u64, + 0xAFEBFF0BCB24AAFF'u64, 0xDBE6FECEBDEDD5BF'u64, 0x89705F4136B4A598'u64, + 0xABCC77118461CEFD'u64, 0xD6BF94D5E57A42BD'u64, 0x8637BD05AF6C69B6'u64, + 0xA7C5AC471B478424'u64, 0xD1B71758E219652C'u64, 0x83126E978D4FDF3C'u64, + 0xA3D70A3D70A3D70B'u64, 0xCCCCCCCCCCCCCCCD'u64, 0x8000000000000000'u64, + 0xA000000000000000'u64, 0xC800000000000000'u64, 0xFA00000000000000'u64, + 0x9C40000000000000'u64, 0xC350000000000000'u64, 0xF424000000000000'u64, + 0x9896800000000000'u64, 0xBEBC200000000000'u64, 0xEE6B280000000000'u64, + 0x9502F90000000000'u64, 0xBA43B74000000000'u64, 0xE8D4A51000000000'u64, + 0x9184E72A00000000'u64, 0xB5E620F480000000'u64, 0xE35FA931A0000000'u64, + 0x8E1BC9BF04000000'u64, 0xB1A2BC2EC5000000'u64, 0xDE0B6B3A76400000'u64, + 0x8AC7230489E80000'u64, 0xAD78EBC5AC620000'u64, 0xD8D726B7177A8000'u64, + 0x878678326EAC9000'u64, 0xA968163F0A57B400'u64, 0xD3C21BCECCEDA100'u64, + 0x84595161401484A0'u64, 0xA56FA5B99019A5C8'u64, 0xCECB8F27F4200F3A'u64, + 0x813F3978F8940985'u64, 0xA18F07D736B90BE6'u64, 0xC9F2C9CD04674EDF'u64, + 0xFC6F7C4045812297'u64, 0x9DC5ADA82B70B59E'u64, 0xC5371912364CE306'u64, + 0xF684DF56C3E01BC7'u64, 0x9A130B963A6C115D'u64, 0xC097CE7BC90715B4'u64, + 0xF0BDC21ABB48DB21'u64, 0x96769950B50D88F5'u64, 0xBC143FA4E250EB32'u64, + 0xEB194F8E1AE525FE'u64, 0x92EFD1B8D0CF37BF'u64, 0xB7ABC627050305AE'u64, + 0xE596B7B0C643C71A'u64, 0x8F7E32CE7BEA5C70'u64, 0xB35DBF821AE4F38C'u64] + sf_Assert(k >= kMin) + sf_Assert(k <= kMax) + return g[k - kMin] + +proc lo32(x: uint64): uint32 {.inline.} = + return cast[uint32](x) + +proc hi32(x: uint64): uint32 {.inline.} = + return cast[uint32](x shr 32) + +when defined(sizeof_Int128): + proc roundToOdd(g: uint64; cp: uint32): uint32 {.inline.} = + let p: uint128 = uint128(g) * cp + let y1: uint32 = lo32(cast[uint64](p shr 64)) + let y0: uint32 = hi32(cast[uint64](p)) + return y1 or uint32(y0 > 1) + +elif defined(vcc) and defined(cpu64): + proc roundToOdd(g: uint64; cpHi: uint32): uint32 {.inline.} = + var p1: uint64 = 0 + var p0: uint64 = umul128(g, cpHi, addr(p1)) + let y1: uint32 = lo32(p1) + let y0: uint32 = hi32(p0) + return y1 or uint32(y0 > 1) + +else: + proc roundToOdd(g: uint64; cp: uint32): uint32 {.inline.} = + let b01: uint64 = uint64(lo32(g)) * cp + let b11: uint64 = uint64(hi32(g)) * cp + let hi: uint64 = b11 + hi32(b01) + let y1: uint32 = hi32(hi) + let y0: uint32 = lo32(hi) + return y1 or uint32(y0 > 1) + +## Returns whether value is divisible by 2^e2 + +proc multipleOfPow2(value: uint32; e2: int32): bool {.inline.} = + sf_Assert(e2 >= 0) + sf_Assert(e2 <= 31) + return (value and ((uint32(1) shl e2) - 1)) == 0 + +type + FloatingDecimal32 {.bycopy.} = object + digits: uint32 ## num_digits <= 9 + exponent: int32 + +proc toDecimal32(ieeeSignificand: uint32; ieeeExponent: uint32): FloatingDecimal32 {. + inline.} = + var c: uint32 + var q: int32 + if ieeeExponent != 0: + c = hiddenBit or ieeeSignificand + q = cast[int32](ieeeExponent) - exponentBias + if 0 <= -q and -q < significandSize and multipleOfPow2(c, -q): + return FloatingDecimal32(digits: c shr -q, exponent: 0'i32) + else: + c = ieeeSignificand + q = 1 - exponentBias + let isEven: bool = (c mod 2 == 0) + let lowerBoundaryIsCloser: bool = (ieeeSignificand == 0 and ieeeExponent > 1) + ## const int32_t qb = q - 2; + let cbl: uint32 = 4 * c - 2 + uint32(lowerBoundaryIsCloser) + let cb: uint32 = 4 * c + let cbr: uint32 = 4 * c + 2 + ## (q * 1262611 ) >> 22 == floor(log_10( 2^q)) + ## (q * 1262611 - 524031) >> 22 == floor(log_10(3/4 2^q)) + sf_Assert(q >= -1500) + sf_Assert(q <= 1500) + let k: int32 = floorDivPow2(q * 1262611 - (if lowerBoundaryIsCloser: 524031 else: 0), 22) + let h: int32 = q + floorLog2Pow10(-k) + 1 + sf_Assert(h >= 1) + sf_Assert(h <= 4) + let pow10: uint64 = computePow10Single(-k) + let vbl: uint32 = roundToOdd(pow10, cbl shl h) + let vb: uint32 = roundToOdd(pow10, cb shl h) + let vbr: uint32 = roundToOdd(pow10, cbr shl h) + let lower: uint32 = vbl + uint32(not isEven) + let upper: uint32 = vbr - uint32(not isEven) + ## See Figure 4 in [1]. + ## And the modifications in Figure 6. + let s: uint32 = vb div 4 + ## NB: 4 * s == vb & ~3 == vb & -4 + if s >= 10: + let sp: uint32 = s div 10 + ## = vb / 40 + let upInside: bool = lower <= 40 * sp + let wpInside: bool = 40 * sp + 40 <= upper + ## if (up_inside || wp_inside) // NB: At most one of u' and w' is in R_v. + if upInside != wpInside: + return FloatingDecimal32(digits: sp + uint32(wpInside), exponent: k + 1) + let uInside: bool = lower <= 4 * s + let wInside: bool = 4 * s + 4 <= upper + if uInside != wInside: + return FloatingDecimal32(digits: s + uint32(wInside), exponent: k) + let mid: uint32 = 4 * s + 2 + ## = 2(s + t) + let roundUp: bool = vb > mid or (vb == mid and (s and 1) != 0) + return FloatingDecimal32(digits: s + uint32(roundUp), exponent: k) + +## ================================================================================================== +## ToChars +## ================================================================================================== + +proc utoa2Digits(buf: var openArray[char]; pos: int; digits: uint32) {.inline.} = + const + digits100: array[200, char] = ['0', '0', '0', '1', '0', '2', '0', '3', '0', '4', '0', '5', + '0', '6', '0', '7', '0', '8', '0', '9', '1', '0', '1', '1', '1', '2', '1', '3', '1', '4', + '1', '5', '1', '6', '1', '7', '1', '8', '1', '9', '2', '0', '2', '1', '2', '2', '2', '3', + '2', '4', '2', '5', '2', '6', '2', '7', '2', '8', '2', '9', '3', '0', '3', '1', '3', '2', + '3', '3', '3', '4', '3', '5', '3', '6', '3', '7', '3', '8', '3', '9', '4', '0', '4', '1', + '4', '2', '4', '3', '4', '4', '4', '5', '4', '6', '4', '7', '4', '8', '4', '9', '5', '0', + '5', '1', '5', '2', '5', '3', '5', '4', '5', '5', '5', '6', '5', '7', '5', '8', '5', '9', + '6', '0', '6', '1', '6', '2', '6', '3', '6', '4', '6', '5', '6', '6', '6', '7', '6', '8', + '6', '9', '7', '0', '7', '1', '7', '2', '7', '3', '7', '4', '7', '5', '7', '6', '7', '7', + '7', '8', '7', '9', '8', '0', '8', '1', '8', '2', '8', '3', '8', '4', '8', '5', '8', '6', + '8', '7', '8', '8', '8', '9', '9', '0', '9', '1', '9', '2', '9', '3', '9', '4', '9', '5', + '9', '6', '9', '7', '9', '8', '9', '9'] + sf_Assert(digits <= 99) + buf[pos] = digits100[2 * digits] + buf[pos+1] = digits100[2 * digits + 1] + +proc trailingZeros2Digits(digits: uint32): int32 {.inline.} = + const + trailingZeros100: array[100, int8] = [2'i8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0] + sf_Assert(digits <= 99) + return trailingZeros100[digits] + +proc printDecimalDigitsBackwards(buf: var openArray[char]; pos: int; output: uint32): int32 {.inline.} = + var output = output + var pos = pos + var tz: int32 = 0 + ## number of trailing zeros removed. + var nd: int32 = 0 + ## number of decimal digits processed. + ## At most 9 digits remaining + if output >= 10000: + let q: uint32 = output div 10000 + let r: uint32 = output mod 10000 + output = q + dec(pos, 4) + if r != 0: + let rH: uint32 = r div 100 + let rL: uint32 = r mod 100 + utoa2Digits(buf, pos, rH) + utoa2Digits(buf, pos + 2, rL) + tz = trailingZeros2Digits(if rL == 0: rH else: rL) + (if rL == 0: 2 else: 0) + else: + tz = 4 + nd = 4 + if output >= 100: + let q: uint32 = output div 100 + let r: uint32 = output mod 100 + output = q + dec(pos, 2) + utoa2Digits(buf, pos, r) + if tz == nd: + inc(tz, trailingZeros2Digits(r)) + inc(nd, 2) + if output >= 100: + let q2: uint32 = output div 100 + let r2: uint32 = output mod 100 + output = q2 + dec(pos, 2) + utoa2Digits(buf, pos, r2) + if tz == nd: + inc(tz, trailingZeros2Digits(r2)) + inc(nd, 2) + sf_Assert(output >= 1) + sf_Assert(output <= 99) + if output >= 10: + let q: uint32 = output + dec(pos, 2) + utoa2Digits(buf, pos, q) + if tz == nd: + inc(tz, trailingZeros2Digits(q)) + else: + let q: uint32 = output + sf_Assert(q >= 1) + sf_Assert(q <= 9) + dec(pos) + buf[pos] = chr(uint32('0') + q) + return tz + +proc decimalLength(v: uint32): int32 {.inline.} = + sf_Assert(v >= 1) + sf_Assert(v <= 999999999'u) + if v >= 100000000'u: + return 9 + if v >= 10000000'u: + return 8 + if v >= 1000000'u: + return 7 + if v >= 100000'u: + return 6 + if v >= 10000'u: + return 5 + if v >= 1000'u: + return 4 + if v >= 100'u: + return 3 + if v >= 10'u: + return 2 + return 1 + +proc formatDigits(buffer: var openArray[char]; pos: int; digits: uint32; decimalExponent: int32; + forceTrailingDotZero: bool = false): int {.inline.} = + const + minFixedDecimalPoint: int32 = -4 + maxFixedDecimalPoint: int32 = 9 + var pos = pos + assert(minFixedDecimalPoint <= -1, "internal error") + assert(maxFixedDecimalPoint >= 1, "internal error") + sf_Assert(digits >= 1) + sf_Assert(digits <= 999999999'u) + sf_Assert(decimalExponent >= -99) + sf_Assert(decimalExponent <= 99) + var numDigits: int32 = decimalLength(digits) + let decimalPoint: int32 = numDigits + decimalExponent + let useFixed: bool = minFixedDecimalPoint <= decimalPoint and + decimalPoint <= maxFixedDecimalPoint + ## Prepare the buffer. + ## Avoid calling memset/memcpy with variable arguments below... + for i in 0..<32: buffer[pos+i] = '0' + assert(minFixedDecimalPoint >= -30, "internal error") + assert(maxFixedDecimalPoint <= 32, "internal error") + var decimalDigitsPosition: int32 + if useFixed: + if decimalPoint <= 0: + ## 0.[000]digits + decimalDigitsPosition = 2 - decimalPoint + else: + ## dig.its + ## digits[000] + decimalDigitsPosition = 0 + else: + ## dE+123 or d.igitsE+123 + decimalDigitsPosition = 1 + var digitsEnd = pos + decimalDigitsPosition + numDigits + let tz: int32 = printDecimalDigitsBackwards(buffer, digitsEnd, digits) + dec(digitsEnd, tz) + dec(numDigits, tz) + ## decimal_exponent += tz; // => decimal_point unchanged. + if useFixed: + if decimalPoint <= 0: + ## 0.[000]digits + buffer[pos+1] = '.' + pos = digitsEnd + elif decimalPoint < numDigits: + ## dig.its + for i in 0..<8: + buffer[i + decimalPoint + 1] = buffer[i + decimalPoint] + buffer[pos+decimalPoint] = '.' + pos = digitsEnd + 1 + else: + ## digits[000] + inc(pos, decimalPoint) + if forceTrailingDotZero: + buffer[pos] = '.' + buffer[pos+1] = '0' + inc(pos, 2) + else: + buffer[pos] = buffer[pos+1] + if numDigits == 1: + ## dE+123 + inc(pos) + else: + ## d.igitsE+123 + buffer[pos+1] = '.' + pos = digitsEnd + let scientificExponent: int32 = decimalPoint - 1 + ## SF_ASSERT(scientific_exponent != 0); + buffer[pos] = 'e' + buffer[pos+1] = if scientificExponent < 0: '-' else: '+' + inc(pos, 2) + let k: uint32 = cast[uint32](if scientificExponent < 0: -scientificExponent else: scientificExponent) + if k < 10: + buffer[pos] = chr(uint32('0') + k) + inc pos + else: + utoa2Digits(buffer, pos, k) + inc(pos, 2) + return pos + +proc float32ToChars*(buffer: var openArray[char]; v: float32; forceTrailingDotZero = false): int {. + inline.} = + let significand: uint32 = physicalSignificand(constructSingle(v)) + let exponent: uint32 = physicalExponent(constructSingle(v)) + var pos = 0 + if exponent != maxIeeeExponent: + ## Finite + buffer[pos] = '-' + inc(pos, signBit(constructSingle(v))) + if exponent != 0 or significand != 0: + ## != 0 + let dec: auto = toDecimal32(significand, exponent) + return formatDigits(buffer, pos, dec.digits, dec.exponent, forceTrailingDotZero) + else: + buffer[pos] = '0' + buffer[pos+1] = '.' + buffer[pos+2] = '0' + buffer[pos+3] = ' ' + inc(pos, if forceTrailingDotZero: 3 else: 1) + return pos + if significand == 0: + buffer[pos] = '-' + inc(pos, signBit(constructSingle(v))) + buffer[pos] = 'i' + buffer[pos+1] = 'n' + buffer[pos+2] = 'f' + buffer[pos+3] = ' ' + return pos + 3 + else: + buffer[pos] = 'n' + buffer[pos+1] = 'a' + buffer[pos+2] = 'n' + buffer[pos+3] = ' ' + return pos + 3 diff --git a/lib/system/strmantle.nim b/lib/system/strmantle.nim index 42ea9d226..1a32a4afe 100644 --- a/lib/system/strmantle.nim +++ b/lib/system/strmantle.nim @@ -164,6 +164,19 @@ proc nimFloatToStr(f: float): string {.compilerproc.} = result = newStringOfCap(8) result.addFloat f +when not defined(nimLegacyAddFloat) and not defined(nimscript) and + not defined(js) and defined(nimHasDragonBox): + import schubfach + +proc nimFloat32ToStr(f: float32): string {.compilerproc.} = + when declared(float32ToChars): + result = newString(65) + let L = float32ToChars(result, f, forceTrailingDotZero=true) + setLen(result, L) + else: + result = newStringOfCap(8) + result.addFloat f + proc c_strtod(buf: cstring, endptr: ptr cstring): float64 {. importc: "strtod", header: "", noSideEffect.} diff --git a/tests/float/tfloat6.nim b/tests/float/tfloat6.nim index c0e6e419f..fa8fc9da8 100644 --- a/tests/float/tfloat6.nim +++ b/tests/float/tfloat6.nim @@ -21,3 +21,8 @@ echo "0.00000_1".parseFloat(), " : ", 1E-6 echo "1_0.00_0001".parseFloat(), " : ", 10.000001 echo "1__00.00_0001".parseFloat(), " : ", 1_00.000001 + +# bug #18148 + +var a = 1.1'f32 +doAssert $a == "1.1", $a # fails From 282d61bafa4f1b06e1328cf8e517f556fb86d592 Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 3 Jun 2021 14:00:53 +0200 Subject: [PATCH 0875/2472] added lib/deps.txt --- lib/deps.txt | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 lib/deps.txt diff --git a/lib/deps.txt b/lib/deps.txt new file mode 100644 index 000000000..2a44b5fe2 --- /dev/null +++ b/lib/deps.txt @@ -0,0 +1,14 @@ +# Dragonbox.nim and Schubfach.nim + +"Dragonbox" is Nim's "float64 to string" algorithm. +"Schubfach" is Nim's "float32 to string" algorithm. These are based on + +https://github.com/abolz/Drachennest/blob/master/src/dragonbox.cc +https://github.com/abolz/Drachennest/blob/master/src/schubfach_32.cc + +commit e6714a39ad331b4489d0b6aaf3968635bff4eb5e + +The `.cc` files were translated by c2nim via `--cpp --keepBodies --nep1` +and then modified to remove the unsafe code. + +We used c2nim as of commit f0469c909d9e2e28d59687e394bf5ac862f561b6. From 9df631a379998f554748697da7fe95ce4463cb83 Mon Sep 17 00:00:00 2001 From: n5m <72841454+n5m@users.noreply.github.com> Date: Thu, 3 Jun 2021 12:25:52 +0000 Subject: [PATCH 0876/2472] reuse algorithm.fill while building SkipTable (#18138) * reuse algorithm.fill while building SkipTable * Update lib/pure/strutils.nim Co-authored-by: Timothee Cour Co-authored-by: Timothee Cour --- lib/pure/strutils.nim | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 3fb67ead1..8e0a6158b 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -72,7 +72,7 @@ runnableExamples: import parseutils from math import pow, floor, log10 -from algorithm import reverse +from algorithm import fill, reverse import std/enumutils from unicode import toLower, toUpper @@ -1779,17 +1779,7 @@ func initSkipTable*(a: var SkipTable, sub: string) {.rtl, extern: "nsuInitSkipTable".} = ## Preprocess table `a` for `sub`. let m = len(sub) - var i = 0 - while i <= 0xff-7: - a[chr(i + 0)] = m - a[chr(i + 1)] = m - a[chr(i + 2)] = m - a[chr(i + 3)] = m - a[chr(i + 4)] = m - a[chr(i + 5)] = m - a[chr(i + 6)] = m - a[chr(i + 7)] = m - i += 8 + fill(a, m) for i in 0 ..< m - 1: a[sub[i]] = m - 1 - i From 3e57c2f7800dbedd0df735a0236f09410f6db11a Mon Sep 17 00:00:00 2001 From: Artem Klevtsov Date: Thu, 3 Jun 2021 19:41:57 +0700 Subject: [PATCH 0877/2472] Improve db_postgres iterators (#18144) * Fix pqSetSingleRowMode case. Add links to the docs * Add missing PGContextVisibility enum * Remove unused PGContextVisibility enum * Improve db_postgres iterators * Fix instantRows with DbColumns. Cosmetics. * Reduce copy&paste in db_postgres * Move pqclear inside loop --- lib/impure/db_postgres.nim | 179 ++++++++++++++++++++++++------------- 1 file changed, 117 insertions(+), 62 deletions(-) diff --git a/lib/impure/db_postgres.nim b/lib/impure/db_postgres.nim index 4edeac2fc..36e035d3d 100644 --- a/lib/impure/db_postgres.nim +++ b/lib/impure/db_postgres.nim @@ -96,7 +96,6 @@ type ## converted to nil. InstantRow* = object ## a handle that can be res: PPGresult ## used to get a row's - line: int ## column text on demand SqlPrepared* = distinct string ## a identifier for the prepared queries proc dbError*(db: DbConn) {.noreturn.} = @@ -184,6 +183,22 @@ proc setupQuery(db: DbConn, stmtName: SqlPrepared, deallocCStringArray(arr) if pqResultStatus(result) != PGRES_TUPLES_OK: dbError(db) +proc setupSingeRowQuery(db: DbConn, query: SqlQuery, + args: varargs[string]) = + if pqsendquery(db, dbFormat(query, args)) != 1: + dbError(db) + if pqSetSingleRowMode(db) != 1: + dbError(db) + +proc setupSingeRowQuery(db: DbConn, stmtName: SqlPrepared, + args: varargs[string]) = + var arr = allocCStringArray(args) + if pqsendqueryprepared(db, stmtName.string, int32(args.len), arr, nil, nil, 0) != 1: + dbError(db) + if pqSetSingleRowMode(db) != 1: + dbError(db) + deallocCStringArray(arr) + proc prepare*(db: DbConn; stmtName: string, query: SqlQuery; nParams: int): SqlPrepared = ## Creates a new `SqlPrepared` statement. Parameter substitution is done @@ -203,49 +218,70 @@ proc setRow(res: PPGresult, r: var Row, line, cols: int32) = else: add(r[col], x) +template fetchRows(db: DbConn): untyped = + var res: PPGresult = nil + while true: + res = pqgetresult(db) + if res == nil: + break + let status = pqresultStatus(res) + if status == PGRES_TUPLES_OK: + discard + elif status != PGRES_SINGLE_TUPLE: + dbError(db) + else: + let L = pqNfields(res) + var result = newRow(L) + setRow(res, result, 0, L) + yield result + pqclear(res) + iterator fastRows*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = ## executes the query and iterates over the result dataset. This is very ## fast, but potentially dangerous: If the for-loop-body executes another ## query, the results can be undefined. For Postgres it is safe though. - var res = setupQuery(db, query, args) - var L = pqnfields(res) - var result = newRow(L) - for i in 0'i32..pqntuples(res)-1: - setRow(res, result, i, L) - yield result - pqclear(res) + setupSingeRowQuery(db, query, args) + fetchRows(db) iterator fastRows*(db: DbConn, stmtName: SqlPrepared, args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = - ## executes the prepared query and iterates over the result dataset. - var res = setupQuery(db, stmtName, args) - var L = pqNfields(res) - var result = newRow(L) - for i in 0'i32..pqNtuples(res)-1: - setRow(res, result, i, L) - yield result - pqClear(res) + ## executes the query and iterates over the result dataset. This is very + ## fast, but potentially dangerous: If the for-loop-body executes another + ## query, the results can be undefined. For Postgres it is safe though. + setupSingeRowQuery(db, stmtName, args) + fetchRows(db) + +template fetchinstantRows(db: DbConn): untyped = + var res: PPGresult = nil + while true: + res = pqgetresult(db) + if res == nil: + break + let status = pqresultStatus(res) + if status == PGRES_TUPLES_OK: + discard + elif status != PGRES_SINGLE_TUPLE: + dbError(db) + else: + yield InstantRow(res: res) + pqclear(res) iterator instantRows*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): InstantRow {.tags: [ReadDbEffect].} = ## same as fastRows but returns a handle that can be used to get column text ## on demand using []. Returned handle is valid only within iterator body. - var res = setupQuery(db, query, args) - for i in 0'i32..pqNtuples(res)-1: - yield InstantRow(res: res, line: i) - pqClear(res) + setupSingeRowQuery(db, query, args) + fetchinstantRows(db) iterator instantRows*(db: DbConn, stmtName: SqlPrepared, args: varargs[string, `$`]): InstantRow {.tags: [ReadDbEffect].} = ## same as fastRows but returns a handle that can be used to get column text ## on demand using []. Returned handle is valid only within iterator body. - var res = setupQuery(db, stmtName, args) - for i in 0'i32..pqNtuples(res)-1: - yield InstantRow(res: res, line: i) - pqClear(res) + setupSingeRowQuery(db, stmtName, args) + fetchinstantRows(db) proc getColumnType(res: PPGresult, col: int) : DbType = ## returns DbType for given column in the row @@ -382,7 +418,7 @@ proc getColumnType(res: PPGresult, col: int) : DbType = of 705: return DbType(kind: DbTypeKind.dbUnknown, name: "unknown") else: return DbType(kind: DbTypeKind.dbUnknown, name: $oid) ## Query the system table pg_type to determine exactly which type is referenced. -proc setColumnInfo(columns: var DbColumns; res: PPGresult; L: int32) = +proc setColumnInfo(columns: var DbColumns; res: PPGresult, L: int32) = setLen(columns, L) for i in 0'i32.. 0: setRow(res, result, 0, L) pqclear(res) +proc getRow*(db: DbConn, query: SqlQuery, + args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = + ## retrieves a single row. If the query doesn't return any rows, this proc + ## will return a Row with empty strings for each column. + let res = setupQuery(db, query, args) + getRow(res) + proc getRow*(db: DbConn, stmtName: SqlPrepared, args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = - var res = setupQuery(db, stmtName, args) - var L = pqNfields(res) - result = newRow(L) - if pqntuples(res) > 0: - setRow(res, result, 0, L) - pqClear(res) + let res = setupQuery(db, stmtName, args) + getRow(res) + +proc getAllRows(res: PPGresult): seq[Row] = + let N = pqntuples(res) + let L = pqnfields(res) + result = newSeqOfCap[Row](N) + var row = newRow(L) + for i in 0'i32..N-1: + setRow(res, row, i, L) + result.add(row) + pqclear(res) proc getAllRows*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): seq[Row] {. tags: [ReadDbEffect].} = ## executes the query and returns the whole result dataset. - result = @[] - for r in fastRows(db, query, args): - result.add(r) + let res = setupQuery(db, query, args) + getAllRows(res) proc getAllRows*(db: DbConn, stmtName: SqlPrepared, args: varargs[string, `$`]): seq[Row] {.tags: [ReadDbEffect].} = ## executes the prepared query and returns the whole result dataset. - result = @[] - for r in fastRows(db, stmtName, args): - result.add(r) + let res = setupQuery(db, stmtName, args) + getAllRows(res) iterator rows*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = @@ -459,18 +515,21 @@ iterator rows*(db: DbConn, stmtName: SqlPrepared, ## same as `fastRows`, but slower and safe. for r in items(getAllRows(db, stmtName, args)): yield r +proc getValue(res: PPGresult): string = + if pqntuples(res) > 0: + var x = pqgetvalue(res, 0, 0) + result = if isNil(x): "" else: $x + else: + result = "" + proc getValue*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): string {. tags: [ReadDbEffect].} = ## executes the query and returns the first column of the first row of the ## result dataset. Returns "" if the dataset contains no rows or the database ## value is NULL. - var res = setupQuery(db, query, args) - if pqntuples(res) > 0: - var x = pqgetvalue(res, 0, 0) - result = if isNil(x): "" else: $x - else: - result = "" + let res = setupQuery(db, query, args) + getValue(res) proc getValue*(db: DbConn, stmtName: SqlPrepared, args: varargs[string, `$`]): string {. @@ -478,12 +537,8 @@ proc getValue*(db: DbConn, stmtName: SqlPrepared, ## executes the query and returns the first column of the first row of the ## result dataset. Returns "" if the dataset contains no rows or the database ## value is NULL. - var res = setupQuery(db, stmtName, args) - if pqntuples(res) > 0: - var x = pqgetvalue(res, 0, 0) - result = if isNil(x): "" else: $x - else: - result = "" + let res = setupQuery(db, stmtName, args) + getValue(res) proc tryInsertID*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): int64 {. From 7ef364a402d3d827f10c893280f8dc7b9ef056f5 Mon Sep 17 00:00:00 2001 From: narimiran Date: Thu, 3 Jun 2021 15:20:42 +0200 Subject: [PATCH 0878/2472] add missing import to asynchttpserver's example --- lib/pure/asynchttpserver.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index fd0cad5ca..8cea7f162 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -18,7 +18,7 @@ runnableExamples("-r:off"): # This example will create an HTTP server on an automatically chosen port. # It will respond to all requests with a `200 OK` response code and "Hello World" # as the response body. - import std/asyncdispatch + import std/asyncdispatch, asynchttpserver proc main {.async.} = var server = newAsyncHttpServer() proc cb(req: Request) {.async.} = From 0aa8b793a5f5ef58afe5e8aa96b7646a8afde765 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 3 Jun 2021 16:27:34 +0200 Subject: [PATCH 0879/2472] clarify what a 'monotonic' timestamp is (#18163) --- lib/std/monotimes.nim | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/std/monotimes.nim b/lib/std/monotimes.nim index 8f6aa5b66..3bb4fa141 100644 --- a/lib/std/monotimes.nim +++ b/lib/std/monotimes.nim @@ -10,7 +10,7 @@ ##[ The `std/monotimes` module implements monotonic timestamps. A monotonic timestamp represents the time that has passed since some system defined -point in time. The monotonic timestamps are guaranteed to always increase, +point in time. The monotonic timestamps are guaranteed not to decrease, meaning that that the following is guaranteed to work: ]## @@ -18,9 +18,8 @@ runnableExamples: import std/os let a = getMonoTime() - sleep(10) let b = getMonoTime() - assert a < b + assert a <= b ##[ This is not guaranteed for the `times.Time` type! This means that the From d31cbfd167c0bf2fcf2b1e6d401a20a16acaaebb Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 3 Jun 2021 22:44:11 +0800 Subject: [PATCH 0880/2472] Revert "add missing import to asynchttpserver's example" (#18164) This reverts commit 7ef364a402d3d827f10c893280f8dc7b9ef056f5. --- lib/pure/asynchttpserver.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index 8cea7f162..fd0cad5ca 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -18,7 +18,7 @@ runnableExamples("-r:off"): # This example will create an HTTP server on an automatically chosen port. # It will respond to all requests with a `200 OK` response code and "Hello World" # as the response body. - import std/asyncdispatch, asynchttpserver + import std/asyncdispatch proc main {.async.} = var server = newAsyncHttpServer() proc cb(req: Request) {.async.} = From 06232b7f2e230a46beea982f172087e9da51814a Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 3 Jun 2021 17:12:45 +0200 Subject: [PATCH 0881/2472] fixes #18058 (#18162) --- compiler/commands.nim | 2 ++ compiler/msgs.nim | 12 ++++-------- compiler/options.nim | 1 + doc/advopt.txt | 2 ++ tests/misc/trunner.nim | 4 ++-- tests/osproc/treadlines.nim | 4 ++-- 6 files changed, 13 insertions(+), 12 deletions(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index 605e2930c..994b224f1 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -919,6 +919,8 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; excl(conf.notes, hintProcessing) excl(conf.mainPackageNotes, hintProcessing) else: localError(conf, info, "expected: dots|filenames|off, got: $1" % arg) + of "unitsep": + conf.unitSep = if switchOn(arg): "\31" else: "" of "listfullpaths": # xxx in future work, use `warningDeprecated` conf.filenameOption = if switchOn(arg): foAbs else: foCanonical diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 6a8c45db5..053b5c928 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -289,10 +289,6 @@ proc `??`* (conf: ConfigRef; info: TLineInfo, filename: string): bool = # only for debugging purposes result = filename in toFilename(conf, info) -const - UnitSep = "\31" - # this needs care to avoid issues similar to https://github.com/nim-lang/Nim/issues/17853 - type MsgFlag* = enum ## flags altering msgWriteln behavior msgStdout, ## force writing to stdout, even stderr is default @@ -309,7 +305,7 @@ proc msgWriteln*(conf: ConfigRef; s: string, flags: MsgFlags = {}) = ## This is used for 'nim dump' etc. where we don't have nimsuggest ## support. #if conf.cmd == cmdIdeTools and optCDebug notin gGlobalOptions: return - let sep = if msgNoUnitSep notin flags: UnitSep else: "" + let sep = if msgNoUnitSep notin flags: conf.unitSep else: "" if not isNil(conf.writelnHook) and msgSkipHook notin flags: conf.writelnHook(s & sep) elif optStdout in conf.globalOptions or msgStdout in flags: @@ -409,7 +405,7 @@ proc quit(conf: ConfigRef; msg: TMsgKind) {.gcsafe.} = styledMsgWriteln(fgRed, """ No stack traceback available To create a stacktrace, rerun compilation with './koch temp $1 ', see $2 for details""" % - [conf.command, "intern.html#debugging-the-compiler".createDocLink], UnitSep) + [conf.command, "intern.html#debugging-the-compiler".createDocLink], conf.unitSep) quit 1 proc handleError(conf: ConfigRef; msg: TMsgKind, eh: TErrorHandling, s: string) = @@ -550,13 +546,13 @@ proc liMessage*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string, msgWrite(conf, ".") else: styledMsgWriteln(styleBright, loc, resetStyle, color, title, resetStyle, s, KindColor, kindmsg, - resetStyle, conf.getSurroundingSrc(info), UnitSep) + resetStyle, conf.getSurroundingSrc(info), conf.unitSep) if hintMsgOrigin in conf.mainPackageNotes: # xxx needs a bit of refactoring to honor `conf.filenameOption` styledMsgWriteln(styleBright, toFileLineCol(info2), resetStyle, " compiler msg initiated here", KindColor, KindFormat % $hintMsgOrigin, - resetStyle, UnitSep) + resetStyle, conf.unitSep) handleError(conf, msg, eh, s) template rawMessage*(conf: ConfigRef; msg: TMsgKind, args: openArray[string]) = diff --git a/compiler/options.nim b/compiler/options.nim index 9cbb747c9..e6c667d92 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -283,6 +283,7 @@ type arcToExpand*: StringTableRef m*: MsgConfig filenameOption*: FilenameOption # how to render paths in compiler messages + unitSep*: string evalTemplateCounter*: int evalMacroCounter*: int exitcode*: int8 diff --git a/doc/advopt.txt b/doc/advopt.txt index ea3370880..e627d4768 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -41,6 +41,8 @@ Advanced options: defaults to `abs` (absolute) --processing:dots|filenames|off show files as they're being processed by nim compiler + --unitsep:on|off use the ASCII unit separator (31) between error + messages, useful for IDE-like tooling --declaredLocs:on|off show declaration locations in messages --spellSuggest|:num show at most `num >= 0` spelling suggestions on typos. if `num` is not specified (or `auto`), return diff --git a/tests/misc/trunner.nim b/tests/misc/trunner.nim index 3297b3a24..b6b302836 100644 --- a/tests/misc/trunner.nim +++ b/tests/misc/trunner.nim @@ -249,7 +249,7 @@ tests/newconfig/bar/mfoo.nims""".splitLines var expected = "" for a in files: let b = dir / a - expected.add &"Hint: used config file '{b}' [Conf]\31\n" + expected.add &"Hint: used config file '{b}' [Conf]\n" doAssert outp.endsWith expected, outp & "\n" & expected block: # mfoo2.customext @@ -257,7 +257,7 @@ tests/newconfig/bar/mfoo.nims""".splitLines let cmd = fmt"{nim} e --hint:conf {filename}" let (outp, exitCode) = execCmdEx(cmd, options = {poStdErrToStdOut}) doAssert exitCode == 0 - var expected = &"Hint: used config file '{filename}' [Conf]\31\n" + var expected = &"Hint: used config file '{filename}' [Conf]\n" doAssert outp.endsWith "123" & "\n" & expected diff --git a/tests/osproc/treadlines.nim b/tests/osproc/treadlines.nim index 200a7c299..bb6a7f129 100644 --- a/tests/osproc/treadlines.nim +++ b/tests/osproc/treadlines.nim @@ -1,7 +1,7 @@ discard """ output: ''' -Error: cannot open 'a.nim'\31 -Error: cannot open 'b.nim'\31 +Error: cannot open 'a.nim' +Error: cannot open 'b.nim' ''' targets: "c" """ From 28f2abe1a20a7f7f758b42c233deb7daa78406b1 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 3 Jun 2021 20:55:41 +0200 Subject: [PATCH 0882/2472] fixes #18112 (#18165) --- compiler/docgen.nim | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 18c568ba0..dc2a63f48 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -20,7 +20,7 @@ import from uri import encodeUrl from std/private/globs import nativeToUnixPath - +from nodejs import findNodeJs const exportSection = skField @@ -430,7 +430,9 @@ proc runAllExamples(d: PDoc) = "rdoccmd", group.rdoccmd, "docCmd", group.docCmd, ] - if os.execShellCmd(cmd) != 0: + if d.conf.backend == backendJs and findNodeJs() == "": + discard "ignore JS runnableExample" + elif os.execShellCmd(cmd) != 0: d.conf.quitOrRaise "[runnableExamples] failed: generated file: '$1' group: '$2' cmd: $3" % [outp.string, group[].prettyString, cmd] else: # keep generated source file `outp` to allow inspection. From d91d78f8aed2b1b59effa3b98765aaf675d7e4c3 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 3 Jun 2021 13:25:36 -0700 Subject: [PATCH 0883/2472] changelog for --unitsep (#18167) --- changelog.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/changelog.md b/changelog.md index a23aedbdf..06251abfc 100644 --- a/changelog.md +++ b/changelog.md @@ -61,9 +61,6 @@ - Removed `.travis.yml`, `appveyor.yml.disabled`, `.github/workflows/ci.yml.disabled`. -- Nim compiler now adds ASCII unit separator `\31` before a newline for every generated - message (potentially multiline), so tooling can tell when messages start and end. - - `random.initRand(seed)` now produces non-skewed values for the 1st call to `rand()` after initialization with a small (< 30000) seed. Use `-d:nimLegacyRandomInitRand` to restore previous behavior for a transition time, see PR #17467. @@ -395,6 +392,9 @@ - Added `--processing:dots|filenames|off` which customizes `hintProcessing` +- Added `--unitsep:on|off` to control whether to add ASCII unit separator `\31` before a newline + for every generated message (potentially multiline), so tooling can tell when messages start and end. + - Source+Edit links now appear on top of every docgen'd page when `nim doc --git.url:url ...` is given. From 654a20166eabb614cbf0379c67115981f0532079 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 3 Jun 2021 13:29:45 -0700 Subject: [PATCH 0884/2472] simplify extccomp.nim json logic via jsonutils; fix #18084 (#18100) * simplify extccomp.nim json logic via jsonutils * fix #18084 * simplify further * workaround for bootstrap that can be removed after updating csources_v1 >= 1.2 --- compiler/extccomp.nim | 256 +++++++++++++----------------------------- compiler/nimconf.nim | 11 +- compiler/options.nim | 4 +- lib/std/jsonutils.nim | 18 ++- 4 files changed, 99 insertions(+), 190 deletions(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 1cea9edeb..ea2b09866 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -14,7 +14,7 @@ import ropes, platform, condsyms, options, msgs, lineinfos, pathutils -import os, strutils, osproc, std/sha1, streams, sequtils, times, strtabs, json +import std/[os, strutils, osproc, sha1, streams, sequtils, times, strtabs, json, jsonutils, sugar] type TInfoCCProp* = enum # properties of the C compiler: @@ -942,195 +942,91 @@ proc jsonBuildInstructionsFile*(conf: ConfigRef): AbsoluteFile = # works out of the box with `hashMainCompilationParams`. result = getNimcacheDir(conf) / conf.outFile.changeFileExt("json") +const cacheVersion = "D20210525T193831" # update when `BuildCache` spec changes +type BuildCache = object + cacheVersion: string + outputFile: string + compile: seq[(string, string)] + link: seq[string] + linkcmd: string + extraCmds: seq[string] + configFiles: seq[string] # the hash shouldn't be needed + stdinInput: bool + projectIsCmd: bool + cmdInput: string + currentDir: string + cmdline: string + depfiles: seq[(string, string)] + nimexe: string + proc writeJsonBuildInstructions*(conf: ConfigRef) = - # xxx use std/json instead, will result in simpler, more maintainable code. - template lit(x: string) = f.write x - template str(x: string) = - buf.setLen 0 - escapeJson(x, buf) - f.write buf - - proc cfiles(conf: ConfigRef; f: File; buf: var string; clist: CfileList, isExternal: bool) = - var comma = false - for i, it in clist: - if CfileFlag.Cached in it.flags: continue - let compileCmd = getCompileCFileCmd(conf, it) - if comma: lit ",\L" else: comma = true - lit "[" - str it.cname.string - lit ", " - str compileCmd - lit "]" - - proc linkfiles(conf: ConfigRef; f: File; buf, objfiles: var string; clist: CfileList; - llist: seq[string]) = - var pastStart = false - template impl(path) = - let path2 = quoteShell(path) - objfiles.add(' ') - objfiles.add(path2) - if pastStart: lit ",\L" - str path2 - pastStart = true - for it in llist: - let objfile = if noAbsolutePaths(conf): it.extractFilename else: it - impl(addFileExt(objfile, CC[conf.cCompiler].objExt)) - for it in clist: - impl(it.obj) - lit "\L" - - proc depfiles(conf: ConfigRef; f: File; buf: var string) = - var i = 0 - for it in conf.m.fileInfos: + var linkFiles = collect(for it in conf.externalToLink: + var it = it + if conf.noAbsolutePaths: it = it.extractFilename + it.addFileExt(CC[conf.cCompiler].objExt)) + for it in conf.toCompile: linkFiles.add it.obj.string + var bcache = BuildCache( + cacheVersion: cacheVersion, + outputFile: conf.absOutFile.string, + compile: collect(for i, it in conf.toCompile: + if CfileFlag.Cached notin it.flags: (it.cname.string, getCompileCFileCmd(conf, it))), + link: linkFiles, + linkcmd: getLinkCmd(conf, conf.absOutFile, linkFiles.quoteShellCommand), + extraCmds: getExtraCmds(conf, conf.absOutFile), + stdinInput: conf.projectIsStdin, + projectIsCmd: conf.projectIsCmd, + cmdInput: conf.cmdInput, + configFiles: conf.configFiles.mapIt(it.string), + currentDir: getCurrentDir()) + if optRun in conf.globalOptions or isDefined(conf, "nimBetterRun"): + bcache.cmdline = conf.commandLine + bcache.depfiles = collect(for it in conf.m.fileInfos: let path = it.fullPath.string if isAbsolute(path): # TODO: else? - if i > 0: lit "],\L" - lit "[" - str path - lit ", " - str $secureHashFile(path) - inc i - lit "]\L" - - - var buf = newStringOfCap(50) - let jsonFile = conf.jsonBuildInstructionsFile - conf.jsonBuildFile = jsonFile - let output = conf.absOutFile - - var f: File - if open(f, jsonFile.string, fmWrite): - lit "{\L" - lit "\"outputFile\": " - str $output - - lit ",\L\"compile\":[\L" - cfiles(conf, f, buf, conf.toCompile, false) - lit "],\L\"link\":[\L" - var objfiles = "" - # XXX add every file here that is to link - linkfiles(conf, f, buf, objfiles, conf.toCompile, conf.externalToLink) - - lit "],\L\"linkcmd\": " - str getLinkCmd(conf, output, objfiles) - - lit ",\L\"extraCmds\": " - lit $(%* getExtraCmds(conf, conf.absOutFile)) - - lit ",\L\"stdinInput\": " - lit $(%* conf.projectIsStdin) - lit ",\L\"projectIsCmd\": " - lit $(%* conf.projectIsCmd) - lit ",\L\"cmdInput\": " - lit $(%* conf.cmdInput) - lit ",\L\"currentDir\": " - lit $(%* getCurrentDir()) - - if optRun in conf.globalOptions or isDefined(conf, "nimBetterRun"): - lit ",\L\"cmdline\": " - str conf.commandLine - lit ",\L\"depfiles\":[\L" - depfiles(conf, f, buf) - lit "],\L\"nimexe\": \L" - str hashNimExe() - lit "\L" - - lit "\L}\L" - close(f) + (path, $secureHashFile(path))) + bcache.nimexe = hashNimExe() + conf.jsonBuildFile = conf.jsonBuildInstructionsFile + conf.jsonBuildFile.string.writeFile(bcache.toJson.pretty) proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile): bool = - if not fileExists(jsonFile): return true - if not fileExists(conf.absOutFile): return true - result = false - try: - let data = json.parseFile(jsonFile.string) - for key in "depfiles cmdline stdinInput currentDir".split: - if not data.hasKey(key): return true - if getCurrentDir() != data["currentDir"].getStr: - # fixes bug #16271 - # Note that simply comparing `expandFilename(projectFile)` would - # not be sufficient in case other flags depend implicitly on `getCurrentDir`, - # and would require much more care. Simply re-compiling is safer for now. - # A better strategy for future work would be to cache (with an LRU cache) - # the N most recent unique build instructions, as done with `rdmd`, - # which is both robust and avoids recompilation when switching back and forth - # between projects, see https://github.com/timotheecour/Nim/issues/199 - return true - let oldCmdLine = data["cmdline"].getStr - if conf.commandLine != oldCmdLine: - return true - if hashNimExe() != data["nimexe"].getStr: - return true - let stdinInput = data["stdinInput"].getBool - let projectIsCmd = data["projectIsCmd"].getBool - if conf.projectIsStdin or stdinInput: - # could optimize by returning false if stdin input was the same, - # but I'm not sure how to get full stding input - return true - - if conf.projectIsCmd or projectIsCmd: - if not (conf.projectIsCmd and projectIsCmd): return true - if not data.hasKey("cmdInput"): return true - let cmdInput = data["cmdInput"].getStr - if cmdInput != conf.cmdInput: return true - - let depfilesPairs = data["depfiles"] - doAssert depfilesPairs.kind == JArray - for p in depfilesPairs: - doAssert p.kind == JArray - # >= 2 for forwards compatibility with potential later .json files: - doAssert p.len >= 2 - let depFilename = p[0].getStr - let oldHashValue = p[1].getStr - let newHashValue = $secureHashFile(depFilename) - if oldHashValue != newHashValue: - return true + if not fileExists(jsonFile) or not fileExists(conf.absOutFile): return true + var bcache: BuildCache + try: bcache.fromJson(jsonFile.string.parseFile) except IOError, OSError, ValueError: - echo "Warning: JSON processing failed: ", getCurrentExceptionMsg() - result = true + stderr.write "Warning: JSON processing failed for $#: $#\n" % [jsonFile.string, getCurrentExceptionMsg()] + return true + if bcache.currentDir != getCurrentDir() or # fixes bug #16271 + bcache.configFiles != conf.configFiles.mapIt(it.string) or + bcache.cacheVersion != cacheVersion or bcache.outputFile != conf.absOutFile.string or + bcache.cmdline != conf.commandLine or bcache.nimexe != hashNimExe() or + bcache.projectIsCmd != conf.projectIsCmd or conf.cmdInput != bcache.cmdInput: return true + if bcache.stdinInput or conf.projectIsStdin: return true + # xxx optimize by returning false if stdin input was the same + for (file, hash) in bcache.depfiles: + if $secureHashFile(file) != hash: return true proc runJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile) = - try: - let data = json.parseFile(jsonFile.string) - let output = data["outputFile"].getStr - createDir output.parentDir - let outputCurrent = $conf.absOutFile - if output != outputCurrent: - # previously, any specified output file would be silently ignored; - # simply copying won't work in some cases, for example with `extraCmds`, - # so we just make it an error, user should use same command for jsonscript - # as was used with --compileOnly. - globalError(conf, gCmdLineInfo, "jsonscript command outputFile '$1' must match '$2' which was specified during --compileOnly, see \"outputFile\" entry in '$3' " % [outputCurrent, output, jsonFile.string]) - - let toCompile = data["compile"] - doAssert toCompile.kind == JArray - var cmds: TStringSeq - var prettyCmds: TStringSeq - let prettyCb = proc (idx: int) = writePrettyCmdsStderr(prettyCmds[idx]) - for c in toCompile: - doAssert c.kind == JArray - doAssert c.len >= 2 - - cmds.add(c[1].getStr) - prettyCmds.add displayProgressCC(conf, c[0].getStr, c[1].getStr) - - execCmdsInParallel(conf, cmds, prettyCb) - - let linkCmd = data["linkcmd"] - doAssert linkCmd.kind == JString - execLinkCmd(conf, linkCmd.getStr) - if data.hasKey("extraCmds"): - let extraCmds = data["extraCmds"] - doAssert extraCmds.kind == JArray - for cmd in extraCmds: - doAssert cmd.kind == JString, $cmd.kind - let cmd2 = cmd.getStr - execExternalProgram(conf, cmd2, hintExecuting) - + var bcache: BuildCache + try: bcache.fromJson(jsonFile.string.parseFile) except: let e = getCurrentException() - conf.quitOrRaise "\ncaught exception:\n" & e.msg & "\nstacktrace:\n" & e.getStackTrace() & - "error evaluating JSON file: " & jsonFile.string + conf.quitOrRaise "\ncaught exception:\n$#\nstacktrace:\n$#error evaluating JSON file: $#" % + [e.msg, e.getStackTrace(), jsonFile.string] + let output = bcache.outputFile + createDir output.parentDir + let outputCurrent = $conf.absOutFile + if output != outputCurrent or bcache.cacheVersion != cacheVersion: + globalError(conf, gCmdLineInfo, + "jsonscript command outputFile '$1' must match '$2' which was specified during --compileOnly, see \"outputFile\" entry in '$3' " % + [outputCurrent, output, jsonFile.string]) + var cmds, prettyCmds: TStringSeq + let prettyCb = proc (idx: int) = writePrettyCmdsStderr(prettyCmds[idx]) + for (name, cmd) in bcache.compile: + cmds.add cmd + prettyCmds.add displayProgressCC(conf, name, cmd) + execCmdsInParallel(conf, cmds, prettyCb) + execLinkCmd(conf, bcache.linkcmd) + for cmd in bcache.extraCmds: execExternalProgram(conf, cmd, hintExecuting) proc genMappingFiles(conf: ConfigRef; list: CfileList): Rope = for it in list: diff --git a/compiler/nimconf.nim b/compiler/nimconf.nim index b63e5a0ad..94c93a283 100644 --- a/compiler/nimconf.nim +++ b/compiler/nimconf.nim @@ -240,13 +240,10 @@ proc getSystemConfigPath*(conf: ConfigRef; filename: RelativeFile): AbsoluteFile proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen: IdGenerator) = setDefaultLibpath(conf) - - var configFiles = newSeq[AbsoluteFile]() - template readConfigFile(path) = let configPath = path if readConfigFile(configPath, cache, conf): - configFiles.add(configPath) + conf.configFiles.add(configPath) template runNimScriptIfExists(path: AbsoluteFile, isMain = false) = let p = path # eval once @@ -256,7 +253,7 @@ proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen: elif conf.projectIsCmd: s = llStreamOpen(conf.cmdInput) if s == nil and fileExists(p): s = llStreamOpen(p, fmRead) if s != nil: - configFiles.add(p) + conf.configFiles.add(p) runNimScript(cache, p, idgen, freshDefines = false, conf, s) if optSkipSystemConfigFile notin conf.globalOptions: @@ -295,12 +292,12 @@ proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen: let scriptFile = conf.projectFull.changeFileExt("nims") let scriptIsProj = scriptFile == conf.projectFull template showHintConf = - for filename in configFiles: + for filename in conf.configFiles: # delayed to here so that `hintConf` is honored rawMessage(conf, hintConf, filename.string) if conf.cmd == cmdNimscript: showHintConf() - configFiles.setLen 0 + conf.configFiles.setLen 0 if conf.cmd != cmdIdeTools: if conf.cmd == cmdNimscript: runNimScriptIfExists(conf.projectFull, isMain = true) diff --git a/compiler/options.nim b/compiler/options.nim index e6c667d92..afe8b0fc7 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -308,7 +308,7 @@ type ## should be run ideCmd*: IdeCmd oldNewlines*: bool - cCompiler*: TSystemCC + cCompiler*: TSystemCC # the used compiler modifiedyNotes*: TNoteKinds # notes that have been set/unset from either cmdline/configs cmdlineNotes*: TNoteKinds # notes that have been set/unset from cmdline foreignPackageNotes*: TNoteKinds @@ -353,7 +353,7 @@ type docRoot*: string ## see nim --fullhelp for --docRoot docCmd*: string ## see nim --fullhelp for --docCmd - # the used compiler + configFiles*: seq[AbsoluteFile] # config files (cfg,nims) cIncludes*: seq[AbsoluteDir] # directories to search for included files cLibs*: seq[AbsoluteDir] # directories to search for lib files cLinkedLibs*: seq[string] # libraries to link diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim index e33ae4401..60d78fea5 100644 --- a/lib/std/jsonutils.nim +++ b/lib/std/jsonutils.nim @@ -34,6 +34,23 @@ import macros from enumutils import symbolName from typetraits import OrdinalEnum +when not defined(nimFixedForwardGeneric): + # xxx remove pending csources_v1 update >= 1.2.0 + proc to[T](node: JsonNode, t: typedesc[T]): T = + when T is string: node.getStr + elif T is bool: node.getBool + else: static: doAssert false, $T # support as needed (only needed during bootstrap) + proc isNamedTuple(T: typedesc): bool = # old implementation + when T isnot tuple: result = false + else: + var t: T + for name, _ in t.fieldPairs: + when name == "Field0": return compiles(t.Field0) + else: return true + return false +else: + proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} + type Joptions* = object # xxx rename FromJsonOptions ## Options controlling the behavior of `fromJson`. @@ -61,7 +78,6 @@ proc initToJsonOptions*(): ToJsonOptions = ## initializes `ToJsonOptions` with sane options. ToJsonOptions(enumMode: joptEnumOrd, jsonNodeMode: joptJsonNodeAsRef) -proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} proc distinctBase(T: typedesc): typedesc {.magic: "TypeTrait".} template distinctBase[T](a: T): untyped = distinctBase(typeof(a))(a) From 7e21218a07f919d56cd0974c657aefb11574da26 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Fri, 4 Jun 2021 17:16:51 +0300 Subject: [PATCH 0885/2472] follow-up #18013 - inline syntax highlighting (#18166) --- doc/destructors.rst | 2 +- doc/docgen.rst | 2 +- doc/drnim.rst | 4 +- doc/estp.rst | 2 +- doc/hcr.rst | 31 +++++++++------ doc/intern.rst | 2 +- doc/koch.rst | 27 +++++++------ doc/lib.rst | 12 +++--- doc/manual.rst | 48 ++++++++++++------------ doc/manual_experimental.rst | 20 +++++----- doc/manual_experimental_strictnotnil.rst | 26 ++++++++----- doc/nep1.rst | 11 +++--- doc/nimgrep.rst | 31 ++++++++------- doc/niminst.rst | 24 ++++++------ doc/nims.rst | 45 +++++++++++----------- doc/nimsuggest.rst | 37 +++++++++--------- doc/packaging.rst | 2 +- doc/regexprs.txt | 4 +- doc/testament.rst | 2 +- doc/tools.rst | 15 ++++---- 20 files changed, 183 insertions(+), 164 deletions(-) diff --git a/doc/destructors.rst b/doc/destructors.rst index 3cd4cd925..c99606b95 100644 --- a/doc/destructors.rst +++ b/doc/destructors.rst @@ -5,8 +5,8 @@ Nim Destructors and Move Semantics :Authors: Andreas Rumpf :Version: |nimversion| +.. default-role:: code .. include:: rstcommon.rst -.. default-role:: nim .. contents:: diff --git a/doc/docgen.rst b/doc/docgen.rst index 86352a989..e9ff7f1e3 100644 --- a/doc/docgen.rst +++ b/doc/docgen.rst @@ -5,8 +5,8 @@ :Author: Erik O'Leary :Version: |nimversion| +.. default-role:: code .. include:: rstcommon.rst -.. default-role:: Nim .. contents:: diff --git a/doc/drnim.rst b/doc/drnim.rst index 08de7cd6d..070cd1787 100644 --- a/doc/drnim.rst +++ b/doc/drnim.rst @@ -5,8 +5,8 @@ :Author: Andreas Rumpf :Version: |nimversion| +.. default-role:: code .. include:: rstcommon.rst -.. default-role:: Nim .. contents:: @@ -43,7 +43,7 @@ your code. Installation ============ -Run `koch drnim`:option:, the executable will afterwards be +Run `koch drnim`:cmd:, the executable will afterwards be in ``$nim/bin/drnim``. diff --git a/doc/estp.rst b/doc/estp.rst index 43128bef7..550194d2d 100644 --- a/doc/estp.rst +++ b/doc/estp.rst @@ -2,8 +2,8 @@ Embedded Stack Trace Profiler (ESTP) User Guide =================================================== +.. default-role:: code .. include:: rstcommon.rst -.. default-role:: Nim :Author: Andreas Rumpf :Version: |nimversion| diff --git a/doc/hcr.rst b/doc/hcr.rst index 574c4f39d..dd25e39b3 100644 --- a/doc/hcr.rst +++ b/doc/hcr.rst @@ -1,9 +1,10 @@ -.. default-role:: code - =================================== Hot code reloading =================================== +.. default-role:: code +.. include:: rstcommon.rst + The `hotCodeReloading`:idx: option enables special compilation mode where changes in the code can be applied automatically to a running program. The code reloading happens at the granularity of an individual module. @@ -22,7 +23,7 @@ during development resides. In this example, we use SDL2 to create a window and we reload the logic code when `F9` is pressed. The important lines are marked with `#***`. -To install SDL2 you can use `nimble install sdl2`. +To install SDL2 you can use `nimble install sdl2`:cmd:. .. code-block:: nim @@ -98,34 +99,40 @@ To install SDL2 you can use `nimble install sdl2`. main() -Compile this example via:: +Compile this example via: +```cmd nim c --hotcodereloading:on mymain.nim +``` Now start the program and KEEP it running! -:: - +.. code:: cmd # Unix: mymain & # or Windows (click on the .exe) mymain.exe # edit -For example, change the line:: +For example, change the line: +```nim discard renderer.setDrawColor(255, 128, 128, 0) +``` -into:: +into: +```nim discard renderer.setDrawColor(255, 255, 128, 0) +``` (This will change the color of the rectangle.) Then recompile the project, but do not restart or quit the mymain.exe program! -:: +```cmd nim c --hotcodereloading:on mymain.nim +``` Now give the `mymain` SDL window the focus, press F9, and watch the updated version of the program. @@ -188,15 +195,15 @@ Native code targets =================== Native projects using the hot code reloading option will be implicitly -compiled with the `-d:useNimRtl` option and they will depend on both +compiled with the `-d:useNimRtl`:option: option and they will depend on both the `nimrtl` library and the `nimhcr` library which implements the hot code reloading run-time. Both libraries can be found in the `lib` folder of Nim and can be compiled into dynamic libraries to satisfy runtime demands of the example code above. An example of compiling -`nimhcr.nim` and `nimrtl.nim` when the source dir of Nim is installed +``nimhcr.nim`` and ``nimrtl.nim`` when the source dir of Nim is installed with choosenim follows. -:: +.. code:: console # Unix/MacOS # Make sure you are in the directory containing your .nim files diff --git a/doc/intern.rst b/doc/intern.rst index 2f83e0d06..ab74c4a3a 100644 --- a/doc/intern.rst +++ b/doc/intern.rst @@ -6,8 +6,8 @@ :Author: Andreas Rumpf :Version: |nimversion| +.. default-role:: code .. include:: rstcommon.rst -.. default-role:: Nim .. contents:: "Abstraction is layering ignorance on top of reality." -- Richard Gabriel diff --git a/doc/koch.rst b/doc/koch.rst index 1eb02d785..b8d85dff4 100644 --- a/doc/koch.rst +++ b/doc/koch.rst @@ -1,11 +1,11 @@ -.. default-role:: code - =============================== Nim maintenance script =============================== :Version: |nimversion| +.. default-role:: code +.. include:: rstcommon.rst .. contents:: .. raw:: html @@ -19,7 +19,7 @@ Introduction The `koch`:idx: program is Nim's maintenance script. It is a replacement for make and shell scripting with the advantage that it is much more portable. -The word *koch* means *cook* in German. `koch` is used mainly to build the +The word *koch* means *cook* in German. `koch`:cmd: is used mainly to build the Nim compiler, but it can also be used for other tasks. This document describes the supported commands and their options. @@ -33,12 +33,11 @@ boot command The `boot`:idx: command bootstraps the compiler, and it accepts different options: --d:release - By default a debug version is created, passing this option will +-d:release By default a debug version is created, passing this option will force a release build, which is much faster and should be preferred unless you are debugging the compiler. --d:nimUseLinenoise - Use the linenoise library for interactive mode (not needed on Windows). +-d:nimUseLinenoise Use the linenoise library for interactive mode + (not needed on Windows). After compilation is finished you will hopefully end up with the nim compiler in the `bin` directory. You can add Nim's `bin` directory to @@ -56,16 +55,16 @@ temp command ------------ The temp command builds the Nim compiler but with a different final name -(`nim_temp`), so it doesn't overwrite your normal compiler. You can use +(`nim_temp`:cmd:), so it doesn't overwrite your normal compiler. You can use this command to test different options, the same you would issue for the `boot command <#commands-boot-command>`_. test command ------------ -The `test`:idx: command can also be invoked with the alias `tests`. This -command will compile and run `testament/tester.nim`, which is the main -driver of Nim's test suite. You can pass options to the `test` command, +The `test`:idx: command can also be invoked with the alias `tests`:option:. This +command will compile and run ``testament/tester.nim``, which is the main +driver of Nim's test suite. You can pass options to the `test`:option: command, they will be forwarded to the tester. See its source code for available options. @@ -74,13 +73,13 @@ web command The `web`:idx: command converts the documentation in the `doc` directory from rst to HTML. It also repeats the same operation but places the result in -the `web/upload` which can be used to update the website at +the ``web/upload`` which can be used to update the website at https://nim-lang.org. By default, the documentation will be built in parallel using the number of available CPU cores. If any documentation build sub-commands fail, they will be rerun in serial fashion so that meaningful error output can be gathered for -inspection. The `--parallelBuild:n` switch or configuration option can be +inspection. The `--parallelBuild:n`:option: switch or configuration option can be used to force a specific number of parallel jobs or run everything serially from the start (`n == 1`). @@ -89,4 +88,4 @@ pdf command The `pdf`:idx: command builds PDF versions of Nim documentation: Manual, Tutorial and a few other documents. To run it one needs to -`install Latex/pdflatex `_ first. +`install Latex/xelatex `_ first. diff --git a/doc/lib.rst b/doc/lib.rst index f09fe16b2..988fbf6f0 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -1,5 +1,3 @@ -.. default-role:: code - ==================== Nim Standard Library ==================== @@ -7,11 +5,13 @@ Nim Standard Library :Author: Andreas Rumpf :Version: |nimversion| +.. default-role:: code +.. include:: rstcommon.rst .. contents:: Nim's library is divided into *pure libraries*, *impure libraries*, and *wrappers*. -Pure libraries do not depend on any external `*.dll` or `lib*.so` binary +Pure libraries do not depend on any external ``*.dll`` or ``lib*.so`` binary while impure libraries do. A wrapper is an impure library that is a very low-level interface to a C library. @@ -39,11 +39,11 @@ Automatic imports * `threads `_ Basic Nim thread support. **Note:** This is part of the system module. Do not - import it explicitly. Enabled with `--threads:on`. + import it explicitly. Enabled with `--threads:on`:option:. * `channels `_ Nim message passing support for threads. **Note:** This is part of the - system module. Do not import it explicitly. Enabled with `--threads:on`. + system module. Do not import it explicitly. Enabled with `--threads:on`:option:. Core @@ -366,7 +366,7 @@ Parsers * `parsecfg `_ The `parsecfg` module implements a high-performance configuration file - parser. The configuration file's syntax is similar to the Windows `.ini` + parser. The configuration file's syntax is similar to the Windows ``.ini`` format, but much more powerful, as it is not a line based parser. String literals, raw string literals, and triple quote string literals are supported as in the Nim programming language. diff --git a/doc/manual.rst b/doc/manual.rst index ff34b62ca..867db245b 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -572,7 +572,8 @@ In the following examples, `-1` is a single token: "abc";-1 -In the following examples, `-1` is parsed as two separate tokens (as `- 1`): +In the following examples, `-1` is parsed as two separate tokens +(as `-`:tok: `1`:tok:): .. code-block:: nim @@ -2509,7 +2510,7 @@ of the argument. range. 3. Generic match: `f` is a generic type and `a` matches, for instance `a` is `int` and `f` is a generic (constrained) parameter - type (like in `[T]` or `[T: int|char]`. + type (like in `[T]` or `[T: int|char]`). 4. Subrange or subtype match: `a` is a `range[T]` and `T` matches `f` exactly. Or: `a` is a subtype of `f`. 5. Integral conversion match: `a` is convertible to `f` and `f` and `a` @@ -3840,7 +3841,7 @@ Type bound operators currently include: (some of which are still implementation defined and not yet documented). For more details on some of those procs, see -`lifetimeminustracking-hooks `_. +`Lifetime-tracking hooks `_. Nonoverloadable builtins ------------------------ @@ -5283,7 +5284,7 @@ instantiations cross multiple different modules: In module B has an `init` proc from module C in its scope that is not taken into account when `genericB` is instantiated which leads to the -instantiation of `genericA`. The solution is to `forward`:idx these +instantiation of `genericA`. The solution is to `forward`:idx: these symbols by a `bind` statement inside `genericB`. @@ -7305,19 +7306,19 @@ Produces: supplied pattern to denote the concrete type parameters of the generic type. See the usage of the apostrophe operator in proc patterns for more details. -.. code-block:: nim + .. code-block:: nim - type - VectorIterator {.importcpp: "std::vector<'0>::iterator".} [T] = object + type + VectorIterator {.importcpp: "std::vector<'0>::iterator".} [T] = object - var x: VectorIterator[cint] + var x: VectorIterator[cint] -Produces: + Produces: -.. code-block:: C + .. code-block:: C - std::vector::iterator x; + std::vector::iterator x; ImportJs pragma @@ -7444,7 +7445,7 @@ compile-time define pragmas --------------------------- The pragmas listed here can be used to optionally accept values from -the -d/--define option at compile time. +the `-d/--define`:option: option at compile time. The implementation currently provides the following possible options (various others may be added later). @@ -7461,7 +7462,7 @@ pragma description const FooBar {.intdefine.}: int = 5 echo FooBar -:: +.. code:: cmd nim c -d:FooBar=42 foobar.nim In the above example, providing the `-d`:option: flag causes the symbol @@ -7480,7 +7481,8 @@ pragma pragma ------------- The `pragma` pragma can be used to declare user-defined pragmas. This is -useful because Nim's templates and macros do not affect pragmas. User-defined pragmas are in a different module-wide scope than all other symbols. +useful because Nim's templates and macros do not affect pragmas. +User-defined pragmas are in a different module-wide scope than all other symbols. They cannot be imported from a module. Example: @@ -7553,18 +7555,18 @@ More examples with custom pragmas: - Better serialization/deserialization control: -.. code-block:: nim - type MyObj = object - a {.dontSerialize.}: int - b {.defaultDeserialize: 5.}: int - c {.serializationKey: "_c".}: string + .. code-block:: nim + type MyObj = object + a {.dontSerialize.}: int + b {.defaultDeserialize: 5.}: int + c {.serializationKey: "_c".}: string - Adopting type for gui inspector in a game engine: -.. code-block:: nim - type MyComponent = object - position {.editable, animatable.}: Vector3 - alpha {.editRange: [0.0..1.0], animatable.}: float32 + .. code-block:: nim + type MyComponent = object + position {.editable, animatable.}: Vector3 + alpha {.editRange: [0.0..1.0], animatable.}: float32 Macro pragmas diff --git a/doc/manual_experimental.rst b/doc/manual_experimental.rst index 51037b509..2c1043ecb 100644 --- a/doc/manual_experimental.rst +++ b/doc/manual_experimental.rst @@ -1,5 +1,3 @@ -.. default-role:: code - ========================= Nim Experimental Features ========================= @@ -7,6 +5,8 @@ Nim Experimental Features :Authors: Andreas Rumpf :Version: |nimversion| +.. default-role:: code +.. include:: rstcommon.rst .. contents:: @@ -15,7 +15,7 @@ About this document This document describes features of Nim that are to be considered experimental. Some of these are not covered by the `.experimental` pragma or -`--experimental` switch because they are already behind a special syntax and +`--experimental`:option: switch because they are already behind a special syntax and one may want to use Nim libraries using these features without using them oneself. @@ -1109,7 +1109,7 @@ The `parameter constraint`:idx: expression can use the operators `|` (or), Predicate Meaning =================== ===================================================== `atom` The matching node has no children. -`lit` The matching node is a literal like "abc", 12. +`lit` The matching node is a literal like `"abc"`, `12`. `sym` The matching node must be a symbol (a bound identifier). `ident` The matching node must be an identifier (an unbound @@ -1185,7 +1185,7 @@ constant folding, so the following does not work: The reason is that the compiler already transformed the 1 into "1" for the `echo` statement. However, a term rewriting macro should not change the -semantics anyway. In fact they can be deactivated with the `--patterns:off` +semantics anyway. In fact they can be deactivated with the `--patterns:off`:option: command line option or temporarily with the `patterns` pragma. @@ -1779,7 +1779,7 @@ Noalias annotation ================== Since version 1.4 of the Nim compiler, there is a `.noalias` annotation for variables -and parameters. It is mapped directly to C/C++'s `restrict` keyword and means that +and parameters. It is mapped directly to C/C++'s `restrict`:c: keyword and means that the underlying pointer is pointing to a unique location in memory, no other aliases to this location exist. It is *unchecked* that this alias restriction is followed, if the restriction is violated, the backend optimizer is free to miscompile the code. @@ -1832,8 +1832,8 @@ the `view types section <#view-types-algorithm>`_. View types ========== -**Note**: `--experimental:views` is more effective -with `--experimental:strictFuncs`. +**Note**: `--experimental:views`:option: is more effective +with `--experimental:strictFuncs`:option:. A view type is a type that is or contains one of the following types: @@ -1981,8 +1981,8 @@ The scope of the view does not matter: The analysis requires as much precision about mutations as is reasonably obtainable, so it is more effective with the experimental `strict funcs <#strict-funcs>`_ -feature. In other words `--experimental:views` works better -with `--experimental:strictFuncs`. +feature. In other words `--experimental:views`:option: works better +with `--experimental:strictFuncs`:option:. The analysis is currently control flow insensitive: diff --git a/doc/manual_experimental_strictnotnil.rst b/doc/manual_experimental_strictnotnil.rst index cb3bfeca9..b14e5f9f3 100644 --- a/doc/manual_experimental_strictnotnil.rst +++ b/doc/manual_experimental_strictnotnil.rst @@ -1,8 +1,9 @@ -.. default-role:: code - Strict not nil checking ========================= +.. default-role:: code +.. include:: rstcommon.rst + **Note:** This feature is experimental, you need to enable it with .. code-block:: nim @@ -10,7 +11,7 @@ Strict not nil checking or -.. code-block:: bash +.. code-block:: cmd nim c --experimental:strictNotNil In the second case it would check builtin and imported modules as well. @@ -60,14 +61,16 @@ You can annotate a type where nil isn't a valid value with `not nil`. If a type can include `nil` as a valid value, dereferencing values of the type -is checked by the compiler: if a value which might be nil is derefenced, this produces a warning by default, you can turn this into an error using the compiler options `--warningAsError:strictNotNil` +is checked by the compiler: if a value which might be nil is derefenced, this +produces a warning by default, you can turn this into an error using +the compiler options `--warningAsError:strictNotNil`:option:. If a type is nilable, you should dereference its values only after a `isNil` or equivalent check. local turn on/off --------------------- -You can still turn off nil checking on function/module level by using a `{.strictNotNil: off}.` pragma. +You can still turn off nil checking on function/module level by using a `{.strictNotNil: off.}` pragma. Note: test that/TODO for code/manual. nilability state @@ -75,11 +78,14 @@ nilability state Currently a nilable value can be `Safe`, `MaybeNil` or `Nil` : we use internally `Parent` and `Unreachable` but this is an implementation detail(a parent layer has the actual nilability). -`Safe` means it shouldn't be nil at that point: e.g. after assignment to a non-nil value or `not a.isNil` check -`MaybeNil` means it might be nil, but it might not be nil: e.g. an argument, a call argument or a value after an `if` and `else`. -`Nil` means it should be nil at that point; e.g. after an assignment to `nil` or a `.isNil` check. - -`Unreachable` means it shouldn't be possible to access this in this branch: so we do generate a warning as well. +- `Safe` means it shouldn't be nil at that point: e.g. after assignment to + a non-nil value or `not a.isNil` check +- `MaybeNil` means it might be nil, but it might not be nil: e.g. an argument, + a call argument or a value after an `if` and `else`. +- `Nil` means it should be nil at that point; e.g. after an assignment to + `nil` or a `.isNil` check. +- `Unreachable` means it shouldn't be possible to access this in this branch: + so we do generate a warning as well. We show an error for each dereference (`[]`, `.field`, `[index]` `()` etc) which is of a tracked expression which is in `MaybeNil` or `Nil` state. diff --git a/doc/nep1.rst b/doc/nep1.rst index 8687725bb..fc69bab69 100644 --- a/doc/nep1.rst +++ b/doc/nep1.rst @@ -1,11 +1,12 @@ -.. default-role:: code - ========================================================== Nim Enhancement Proposal #1 - Standard Library Style Guide ========================================================== + :Author: Clay Sweetser, Dominik Picheta :Version: |nimversion| +.. default-role:: code +.. include:: rstcommon.rst .. contents:: @@ -139,7 +140,7 @@ Naming Conventions - rotate and rotated - When the 'returns transformed copy' version already exists like `strutils.replace` - an in-place version should get an `-In` suffix (`replaceIn` for this example). + an in-place version should get an ``-In`` suffix (`replaceIn` for this example). - Use `subjectVerb`, not `verbSubject`, e.g.: `fileExists`, not `existsFile`. @@ -225,8 +226,8 @@ indentation indent Coding Conventions ------------------ -- The 'return' statement should ideally be used when its control-flow properties - are required. Use a procedure's implicit 'result' variable whenever possible. +- The `return` statement should ideally be used when its control-flow properties + are required. Use a procedure's implicit `result` variable whenever possible. This improves readability. .. code-block:: nim diff --git a/doc/nimgrep.rst b/doc/nimgrep.rst index 52fc4d62a..6088a4c45 100644 --- a/doc/nimgrep.rst +++ b/doc/nimgrep.rst @@ -1,14 +1,11 @@ -.. default-role:: code - ========================= nimgrep User's manual ========================= -.. default-role:: literal - :Author: Andreas Rumpf :Version: 1.6.0 +.. default-role:: option .. contents:: Nimgrep is a command line tool for search and replace tasks. It can search for @@ -23,11 +20,12 @@ Apart from that it is a generic text manipulation tool. Installation ============ -Compile nimgrep with the command:: +Compile nimgrep with the command: +.. code:: cmd nim c -d:release tools/nimgrep.nim -And copy the executable somewhere in your `$PATH`. +And copy the executable somewhere in your ``$PATH``. Command line switches @@ -40,31 +38,32 @@ Examples All examples below use default PCRE Regex patterns: -+ To search recursively in Nim files using style-insensitive identifiers:: ++ To search recursively in Nim files using style-insensitive identifiers: - --recursive --ext:'nim|nims' --ignoreStyle + .. code:: cmd + nimgrep --recursive --ext:'nim|nims' --ignoreStyle # short: -r --ext:'nim|nims' -y .. Note:: we used `'` quotes to avoid special treatment of `|` symbol for shells like Bash + To exclude version control directories (Git, Mercurial=hg, Subversion=svn) - from the search:: + from the search: - --excludeDir:'^\.git$' --excludeDir:'^\.hg$' --excludeDir:'^\.svn$' + .. code:: cmd + nimgrep --excludeDir:'^\.git$' --excludeDir:'^\.hg$' --excludeDir:'^\.svn$' # short: --ed:'^\.git$' --ed:'^\.hg$' --ed:'^\.svn$' + To search only in paths containing the `tests` sub-directory recursively:: - --recursive --includeDir:'(^|/)tests($|/)' + .. code:: cmd + nimgrep --recursive --includeDir:'(^|/)tests($|/)' # short: -r --id:'(^|/)tests($|/)' - .. Attention:: note the subtle difference between `--excludeDir` and - `--includeDir`: the former is applied to relative directory entries + .. Attention:: note the subtle difference between `--excludeDir`:option: and + `--includeDir`:option:\: the former is applied to relative directory entries and the latter is applied to the whole paths + Nimgrep can search multi-line, e.g. to find files containing `import` - and then `strutils` use:: - - 'import(.|\n)*?strutils' + and then `strutils` use pattern `'import(.|\n)*?strutils'`:option:. diff --git a/doc/niminst.rst b/doc/niminst.rst index 3ccb47cc8..8fff1f0e8 100644 --- a/doc/niminst.rst +++ b/doc/niminst.rst @@ -1,5 +1,3 @@ -.. default-role:: code - ========================= niminst User's manual ========================= @@ -7,6 +5,8 @@ :Author: Andreas Rumpf :Version: |nimversion| +.. default-role:: code +.. include:: rstcommon.rst .. contents:: Introduction @@ -34,7 +34,7 @@ configuration file. Here's an example of how the syntax looks like: The value of a key-value pair can reference user-defined variables via the `$variable` notation: They can be defined in the command line with the -`--var:name=value` switch. This is useful to not hard-coding the +`--var:name=value`:option: switch. This is useful to not hard-coding the program's version number into the configuration file, for instance. It follows a description of each possible section and how it affects the @@ -91,10 +91,10 @@ Documentation section The `documentation` section supports the `files` key. Listed files will be installed into the OS's native documentation directory -(which might be `$appdir/doc`). +(which might be ``$appdir/doc``). There is a `start` key which determines whether the Windows installer -generates a link to e.g. the `index.html` of your documentation. +generates a link to e.g. the ``index.html`` of your documentation. Other section @@ -125,9 +125,9 @@ Other possible options are: ==================== ======================================================= Key description ==================== ======================================================= -`BinPath` paths to add to the Windows `%PATH%` environment +`BinPath` paths to add to the Windows `%PATH%` environment variable. Example: ``BinPath: r"bin;dist\mingw\bin"`` -`InnoSetup` boolean flag whether an Inno Setup installer should be +`InnoSetup` boolean flag whether an Inno Setup installer should be generated for Windows. Example: `InnoSetup: "Yes"` ==================== ======================================================= @@ -137,7 +137,7 @@ UnixBin section The `UnixBin` section currently only supports the `files` key. Listed files will be installed into the OS's native bin directory -(e.g. `/usr/local/bin`). The exact location depends on the +(e.g. ``/usr/local/bin``). The exact location depends on the installation path the user specifies when running the `install.sh` script. @@ -165,9 +165,9 @@ Possible options are: ==================== ======================================================= Key description ==================== ======================================================= -`path` Path to Inno Setup. +`path` Path to Inno Setup. Example: ``path = r"c:\inno setup 5\iscc.exe"`` -`flags` Flags to pass to Inno Setup. +`flags` Flags to pass to Inno Setup. Example: `flags = "/Q"` ==================== ======================================================= @@ -180,8 +180,8 @@ Possible options are: ==================== ======================================================= Key description ==================== ======================================================= -`path` Path to the C compiler. -`flags` Flags to pass to the C Compiler. +`path` Path to the C compiler. +`flags` Flags to pass to the C Compiler. Example: `flags = "-w"` ==================== ======================================================= diff --git a/doc/nims.rst b/doc/nims.rst index f81637d73..5ae1d7baa 100644 --- a/doc/nims.rst +++ b/doc/nims.rst @@ -1,32 +1,33 @@ -.. default-role:: code - ================================ NimScript ================================ +.. default-role:: code +.. include:: rstcommon.rst + Strictly speaking, `NimScript` is the subset of Nim that can be evaluated by Nim's builtin virtual machine (VM). This VM is used for Nim's compiletime function evaluation features. -The `nim` executable processes the `.nims` configuration files in +The `nim`:cmd: executable processes the ``.nims`` configuration files in the following directories (in this order; later files overwrite previous settings): 1) If environment variable `XDG_CONFIG_HOME` is defined, - `$XDG_CONFIG_HOME/nim/config.nims` or - `~/.config/nim/config.nims` (POSIX) or - `%APPDATA%/nim/config.nims` (Windows). This file can be skipped - with the `--skipUserCfg` command line option. -2) `$parentDir/config.nims` where `$parentDir` stands for any + ``$XDG_CONFIG_HOME/nim/config.nims`` or + ``~/.config/nim/config.nims`` (POSIX) or + ``%APPDATA%/nim/config.nims`` (Windows). This file can be skipped + with the `--skipUserCfg`:option: command line option. +2) ``$parentDir/config.nims`` where ``$parentDir`` stands for any parent directory of the project file's path. These files can be - skipped with the `--skipParentCfg` command line option. -3) `$projectDir/config.nims` where `$projectDir` stands for the - project's path. This file can be skipped with the `--skipProjCfg` + skipped with the `--skipParentCfg`:option: command line option. +3) ``$projectDir/config.nims`` where ``$projectDir`` stands for the + project's path. This file can be skipped with the `--skipProjCfg`:option: command line option. 4) A project can also have a project specific configuration file named - `$project.nims` that resides in the same directory as - `$project.nim`. This file can be skipped with the same - `--skipProjCfg` command line option. + ``$project.nims`` that resides in the same directory as + ``$project.nim``. This file can be skipped with the same + `--skipProjCfg`:option: command line option. For available procs and implementation details see `nimscript `_. @@ -111,8 +112,8 @@ See also: NimScript as a configuration file ================================= -A command-line switch `--FOO` is written as `switch("FOO")` in -NimScript. Similarly, command-line `--FOO:VAL` translates to +A command-line switch `--FOO`:option: is written as `switch("FOO")` in +NimScript. Similarly, command-line `--FOO:VAL`:option: translates to `switch("FOO", "VAL")`. Here are few examples of using the `switch` proc: @@ -125,7 +126,7 @@ Here are few examples of using the `switch` proc: # command-line: --forceBuild switch("forceBuild") -NimScripts also support `--` templates for convenience, which look +NimScripts also support `--`:option: templates for convenience, which look like command-line switches written as-is in the NimScript file. So the above example can be rewritten as: @@ -136,7 +137,7 @@ above example can be rewritten as: **Note**: In general, the *define* switches can also be set in NimScripts using `switch` or `--`, as shown in above -examples. Only the `release` define (`-d:release`) cannot be set +examples. Only the `release` define (`-d:release`:option:) cannot be set in NimScripts. @@ -145,7 +146,7 @@ NimScript as a build tool The `task` template that the `system` module defines allows a NimScript file to be used as a build tool. The following example defines a -task `build` that is an alias for the `c` command: +task `build` that is an alias for the `c`:option: command: .. code-block:: nim task build, "builds an example": @@ -159,7 +160,7 @@ Task Description ========= =================================================== `help` List all the available NimScript tasks along with their docstrings. `build` Build the project with the required - backend (`c`, `cpp` or `js`). + backend (`c`:option:, `cpp`:option: or `js`:option:). `tests` Runs the tests belonging to the project. `bench` Runs benchmarks belonging to the project. ========= =================================================== @@ -180,7 +181,7 @@ Standalone NimScript ==================== NimScript can also be used directly as a portable replacement for Bash and -Batch files. Use `nim myscript.nims` to run `myscript.nims`. For example, +Batch files. Use `nim myscript.nims`:cmd: to run ``myscript.nims``. For example, installation of Nimble could be accomplished with this simple script: .. code-block:: nim @@ -199,7 +200,7 @@ installation of Nimble could be accomplished with this simple script: mvFile "nimble" & $id & "/src/nimble".toExe, "bin/nimble".toExe On Unix, you can also use the shebang `#!/usr/bin/env nim`, as long as your filename -ends with `.nims`: +ends with ``.nims``: .. code-block:: nim diff --git a/doc/nimsuggest.rst b/doc/nimsuggest.rst index 509f72e8a..82693da0e 100644 --- a/doc/nimsuggest.rst +++ b/doc/nimsuggest.rst @@ -1,5 +1,3 @@ -.. default-role:: code - ================================ Nim IDE Integration Guide ================================ @@ -7,14 +5,16 @@ :Author: Unknown :Version: |nimversion| +.. default-role:: code +.. include:: rstcommon.rst .. contents:: Nim differs from many other compilers in that it is really fast, and being so fast makes it suited to provide external queries for text editors about the source code being written. Through the -`nimsuggest` tool, any IDE -can query a `.nim` source file and obtain useful information like +`nimsuggest`:cmd: tool, any IDE +can query a ``.nim`` source file and obtain useful information like definition of symbols or suggestions for completion. This document will guide you through the available options. If you @@ -27,29 +27,30 @@ already available. Installation ============ -Nimsuggest is part of Nim's core. Build it via:: +Nimsuggest is part of Nim's core. Build it via: +.. code:: cmd koch nimsuggest Nimsuggest invocation ===================== -Run it via `nimsuggest --stdin --debug myproject.nim`. Nimsuggest is a +Run it via `nimsuggest --stdin --debug myproject.nim`:cmd:. Nimsuggest is a server that takes queries that are related to `myproject`. There is some -support so that you can throw random `.nim` files which are not part +support so that you can throw random ``.nim`` files which are not part of `myproject` at Nimsuggest too, but usually the query refer to modules/files that are part of `myproject`. -`--stdin` means that Nimsuggest reads the query from `stdin`. This is great +`--stdin`:option: means that Nimsuggest reads the query from `stdin`. This is great for testing things out and playing with it but for an editor communication via sockets is more reasonable so that is the default. It listens to port 6000 by default. -Nimsuggest is basically a frontend for the nim compiler so `--path` flags and +Nimsuggest is basically a frontend for the nim compiler so `--path`:option: flags and `config files `_ can be used to specify additional dependencies like -`nimsuggest --stdin --debug --path:"dependencies" myproject.nim`. +`nimsuggest --stdin --debug --path:"dependencies" myproject.nim`:cmd:. Specifying the location of the query @@ -60,25 +61,25 @@ cryptic 3 letter "command" `def` or `con` or `sug` or `use` followed by a location. A query location consists of: -`file.nim` +``file.nim`` This is the name of the module or include file the query refers to. -`dirtyfile.nim` +``dirtyfile.nim`` This is optional. The `file` parameter is enough for static analysis, but IDEs tend to have *unsaved buffers* where the user may still be in the middle of typing a line. In such situations the IDE can save the current contents to a temporary file and then use the - `dirtyfile.nim` option to tell Nimsuggest that `foobar.nim` should - be taken from `temporary/foobar.nim`. + ``dirtyfile.nim`` option to tell Nimsuggest that ``foobar.nim`` should + be taken from ``temporary/foobar.nim``. -`line` +``line`` An integer with the line you are going to query. For the compiler lines start at **1**. -`col` +``col`` An integer with the column you are going to query. For the compiler columns start at **0**. @@ -149,9 +150,9 @@ tab characters (``\t``). The values of each column are: 1. Three characters indicating the type of returned answer (e.g. `def` for definition, `sug` for suggestion, etc). 2. Type of the symbol. This can be `skProc`, `skLet`, and just - about any of the enums defined in the module `compiler/ast.nim`. + about any of the enums defined in the module ``compiler/ast.nim``. 3. Fully qualified path of the symbol. If you are querying a symbol - defined in the `proj.nim` file, this would have the form + defined in the ``proj.nim`` file, this would have the form `proj.symbolName`. 4. Type/signature. For variables and enums this will contain the type of the symbol, for procs, methods and templates this will diff --git a/doc/packaging.rst b/doc/packaging.rst index ecde73f61..ecb17e482 100644 --- a/doc/packaging.rst +++ b/doc/packaging.rst @@ -45,7 +45,7 @@ The Debian package ships bash and ksh completion and manpages that can be reused Hints on the build process: -:: +.. code:: cmd # build from C sources and then using koch ./build.sh --os $os_type --cpu $cpu_arch diff --git a/doc/regexprs.txt b/doc/regexprs.txt index 9ec08b810..f5544cc28 100644 --- a/doc/regexprs.txt +++ b/doc/regexprs.txt @@ -47,9 +47,11 @@ full documentation of Perl's regular expressions. Because the backslash ``\`` is a meta character both in the Nim programming language and in regular expressions, it is strongly recommended that one uses the *raw* strings of Nim, so that -backslashes are interpreted by the regular expression engine:: +backslashes are interpreted by the regular expression engine: +```nim r"\S" # matches any character that is not whitespace +``` A regular expression is a pattern that is matched against a subject string from left to right. Most characters stand for themselves in a pattern, and diff --git a/doc/testament.rst b/doc/testament.rst index ed1bd63a1..5590cc6d7 100644 --- a/doc/testament.rst +++ b/doc/testament.rst @@ -2,8 +2,8 @@ Testament =========== +.. default-role:: code .. include:: rstcommon.rst -.. default-role:: nim .. contents:: Testament is an advanced automatic unittests runner for Nim tests, is used for the development of Nim itself, diff --git a/doc/tools.rst b/doc/tools.rst index e0044e1ca..0de4ac914 100644 --- a/doc/tools.rst +++ b/doc/tools.rst @@ -1,9 +1,10 @@ -.. default-role:: code - ======================== Tools available with Nim ======================== +.. default-role:: code +.. include:: rstcommon.rst + The standard distribution ships with the following tools: - | `Hot code reloading `_ @@ -11,11 +12,11 @@ The standard distribution ships with the following tools: document explaining how it works. - | `Documentation generator `_ - | The builtin document generator `nim doc` generates HTML documentation - from `.nim` source files. + | The builtin document generator `nim doc`:cmd: generates HTML documentation + from ``.nim`` source files. - | `Nimsuggest for IDE support `_ - | Through the `nimsuggest` tool, any IDE can query a `.nim` source file + | Through the `nimsuggest`:cmd: tool, any IDE can query a ``.nim`` source file and obtain useful information like the definition of symbols or suggestions for completion. @@ -29,11 +30,11 @@ The standard distribution ships with the following tools: | Nim search and replace utility. - | nimpretty - | `nimpretty` is a Nim source code beautifier, + | `nimpretty`:cmd: is a Nim source code beautifier, to format code according to the official style guide. - | `testament `_ - | `testament` is an advanced automatic *unittests runner* for Nim tests, + | `testament`:cmd: is an advanced automatic *unittests runner* for Nim tests, is used for the development of Nim itself, offers process isolation for your tests, it can generate statistics about test cases, supports multiple targets (C, JS, etc), `simulated Dry-Runs `_, From 5423915e5ade14fb98c2aba28f269311506960e5 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 4 Jun 2021 16:20:47 +0200 Subject: [PATCH 0886/2472] real bugfix for #17170 (#18171) * real bugfix for #17170 * better fix --- compiler/closureiters.nim | 10 ++++++---- compiler/injectdestructors.nim | 2 ++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/compiler/closureiters.nim b/compiler/closureiters.nim index adc5a4873..a8da7485e 100644 --- a/compiler/closureiters.nim +++ b/compiler/closureiters.nim @@ -414,8 +414,11 @@ proc exprToStmtList(n: PNode): tuple[s, res: PNode] = proc newEnvVarAsgn(ctx: Ctx, s: PSym, v: PNode): PNode = - result = newTree(nkFastAsgn, ctx.newEnvVarAccess(s), v) - result.info = v.info + if isEmptyType(v.typ): + result = v + else: + result = newTree(nkFastAsgn, ctx.newEnvVarAccess(s), v) + result.info = v.info proc addExprAssgn(ctx: Ctx, output, input: PNode, sym: PSym) = if input.kind == nkStmtListExpr: @@ -427,8 +430,7 @@ proc addExprAssgn(ctx: Ctx, output, input: PNode, sym: PSym) = proc convertExprBodyToAsgn(ctx: Ctx, exprBody: PNode, res: PSym): PNode = result = newNodeI(nkStmtList, exprBody.info) - if exprBody.typ != nil: - ctx.addExprAssgn(result, exprBody, res) + ctx.addExprAssgn(result, exprBody, res) proc newNotCall(g: ModuleGraph; e: PNode): PNode = result = newTree(nkCall, newSymNode(g.getSysMagic(e.info, "not", mNot), e.info), e) diff --git a/compiler/injectdestructors.nim b/compiler/injectdestructors.nim index 48a852d26..8ef08adeb 100644 --- a/compiler/injectdestructors.nim +++ b/compiler/injectdestructors.nim @@ -364,6 +364,7 @@ proc genMarkCyclic(c: var Con; result, dest: PNode) = proc genCopyNoCheck(c: var Con; dest, ri: PNode): PNode = let t = dest.typ.skipTypes({tyGenericInst, tyAlias, tySink}) result = c.genOp(t, attachedAsgn, dest, ri) + assert ri.typ != nil proc genCopy(c: var Con; dest, ri: PNode): PNode = let t = dest.typ @@ -371,6 +372,7 @@ proc genCopy(c: var Con; dest, ri: PNode): PNode = # try to improve the error message here: c.checkForErrorPragma(t, ri, "=copy") result = c.genCopyNoCheck(dest, ri) + assert ri.typ != nil proc genDiscriminantAsgn(c: var Con; s: var Scope; n: PNode): PNode = # discriminator is ordinal value that doesn't need sink destroy From a77360da5bad40d22e942cdd5cb9614c3548bf34 Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 5 Jun 2021 03:41:13 +0800 Subject: [PATCH 0887/2472] workaround bug in sharedtables.withValue to unblock frozen CI on OSX (#18172) --- lib/pure/collections/sharedtables.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/pure/collections/sharedtables.nim b/lib/pure/collections/sharedtables.nim index 2788ec785..a85674df9 100644 --- a/lib/pure/collections/sharedtables.nim +++ b/lib/pure/collections/sharedtables.nim @@ -108,10 +108,14 @@ template withValue*[A, B](t: var SharedTable[A, B], key: A, table.withValue("a", value): value[] = "m" + var flag = false table.withValue("d", value): discard value doAssert false do: # if "d" notin table + flag = true + + if flag: table["d"] = "n" assert table.mget("a") == "m" From cc7ec5a6a42c2aead2c411228f023cbeff4220de Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 4 Jun 2021 15:32:46 -0700 Subject: [PATCH 0888/2472] fix #7295: use move(result) inside template to avoid copy with --gc:refc (#18168) --- lib/pure/collections/sequtils.nim | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index e937b8394..6facfcc32 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -1014,27 +1014,27 @@ template applyIt*(varSeq, op: untyped) = template newSeqWith*(len: int, init: untyped): untyped = - ## Creates a new sequence of length `len`, calling `init` to initialize - ## each value of the sequence. - ## - ## Useful for creating "2D" sequences - sequences containing other sequences - ## or to populate fields of the created sequence. + ## Creates a new `seq` of length `len`, calling `init` to initialize + ## each value of the seq. ## + ## Useful for creating "2D" seqs - seqs containing other seqs + ## or to populate fields of the created seq. runnableExamples: - ## Creates a sequence containing 5 bool sequences, each of length of 3. + ## Creates a seq containing 5 bool seqs, each of length of 3. var seq2D = newSeqWith(5, newSeq[bool](3)) assert seq2D.len == 5 assert seq2D[0].len == 3 assert seq2D[4][2] == false - ## Creates a sequence of 20 random numbers from 1 to 10 - import random - var seqRand = newSeqWith(20, rand(10)) + ## Creates a seq with random numbers + import std/random + var seqRand = newSeqWith(20, rand(1.0)) + assert seqRand[0] != seqRand[1] var result = newSeq[typeof(init)](len) for i in 0 ..< len: result[i] = init - result + move(result) # refs bug #7295 func mapLitsImpl(constructor: NimNode; op: NimNode; nested: bool; filter = nnkLiterals): NimNode = From a2b60812568b5b90af9bb15c50873a726dd49b94 Mon Sep 17 00:00:00 2001 From: Euan Date: Fri, 4 Jun 2021 23:46:57 +0100 Subject: [PATCH 0889/2472] Ref #18177 - lbgc.so.5.0 on OpenBSD 6.9 (#18180) --- lib/system.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/system.nim b/lib/system.nim index 942d29fe8..cae0ec3e2 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1146,7 +1146,7 @@ when defined(boehmgc): elif defined(macosx): const boehmLib = "libgc.dylib" elif defined(openbsd): - const boehmLib = "libgc.so.4.0" + const boehmLib = "libgc.so.(4|5).0" elif defined(freebsd): const boehmLib = "libgc-threaded.so.1" else: From 3cc547f2dfc3d2a981ccde38adb1f3deeb9715c2 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 4 Jun 2021 21:58:26 -0700 Subject: [PATCH 0890/2472] macros.treeRepr + friends: collapse SymChoice (#18072) * macros.treeRepr + friends: collapse SymChoice * make repr+friends work with invalid symchoice nodes * address comment --- changelog.md | 3 + lib/core/macros.nim | 63 +++++++++++++------- tests/macros/tdumpast.nim | 122 +++++++++++++++++++++++++++++++------- tests/stdlib/tmacros.nim | 6 ++ 4 files changed, 151 insertions(+), 43 deletions(-) diff --git a/changelog.md b/changelog.md index 06251abfc..1050e4916 100644 --- a/changelog.md +++ b/changelog.md @@ -81,6 +81,9 @@ - Deprecated `proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T]` in `std/algorithm`. +- In `std/macros`, `treeRepr,lispRepr,astGenRepr` now represent SymChoice nodes in a collapsed way, + use `-d:nimLegacyMacrosCollapseSymChoice` to get previous behavior. + - The configuration subsystem now allows for `-d:release` and `-d:danger` to work as expected. The downside is that these defines now have custom logic that doesn't apply for other defines. diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 89f90a23b..37f855d94 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -856,6 +856,29 @@ proc nestList*(op: NimNode; pack: NimNode; init: NimNode): NimNode = for i in countdown(pack.len - 1, 0): result = newCall(op, pack[i], result) +proc eqIdent*(a: string; b: string): bool {.magic: "EqIdent", noSideEffect.} + ## Style insensitive comparison. + +proc eqIdent*(a: NimNode; b: string): bool {.magic: "EqIdent", noSideEffect.} + ## Style insensitive comparison. `a` can be an identifier or a + ## symbol. `a` may be wrapped in an export marker + ## (`nnkPostfix`) or quoted with backticks (`nnkAccQuoted`), + ## these nodes will be unwrapped. + +proc eqIdent*(a: string; b: NimNode): bool {.magic: "EqIdent", noSideEffect.} + ## Style insensitive comparison. `b` can be an identifier or a + ## symbol. `b` may be wrapped in an export marker + ## (`nnkPostfix`) or quoted with backticks (`nnkAccQuoted`), + ## these nodes will be unwrapped. + +proc eqIdent*(a: NimNode; b: NimNode): bool {.magic: "EqIdent", noSideEffect.} + ## Style insensitive comparison. `a` and `b` can be an + ## identifier or a symbol. Both may be wrapped in an export marker + ## (`nnkPostfix`) or quoted with backticks (`nnkAccQuoted`), + ## these nodes will be unwrapped. + +const collapseSymChoice = not defined(nimLegacyMacrosCollapseSymChoice) + proc treeTraverse(n: NimNode; res: var string; level = 0; isLisp = false, indented = false) {.benign.} = if level > 0: if indented: @@ -883,8 +906,21 @@ proc treeTraverse(n: NimNode; res: var string; level = 0; isLisp = false, indent res.add(" " & $n.strVal.newLit.repr) of nnkNone: assert false + elif n.kind in {nnkOpenSymChoice, nnkClosedSymChoice} and collapseSymChoice: + res.add(" " & $n.len) + if n.len > 0: + var allSameSymName = true + for i in 0.. 0: + res.add(" " & n[0].strVal.newLit.repr) else: res.add(".newTree(") for j in 0..`_. return node.copyNimTree() -proc eqIdent*(a: string; b: string): bool {.magic: "EqIdent", noSideEffect.} - ## Style insensitive comparison. - -proc eqIdent*(a: NimNode; b: string): bool {.magic: "EqIdent", noSideEffect.} - ## Style insensitive comparison. `a` can be an identifier or a - ## symbol. `a` may be wrapped in an export marker - ## (`nnkPostfix`) or quoted with backticks (`nnkAccQuoted`), - ## these nodes will be unwrapped. - -proc eqIdent*(a: string; b: NimNode): bool {.magic: "EqIdent", noSideEffect.} - ## Style insensitive comparison. `b` can be an identifier or a - ## symbol. `b` may be wrapped in an export marker - ## (`nnkPostfix`) or quoted with backticks (`nnkAccQuoted`), - ## these nodes will be unwrapped. - -proc eqIdent*(a: NimNode; b: NimNode): bool {.magic: "EqIdent", noSideEffect.} - ## Style insensitive comparison. `a` and `b` can be an - ## identifier or a symbol. Both may be wrapped in an export marker - ## (`nnkPostfix`) or quoted with backticks (`nnkAccQuoted`), - ## these nodes will be unwrapped. - proc expectIdent*(n: NimNode, name: string) {.since: (1,1).} = ## Check that `eqIdent(n,name)` holds true. If this is not the ## case, compilation aborts with an error message. This is useful diff --git a/tests/macros/tdumpast.nim b/tests/macros/tdumpast.nim index b6bcbe8f0..484b3c2f3 100644 --- a/tests/macros/tdumpast.nim +++ b/tests/macros/tdumpast.nim @@ -2,34 +2,69 @@ import macros -template plus(a, b: untyped): untyped {.dirty} = - a + b +block: + template plus(a, b: untyped): untyped {.dirty} = + a + b -macro call(e: untyped): untyped = - result = newCall("foo", newStrLitNode("bar")) + macro call(e: untyped): untyped = + result = newCall("foo", newStrLitNode("bar")) -macro dumpAST(n: untyped): untyped = - # dump AST as a side-effect and return the inner node - let n = callsite() - echo n.lispRepr - echo n.treeRepr + macro dumpAST(n: untyped): string = + var msg = "" + msg.add "lispRepr:\n" & n.lispRepr & "\n" + msg.add "treeRepr:\n" & n.treeRepr & "\n" - var plusAst = getAst(plus(1, 2)) - echo plusAst.lispRepr + var plusAst = getAst(plus(1, 2)) + msg.add "lispRepr:\n" & n.lispRepr & "\n" - var callAst = getAst(call(4)) - echo callAst.lispRepr + var callAst = getAst(call(4)) + msg.add "callAst.lispRepr:\n" & callAst.lispRepr & "\n" - var e = parseExpr("foo(bar + baz)") - echo e.lispRepr + var e = parseExpr("foo(bar + baz)") + msg.add "e.lispRepr:\n" & e.lispRepr & "\n" + result = msg.newLit - result = n[1] + let a = dumpAST: + proc add(x, y: int): int = + return x + y + const foo = 3 -dumpAST: - proc add(x, y: int): int = - return x + y - - proc sub(x, y: int): int = return x - y + doAssert a == """ +lispRepr: +(StmtList (ProcDef (Ident "add") (Empty) (Empty) (FormalParams (Ident "int") (IdentDefs (Ident "x") (Ident "y") (Ident "int") (Empty))) (Empty) (Empty) (StmtList (ReturnStmt (Infix (Ident "+") (Ident "x") (Ident "y"))))) (ConstSection (ConstDef (Ident "foo") (Empty) (IntLit 3)))) +treeRepr: +StmtList + ProcDef + Ident "add" + Empty + Empty + FormalParams + Ident "int" + IdentDefs + Ident "x" + Ident "y" + Ident "int" + Empty + Empty + Empty + StmtList + ReturnStmt + Infix + Ident "+" + Ident "x" + Ident "y" + ConstSection + ConstDef + Ident "foo" + Empty + IntLit 3 +lispRepr: +(StmtList (ProcDef (Ident "add") (Empty) (Empty) (FormalParams (Ident "int") (IdentDefs (Ident "x") (Ident "y") (Ident "int") (Empty))) (Empty) (Empty) (StmtList (ReturnStmt (Infix (Ident "+") (Ident "x") (Ident "y"))))) (ConstSection (ConstDef (Ident "foo") (Empty) (IntLit 3)))) +callAst.lispRepr: +(Call (Ident "foo") (StrLit "bar")) +e.lispRepr: +(Call (Ident "foo") (Infix (Ident "+") (Ident "bar") (Ident "baz"))) +""" macro fun() = let n = quote do: @@ -79,3 +114,48 @@ block: # repr doAssert repr2((1, 2, 3.0)) == "(1, 2, 3.0)" doAssert repr2((1)) == "(1)" doAssert repr2((1+2)) == "(1 + 2)" + +block: # treeRepr + macro treeRepr2(a: untyped): string = newLit a.treeRepr + macro treeRepr3(a: typed): string = newLit a.treeRepr + + doAssert treeRepr2(1+1 == 2) == """ +Infix + Ident "==" + Infix + Ident "+" + IntLit 1 + IntLit 1 + IntLit 2""" + + proc baz() = discard + proc baz(a: int) = discard + proc baz(a: float) = discard + + doAssert treeRepr3(baz()) == """ +Call + Sym "baz"""" + + let a = treeRepr3(block: + proc bar(a: auto) = baz()) + doAssert a == """ +BlockStmt + Empty + ProcDef + Sym "bar" + Empty + GenericParams + Sym "a:type" + FormalParams + Empty + IdentDefs + Sym "a" + Sym "auto" + Empty + Empty + Bracket + Empty + Empty + StmtList + Call + OpenSymChoice 3 "baz"""" diff --git a/tests/stdlib/tmacros.nim b/tests/stdlib/tmacros.nim index 52ea1e898..18cfdc75c 100644 --- a/tests/stdlib/tmacros.nim +++ b/tests/stdlib/tmacros.nim @@ -1,3 +1,9 @@ +#[ +xxx macros tests need to be reorganized to makes sure each API is tested once +See also: + tests/macros/tdumpast.nim for treeRepr + friends +]# + import std/macros block: # hasArgOfName From 295429f42595ce5c707456e657e4de6b2734df42 Mon Sep 17 00:00:00 2001 From: Rainbow Asteroids Date: Sat, 5 Jun 2021 03:07:26 -0400 Subject: [PATCH 0891/2472] add `dom.scrollIntoView` with options; refs #18093 (#18181) --- changelog.md | 2 ++ lib/js/dom.nim | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/changelog.md b/changelog.md index 1050e4916..7619ee6a3 100644 --- a/changelog.md +++ b/changelog.md @@ -335,6 +335,8 @@ - Added setCurrentException for JS backend. +- Added `dom.scrollIntoView` proc with options + ## Language changes - `nimscript` now handles `except Exception as e`. diff --git a/lib/js/dom.nim b/lib/js/dom.nim index 16fc96b68..2b2332020 100644 --- a/lib/js/dom.nim +++ b/lib/js/dom.nim @@ -1317,6 +1317,11 @@ type ready*: FontFaceSetReady onloadingdone*: proc(event: Event) + ScrollIntoViewOptions* = object + behavior*: cstring + `block`*: cstring + inline*: cstring + since (1, 3): type DomParser* = ref object @@ -1538,6 +1543,7 @@ proc removeAttribute*(n: Node, attr: cstring) proc removeAttributeNode*(n, attr: Node) proc replaceData*(n: Node, start, len: int, text: cstring) proc scrollIntoView*(n: Node) +proc scrollIntoView*(n: Node, options: ScrollIntoViewOptions) proc setAttribute*(n: Node, name, value: cstring) proc setAttributeNode*(n: Node, attr: Node) proc querySelector*(n: Node, selectors: cstring): Element From 9c0666e0bbb7af69e2d87e7e81302ed3d3f06412 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Sat, 5 Jun 2021 10:12:37 +0300 Subject: [PATCH 0892/2472] rst: remove `roSkipPounds` option (#18175) --- lib/packages/docutils/rst.nim | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index dae692fb7..336caa2df 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -201,8 +201,6 @@ from highlite import SourceLanguage, getSourceLanguage type RstParseOption* = enum ## options for the RST parser - roSkipPounds, ## skip ``#`` at line beginning (documentation - ## embedded in Nim comments) roSupportSmilies, ## make the RST parser support smilies like ``:)`` roSupportRawDirective, ## support the ``raw`` directive (don't support ## it for sandboxing) @@ -307,7 +305,6 @@ type buf*: cstring bufpos*: int line*, col*, baseIndent*: int - skipPounds*: bool adornmentLine*: bool escapeNext*: bool @@ -380,9 +377,6 @@ proc getIndentAux(L: var Lexer, start: int): int = else: inc pos elif L.buf[pos] == '\n': inc pos - if L.skipPounds: - if L.buf[pos] == '#': inc pos - if L.buf[pos] == '#': inc pos while true: case L.buf[pos] of ' ', '\v', '\f': @@ -447,26 +441,13 @@ proc rawGetTok(L: var Lexer, tok: var Token) = inc L.col tok.col = max(tok.col - L.baseIndent, 0) -proc getTokens(buffer: string, skipPounds: bool, tokens: var TokenSeq): int = +proc getTokens(buffer: string, tokens: var TokenSeq): int = var L: Lexer var length = tokens.len L.buf = cstring(buffer) L.line = 0 # skip UTF-8 BOM if L.buf[0] == '\xEF' and L.buf[1] == '\xBB' and L.buf[2] == '\xBF': inc L.bufpos, 3 - L.skipPounds = skipPounds - if skipPounds: - if L.buf[L.bufpos] == '#': - inc L.bufpos - inc result - if L.buf[L.bufpos] == '#': - inc L.bufpos - inc result - L.baseIndent = 0 - while L.buf[L.bufpos] == ' ': - inc L.bufpos - inc L.baseIndent - inc result while true: inc length setLen(tokens, length) @@ -2123,7 +2104,7 @@ proc parseSimpleTable(p: var RstParser): PRstNode = q.col = cols[j] q.line = line - 1 q.filename = p.filename - q.col += getTokens(row[j], false, q.tok) + q.col += getTokens(row[j], q.tok) b = newRstNode(rnTableDataCell) b.add(parseDoc(q)) a.add(b) @@ -2176,7 +2157,7 @@ proc parseMarkdownTable(p: var RstParser): PRstNode = q.col = p.col q.line = currentTok(p).line - 1 q.filename = p.filename - q.col += getTokens(getColContents(p, row[j]), false, q.tok) + q.col += getTokens(getColContents(p, row[j]), q.tok) b.add(parseDoc(q)) a.add(b) result.add(a) @@ -2584,7 +2565,6 @@ proc dirInclude(p: var RstParser): PRstNode = q.filename = path q.col += getTokens( inputString[startPosition..endPosition].strip(), - false, q.tok) # workaround a GCC bug; more like the interior pointer bug? #if find(q.tok[high(q.tok)].symbol, "\0\x01\x02") > 0: @@ -2948,7 +2928,7 @@ proc rstParse*(text, filename: string, initParser(p, newSharedState(options, findFile, msgHandler)) p.filename = filename p.line = line - p.col = column + getTokens(text, roSkipPounds in options, p.tok) + p.col = column + getTokens(text, p.tok) let unresolved = parseDoc(p) countTitles(p, unresolved) orderFootnotes(p) From 9c6259e55717acbc954140ef2bb0ca363971c8c7 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 5 Jun 2021 00:45:37 -0700 Subject: [PATCH 0893/2472] up to 20x faster jsonutils deserialization (#18183) * up to 20x faster jsonutils deserialization * noinline --- lib/std/jsonutils.nim | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim index 60d78fea5..c141985d3 100644 --- a/lib/std/jsonutils.nim +++ b/lib/std/jsonutils.nim @@ -128,14 +128,14 @@ macro initCaseObject(T: typedesc, fun: untyped): untyped = `fun`(`key2`, typedesc[`typ`]) result.add newTree(nnkExprColonExpr, key, val) -proc checkJsonImpl(cond: bool, condStr: string, msg = "") = - if not cond: - # just pick 1 exception type for simplicity; other choices would be: - # JsonError, JsonParser, JsonKindError - raise newException(ValueError, msg) +proc raiseJsonException(condStr: string, msg: string) {.noinline.} = + # just pick 1 exception type for simplicity; other choices would be: + # JsonError, JsonParser, JsonKindError + raise newException(ValueError, condStr & " failed: " & msg) template checkJson(cond: untyped, msg = "") = - checkJsonImpl(cond, astToStr(cond), msg) + if not cond: + raiseJsonException(astToStr(cond), msg) proc hasField[T](obj: T, field: string): bool = for k, _ in fieldPairs(obj): From b9843b156dfc16f1f99cbee8fc8e55b7abd6c21b Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 5 Jun 2021 00:47:28 -0700 Subject: [PATCH 0894/2472] fix #18178, set a CI timeout for github action pipelines instead of the 6 hour default (#18179) --- .github/workflows/ci_docs.yml | 1 + .github/workflows/ci_packages.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/ci_docs.yml b/.github/workflows/ci_docs.yml index ddef94fbe..6ae228d2d 100644 --- a/.github/workflows/ci_docs.yml +++ b/.github/workflows/ci_docs.yml @@ -44,6 +44,7 @@ jobs: name: ${{ matrix.target }} runs-on: ${{ matrix.os }} + timeout-minutes: 60 # refs bug #18178 steps: - name: 'Checkout' diff --git a/.github/workflows/ci_packages.yml b/.github/workflows/ci_packages.yml index aeb29e147..529e6e8b6 100644 --- a/.github/workflows/ci_packages.yml +++ b/.github/workflows/ci_packages.yml @@ -11,6 +11,7 @@ jobs: batch: ["allowed_failures", "0_3", "1_3", "2_3"] # list of `index_num` name: '${{ matrix.os }} (batch: ${{ matrix.batch }})' runs-on: ${{ matrix.os }} + timeout-minutes: 60 # refs bug #18178 env: NIM_TEST_PACKAGES: "1" NIM_TESTAMENT_BATCH: ${{ matrix.batch }} From 9106ab7db19e22782c544c6362741dff4142820e Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 6 Jun 2021 00:53:22 -0700 Subject: [PATCH 0895/2472] make changeDetectedViaJsonBuildInstructions warning less verbose when json file invalid (eg due to spec change) (#18197) --- compiler/extccomp.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index ea2b09866..831acae68 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -993,7 +993,7 @@ proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; jsonFile: Absolute var bcache: BuildCache try: bcache.fromJson(jsonFile.string.parseFile) except IOError, OSError, ValueError: - stderr.write "Warning: JSON processing failed for $#: $#\n" % [jsonFile.string, getCurrentExceptionMsg()] + stderr.write "Warning: JSON processing failed for: $#\n" % jsonFile.string return true if bcache.currentDir != getCurrentDir() or # fixes bug #16271 bcache.configFiles != conf.configFiles.mapIt(it.string) or From e7816a3e77ab0f821d66667dfc448f1898199f59 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Sun, 6 Jun 2021 10:55:20 +0300 Subject: [PATCH 0896/2472] update docgen documentation (#18192) * update docgen documentation * --fullhelp * Update doc/docgen.rst Co-authored-by: Timothee Cour * [skip ci] fix typos Co-authored-by: Timothee Cour --- changelog.md | 7 +++ doc/advopt.txt | 3 +- doc/docgen.rst | 123 ++++++++++++++++++++++++++++++------------------- 3 files changed, 84 insertions(+), 49 deletions(-) diff --git a/changelog.md b/changelog.md index 7619ee6a3..f141ea95a 100644 --- a/changelog.md +++ b/changelog.md @@ -449,6 +449,13 @@ ## Tool changes +- Latex doc generation is revised: output `.tex` files should be compiled + by `xelatex` (not by `pdflatex` as before). Now default Latex settings + provide support for Unicode and do better job for avoiding margin overflows. + +- Implemented `doc2tex` compiler command which converts documentation in + `.nim` files to Latex. + - The rst parser now supports markdown table syntax. Known limitations: - cell alignment is not supported, i.e. alignment annotations in a delimiter diff --git a/doc/advopt.txt b/doc/advopt.txt index e627d4768..e5b4c9568 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -6,7 +6,8 @@ Advanced commands: //e run a Nimscript file //rst2html convert a reStructuredText file to HTML use `--docCmd:skip` to skip compiling snippets - //rst2tex convert a reStructuredText file to TeX + //rst2tex convert a reStructuredText file to LaTeX + //doc2tex extract the documentation to a LaTeX file //jsondoc extract the documentation to a json file //ctags create a tags file //buildIndex build an index for the whole documentation diff --git a/doc/docgen.rst b/doc/docgen.rst index e9ff7f1e3..2074ee06f 100644 --- a/doc/docgen.rst +++ b/doc/docgen.rst @@ -14,12 +14,22 @@ Introduction ============ This document describes the `documentation generation tools`:idx: built into -the `Nim compiler `_, which can generate HTML and JSON output -from input .nim files and projects, as well as HTML and LaTeX from input RST +the `Nim compiler `_, which can generate HTML, Latex and JSON output +from input ``.nim`` files and projects, as well as HTML and LaTeX from input RST (reStructuredText) files. The output documentation will include the module dependencies (`import`), any top-level documentation comments (`##`), and exported symbols (`*`), including procedures, types, and variables. +=================== ====================== ============ ============== +command runs on... input format output format +=================== ====================== ============ ============== +`nim doc`:cmd: documentation comments ``.nim`` ``.html`` HTML +`nim doc2tex`:cmd: ″ ″ ``.tex`` LaTeX +`nim jsondoc`:cmd: ″ ″ ``.json`` JSON +`nim rst2html`:cmd: standalone rst files ``.rst`` ``.html`` HTML +`nim rst2tex`:cmd: ″ ″ ``.tex`` LaTeX +=================== ====================== ============ ============== + Quick start ----------- @@ -53,7 +63,8 @@ Any comments which are preceded by a double-hash (`##`), are interpreted as documentation. Comments are parsed as RST (see `reference `_), providing Nim module authors the ability to easily generate richly formatted -documentation with only their well-documented code. +documentation with only their well-documented code! +Basic Markdown syntax is also supported inside the doc comments. Example: @@ -99,32 +110,22 @@ won't influence RST formatting. ## ## Paragraph. -Nim file input ------------------ - -The following examples will generate documentation for the below contrived -*Nim* module, aptly named 'sample.nim' - -sample.nim: - -.. code-block:: nim - ## This module is a sample. - - import std/strutils - - proc helloWorld*(times: int) = - ## Takes an integer and outputs - ## as many "hello world!"s - - for i in 0 .. times-1: - echo "hello world!" - - helloWorld(5) - Document Types ============== +Example of Nim file input +------------------------- + +The following examples will generate documentation for this sample +*Nim* module, aptly named ``doc/docgen_sample.nim``: + +.. code:: nim + :file: docgen_sample.nim + +All the below commands save their output to ``htmldocs`` directory relative to +the directory of file; +hence the output for this sample will be in ``doc/htmldocs``. HTML ---- @@ -138,7 +139,7 @@ The `doc`:option: command: .. code:: cmd - nim doc sample + nim doc docgen_sample.nim Partial Output:: ... @@ -149,6 +150,27 @@ The full output can be seen here: `docgen_sample.html `_. It runs after semantic checking and includes pragmas attached implicitly by the compiler. +LaTeX +----- + +LaTeX files are intended to be converted to PDF, especially for offline +reading or making hard copies. (LaTeX output is oftentimes better than +HTML -> PDF conversion). + +The `doc2tex`:option: command: + +.. code:: cmd + + nim doc2tex docgen_sample.nim + cd htmldocs + xelatex docgen_sample.tex + xelatex docgen_sample.tex + # It is usually necessary to run `xelatex` 2 times (or even 3 times for + # large documents) to get all labels generated. + # That depends on this warning in the end of `xelatex` output: + # LaTeX Warning: Label(s) may have changed. Rerun to get cross-references right. + +The output is ``docgen_sample.pdf``. JSON ---- @@ -163,7 +185,7 @@ The `jsondoc`:option: command: .. code:: cmd - nim jsondoc sample + nim jsondoc docgen_sample.nim Output:: { @@ -189,7 +211,7 @@ The `jsondoc0`:option: command: .. code:: cmd - nim jsondoc0 sample + nim jsondoc0 docgen_sample.nim Output:: [ @@ -204,7 +226,7 @@ Output:: } ] -Note that the `jsondoc`:option: command outputs it's JSON without pretty-printing it, +Note that the `jsondoc`:option: command outputs its JSON without pretty-printing it, while `jsondoc0`:option: outputs pretty-printed JSON. Related Options @@ -217,9 +239,10 @@ Project switch nim doc --project filename.nim -This will recursively generate documentation of all nim modules imported +This will recursively generate documentation of all Nim modules imported into the input module that belong to the Nimble package that ``filename.nim`` -belongs to. +belongs to. The index files and the corresponding ``theindex.html`` will +also be generated. Index switch @@ -252,12 +275,13 @@ documented item in your source code pointing to the implementation of that item on a GitHub repository. You can click the link to see the implementation of the item. -The `git.commit`:option: switch overrides the hardcoded `devel` branch in config/nimdoc.cfg. +The `git.commit`:option: switch overrides the hardcoded `devel` branch in +``config/nimdoc.cfg``. This is useful to link to a different branch e.g. `--git.commit:master`:option:, or to a tag e.g. `--git.commit:1.2.3`:option: or a commit. Source URLs are generated as ``href="${url}/tree/${commit}/${path}#L${line}"`` -by default and this compatible with GitHub but not with GitLab. +by default and thus compatible with GitHub but not with GitLab. Similarly, `git.devel`:option: switch overrides the hardcoded `devel` branch for the `Edit` link which is also useful if you have a different working @@ -282,9 +306,10 @@ Other Input Formats The *Nim compiler* also has support for RST (reStructuredText) files with the `rst2html`:option: and `rst2tex`:option: commands. Documents like this one are -initially written in a dialect of RST which adds support for nim source code +initially written in a dialect of RST which adds support for Nim source code highlighting with the ``.. code-block:: nim`` prefix. ``code-block`` also -supports highlighting of C++ and some other c-like languages. +supports highlighting of a few other languages supported by the +`packages/docutils/highlite module `_. Usage: @@ -333,16 +358,16 @@ if they don't have any parameters. If there are parameters, they are represented by their types and will be comma-separated. To the plain symbol a suffix may be added depending on the type of the callable: -------------- -------------- -Callable type Suffix -------------- -------------- -proc *empty string* -macro ``.m`` -method ``.e`` -iterator ``.i`` -template ``.t`` -converter ``.c`` -------------- -------------- +============== ============== +Callable type Suffix +============== ============== +`proc`, `func` *empty string* +`macro` ``.m`` +`method` ``.e`` +`iterator` ``.i`` +`template` ``.t`` +`converter` ``.c`` +============== ============== The relationship of type to suffix is made by the proc `complexName` in the ``compiler/docgen.nim`` file. Here are some examples of complex names for @@ -422,10 +447,12 @@ final index, and TOC entries found in ``.nim`` files are discarded. Additional resources ==================== -`Nim Compiler User Guide `_ +* `Nim Compiler User Guide `_ -`RST Quick Reference -`_ +* Documentation for `rst module `_ -- Nim RST/Markdown parser. + +* `RST Quick Reference + `_ The output for HTML and LaTeX comes from the ``config/nimdoc.cfg`` and ``config/nimdoc.tex.cfg`` configuration files. You can add and modify these From e7aa41ff3f6820d8ada3fd650d523c7f7d584fda Mon Sep 17 00:00:00 2001 From: slangmgh <37659406+slangmgh@users.noreply.github.com> Date: Mon, 7 Jun 2021 17:42:32 +0800 Subject: [PATCH 0897/2472] fix 18186 with adding importc _umul128 (#18200) --- lib/system/dragonbox.nim | 3 ++- lib/system/schubfach.nim | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/system/dragonbox.nim b/lib/system/dragonbox.nim index db608c630..eec8749da 100644 --- a/lib/system/dragonbox.nim +++ b/lib/system/dragonbox.nim @@ -867,10 +867,11 @@ when defined(sizeof_Int128): return (hi, lo) elif defined(vcc) and defined(cpu64): + proc umul128(x, y: uint64, z: ptr uint64): uint64 {.importc: "_umul128", header: "".} proc mul128*(x: uint64; y: uint64): uint64x2 {.inline.} = var hi: uint64 = 0 var lo: uint64 = umul128(x, y, addr(hi)) - return (hi, lo) + return uint64x2(hi: hi, lo: lo) else: proc lo32*(x: uint64): uint32 {.inline.} = diff --git a/lib/system/schubfach.nim b/lib/system/schubfach.nim index c344b7425..94d3d3c25 100644 --- a/lib/system/schubfach.nim +++ b/lib/system/schubfach.nim @@ -147,6 +147,7 @@ when defined(sizeof_Int128): return y1 or uint32(y0 > 1) elif defined(vcc) and defined(cpu64): + proc umul128(x, y: uint64, z: ptr uint64): uint64 {.importc: "_umul128", header: "".} proc roundToOdd(g: uint64; cpHi: uint32): uint32 {.inline.} = var p1: uint64 = 0 var p0: uint64 = umul128(g, cpHi, addr(p1)) From 2ec52faae5ba5c233f977a8a4b997734c4ad83e7 Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Mon, 7 Jun 2021 10:44:43 +0100 Subject: [PATCH 0898/2472] koch: print help if no command is given (#18199) --- koch.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/koch.nim b/koch.nim index 3d7276aee..665b5e042 100644 --- a/koch.nim +++ b/koch.nim @@ -708,4 +708,5 @@ when isMainModule: of "branchdone": branchDone() else: showHelp(success = false) break - of cmdEnd: break + of cmdEnd: + showHelp(success = false) From 429b1286325e7b6cf86ed4e29cea3e11a8c0e2df Mon Sep 17 00:00:00 2001 From: Bung Date: Mon, 7 Jun 2021 19:57:42 +0800 Subject: [PATCH 0899/2472] change mimedb stroe stringtable to orderedtable (#18065) * change mimedb stroe stringtable to orderedtable * Update lib/pure/mimetypes.nim Co-authored-by: Andreas Rumpf --- lib/pure/mimetypes.nim | 7 ++++--- tests/stdlib/tmimetypes.nim | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/pure/mimetypes.nim b/lib/pure/mimetypes.nim index c10739fec..6ae63a0ae 100644 --- a/lib/pure/mimetypes.nim +++ b/lib/pure/mimetypes.nim @@ -26,12 +26,12 @@ runnableExamples: doAssert m.getMimetype("fakext") == "text/fakelang" doAssert m.getMimetype("FaKeXT") == "text/fakelang" -import strtabs +import tables from strutils import startsWith, toLowerAscii, strip type MimeDB* = object - mimes: StringTableRef + mimes: OrderedTableRef[string, string] const mimes* = { "123": "application/vnd.lotus-1-2-3", @@ -1903,7 +1903,8 @@ const mimes* = { func newMimetypes*(): MimeDB = ## Creates a new Mimetypes database. The database will contain the most ## common mimetypes. - result.mimes = mimes.newStringTable() + {.cast(noSideEffect).}: + result.mimes = mimes.newOrderedTable() func getMimetype*(mimedb: MimeDB, ext: string, default = "text/plain"): string = ## Gets mimetype which corresponds to `ext`. Returns `default` if `ext` diff --git a/tests/stdlib/tmimetypes.nim b/tests/stdlib/tmimetypes.nim index cd41e614b..93c20d4a3 100644 --- a/tests/stdlib/tmimetypes.nim +++ b/tests/stdlib/tmimetypes.nim @@ -6,6 +6,7 @@ import std/mimetypes template main() = var m = newMimetypes() doAssert m.getMimetype("mp4") == "video/mp4" + doAssert m.getExt("application/json") == "json" # see also `runnableExamples`. # xxx we should have a way to avoid duplicating code between runnableExamples and tests From 5bad022d58e21f0da455fbd7d421cd26a2b5c812 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 7 Jun 2021 21:32:37 +0800 Subject: [PATCH 0900/2472] alternative to #18185 (#18206) --- lib/core/rlocks.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/core/rlocks.nim b/lib/core/rlocks.nim index 5f155262a..a1ee0a6ec 100644 --- a/lib/core/rlocks.nim +++ b/lib/core/rlocks.nim @@ -11,7 +11,10 @@ when not compileOption("threads") and not defined(nimdoc): - {.error: "Rlocks requires --threads:on option.".} + when false: + # make rlocks modlue consistent with locks module, + # so they can replace each other seamlessly. + {.error: "Rlocks requires --threads:on option.".} const insideRLocksModule = true include "system/syslocks" From 21f3b8539a2e440f7cb69502721b6c56070e122f Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 7 Jun 2021 17:22:35 +0200 Subject: [PATCH 0901/2472] fixes #18088 (#18209) --- lib/pure/nativesockets.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index 605f62321..124afce56 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -438,7 +438,7 @@ proc getHostname*(): string {.tags: [ReadIOEffect].} = ## Returns the local hostname (not the FQDN) # https://tools.ietf.org/html/rfc1035#section-2.3.1 # https://tools.ietf.org/html/rfc2181#section-11 - const size = 64 + const size = 256 result = newString(size) when useWinVersion: let success = winlean.gethostname(result, size) From 51ab7ccec1ffa21cef67f1a75dea889653751cfc Mon Sep 17 00:00:00 2001 From: drtheuns Date: Mon, 7 Jun 2021 18:40:09 +0200 Subject: [PATCH 0902/2472] Fix JS error on index page and detect dark mode (#18191) * Fix JS error on index page and detect dark mode The theindex.html page doesn't have a dark mode switch so the main function will error because `toggleSwitch` is not defined. Checks have been added to prevent this from happening. Also add automatic detection of system settings for dark-mode. This could also be done with pure css, but then the dark mode variable declarations would have to be duplicated to work with the switch so I went with this approach. * Fix nimdoc tests * Fix rst2html tests --- config/nimdoc.cfg | 13 +++++++++---- nimdoc/rst2html/expected/rst_examples.html | 13 +++++++++---- nimdoc/test_out_index_dot_html/expected/index.html | 13 +++++++++---- .../test_out_index_dot_html/expected/theindex.html | 13 +++++++++---- .../testproject/expected/subdir/subdir_b/utils.html | 13 +++++++++---- nimdoc/testproject/expected/testproject.html | 13 +++++++++---- nimdoc/testproject/expected/theindex.html | 13 +++++++++---- 7 files changed, 63 insertions(+), 28 deletions(-) diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index ed1b346a2..cdf49197d 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -252,7 +252,6 @@ function main() { } } - const toggleSwitch = document.querySelector('.theme-switch input[type="checkbox"]'); function switchTheme(e) { if (e.target.checked) { document.documentElement.setAttribute('data-theme', 'dark'); @@ -263,13 +262,19 @@ function main() { } } - toggleSwitch.addEventListener('change', switchTheme, false); + const toggleSwitch = document.querySelector('.theme-switch input[type="checkbox"]'); + if (toggleSwitch !== null) { + toggleSwitch.addEventListener('change', switchTheme, false); + } - const currentTheme = localStorage.getItem('theme') ? localStorage.getItem('theme') : null; + var currentTheme = localStorage.getItem('theme'); + if (!currentTheme && window.matchMedia('(prefers-color-scheme: dark)').matches) { + currentTheme = 'dark'; + } if (currentTheme) { document.documentElement.setAttribute('data-theme', currentTheme); - if (currentTheme === 'dark') { + if (currentTheme === 'dark' && toggleSwitch !== null) { toggleSwitch.checked = true; } } diff --git a/nimdoc/rst2html/expected/rst_examples.html b/nimdoc/rst2html/expected/rst_examples.html index 5c434193e..68b796be7 100644 --- a/nimdoc/rst2html/expected/rst_examples.html +++ b/nimdoc/rst2html/expected/rst_examples.html @@ -34,7 +34,6 @@ function main() { } } - const toggleSwitch = document.querySelector('.theme-switch input[type="checkbox"]'); function switchTheme(e) { if (e.target.checked) { document.documentElement.setAttribute('data-theme', 'dark'); @@ -45,13 +44,19 @@ function main() { } } - toggleSwitch.addEventListener('change', switchTheme, false); + const toggleSwitch = document.querySelector('.theme-switch input[type="checkbox"]'); + if (toggleSwitch !== null) { + toggleSwitch.addEventListener('change', switchTheme, false); + } - const currentTheme = localStorage.getItem('theme') ? localStorage.getItem('theme') : null; + var currentTheme = localStorage.getItem('theme'); + if (!currentTheme && window.matchMedia('(prefers-color-scheme: dark)').matches) { + currentTheme = 'dark'; + } if (currentTheme) { document.documentElement.setAttribute('data-theme', currentTheme); - if (currentTheme === 'dark') { + if (currentTheme === 'dark' && toggleSwitch !== null) { toggleSwitch.checked = true; } } diff --git a/nimdoc/test_out_index_dot_html/expected/index.html b/nimdoc/test_out_index_dot_html/expected/index.html index bac39fd33..fac758c9b 100644 --- a/nimdoc/test_out_index_dot_html/expected/index.html +++ b/nimdoc/test_out_index_dot_html/expected/index.html @@ -34,7 +34,6 @@ function main() { } } - const toggleSwitch = document.querySelector('.theme-switch input[type="checkbox"]'); function switchTheme(e) { if (e.target.checked) { document.documentElement.setAttribute('data-theme', 'dark'); @@ -45,13 +44,19 @@ function main() { } } - toggleSwitch.addEventListener('change', switchTheme, false); + const toggleSwitch = document.querySelector('.theme-switch input[type="checkbox"]'); + if (toggleSwitch !== null) { + toggleSwitch.addEventListener('change', switchTheme, false); + } - const currentTheme = localStorage.getItem('theme') ? localStorage.getItem('theme') : null; + var currentTheme = localStorage.getItem('theme'); + if (!currentTheme && window.matchMedia('(prefers-color-scheme: dark)').matches) { + currentTheme = 'dark'; + } if (currentTheme) { document.documentElement.setAttribute('data-theme', currentTheme); - if (currentTheme === 'dark') { + if (currentTheme === 'dark' && toggleSwitch !== null) { toggleSwitch.checked = true; } } diff --git a/nimdoc/test_out_index_dot_html/expected/theindex.html b/nimdoc/test_out_index_dot_html/expected/theindex.html index aa9ca2645..34ddf8f6a 100644 --- a/nimdoc/test_out_index_dot_html/expected/theindex.html +++ b/nimdoc/test_out_index_dot_html/expected/theindex.html @@ -34,7 +34,6 @@ function main() { } } - const toggleSwitch = document.querySelector('.theme-switch input[type="checkbox"]'); function switchTheme(e) { if (e.target.checked) { document.documentElement.setAttribute('data-theme', 'dark'); @@ -45,13 +44,19 @@ function main() { } } - toggleSwitch.addEventListener('change', switchTheme, false); + const toggleSwitch = document.querySelector('.theme-switch input[type="checkbox"]'); + if (toggleSwitch !== null) { + toggleSwitch.addEventListener('change', switchTheme, false); + } - const currentTheme = localStorage.getItem('theme') ? localStorage.getItem('theme') : null; + var currentTheme = localStorage.getItem('theme'); + if (!currentTheme && window.matchMedia('(prefers-color-scheme: dark)').matches) { + currentTheme = 'dark'; + } if (currentTheme) { document.documentElement.setAttribute('data-theme', currentTheme); - if (currentTheme === 'dark') { + if (currentTheme === 'dark' && toggleSwitch !== null) { toggleSwitch.checked = true; } } diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html index 826cad02a..6aa71e8c2 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html @@ -34,7 +34,6 @@ function main() { } } - const toggleSwitch = document.querySelector('.theme-switch input[type="checkbox"]'); function switchTheme(e) { if (e.target.checked) { document.documentElement.setAttribute('data-theme', 'dark'); @@ -45,13 +44,19 @@ function main() { } } - toggleSwitch.addEventListener('change', switchTheme, false); + const toggleSwitch = document.querySelector('.theme-switch input[type="checkbox"]'); + if (toggleSwitch !== null) { + toggleSwitch.addEventListener('change', switchTheme, false); + } - const currentTheme = localStorage.getItem('theme') ? localStorage.getItem('theme') : null; + var currentTheme = localStorage.getItem('theme'); + if (!currentTheme && window.matchMedia('(prefers-color-scheme: dark)').matches) { + currentTheme = 'dark'; + } if (currentTheme) { document.documentElement.setAttribute('data-theme', currentTheme); - if (currentTheme === 'dark') { + if (currentTheme === 'dark' && toggleSwitch !== null) { toggleSwitch.checked = true; } } diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html index 513dd507b..65d9f83f5 100644 --- a/nimdoc/testproject/expected/testproject.html +++ b/nimdoc/testproject/expected/testproject.html @@ -34,7 +34,6 @@ function main() { } } - const toggleSwitch = document.querySelector('.theme-switch input[type="checkbox"]'); function switchTheme(e) { if (e.target.checked) { document.documentElement.setAttribute('data-theme', 'dark'); @@ -45,13 +44,19 @@ function main() { } } - toggleSwitch.addEventListener('change', switchTheme, false); + const toggleSwitch = document.querySelector('.theme-switch input[type="checkbox"]'); + if (toggleSwitch !== null) { + toggleSwitch.addEventListener('change', switchTheme, false); + } - const currentTheme = localStorage.getItem('theme') ? localStorage.getItem('theme') : null; + var currentTheme = localStorage.getItem('theme'); + if (!currentTheme && window.matchMedia('(prefers-color-scheme: dark)').matches) { + currentTheme = 'dark'; + } if (currentTheme) { document.documentElement.setAttribute('data-theme', currentTheme); - if (currentTheme === 'dark') { + if (currentTheme === 'dark' && toggleSwitch !== null) { toggleSwitch.checked = true; } } diff --git a/nimdoc/testproject/expected/theindex.html b/nimdoc/testproject/expected/theindex.html index e6efee986..55f7f33c9 100644 --- a/nimdoc/testproject/expected/theindex.html +++ b/nimdoc/testproject/expected/theindex.html @@ -34,7 +34,6 @@ function main() { } } - const toggleSwitch = document.querySelector('.theme-switch input[type="checkbox"]'); function switchTheme(e) { if (e.target.checked) { document.documentElement.setAttribute('data-theme', 'dark'); @@ -45,13 +44,19 @@ function main() { } } - toggleSwitch.addEventListener('change', switchTheme, false); + const toggleSwitch = document.querySelector('.theme-switch input[type="checkbox"]'); + if (toggleSwitch !== null) { + toggleSwitch.addEventListener('change', switchTheme, false); + } - const currentTheme = localStorage.getItem('theme') ? localStorage.getItem('theme') : null; + var currentTheme = localStorage.getItem('theme'); + if (!currentTheme && window.matchMedia('(prefers-color-scheme: dark)').matches) { + currentTheme = 'dark'; + } if (currentTheme) { document.documentElement.setAttribute('data-theme', currentTheme); - if (currentTheme === 'dark') { + if (currentTheme === 'dark' && toggleSwitch !== null) { toggleSwitch.checked = true; } } From 47acc80f4e05053d2653c7218434bc7fad880741 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 9 Jun 2021 17:33:19 +0200 Subject: [PATCH 0903/2472] make strict funcs analysis smarter (#18219) * make strict funcs analysis smarter: varParam[i] = v is different from varParam[i][] = v * added a test case * Update compiler/varpartitions.nim Co-authored-by: Clyybber --- compiler/varpartitions.nim | 87 +++++++++++++++---------- tests/effects/tfuncs_cannot_mutate2.nim | 27 ++++++++ tests/effects/tstrict_funcs.nim | 17 +++++ 3 files changed, 95 insertions(+), 36 deletions(-) create mode 100644 tests/effects/tfuncs_cannot_mutate2.nim diff --git a/compiler/varpartitions.nim b/compiler/varpartitions.nim index 810ce8586..9cf1e38ad 100644 --- a/compiler/varpartitions.nim +++ b/compiler/varpartitions.nim @@ -175,12 +175,18 @@ proc root(v: var Partitions; start: int): int = v.s[it].con = Connection(kind: dependsOn, parent: result) it = next -proc potentialMutation(v: var Partitions; s: PSym; info: TLineInfo) = +proc potentialMutation(v: var Partitions; s: PSym; level: int; info: TLineInfo) = let id = variableId(v, s) if id >= 0: let r = root(v, id) - let flags = if s.kind == skParam and isConstParam(s): - {isMutated, isMutatedDirectly} + let flags = if s.kind == skParam: + if isConstParam(s): + {isMutated, isMutatedDirectly} + elif s.typ.kind == tyVar and level <= 1: + # varParam[i] = v is different from varParam[i][] = v + {} + else: + {isMutated} else: {isMutated} @@ -339,36 +345,44 @@ proc pathExpr(node: PNode; owner: PSym): PNode = if result == nil and borrowFromConstExpr(n): result = n -proc allRoots(n: PNode; result: var seq[PSym]; followDotExpr = true) = +const + RootEscapes = 1000 # in 'p(r)' we don't know what p does to our poor root. + # so we assume a high level of indirections + +proc allRoots(n: PNode; result: var seq[(PSym, int)]; level: int) = case n.kind of nkSym: if n.sym.kind in {skParam, skVar, skTemp, skLet, skResult, skForVar}: - result.add(n.sym) + result.add((n.sym, level)) - of nkDotExpr, nkDerefExpr, nkBracketExpr, nkHiddenDeref, - nkCheckedFieldExpr, nkAddr, nkHiddenAddr: - if followDotExpr: - allRoots(n[0], result, followDotExpr) + of nkDerefExpr, nkHiddenDeref: + allRoots(n[0], result, level+1) + of nkBracketExpr, nkDotExpr, nkCheckedFieldExpr, nkAddr, nkHiddenAddr: + allRoots(n[0], result, level) of nkExprEqExpr, nkExprColonExpr, nkHiddenStdConv, nkHiddenSubConv, nkConv, nkStmtList, nkStmtListExpr, nkBlockStmt, nkBlockExpr, nkCast, nkObjUpConv, nkObjDownConv: if n.len > 0: - allRoots(n.lastSon, result, followDotExpr) + allRoots(n.lastSon, result, level) of nkCaseStmt, nkObjConstr: for i in 1.. 1: - allRoots(n[1], result, followDotExpr) + # XXX We really need the unwritten RFC here and distinguish between + # proc `[]`(x: var Container): var T # resizes the container + # and + # proc `[]`(x: Container): var T # only allows for slot mutation + allRoots(n[1], result, RootEscapes) else: let m = getMagic(n) case m @@ -387,12 +401,12 @@ proc allRoots(n: PNode; result: var seq[PSym]; followDotExpr = true) = let paramType = typ.n[i].typ if not paramType.isCompileTimeOnly and not typ.sons[0].isEmptyType and canAlias(paramType, typ.sons[0]): - allRoots(it, result, followDotExpr) + allRoots(it, result, RootEscapes) else: - allRoots(it, result, followDotExpr) + allRoots(it, result, RootEscapes) of mSlice: - allRoots(n[1], result, followDotExpr) + allRoots(n[1], result, level+1) else: discard "harmless operation" else: @@ -466,10 +480,10 @@ proc destMightOwn(c: var Partitions; dest: var VarIndex; n: PNode) = dest.flags.incl ownsData elif n.typ.kind in {tyLent, tyVar}: # we know the result is derived from the first argument: - var roots: seq[PSym] - allRoots(n[1], roots) + var roots: seq[(PSym, int)] + allRoots(n[1], roots, RootEscapes) for r in roots: - connect(c, dest.sym, r, n[1].info) + connect(c, dest.sym, r[0], n[1].info) else: let magic = if n[0].kind == nkSym: n[0].sym.magic else: mNone @@ -582,19 +596,19 @@ proc deps(c: var Partitions; dest, src: PNode) = if borrowChecking in c.goals: borrowingAsgn(c, dest, src) - var targets, sources: seq[PSym] - allRoots(dest, targets) - allRoots(src, sources) + var targets, sources: seq[(PSym, int)] + allRoots(dest, targets, 0) + allRoots(src, sources, 0) let destIsComplex = containsPointer(dest.typ) for t in targets: if dest.kind != nkSym and c.inNoSideEffectSection == 0: - potentialMutation(c, t, dest.info) + potentialMutation(c, t[0], t[1], dest.info) if destIsComplex: for s in sources: - connect(c, t, s, dest.info) + connect(c, t[0], s[0], dest.info) if cursorInference in c.goals and src.kind != nkEmpty: let d = pathExpr(dest, c.owner) @@ -602,7 +616,8 @@ proc deps(c: var Partitions; dest, src: PNode) = let vid = variableId(c, d.sym) if vid >= 0: destMightOwn(c, c.s[vid], src) - for s in sources: + for source in sources: + let s = source[0] if s == d.sym: discard "assignments like: it = it.next are fine" elif {sfGlobal, sfThread} * s.flags != {} or hasDisabledAsgn(c.g, s.typ): @@ -638,9 +653,9 @@ proc potentialMutationViaArg(c: var Partitions; n: PNode; callee: PType) = if constParameters in c.goals and tfNoSideEffect in callee.flags: discard "we know there are no hidden mutations through an immutable parameter" elif c.inNoSideEffectSection == 0 and containsPointer(n.typ): - var roots: seq[PSym] - allRoots(n, roots) - for r in roots: potentialMutation(c, r, n.info) + var roots: seq[(PSym, int)] + allRoots(n, roots, RootEscapes) + for r in roots: potentialMutation(c, r[0], r[1], n.info) proc traverse(c: var Partitions; n: PNode) = inc c.abstractTime @@ -682,12 +697,12 @@ proc traverse(c: var Partitions; n: PNode) = if i < L: let paramType = parameters[i].skipTypes({tyGenericInst, tyAlias}) if not paramType.isCompileTimeOnly and paramType.kind in {tyVar, tySink, tyOwned}: - var roots: seq[PSym] - allRoots(it, roots) + var roots: seq[(PSym, int)] + allRoots(it, roots, RootEscapes) if paramType.kind == tyVar: if c.inNoSideEffectSection == 0: - for r in roots: potentialMutation(c, r, it.info) - for r in roots: noCursor(c, r) + for r in roots: potentialMutation(c, r[0], r[1], it.info) + for r in roots: noCursor(c, r[0]) if borrowChecking in c.goals: # a call like 'result.add toOpenArray()' can also be a borrow @@ -703,10 +718,10 @@ proc traverse(c: var Partitions; n: PNode) = when false: # XXX investigate if this is required, it doesn't look # like it is! - var roots: seq[PSym] - allRoots(n[0], roots) + var roots: seq[(PSym, int)] + allRoots(n[0], roots, RootEscapes) for r in roots: - potentialMutation(c, r, it.info) + potentialMutation(c, r[0], r[1], it.info) of nkTupleConstr, nkBracket: for child in n: traverse(c, child) diff --git a/tests/effects/tfuncs_cannot_mutate2.nim b/tests/effects/tfuncs_cannot_mutate2.nim new file mode 100644 index 000000000..d5082e57b --- /dev/null +++ b/tests/effects/tfuncs_cannot_mutate2.nim @@ -0,0 +1,27 @@ +discard """ + errormsg: "'copy' can have side effects" + nimout: '''an object reachable from 'y' is potentially mutated +tfuncs_cannot_mutate2.nim(15, 7) the mutation is here +tfuncs_cannot_mutate2.nim(13, 10) is the statement that connected the mutation to the parameter +''' +""" + +{.experimental: "strictFuncs".} + +func copy[T](x: var openArray[T]; y: openArray[T]) = + for i in 0..high(x): + x[i] = y[i] + + x[0].a = nil + +type + R = ref object + a, b: R + data: string + +proc main = + var a, b: array[3, R] + b = [R(data: "a"), R(data: "b"), R(data: "c")] + copy a, b + +main() diff --git a/tests/effects/tstrict_funcs.nim b/tests/effects/tstrict_funcs.nim index 044bc7ee1..9d20f5d7e 100644 --- a/tests/effects/tstrict_funcs.nim +++ b/tests/effects/tstrict_funcs.nim @@ -27,3 +27,20 @@ block: var x = @[0, 1] let z = x &&& 2 + + +func copy[T](x: var openArray[T]; y: openArray[T]) = + for i in 0..high(x): + x[i] = y[i] + +type + R = ref object + a, b: R + data: string + +proc main = + var a, b: array[3, R] + b = [R(data: "a"), R(data: "b"), R(data: "c")] + copy a, b + +main() From 19918ceb2b099440cd3e43b6d7fff313de6a84a2 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 9 Jun 2021 14:46:00 -0700 Subject: [PATCH 0904/2472] update .github/ISSUE_TEMPLATE/bug_report.md, mention PRs, contributing, devel branch (#18224) --- .github/ISSUE_TEMPLATE/bug_report.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 9571896d8..807e50d35 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -7,6 +7,8 @@ assignees: '' --- +(Consider writing a PR targetting devel branch after filing this, see [contributing.html](https://nim-lang.github.io/Nim/contributing.html)). + Function `echo` outputs the wrong string. ### Example From 79ded694d7f913755dc7261c7468f04fe421e0b3 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 9 Jun 2021 22:00:34 -0700 Subject: [PATCH 0905/2472] avoid re-exporting options from std/wrapnils (#18222) --- lib/std/wrapnils.nim | 2 +- tests/stdlib/twrapnils.nim | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/std/wrapnils.nim b/lib/std/wrapnils.nim index 708faf4cf..ed0a79d79 100644 --- a/lib/std/wrapnils.nim +++ b/lib/std/wrapnils.nim @@ -25,7 +25,6 @@ runnableExamples: assert (?.f2.x2.x2).x3 == nil # this terminates ?. early from options import Option, isSome, get, option, unsafeGet, UnpackDefect -export options.get, options.isSome, options.isNone template fakeDot*(a: Option, b): untyped = ## See top-level example. @@ -90,6 +89,7 @@ macro `?.`*(a: untyped): auto = macro `??.`*(a: untyped): Option = ## Same as `?.` but returns an `Option`. runnableExamples: + import std/options type Foo = ref object x1: ref int x2: int diff --git a/tests/stdlib/twrapnils.nim b/tests/stdlib/twrapnils.nim index af0978762..956204355 100644 --- a/tests/stdlib/twrapnils.nim +++ b/tests/stdlib/twrapnils.nim @@ -1,4 +1,5 @@ import std/wrapnils +from std/options import get, isSome proc checkNotZero(x: float): float = doAssert x != 0 From 0a4858dc59b1464d4057822d60e888fa4403223e Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 10 Jun 2021 09:25:51 +0200 Subject: [PATCH 0906/2472] fixes #18220 (#18227) --- lib/pure/hashes.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/hashes.nim b/lib/pure/hashes.nim index 46b4fc948..1c49b0317 100644 --- a/lib/pure/hashes.nim +++ b/lib/pure/hashes.nim @@ -509,7 +509,7 @@ proc hashIgnoreCase*(sBuf: string, sPos, ePos: int): Hash = h = h !& ord(c) result = !$h -proc hash*[T: tuple | object | proc](x: T): Hash {.inline.} = +proc hash*[T: tuple | object | proc](x: T): Hash = ## Efficient `hash` overload. runnableExamples: # for `tuple|object`, `hash` must be defined for each component of `x`. From 8a9b20579e4afde6ccd515b9840ab2c68d0fe9e6 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 10 Jun 2021 01:27:12 -0700 Subject: [PATCH 0907/2472] update changelog for wrapnils (#18228) --- changelog.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/changelog.md b/changelog.md index f141ea95a..cc6f826ee 100644 --- a/changelog.md +++ b/changelog.md @@ -285,9 +285,6 @@ - Added `std/strbasics` for high performance string operations. Added `strip`, `setSlice`, `add(a: var string, b: openArray[char])`. - -- Added to `wrapnils` an option-like API via `??.`, `isSome`, `get`. - - `std/options` changed `$some(3)` to `"some(3)"` instead of `"Some(3)"` and `$none(int)` to `"none(int)"` instead of `"None[int]"`. From 7bf0404dd86ff094ebd3d5b4a39bc4c52b442593 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=B6hlich=20A?= Date: Thu, 10 Jun 2021 14:28:00 +0200 Subject: [PATCH 0908/2472] #18216 make moveDir work across partitions on windows (#18223) * return false if AccessDeniedError in tryMoveFSObject - fixes #18216 * add moveDir & moveFile tests * rename `isMoveDir` parameter to `isDir` --- lib/pure/os.nim | 44 ++++++++++++++++++++++++------------------- tests/stdlib/tos.nim | 45 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 19 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index c48d0d84f..4dd5c8d8b 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -2009,28 +2009,32 @@ proc removeFile*(file: string) {.rtl, extern: "nos$1", tags: [WriteDirEffect], n if not tryRemoveFile(file): raiseOSError(osLastError(), file) -proc tryMoveFSObject(source, dest: string): bool {.noWeirdTarget.} = - ## Moves a file or directory from `source` to `dest`. +proc tryMoveFSObject(source, dest: string, isDir: bool): bool {.noWeirdTarget.} = + ## Moves a file (or directory if `isDir` is true) from `source` to `dest`. ## - ## Returns false in case of `EXDEV` error. + ## Returns false in case of `EXDEV` error or `AccessDeniedError` on windows (if `isDir` is true). ## In case of other errors `OSError` is raised. ## Returns true in case of success. when defined(windows): when useWinUnicode: let s = newWideCString(source) let d = newWideCString(dest) - if moveFileExW(s, d, MOVEFILE_COPY_ALLOWED or MOVEFILE_REPLACE_EXISTING) == 0'i32: raiseOSError(osLastError(), $(source, dest)) + result = moveFileExW(s, d, MOVEFILE_COPY_ALLOWED or MOVEFILE_REPLACE_EXISTING) != 0'i32 else: - if moveFileExA(source, dest, MOVEFILE_COPY_ALLOWED or MOVEFILE_REPLACE_EXISTING) == 0'i32: raiseOSError(osLastError(), $(source, dest)) + result = moveFileExA(source, dest, MOVEFILE_COPY_ALLOWED or MOVEFILE_REPLACE_EXISTING) != 0'i32 else: - if c_rename(source, dest) != 0'i32: - let err = osLastError() - if err == EXDEV.OSErrorCode: - return false + result = c_rename(source, dest) == 0'i32 + + if not result: + let err = osLastError() + let isAccessDeniedError = + when defined(windows): + const AccessDeniedError = OSErrorCode(5) + isDir and err == AccessDeniedError else: - # see whether `strerror(errno)` is redundant with what raiseOSError already shows - raiseOSError(err, $(source, dest, strerror(errno))) - return true + err == EXDEV.OSErrorCode + if not isAccessDeniedError: + raiseOSError(err, $(source, dest)) proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect], noWeirdTarget.} = @@ -2051,8 +2055,10 @@ proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", ## * `removeFile proc <#removeFile,string>`_ ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ - if not tryMoveFSObject(source, dest): - when not defined(windows): + if not tryMoveFSObject(source, dest, isDir = false): + when defined(windows): + doAssert false + else: # Fallback to copy & del copyFile(source, dest, {cfSymlinkAsIs}) try: @@ -2060,6 +2066,7 @@ proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", except: discard tryRemoveFile(dest) raise + proc exitStatusLikeShell*(status: cint): cint = ## Converts exit code from `c_system` into a shell exit code. @@ -2614,11 +2621,10 @@ proc moveDir*(source, dest: string) {.tags: [ReadIOEffect, WriteIOEffect], noWei ## * `removeDir proc <#removeDir,string>`_ ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ ## * `createDir proc <#createDir,string>`_ - if not tryMoveFSObject(source, dest): - when not defined(windows): - # Fallback to copy & del - copyDir(source, dest) - removeDir(source) + if not tryMoveFSObject(source, dest, isDir = true): + # Fallback to copy & del + copyDir(source, dest) + removeDir(source) proc createHardlink*(src, dest: string) {.noWeirdTarget.} = ## Create a hard link at `dest` which points to the item specified diff --git a/tests/stdlib/tos.nim b/tests/stdlib/tos.nim index b9769d05c..a4f030dde 100644 --- a/tests/stdlib/tos.nim +++ b/tests/stdlib/tos.nim @@ -261,6 +261,51 @@ block fileOperations: removeDir(dname) +block: # moveFile + let tempDir = getTempDir() / "D20210609T151608" + createDir(tempDir) + defer: removeDir(tempDir) + + writeFile(tempDir / "a.txt", "") + moveFile(tempDir / "a.txt", tempDir / "b.txt") + doAssert not fileExists(tempDir / "a.txt") + doAssert fileExists(tempDir / "b.txt") + removeFile(tempDir / "b.txt") + + createDir(tempDir / "moveFile_test") + writeFile(tempDir / "moveFile_test/a.txt", "") + moveFile(tempDir / "moveFile_test/a.txt", tempDir / "moveFile_test/b.txt") + doAssert not fileExists(tempDir / "moveFile_test/a.txt") + doAssert fileExists(tempDir / "moveFile_test/b.txt") + removeDir(tempDir / "moveFile_test") + + createDir(tempDir / "moveFile_test") + writeFile(tempDir / "a.txt", "") + moveFile(tempDir / "a.txt", tempDir / "moveFile_test/b.txt") + doAssert not fileExists(tempDir / "a.txt") + doAssert fileExists(tempDir / "moveFile_test/b.txt") + removeDir(tempDir / "moveFile_test") + +block: # moveDir + let tempDir = getTempDir() / "D20210609T161443" + createDir(tempDir) + defer: removeDir(tempDir) + + createDir(tempDir / "moveDir_test") + moveDir(tempDir / "moveDir_test/", tempDir / "moveDir_test_dest") + doAssert not dirExists(tempDir / "moveDir_test") + doAssert dirExists(tempDir / "moveDir_test_dest") + removeDir(tempDir / "moveDir_test_dest") + + createDir(tempDir / "moveDir_test") + writeFile(tempDir / "moveDir_test/a.txt", "") + moveDir(tempDir / "moveDir_test", tempDir / "moveDir_test_dest") + doAssert not dirExists(tempDir / "moveDir_test") + doAssert not fileExists(tempDir / "moveDir_test/a.txt") + doAssert dirExists(tempDir / "moveDir_test_dest") + doAssert fileExists(tempDir / "moveDir_test_dest/a.txt") + removeDir(tempDir / "moveDir_test_dest") + import times block modificationTime: # Test get/set modification times From 3481ff6172a2e11c990cc67a7fd74a2f0a2445d2 Mon Sep 17 00:00:00 2001 From: Kaushal Modi Date: Thu, 10 Jun 2021 08:29:03 -0400 Subject: [PATCH 0909/2472] Update nims.rst to reflect that -d:release is now allowed in config.nims (#18221) `-d:release` and `-d:danger` started working in `config.nims` after https://github.com/nim-lang/Nim/commit/df429fa28772e077faa30dd6e3a701abf48c7669 . --- doc/nims.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/nims.rst b/doc/nims.rst index 5ae1d7baa..4141c3bdc 100644 --- a/doc/nims.rst +++ b/doc/nims.rst @@ -121,8 +121,8 @@ Here are few examples of using the `switch` proc: .. code-block:: nim # command-line: --opt:size switch("opt", "size") - # command-line: --define:foo or -d:foo - switch("define", "foo") + # command-line: --define:release or -d:release + switch("define", "release") # command-line: --forceBuild switch("forceBuild") @@ -132,13 +132,13 @@ above example can be rewritten as: .. code-block:: nim --opt:size - --define:foo + --define:release --forceBuild **Note**: In general, the *define* switches can also be set in -NimScripts using `switch` or `--`, as shown in above -examples. Only the `release` define (`-d:release`:option:) cannot be set -in NimScripts. +NimScripts using `switch` or `--`, as shown in above examples. Few +`define` switches such as `-d:strip`:option:, `-d:lto`:option: and +`-d:lto_incremental`:option: cannot be set in NimScripts. NimScript as a build tool From 2ea7287217ffe3ad6c3790ff1233845de2aedcf4 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 10 Jun 2021 16:49:17 +0200 Subject: [PATCH 0910/2472] view types: spec changes (#18226) * view types: spec changes * Update doc/manual_experimental.rst Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Update doc/manual_experimental.rst Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- compiler/varpartitions.nim | 5 +++-- doc/manual_experimental.rst | 31 ++++++++++++++----------------- tests/views/tdont_mutate.nim | 9 ++++----- 3 files changed, 21 insertions(+), 24 deletions(-) diff --git a/compiler/varpartitions.nim b/compiler/varpartitions.nim index 9cf1e38ad..04c7c629d 100644 --- a/compiler/varpartitions.nim +++ b/compiler/varpartitions.nim @@ -572,7 +572,7 @@ proc borrowingAsgn(c: var Partitions; dest, src: PNode) = borrowFrom(c, dest.sym, src) elif dest.kind in {nkHiddenDeref, nkDerefExpr, nkBracketExpr}: case directViewType(dest[0].typ) - of mutableView: + of mutableView, immutableView: # we do not borrow, but we use the view to mutate the borrowed # location: let viewOrigin = pathExpr(dest, c.owner) @@ -580,12 +580,13 @@ proc borrowingAsgn(c: var Partitions; dest, src: PNode) = let vid = variableId(c, viewOrigin.sym) if vid >= 0: c.s[vid].flags.incl viewDoesMutate - of immutableView: + #[of immutableView: if dest.kind == nkBracketExpr and dest[0].kind == nkHiddenDeref and mutableParameter(dest[0][0]): discard "remains a mutable location anyhow" else: localError(c.g.config, dest.info, "attempt to mutate a borrowed location from an immutable view") + ]# of noView: discard "nothing to do" proc containsPointer(t: PType): bool = diff --git a/doc/manual_experimental.rst b/doc/manual_experimental.rst index 2c1043ecb..3157708da 100644 --- a/doc/manual_experimental.rst +++ b/doc/manual_experimental.rst @@ -475,8 +475,8 @@ Not nil annotation **Note:** This is an experimental feature. It can be enabled with `{.experimental: "notnil"}`. -All types for which `nil` is a valid value can be annotated with the `not -nil` annotation to exclude `nil` as a valid value: +All types for which `nil` is a valid value can be annotated with the +`not nil` annotation to exclude `nil` as a valid value: .. code-block:: nim {.experimental: "notnil"} @@ -1837,8 +1837,7 @@ with `--experimental:strictFuncs`:option:. A view type is a type that is or contains one of the following types: -- `var T` (mutable view into `T`) -- `lent T` (immutable view into `T`) +- `lent T` (view into `T`) - `openArray[T]` (pair of (pointer to array of `T`, size)) For example: @@ -1846,10 +1845,9 @@ For example: .. code-block:: nim type - View1 = var int - View2 = openArray[byte] - View3 = lent string - View4 = Table[openArray[char], int] + View1 = openArray[byte] + View2 = lent string + View3 = Table[openArray[char], int] Exceptions to this rule are types constructed via `ptr` or `proc`. @@ -1860,11 +1858,11 @@ For example, the following types are **not** view types: type NotView1 = proc (x: openArray[int]) NotView2 = ptr openArray[char] - NotView3 = ptr array[4, var int] + NotView3 = ptr array[4, lent int] -A *mutable* view type is a type that is or contains a `var T` type. -An *immutable* view type is a view type that is not a mutable view type. +The mutability aspect of a view type is not part of the type but part +of the locations it's derived from. More on this later. A *view* is a symbol (a let, var, const, etc.) that has a view type. @@ -1948,11 +1946,14 @@ details about how this is done for `var T`. A mutable view can borrow from a mutable location, an immutable view can borrow from both a mutable or an immutable location. +If a view borrows from a mutable location, the view can be used to update the +location. Otherwise it cannot be used for mutations. + The *duration* of a borrow is the span of commands beginning from the assignment to the view and ending with the last usage of the view. For the duration of the borrow operation, no mutations to the borrowed locations -may be performed except via the potentially mutable view that borrowed from the +may be performed except via the view that borrowed from the location. The borrowed location is said to be *sealed* during the borrow. .. code-block:: nim @@ -2064,11 +2065,7 @@ and `b` the location that is borrowed from. - The lifetime of `v` must not exceed `b`'s lifetime. Note: The lifetime of a parameter is the complete proc body. -- If `v` is a mutable view and `v` is used to actually mutate the - borrowed location, then `b` has to be a mutable location. - Note: If it is not actually used for mutation, borrowing a mutable view from an - immutable location is allowed! This allows for many important idioms and will be - justified in an upcoming RFC. +- If `v` is used for a mutation, `b` must be a mutable location too. - During `v`'s lifetime, `G(b)` can only be modified by `v` (and only if `v` is a mutable view). - If `v` is `result` then `b` has to be a location derived from the first diff --git a/tests/views/tdont_mutate.nim b/tests/views/tdont_mutate.nim index d243c7c7a..43acaaf71 100644 --- a/tests/views/tdont_mutate.nim +++ b/tests/views/tdont_mutate.nim @@ -9,8 +9,9 @@ import tables const Whitespace = {' ', '\t', '\n', '\r'} -proc split*(s: string, seps: set[char] = Whitespace, - maxsplit: int = -1): Table[int, openArray[char]] = +proc split*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): Table[int, openArray[char]] #[tt.Error + 'result' borrows from the immutable location 's' and attempts to mutate it + ]# = var last = 0 var splits = maxsplit result = initTable[int, openArray[char]]() @@ -22,9 +23,7 @@ proc split*(s: string, seps: set[char] = Whitespace, if splits == 0: last = len(s) result[first] = toOpenArray(s, first, last-1) - result[first][0] = 'c' #[tt.Error - attempt to mutate a borrowed location from an immutable view - ]# + result[first][0] = 'c' if splits == 0: break dec(splits) From f65f760dee2d4e19be826c256be50d34f8873d48 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 10 Jun 2021 18:19:20 +0200 Subject: [PATCH 0911/2472] fixes #15884 (#18230) * fixes #15884 * micro optimization --- compiler/renderer.nim | 2 ++ compiler/sem.nim | 9 +++++++++ compiler/semexprs.nim | 2 +- compiler/semstmts.nim | 2 +- compiler/varpartitions.nim | 6 +++++- 5 files changed, 18 insertions(+), 3 deletions(-) diff --git a/compiler/renderer.nim b/compiler/renderer.nim index 8bc3c3306..ac3f47948 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -1238,6 +1238,8 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext, fromStmtList = false) = putWithSpace(g, tkBind, "bind") gsub(g, n, 0) of nkCheckedFieldExpr, nkHiddenAddr, nkHiddenDeref, nkStringToCString, nkCStringToString: + if renderIds in g.flags: + putWithSpace(g, tkAddr, $n.kind) gsub(g, n, 0) of nkLambda: putWithSpace(g, tkProc, "proc") diff --git a/compiler/sem.nim b/compiler/sem.nim index 72b8dfe10..879f6efc9 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -100,6 +100,15 @@ proc fitNode(c: PContext, formal: PType, arg: PNode; info: TLineInfo): PNode = else: result = fitNodePostMatch(c, formal, result) +proc fitNodeForLocalVar(c: PContext, formal: PType, arg: PNode; info: TLineInfo): PNode = + let a = fitNode(c, formal, arg, info) + if formal.kind in {tyVar, tyLent}: + #classifyViewType(formal) != noView: + result = newNodeIT(nkHiddenAddr, a.info, formal) + result.add a + else: + result = a + proc inferWithMetatype(c: PContext, formal: PType, arg: PNode, coerceDistincts = false): PNode diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index c839d04d9..ba8b93ae6 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1643,7 +1643,7 @@ proc takeImplicitAddr(c: PContext, n: PNode; isLent: bool): PNode = proc asgnToResultVar(c: PContext, n, le, ri: PNode) {.inline.} = if le.kind == nkHiddenDeref: var x = le[0] - if (x.typ.kind in {tyVar, tyLent} or classifyViewType(x.typ) != noView) and x.kind == nkSym and x.sym.kind == skResult: + if x.kind == nkSym and x.sym.kind == skResult and (x.typ.kind in {tyVar, tyLent} or classifyViewType(x.typ) != noView): n[0] = x # 'result[]' --> 'result' n[1] = takeImplicitAddr(c, ri, x.typ.kind == tyLent) x.typ.flags.incl tfVarIsPtr diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index a2401b376..8f87ce83c 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -526,7 +526,7 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = else: # BUGFIX: ``fitNode`` is needed here! # check type compatibility between def.typ and typ - def = fitNode(c, typ, def, def.info) + def = fitNodeForLocalVar(c, typ, def, def.info) #changeType(def.skipConv, typ, check=true) else: typ = def.typ.skipTypes({tyStatic, tySink}).skipIntLit(c.idgen) diff --git a/compiler/varpartitions.nim b/compiler/varpartitions.nim index 04c7c629d..697dd2b1d 100644 --- a/compiler/varpartitions.nim +++ b/compiler/varpartitions.nim @@ -519,13 +519,17 @@ const proc isConstSym(s: PSym): bool = result = s.kind in {skConst, skLet} or isConstParam(s) +proc toString(n: PNode): string = + if n.kind == nkEmpty: result = "" + else: result = $n + proc borrowFrom(c: var Partitions; dest: PSym; src: PNode) = const url = "see https://nim-lang.github.io/Nim/manual_experimental.html#view-types-algorithm-path-expressions for details" let s = pathExpr(src, c.owner) if s == nil: - localError(c.g.config, src.info, "cannot borrow from " & $src & ", it is not a path expression; " & url) + localError(c.g.config, src.info, "cannot borrow from " & src.toString & ", it is not a path expression; " & url) elif s.kind == nkSym: if dest.kind == skResult: if s.sym.kind != skParam or s.sym.position != 0: From 6b97889f44d06f6649b6547a4ffdb18b94d6e578 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 11 Jun 2021 00:26:58 +0800 Subject: [PATCH 0912/2472] fix #9437(fix `re.replace` wrong behaviour) (#17546) * fix nim js cmp fails at CT * fix --- lib/impure/re.nim | 20 ++++++++++++++++++-- tests/stdlib/tre.nim | 9 ++++++--- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/lib/impure/re.nim b/lib/impure/re.nim index 40860cc88..78ea7b002 100644 --- a/lib/impure/re.nim +++ b/lib/impure/re.nim @@ -199,6 +199,18 @@ proc findBounds*(s: string, pattern: Regex, ## `(-1,0)` is returned. result = findBounds(cstring(s), pattern, matches, start, s.len) +proc findBoundsImpl(buf: cstring, pattern: Regex, + start = 0, bufSize = 0, flags = 0): tuple[first, last: int] = + var rtarray = initRtArray[cint](3) + let rawMatches = rtarray.getRawData + let res = pcre.exec(pattern.h, pattern.e, buf, bufSize.cint, start.cint, flags.int32, + cast[ptr cint](rawMatches), 3) + + if res < 0'i32: + result = (-1, 0) + else: + result = (int(rawMatches[0]), int(rawMatches[1]-1)) + proc findBounds*(buf: cstring, pattern: Regex, start = 0, bufSize: int): tuple[first, last: int] = ## returns the `first` and `last` position of `pattern` in `buf`, @@ -433,12 +445,16 @@ proc replace*(s: string, sub: Regex, by = ""): string = doAssert "var1=key; var2=key2".replace(re"(\w+)=(\w+)", "?") == "?; ?" result = "" var prev = 0 + var flags = int32(0) while prev < s.len: - var match = findBounds(s, sub, prev) + var match = findBoundsImpl(s.cstring, sub, prev, s.len, flags) + flags = 0 if match.first < 0: break add(result, substr(s, prev, match.first-1)) add(result, by) - if match.last + 1 == prev: break + if match.first > match.last: + # 0-len match + flags = pcre.NOTEMPTY_ATSTART prev = match.last + 1 add(result, substr(s, prev)) diff --git a/tests/stdlib/tre.nim b/tests/stdlib/tre.nim index ea1b5af32..2857c6c9e 100644 --- a/tests/stdlib/tre.nim +++ b/tests/stdlib/tre.nim @@ -99,10 +99,13 @@ proc testAll() = accum.add($x) doAssert(accum == @["a","b","c"]) - block: - # bug #9306 + block: # bug #9306 doAssert replace("bar", re"^", "foo") == "foobar" - doAssert replace("foo", re"", "-") == "-foo" doAssert replace("foo", re"$", "bar") == "foobar" + + block: # bug #9437 + doAssert replace("foo", re"", "-") == "-f-o-o-" + doAssert replace("ooo", re"o", "-") == "---" + testAll() From c64d9176190b3b691391f1645873dd4ca5b09933 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 10 Jun 2021 23:36:06 -0700 Subject: [PATCH 0913/2472] handle tyUserTypeClassInst in addDeclaredLoc (#18236) --- compiler/astmsgs.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/astmsgs.nim b/compiler/astmsgs.nim index 5be8dc38f..924f0ddea 100644 --- a/compiler/astmsgs.nim +++ b/compiler/astmsgs.nim @@ -13,7 +13,7 @@ proc addDeclaredLoc*(result: var string, conf: ConfigRef; typ: PType) = # xxx figure out how to resolve `tyGenericParam`, e.g. for # proc fn[T](a: T, b: T) = discard # fn(1.1, "a") - let typ = typ.skipTypes(abstractInst + {tyStatic} - {tyRange}) + let typ = typ.skipTypes(abstractInst + {tyStatic, tyUserTypeClassInst} - {tyRange}) result.add " [$1" % typ.kind.toHumanStr if typ.sym != nil: result.add " declared in " & toFileLineCol(conf, typ.sym.info) From 57609902c4438da2bcc156296128591a9906e922 Mon Sep 17 00:00:00 2001 From: Miran Date: Fri, 11 Jun 2021 10:07:23 +0200 Subject: [PATCH 0914/2472] use more meaningful name than "workaround14447" (#18237) --- compiler/lambdalifting.nim | 2 +- lib/system/iterators.nim | 2 +- testament/important_packages.nim | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 6f43649a1..c555cedfe 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -292,7 +292,7 @@ proc markAsClosure(g: ModuleGraph; owner: PSym; n: PNode) = if illegalCapture(s): localError(g.config, n.info, ("'$1' is of type <$2> which cannot be captured as it would violate memory" & - " safety, declared here: $3; using '-d:nimWorkaround14447' helps in some cases") % + " safety, declared here: $3; using '-d:nimNoLentIterators' helps in some cases") % [s.name.s, typeToString(s.typ), g.config$s.info]) elif not (owner.typ.callConv == ccClosure or owner.typ.callConv == ccNimCall and tfExplicitCallConv notin owner.typ.flags): localError(g.config, n.info, "illegal capture '$1' because '$2' has the calling convention: <$3>" % diff --git a/lib/system/iterators.nim b/lib/system/iterators.nim index 0f79970b8..f23f7aa11 100644 --- a/lib/system/iterators.nim +++ b/lib/system/iterators.nim @@ -1,4 +1,4 @@ -when defined(nimHasLentIterators) and not defined(nimWorkaround14447): +when defined(nimHasLentIterators) and not defined(nimNoLentIterators): template lent2(T): untyped = lent T else: template lent2(T): untyped = T diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 975dd63ad..3d3c340eb 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -156,5 +156,5 @@ pkg "winim", allowFailure = true pkg "with" pkg "ws" pkg "yaml", "nim build" -pkg "zero_functional", "nim c -r -d:nimWorkaround14447 test.nim" +pkg "zero_functional", "nim c -r -d:nimNoLentIterators test.nim" pkg "zippy" From 2ca6169ba10dca704712b59339a24208e30e79ac Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 11 Jun 2021 11:09:00 +0200 Subject: [PATCH 0915/2472] added a test case ensuring exception inference continues to work --- tests/exception/texception_inference.nim | 32 ++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 tests/exception/texception_inference.nim diff --git a/tests/exception/texception_inference.nim b/tests/exception/texception_inference.nim new file mode 100644 index 000000000..7dd01cca1 --- /dev/null +++ b/tests/exception/texception_inference.nim @@ -0,0 +1,32 @@ +discard """ + output: '''good''' + cmd: "nim c --gc:orc -d:release $file" +""" + +type + Raising[T, E] = object + +proc foo[T, Errors](x: proc (x: Raising[T, Errors])) {.raises: Errors.} = + discard + +proc callback(x: Raising[int, ValueError]) = + echo "callback" + +proc xy() {.raises: [ValueError].} = + foo callback + +proc x[E]() {.raises: [E, IOError].} = + raise newException(E, "text here") + +try: + x[ValueError]() +except ValueError: + echo "good" + +proc callback2(x: Raising[int, IOError]) = + discard + +proc foo2[T, OtherErrors](x: proc(x: Raising[T, OtherErrors])) {.raises: [ValueError, OtherErrors].} = + discard + +foo2 callback2 From 897e50d5fe274850d4772612a90b325e33cca8a2 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 12 Jun 2021 12:35:12 -0700 Subject: [PATCH 0916/2472] getType now works with tyInferred (arising from concepts); refs #18220 (#18241) * getType now works with tyInferred (concepts); refs #18220 * avoid cast * add more docs --- compiler/vmdeps.nim | 2 +- tests/macros/tgettype.nim | 100 +++++++++++++++++++++++++++++--------- 2 files changed, 79 insertions(+), 23 deletions(-) diff --git a/compiler/vmdeps.nim b/compiler/vmdeps.nim index b67e2e48a..1afda14b0 100644 --- a/compiler/vmdeps.nim +++ b/compiler/vmdeps.nim @@ -289,7 +289,7 @@ proc mapTypeToAstX(cache: IdentCache; t: PType; info: TLineInfo; of tyNot: result = mapTypeToBracket("not", mNot, t, info) of tyIterable: result = mapTypeToBracket("iterable", mIterableType, t, info) of tyAnything: result = atomicType("anything", mNone) - of tyInferred: assert false + of tyInferred: result = mapTypeToAstX(cache, t.lastSon, info, idgen, inst, allowRecursion) of tyStatic, tyFromExpr: if inst: if t.n != nil: result = t.n.copyTree diff --git a/tests/macros/tgettype.nim b/tests/macros/tgettype.nim index bd70a1c30..d91efe1fe 100644 --- a/tests/macros/tgettype.nim +++ b/tests/macros/tgettype.nim @@ -1,29 +1,85 @@ -discard """ -output: ''' -(ObjectTy (Empty) (Sym "Model") (RecList (Sym "name") (Sym "password"))) -(BracketExpr (Sym "typeDesc") (Sym "User")) -''' -""" -import macros +import std/macros +import stdtest/testutils -type - Model = object of RootObj - User = object of Model - name : string - password : string +# getType -macro testUser: string = - result = newLit(User.getType.lispRepr) +block: + type + Model = object of RootObj + User = object of Model + name : string + password : string -macro testGeneric(T: typedesc[Model]): string= - result = newLit(T.getType.lispRepr) + macro testUser: string = + result = newLit(User.getType.lispRepr) -echo testUser -echo User.testGeneric + macro testGeneric(T: typedesc[Model]): string= + result = newLit(T.getType.lispRepr) -macro assertVoid(e: typed): untyped = - assert(getTypeInst(e).typeKind == ntyVoid) + doAssert testUser == """(ObjectTy (Empty) (Sym "Model") (RecList (Sym "name") (Sym "password")))""" + doAssert User.testGeneric == """(BracketExpr (Sym "typeDesc") (Sym "User"))""" -proc voidProc() = discard + macro assertVoid(e: typed): untyped = + assert(getTypeInst(e).typeKind == ntyVoid) -assertVoid voidProc() + proc voidProc() = discard + + assertVoid voidProc() + +block: + # refs #18220; not an actual solution (yet) but at least shows what's currently + # possible + + type Callable1[R, T, U] = concept fn + fn(default(T)) is R + fn is U + + # note that typetraits.arity doesn't work + macro arity(a: typed): int = + # number of params + # this is not production code! + let a2 = a.getType[1] # this used to crash nim, with: `vmdeps.nim(292, 25) `false`` + newLit a2.len - 1 + + type Callable2[R, T, U] = concept fn + fn(default(T)) is R + fn is U + arity(U) == 2 + + proc map1[T, R, U](a: T, fn: Callable1[R, T, U]): R = + let fn = U(fn) + # `cast[U](fn)` would also work; + # this is currently needed otherwise, sigmatch errors with: + # Error: attempting to call routine: 'fn' + # found 'fn' [param declared in tgettype.nim(53, 28)] + # this can be fixed in future work + fn(a) + + proc map2[T, R, U](a: T, fn: Callable2[R, T, U]): R = + let fn = U(fn) + fn(a) + + proc fn1(a: int, a2 = 'x'): string = $(a, a2, "fn1") + proc fn2(a: int, a2 = "zoo"): string = $(a, a2, "fn2") + proc fn3(a: int, a2 = "zoo2"): string = $(a, a2, "fn3") + proc fn4(a: int): string {.inline.} = $(a, "fn4") + proc fn5(a: int): string = $(a, "fn5") + + assertAll: + # Callable1 + 1.map1(fn1) == """(1, 'x', "fn1")""" + 1.map1(fn2) == """(1, "zoo", "fn2")""" + 1.map1(fn3) == """(1, "zoo", "fn3")""" + # fn3's optional param is not honored, because fn3 and fn2 yield same + # generic instantiation; this is a caveat with this approach + # There are several possible ways to improve things in future work. + 1.map1(fn4) == """(1, "fn4")""" + 1.map1(fn5) == """(1, "fn5")""" + + # Callable2; prevents passing procs with optional params to avoid above + # mentioned caveat, but more restrictive + not compiles(1.map2(fn1)) + not compiles(1.map2(fn2)) + not compiles(1.map2(fn3)) + 1.map2(fn4) == """(1, "fn4")""" + 1.map2(fn5) == """(1, "fn5")""" From c871e22da2ad8f9caf82fdba43fccb7230d726e1 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 12 Jun 2021 22:32:47 -0700 Subject: [PATCH 0917/2472] fix #7717 roundtrip float to string; fix `parseFloat` for js (#18248) * refs #7717 roundtrip float to string * make parseFloat more correct * improve float tests * improve float tests * cleanup --- lib/system/jssys.nim | 99 +++++++++++++++++++---------------------- tests/float/tfloat6.nim | 28 ------------ tests/float/tfloats.nim | 84 ++++++++++++++++++++++++++++++++++ 3 files changed, 130 insertions(+), 81 deletions(-) delete mode 100644 tests/float/tfloat6.nim create mode 100644 tests/float/tfloats.nim diff --git a/lib/system/jssys.nim b/lib/system/jssys.nim index 371cb7962..7cb5652c5 100644 --- a/lib/system/jssys.nim +++ b/lib/system/jssys.nim @@ -716,19 +716,23 @@ proc tenToThePowerOf(b: int): BiggestFloat = const IdentChars = {'a'..'z', 'A'..'Z', '0'..'9', '_'} -# XXX use JS's native way here -proc nimParseBiggestFloat(s: string, number: var BiggestFloat, start = 0): int {. - compilerproc.} = - var - esign = 1.0 - sign = 1.0 - i = start - exponent: int - flags: int - number = 0.0 + +proc parseFloatNative(a: string): float = + let a2 = a.cstring + asm """ + `result` = Number(`a2`); + """ + +#[ +xxx how come code like this doesn't give IndexDefect ? +let z = s[10000] == 'a' +]# +proc nimParseBiggestFloat(s: string, number: var BiggestFloat, start: int): int {.compilerproc.} = + var sign: bool + var i = start if s[i] == '+': inc(i) elif s[i] == '-': - sign = -1.0 + sign = true inc(i) if s[i] == 'N' or s[i] == 'n': if s[i+1] == 'A' or s[i+1] == 'a': @@ -741,51 +745,40 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat, start = 0): int { if s[i+1] == 'N' or s[i+1] == 'n': if s[i+2] == 'F' or s[i+2] == 'f': if s[i+3] notin IdentChars: - number = Inf*sign + number = if sign: -Inf else: Inf return i+3 - start return 0 - while s[i] in {'0'..'9'}: - # Read integer part - flags = flags or 1 - number = number * 10.0 + toFloat(ord(s[i]) - ord('0')) - inc(i) - while s[i] == '_': inc(i) - # Decimal? - if s[i] == '.': - var hd = 1.0 - inc(i) - while s[i] in {'0'..'9'}: - # Read fractional part - flags = flags or 2 - number = number * 10.0 + toFloat(ord(s[i]) - ord('0')) - hd = hd * 10.0 - inc(i) - while s[i] == '_': inc(i) - number = number / hd # this complicated way preserves precision - # Again, read integer and fractional part - if flags == 0: return 0 - # Exponent? - if s[i] in {'e', 'E'}: - inc(i) - if s[i] == '+': - inc(i) - elif s[i] == '-': - esign = -1.0 - inc(i) - if s[i] notin {'0'..'9'}: - return 0 - while s[i] in {'0'..'9'}: - exponent = exponent * 10 + ord(s[i]) - ord('0') - inc(i) - while s[i] == '_': inc(i) - # Calculate Exponent - let hd = tenToThePowerOf(exponent) - if esign > 0.0: number = number * hd - else: number = number / hd - # evaluate sign - number = number * sign - result = i - start + var buf: string + # we could also use an `array[char, N]` buffer to avoid reallocs, or + # use a 2-pass algorithm that first computes the length. + if sign: buf.add '-' + template addInc = + buf.add s[i] + inc(i) + template eatUnderscores = + while s[i] == '_': inc(i) + while s[i] in {'0'..'9'}: # Read integer part + buf.add s[i] + inc(i) + eatUnderscores() + if s[i] == '.': # Decimal? + addInc() + while s[i] in {'0'..'9'}: # Read fractional part + addInc() + eatUnderscores() + # Again, read integer and fractional part + if buf.len == ord(sign): return 0 + if s[i] in {'e', 'E'}: # Exponent? + addInc() + if s[i] == '+': inc(i) + elif s[i] == '-': addInc() + if s[i] notin {'0'..'9'}: return 0 + while s[i] in {'0'..'9'}: + addInc() + eatUnderscores() + number = parseFloatNative(buf) + result = i - start # Workaround for IE, IE up to version 11 lacks 'Math.trunc'. We produce # 'Math.trunc' for Nim's ``div`` and ``mod`` operators: diff --git a/tests/float/tfloat6.nim b/tests/float/tfloat6.nim deleted file mode 100644 index fa8fc9da8..000000000 --- a/tests/float/tfloat6.nim +++ /dev/null @@ -1,28 +0,0 @@ -discard """ - output: ''' -0.000001 : 0.000001 -0.000001 : 0.000001 -0.001 : 0.001 -0.000001 : 0.000001 -0.000001 : 0.000001 -10.000001 : 10.000001 -100.000001 : 100.000001 -''' - disabled: "windows" -""" - -import strutils - -echo "0.00_0001".parseFloat(), " : ", 1E-6 -echo "0.00__00_01".parseFloat(), " : ", 1E-6 -echo "0.0_01".parseFloat(), " : ", 0.001 -echo "0.00_000_1".parseFloat(), " : ", 1E-6 -echo "0.00000_1".parseFloat(), " : ", 1E-6 - -echo "1_0.00_0001".parseFloat(), " : ", 10.000001 -echo "1__00.00_0001".parseFloat(), " : ", 1_00.000001 - -# bug #18148 - -var a = 1.1'f32 -doAssert $a == "1.1", $a # fails diff --git a/tests/float/tfloats.nim b/tests/float/tfloats.nim new file mode 100644 index 000000000..406ddb6d9 --- /dev/null +++ b/tests/float/tfloats.nim @@ -0,0 +1,84 @@ +discard """ + targets: "c cpp js" +""" +# disabled: "windows" + +#[ +xxx merge all or most float tests into this file +]# + +import std/[fenv, math, strutils] + +proc equalsOrNaNs(a, b: float): bool = + if isNaN(a): isNaN(b) + elif a == 0: + b == 0 and signbit(a) == signbit(b) + else: + a == b + +template reject(a) = + doAssertRaises(ValueError): discard parseFloat(a) + +template main = + block: + proc test(a: string, b: float) = + let a2 = a.parseFloat + doAssert equalsOrNaNs(a2, b), $(a, a2, b) + test "0.00_0001", 1E-6 + test "0.00__00_01", 1E-6 + test "0.0_01", 0.001 + test "0.00_000_1", 1E-6 + test "0.00000_1", 1E-6 + test "1_0.00_0001", 10.000001 + test "1__00.00_0001", 1_00.000001 + test "inf", Inf + test "-inf", -Inf + test "-Inf", -Inf + test "-INF", -Inf + test "NaN", NaN + test "-nan", NaN + test ".1", 0.1 + test "-.1", -0.1 + test "-0", -0.0 + when false: # pending bug #18246 + test "-0", -0.0 + test ".1e-1", 0.1e-1 + test "0_1_2_3.0_1_2_3E+0_1_2", 123.0123e12 + test "0_1_2.e-0", 12e0 + test "0_1_2e-0", 12e0 + test "-0e0", -0.0 + test "-0e-0", -0.0 + + reject "a" + reject "" + reject "e1" + reject "infa" + reject "infe1" + reject "_" + reject "1e" + + when false: # gray area; these numbers should probably be invalid + reject "1_" + reject "1_.0" + reject "1.0_" + + block: # bug #18148 + var a = 1.1'f32 + doAssert $a == "1.1", $a # was failing + + block: # bug #7717 + proc test(f: float) = + let f2 = $f + let f3 = parseFloat(f2) + doAssert equalsOrNaNs(f, f3), $(f, f2, f3) + + test 1.0 + epsilon(float64) + test 1000000.0000000123 + test log2(100000.0) + test maximumPositiveValue(float32) + test maximumPositiveValue(float64) + test minimumPositiveValue(float32) + test minimumPositiveValue(float64) + +static: main() +main() From a266c549212d1d6e09dbfa01344edbca8b2f6222 Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Sun, 13 Jun 2021 16:50:04 +0100 Subject: [PATCH 0918/2472] Improve httpClient docs on SSL cert verification (#15201) * Improve httpClient docs on SSL cert verification Cert verification is enabled by default after CVE-2021-29495 * Update httpclient.nim Co-authored-by: Dominik Picheta --- lib/pure/httpclient.nim | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 0dbf8a045..2f9f1913f 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -115,7 +115,7 @@ ## ## SSL/TLS support ## =============== -## This requires the OpenSSL library, fortunately it's widely used and installed +## This requires the OpenSSL library. Fortunately it's widely used and installed ## on many operating systems. httpclient will use SSL automatically if you give ## any of the functions a url with the `https` schema, for example: ## `https://github.com/`. @@ -123,12 +123,25 @@ ## You will also have to compile with `ssl` defined like so: ## `nim c -d:ssl ...`. ## -## Certificate validation is NOT performed by default. -## This will change in the future. +## Certificate validation is performed by default. ## ## A set of directories and files from the `ssl_certs `_ ## module are scanned to locate CA certificates. ## +## Example of setting SSL verification parameters in a new client: +## +## .. code-block:: Nim +## import httpclient +## var client = newHttpClient(sslContext=newContext(verifyMode=CVerifyPeer)) +## +## There are three options for verify mode: +## +## * ``CVerifyNone``: certificates are not verified; +## * ``CVerifyPeer``: certificates are verified; +## * ``CVerifyPeerUseEnvVars``: certificates are verified and the optional +## environment variables SSL_CERT_FILE and SSL_CERT_DIR are also used to +## locate certificates +## ## See `newContext `_ to tweak or disable certificate validation. ## ## Timeouts From 065243dc5988fbfcbed8236e19a588b98d63b199 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 13 Jun 2021 23:21:18 -0700 Subject: [PATCH 0919/2472] followup #17777: declaredloc field error msgs now work with generics (#18259) * followup #17777: declaredloc field error msgs now work with generics * fix tests * cleanup --- compiler/astmsgs.nim | 5 ++++ compiler/semcall.nim | 5 ++-- tests/errmsgs/tundeclared_field.nim | 40 ++++++++++++++++++----------- 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/compiler/astmsgs.nim b/compiler/astmsgs.nim index 924f0ddea..90bdce7cf 100644 --- a/compiler/astmsgs.nim +++ b/compiler/astmsgs.nim @@ -2,6 +2,11 @@ import std/strutils import options, ast, msgs +proc typSym*(t: PType): PSym = + result = t.sym + if result == nil and t.kind == tyGenericInst: # this might need to be refined + result = t[0].sym + proc addDeclaredLoc*(result: var string, conf: ConfigRef; sym: PSym) = result.add " [$1 declared in $2]" % [sym.kind.toHumanStr, toFileLineCol(conf, sym.info)] diff --git a/compiler/semcall.nim b/compiler/semcall.nim index facaa89f7..b56a1dbac 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -326,7 +326,7 @@ proc getMsgDiagnostic(c: PContext, flags: TExprFlags, n, f: PNode): string = let ident = considerQuotedIdent(c, f, n).s if {nfDotField, nfExplicitCall} * n.flags == {nfDotField}: - let sym = n[1].typ.sym + let sym = n[1].typ.typSym var typeHint = "" if sym == nil: # Perhaps we're in a `compiles(foo.bar)` expression, or @@ -337,7 +337,8 @@ proc getMsgDiagnostic(c: PContext, flags: TExprFlags, n, f: PNode): string = discard else: typeHint = " for type " & getProcHeader(c.config, sym) - result = errUndeclaredField % ident & typeHint & " " & result + let suffix = if result.len > 0: " " & result else: "" + result = errUndeclaredField % ident & typeHint & suffix else: if result.len == 0: result = errUndeclaredRoutine % ident else: result = errBadRoutine % [ident, result] diff --git a/tests/errmsgs/tundeclared_field.nim b/tests/errmsgs/tundeclared_field.nim index 2b274ae58..5668050e0 100644 --- a/tests/errmsgs/tundeclared_field.nim +++ b/tests/errmsgs/tundeclared_field.nim @@ -2,38 +2,48 @@ discard """ cmd: '''nim check --hints:off $file''' action: reject nimout: ''' -tundeclared_field.nim(25, 12) Error: undeclared field: 'bad' for type tundeclared_field.A [type declared in tundeclared_field.nim(22, 8)] -tundeclared_field.nim(30, 16) Error: undeclared field: 'bad' for type tundeclared_field.A [type declared in tundeclared_field.nim(28, 8)] -tundeclared_field.nim(36, 4) Error: undeclared field: 'bad' for type tundeclared_field.A [type declared in tundeclared_field.nim(33, 8)] -tundeclared_field.nim(40, 13) Error: cannot instantiate Foo [type declared in tundeclared_field.nim(39, 8)] +tundeclared_field.nim(25, 12) Error: undeclared field: 'bad1' for type tundeclared_field.A [type declared in tundeclared_field.nim(22, 8)] +tundeclared_field.nim(30, 17) Error: undeclared field: 'bad2' for type tundeclared_field.A [type declared in tundeclared_field.nim(28, 8)] +tundeclared_field.nim(36, 4) Error: undeclared field: 'bad3' for type tundeclared_field.A [type declared in tundeclared_field.nim(33, 8)] +tundeclared_field.nim(42, 12) Error: undeclared field: 'bad4' for type tundeclared_field.B [type declared in tundeclared_field.nim(39, 8)] +tundeclared_field.nim(43, 4) Error: undeclared field: 'bad5' for type tundeclared_field.B [type declared in tundeclared_field.nim(39, 8)] +tundeclared_field.nim(44, 23) Error: undeclared field: 'bad6' for type tundeclared_field.B [type declared in tundeclared_field.nim(39, 8)] +tundeclared_field.nim(46, 19) Error: undeclared field: 'bad7' for type tundeclared_field.B [type declared in tundeclared_field.nim(39, 8)] +tundeclared_field.nim(50, 13) Error: cannot instantiate Foo [type declared in tundeclared_field.nim(49, 8)] ''' """ - - - - - - - - +#[ +xxx in future work, generic instantiations (e.g. `B[int]`) should be shown with their instantiation instead of `tundeclared_field.B`, +maybe using TPreferedDesc.preferResolved or preferMixed +]# # line 20 block: type A = object a0: int var a: A - discard a.bad + discard a.bad1 block: type A = object a0: int - var a = A(bad: 0) + var a = A(bad2: 0) block: type A = object a0: int var a: A - a.bad = 0 + a.bad3 = 0 + +block: + type B[T] = object + b0: int + var b: B[int] + discard b.bad4 + b.bad5 = 0 + var b2 = B[int](bad6: 0) + type Bi = B[int] + var b3 = Bi(bad7: 0) block: type Foo[T: SomeInteger] = object From e1e8af535ec195e5749dffa728add95770e5cbd7 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 13 Jun 2021 23:51:40 -0700 Subject: [PATCH 0920/2472] merge BuildMode into SuccessX, remove code duplication w drnim, add useful info to successx, add gc to compilesettings (#18252) * merge BuildMode into SuccessX, add more info * refactor duplicated with drnim * fixup * address comment --- compiler/lineinfos.nim | 7 +++--- compiler/main.nim | 40 ++++++---------------------------- compiler/msgs.nim | 43 +++++++++++++++++++++++++++++++++---- compiler/options.nim | 13 +++++++++-- compiler/vmops.nim | 1 + doc/nimc.rst | 1 - drnim/drnim.nim | 19 +--------------- lib/std/compilesettings.nim | 1 + testament/testament.nim | 3 +-- tests/misc/trunner.nim | 2 +- 10 files changed, 64 insertions(+), 66 deletions(-) diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index bf81aed02..da0a8d623 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -65,7 +65,7 @@ type warnFileChanged = "FileChanged", warnUser = "User", - hintSuccess = "Success", hintSuccessX = "SuccessX", hintBuildMode = "BuildMode", + hintSuccess = "Success", hintSuccessX = "SuccessX", hintCC = "CC", hintLineTooLong = "LineTooLong", hintXDeclaredButNotUsed = "XDeclaredButNotUsed", hintXCannotRaiseY = "XCannotRaiseY", hintConvToBaseNotNeeded = "ConvToBaseNotNeeded", @@ -145,8 +145,7 @@ const warnUser: "$1", hintSuccess: "operation successful: $#", # keep in sync with `testament.isSuccess` - hintSuccessX: "$loc lines; ${sec}s; $mem; proj: $project; out: $output", - hintBuildMode: "$1", + hintSuccessX: "$build\n$loc lines; ${sec}s; $mem; proj: $project; out: $output", hintCC: "CC: $1", hintLineTooLong: "line too long", hintXDeclaredButNotUsed: "'$1' is declared but not used", @@ -200,7 +199,7 @@ proc computeNotesVerbosity(): array[0..3, TNoteKinds] = result[1] = result[2] - {warnProveField, warnProveIndex, warnGcUnsafe, hintPath, hintDependency, hintCodeBegin, hintCodeEnd, hintSource, hintGlobalVar, hintGCStats, hintMsgOrigin, hintPerformance} - result[0] = result[1] - {hintSuccessX, hintBuildMode, hintSuccess, hintConf, + result[0] = result[1] - {hintSuccessX, hintSuccess, hintConf, hintProcessing, hintPattern, hintExecuting, hintLinking, hintCC} const diff --git a/compiler/main.nim b/compiler/main.nim index 71e549ab8..7824e8cb2 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -13,13 +13,14 @@ when not defined(nimcore): {.error: "nimcore MUST be defined for Nim's core tooling".} import - llstream, strutils, os, ast, lexer, syntaxes, options, msgs, - condsyms, times, + std/[strutils, os, times, tables, sha1, with, json], + llstream, ast, lexer, syntaxes, options, msgs, + condsyms, sem, idents, passes, extccomp, - cgen, json, nversion, + cgen, nversion, platform, nimconf, passaux, depends, vm, modules, - modulegraphs, tables, lineinfos, pathutils, vmprofiler, std/[sha1, with] + modulegraphs, lineinfos, pathutils, vmprofiler import ic / [cbackend, integrity, navigator] from ic / ic import rodViewer @@ -387,38 +388,9 @@ proc mainCommand*(graph: ModuleGraph) = rawMessage(conf, errGenerated, "invalid command: " & conf.command) if conf.errorCounter == 0 and conf.cmd notin {cmdTcc, cmdDump, cmdNop}: - # D20210419T170230:here - let mem = - when declared(system.getMaxMem): formatSize(getMaxMem()) & " peakmem" - else: formatSize(getTotalMem()) & " totmem" - let loc = $conf.linesCompiled - let build = if isDefined(conf, "danger"): "Dangerous Release build" - elif isDefined(conf, "release"): "Release build" - else: "***SLOW, DEBUG BUILD***; -d:release makes code run faster." - let sec = formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3) - let project = if conf.filenameOption == foAbs: $conf.projectFull else: $conf.projectName - # xxx honor conf.filenameOption more accurately - var output: string - if optCompileOnly in conf.globalOptions and conf.cmd != cmdJsonscript: - output = $conf.jsonBuildFile - elif conf.outFile.isEmpty and conf.cmd notin {cmdJsonscript} + cmdDocLike + cmdBackends: - # for some cmd we expect a valid absOutFile - output = "unknownOutput" - else: - output = $conf.absOutFile - if conf.filenameOption != foAbs: output = output.AbsoluteFile.extractFilename - # xxx honor filenameOption more accurately if optProfileVM in conf.globalOptions: echo conf.dump(conf.vmProfileData) - rawMessage(conf, hintSuccessX, [ - "loc", loc, - "sec", sec, - "mem", mem, - "project", project, - "output", output, - ]) - if conf.cmd in cmdBackends: - rawMessage(conf, hintBuildMode, build) + genSuccessX(conf) when PrintRopeCacheStats: echo "rope cache stats: " diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 053b5c928..6b005be3d 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -8,10 +8,9 @@ # import - options, strutils, os, tables, ropes, terminal, macros, - lineinfos, pathutils -import std/private/miscdollars -import strutils2 + std/[strutils, os, tables, terminal, macros, times], + std/private/miscdollars, + options, ropes, lineinfos, pathutils, strutils2 type InstantiationInfo* = typeof(instantiationInfo()) template instLoc*(): InstantiationInfo = instantiationInfo(-2, fullPaths = true) @@ -656,3 +655,39 @@ proc uniqueModuleName*(conf: ConfigRef; fid: FileIndex): string = # We mangle upper letters and digits too so that there cannot # be clashes with our special meanings of 'Z' and 'O' result.addInt ord(c) + +proc genSuccessX*(conf: ConfigRef) = + let mem = + when declared(system.getMaxMem): formatSize(getMaxMem()) & " peakmem" + else: formatSize(getTotalMem()) & " totmem" + let loc = $conf.linesCompiled + var build = "" + if conf.cmd in cmdBackends: + build.add "gc: $#; " % $conf.selectedGC + if optThreads in conf.globalOptions: build.add "threads: on; " + build.add "opt: " + if optOptimizeSpeed in conf.options: build.add "speed" + elif optOptimizeSize in conf.options: build.add "size" + else: build.add "none (DEBUG BUILD, `-d:release` generates faster code)" + # pending https://github.com/timotheecour/Nim/issues/752, point to optimization.html + let sec = formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3) + let project = if conf.filenameOption == foAbs: $conf.projectFull else: $conf.projectName + # xxx honor conf.filenameOption more accurately + var output: string + if optCompileOnly in conf.globalOptions and conf.cmd != cmdJsonscript: + output = $conf.jsonBuildFile + elif conf.outFile.isEmpty and conf.cmd notin {cmdJsonscript} + cmdDocLike + cmdBackends: + # for some cmd we expect a valid absOutFile + output = "unknownOutput" + else: + output = $conf.absOutFile + if conf.filenameOption != foAbs: output = output.AbsoluteFile.extractFilename + # xxx honor filenameOption more accurately + rawMessage(conf, hintSuccessX, [ + "build", build, + "loc", loc, + "sec", sec, + "mem", mem, + "project", project, + "output", output, + ]) diff --git a/compiler/options.nim b/compiler/options.nim index afe8b0fc7..1851770d9 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -167,8 +167,17 @@ const type TStringSeq* = seq[string] TGCMode* = enum # the selected GC - gcUnselected, gcNone, gcBoehm, gcRegions, gcArc, gcOrc, - gcMarkAndSweep, gcHooks, gcRefc, gcV2, gcGo + gcUnselected = "unselected" + gcNone = "none" + gcBoehm = "boehm" + gcRegions = "regions" + gcArc = "arc" + gcOrc = "orc" + gcMarkAndSweep = "markAndSweep" + gcHooks = "hooks" + gcRefc = "refc" + gcV2 = "v2" + gcGo = "go" # gcRefc and the GCs that follow it use a write barrier, # as far as usesWriteBarrier() is concerned diff --git a/compiler/vmops.nim b/compiler/vmops.nim index 04356fc76..85729fd59 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -136,6 +136,7 @@ when defined(nimHasInvariant): of ccompilerPath: result = conf.cCompilerPath of backend: result = $conf.backend of libPath: result = conf.libpath.string + of gc: result = $conf.selectedGC proc querySettingSeqImpl(conf: ConfigRef, switch: BiggestInt): seq[string] = template copySeq(field: untyped): untyped = diff --git a/doc/nimc.rst b/doc/nimc.rst index e467a33af..2b9f33cb5 100644 --- a/doc/nimc.rst +++ b/doc/nimc.rst @@ -108,7 +108,6 @@ Source The source line that triggered a diagnostic message. StackTrace Success, SuccessX Successful compilation of a library or a binary. -BuildMode Kind of build: debug, release, danger User UserRaw XDeclaredButNotUsed Unused symbols in the code. diff --git a/drnim/drnim.nim b/drnim/drnim.nim index 1eded533d..eb0d89aa2 100644 --- a/drnim/drnim.nim +++ b/drnim/drnim.nim @@ -1205,24 +1205,7 @@ proc mainCommand(graph: ModuleGraph) = registerPass graph, semPass compileProject(graph) if conf.errorCounter == 0: - # xxx deduplicate with D20210419T170230 - let mem = - when declared(system.getMaxMem): formatSize(getMaxMem()) & " peakmem" - else: formatSize(getTotalMem()) & " totmem" - let loc = $conf.linesCompiled - let build = if isDefined(conf, "danger"): "Dangerous Release build" - elif isDefined(conf, "release"): "Release build" - else: "***SLOW, DEBUG BUILD***; -d:release makes code run faster." - let sec = formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3) - let project = if conf.filenameOption == foAbs: $conf.projectFull else: $conf.projectName - rawMessage(conf, hintSuccessX, [ - "loc", loc, - "sec", sec, - "mem", mem, - "project", project, - "output", "" - ]) - rawMessage(conf, hintBuildMode, build) + genSuccessX(graph.config) proc processCmdLine(pass: TCmdLinePass, cmd: string; config: ConfigRef) = var p = parseopt.initOptParser(cmd) diff --git a/lib/std/compilesettings.nim b/lib/std/compilesettings.nim index 273cfdb57..e1ffc954a 100644 --- a/lib/std/compilesettings.nim +++ b/lib/std/compilesettings.nim @@ -32,6 +32,7 @@ type backend ## the backend (eg: c|cpp|objc|js); both `nim doc --backend:js` ## and `nim js` would imply backend=js libPath ## the absolute path to the stdlib library, i.e. nim's `--lib`, since 1.5.1 + gc ## gc selected MultipleValueSetting* {.pure.} = enum ## \ ## settings resulting in a seq of string values diff --git a/testament/testament.nim b/testament/testament.nim index 789c987c0..ab6620c84 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -108,8 +108,7 @@ proc isSuccess(input: string): bool = # that may appear in user config (eg: `--filenames`). # Passing `XDG_CONFIG_HOME= testament args...` can be used to ignore user config # stored in XDG_CONFIG_HOME, refs https://wiki.archlinux.org/index.php/XDG_Base_Directory - input.startsWith("Hint: ") and - (input.endsWith("[SuccessX]") or input.endsWith("[BuildMode]")) + input.startsWith("Hint: ") and input.endsWith("[SuccessX]") proc getFileDir(filename: string): string = result = filename.splitFile().dir diff --git a/tests/misc/trunner.nim b/tests/misc/trunner.nim index b6b302836..f2ea1d040 100644 --- a/tests/misc/trunner.nim +++ b/tests/misc/trunner.nim @@ -23,7 +23,7 @@ proc isDots(a: string): bool = a.startsWith(".") and a.strip(chars = {'.'}) == "" const - defaultHintsOff = "--hint:successx:off --hint:buildmode:off --hint:exec:off --hint:link:off --hint:cc:off --hint:conf:off --hint:processing:off --hint:QuitCalled:off" + defaultHintsOff = "--hint:successx:off --hint:exec:off --hint:link:off --hint:cc:off --hint:conf:off --hint:processing:off --hint:QuitCalled:off" # useful when you want to turn only some hints on, and some common ones off. # pending https://github.com/timotheecour/Nim/issues/453, simplify to: `--hints:off` nim = getCurrentCompilerExe() From 488acd9d077e8179d35d665ac0591c456bfa93aa Mon Sep 17 00:00:00 2001 From: Saem Ghani Date: Mon, 14 Jun 2021 00:21:33 -0700 Subject: [PATCH 0921/2472] fixes #18235 - proc annotation type macro sym leak (#18249) * fixes #18235 - proc annotation type macro sym leak - also fixed a typo - proc annotations guard symbol exports with shadow scopes - symbol handling is shadow scope aware * test for exporting an existing unexported sym this one is for my homie alaviss. * Special handling not needed in semProcAnnotation * Testcasing * [skip ci] clean-up and add some more comments * [skip ci] rm trailing whitespace Co-authored-by: Clyybber --- compiler/lookups.nim | 27 +++++++++++++++++++++----- compiler/semstmts.nim | 8 ++++---- tests/macros/m18235.nim | 42 +++++++++++++++++++++++++++++++++++++++++ tests/macros/t18235.nim | 18 ++++++++++++++++++ 4 files changed, 86 insertions(+), 9 deletions(-) create mode 100644 tests/macros/m18235.nim create mode 100644 tests/macros/t18235.nim diff --git a/compiler/lookups.nim b/compiler/lookups.nim index 64476d816..a21e35ddc 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -314,6 +314,7 @@ proc addDeclAt*(c: PContext; scope: PScope, sym: PSym) = from ic / ic import addHidden proc addInterfaceDeclAux*(c: PContext, sym: PSym, forceExport = false) = + ## adds symbol to the module for either private or public access. if sfExported in sym.flags or forceExport: # add to interface: if c.module != nil: exportSym(c, sym) @@ -324,10 +325,15 @@ proc addInterfaceDeclAux*(c: PContext, sym: PSym, forceExport = false) = addHidden(c.encoder, c.packedRepr, sym) proc addInterfaceDeclAt*(c: PContext, scope: PScope, sym: PSym) = + ## adds a symbol on the scope and the interface if appropriate addDeclAt(c, scope, sym) - addInterfaceDeclAux(c, sym) + if not scope.isShadowScope: + # adding into a non-shadow scope, we need to handle exports, etc + addInterfaceDeclAux(c, sym) proc addOverloadableSymAt*(c: PContext; scope: PScope, fn: PSym) = + ## adds an symbol to the given scope, will check for and raise errors if it's + ## a redefinition as opposed to an overload. if fn.kind notin OverloadableSyms: internalError(c.config, fn.info, "addOverloadableSymAt") return @@ -338,24 +344,35 @@ proc addOverloadableSymAt*(c: PContext; scope: PScope, fn: PSym) = scope.addSym(fn) proc addInterfaceDecl*(c: PContext, sym: PSym) = - # it adds the symbol to the interface if appropriate + ## adds a decl and the interface if appropriate addDecl(c, sym) - addInterfaceDeclAux(c, sym) + if not c.currentScope.isShadowScope: + addInterfaceDeclAux(c, sym) proc addInterfaceOverloadableSymAt*(c: PContext, scope: PScope, sym: PSym) = - # it adds the symbol to the interface if appropriate + ## adds an overloadable symbol on the scope and the interface if appropriate addOverloadableSymAt(c, scope, sym) - addInterfaceDeclAux(c, sym) + if not scope.isShadowScope: + addInterfaceDeclAux(c, sym) proc openShadowScope*(c: PContext) = + ## opens a shadow scope, just like any other scope except the depth is the + ## same as the parent -- see `isShadowScope`. c.currentScope = PScope(parent: c.currentScope, symbols: newStrTable(), depthLevel: c.scopeDepth) proc closeShadowScope*(c: PContext) = + ## closes the shadow scope, but doesn't merge any of the symbols c.closeScope proc mergeShadowScope*(c: PContext) = + ## close the existing scope and merge in all defined symbols, this will also + ## trigger any export related code if this is into a non-shadow scope. + ## + ## Merges: + ## shadow -> shadow: add symbols to the parent but check for redefinitions etc + ## shadow -> non-shadow: the above, but also handle exports and all that let shadowScope = c.currentScope c.rawCloseScope for sym in shadowScope.symbols: diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 8f87ce83c..d6ad7ab54 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -1844,7 +1844,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, # before compiling the proc params & body, set as current the scope # where the proc was declared - let delcarationScope = c.currentScope + let declarationScope = c.currentScope pushOwner(c, s) openScope(c) @@ -1889,7 +1889,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, var (proto, comesFromShadowScope) = if isAnon: (nil, false) - else: searchForProc(c, delcarationScope, s) + else: searchForProc(c, declarationScope, s) if proto == nil and sfForward in s.flags: ## In cases such as a macro generating a proc with a gensymmed name we ## know `searchForProc` will not find it and sfForward will be set. In @@ -1916,9 +1916,9 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, if not hasProto and sfGenSym notin s.flags: #and not isAnon: if s.kind in OverloadableSyms: - addInterfaceOverloadableSymAt(c, delcarationScope, s) + addInterfaceOverloadableSymAt(c, declarationScope, s) else: - addInterfaceDeclAt(c, delcarationScope, s) + addInterfaceDeclAt(c, declarationScope, s) pragmaCallable(c, s, n, validPragmas) if not hasProto: diff --git a/tests/macros/m18235.nim b/tests/macros/m18235.nim new file mode 100644 index 000000000..6bb4547e0 --- /dev/null +++ b/tests/macros/m18235.nim @@ -0,0 +1,42 @@ +import macros + +# Necessary code to update the AST on a symbol across module boundaries when +# processed by a type macro. Used by a test of a corresponding name of this +# file. + +macro eexport(n: typed): untyped = + result = copyNimTree(n) + # turn exported nnkSym -> nnkPostfix(*, nnkIdent), forcing re-sem + result[0] = nnkPostfix.newTree(ident"*").add: + n.name.strVal.ident + +macro unexport(n: typed): untyped = + result = copyNimTree(n) + # turn nnkSym -> nnkIdent, forcing re-sem and dropping any exported-ness + # that might be present + result[0] = n.name.strVal.ident + +proc foo*() {.unexport.} = discard +proc bar() {.eexport.} = discard + +proc foooof*() {.unexport, eexport, unexport.} = discard +proc barrab() {.eexport, unexport, eexport.} = discard + +macro eexportMulti(n: typed): untyped = + # use the call version of `eexport` macro for one or more decls + result = copyNimTree(n) + for i in 0.. Date: Mon, 14 Jun 2021 08:30:04 +0100 Subject: [PATCH 0922/2472] Enable parallel build in build.sh (#18195) Add help --- tools/niminst/buildsh.nimf | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tools/niminst/buildsh.nimf b/tools/niminst/buildsh.nimf index e13221781..e7afd402e 100644 --- a/tools/niminst/buildsh.nimf +++ b/tools/niminst/buildsh.nimf @@ -21,10 +21,23 @@ do optosname=$2 shift 2 ;; + --parallel) + parallel=$2 + shift 2 + ;; --extraBuildArgs) extraBuildArgs=" $2" shift 2 ;; + -h | --help) + echo "Options:" + echo " --os " + echo " --cpu " + echo " --osname Additional OS specification (used for Android)" + echo " --extraBuildArgs Additional arguments passed to the compiler" + echo " --parallel Multiprocess build. Requires GNU parallel" + exit 0 + ;; --) # End of all options shift break; @@ -39,7 +52,15 @@ do esac done +parallel="${parallel:-0}" CC="${CC:-gcc}" +if [ "$parallel" -gt 1 ]; then + if ! command -v sem > /dev/null; then + echo "Error: GNU parallel is required to use --parallel" + exit 1 + fi + CC="sem -j $parallel --id $$ ${CC}" +fi COMP_FLAGS="${CPPFLAGS:-} ${CFLAGS:-} ?{c.ccompiler.flags}$extraBuildArgs" LINK_FLAGS="${LDFLAGS:-} ?{c.linker.flags}" PS4="" @@ -222,6 +243,9 @@ case $myos in $CC $COMP_FLAGS -Ic_code -c ?{f} -o ?{changeFileExt(f, "o")} # add(linkCmd, " \\\n" & changeFileExt(f, "o")) # end for + if [ "$parallel" -gt 0 ]; then + sem --wait --id $$ + fi $CC -o ?{"$binDir/" & toLowerAscii(c.name)} ?linkCmd $LINK_FLAGS ;; # end for From 0adb47aa15e242983c8251d85367c0fe45fc5f12 Mon Sep 17 00:00:00 2001 From: alaviss Date: Mon, 14 Jun 2021 11:26:12 -0500 Subject: [PATCH 0923/2472] system/excpt: check if the exception is not nil before pop (#18247) In CPS we would consume an exception in the except branch by stashing it into a local then remove the exception from Nim environment so as not to leak it to other code that would be running before the continuation continues However since popCurrentException() assumes that the exception always exist, removing the exception from an except branch will cause a SIGSEGV to happen. This commit fixes that. --- lib/system/excpt.nim | 5 +++-- tests/exception/tsetexceptions.nim | 7 +++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index 5b7d4d49f..72225cf96 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -135,8 +135,9 @@ proc pushCurrentException(e: sink(ref Exception)) {.compilerRtl, inl.} = #showErrorMessage2 "A" proc popCurrentException {.compilerRtl, inl.} = - currException = currException.up - #showErrorMessage2 "B" + if currException != nil: + currException = currException.up + #showErrorMessage2 "B" proc popCurrentExceptionEx(id: uint) {.compilerRtl.} = discard "only for bootstrapping compatbility" diff --git a/tests/exception/tsetexceptions.nim b/tests/exception/tsetexceptions.nim index 557fc1898..386a6ae4c 100644 --- a/tests/exception/tsetexceptions.nim +++ b/tests/exception/tsetexceptions.nim @@ -6,3 +6,10 @@ let ex = newException(CatchableError, "test") setCurrentException(ex) doAssert getCurrentException().msg == ex.msg doAssert getCurrentExceptionMsg() == ex.msg +setCurrentException(nil) + +try: + raise newException(CatchableError, "test2") +except: + setCurrentException(nil) +doAssert getCurrentException() == nil From 8c42f5be0274d8d16910401d7d5a45aa35a63afd Mon Sep 17 00:00:00 2001 From: Clyybber Date: Mon, 14 Jun 2021 19:19:58 +0200 Subject: [PATCH 0924/2472] Small scope refactoring (#18263) * Small scope refactoring * Add test for #10251 * Add inline where appropriate --- compiler/astalgo.nim | 3 ++- compiler/lookups.nim | 36 +++++++++++++++++------------------- compiler/semtypes.nim | 6 ++---- tests/errmsgs/t10251.nim | 12 ++++++++++++ 4 files changed, 33 insertions(+), 24 deletions(-) create mode 100644 tests/errmsgs/t10251.nim diff --git a/compiler/astalgo.nim b/compiler/astalgo.nim index 300089d81..5af8caad8 100644 --- a/compiler/astalgo.nim +++ b/compiler/astalgo.nim @@ -777,9 +777,10 @@ proc strTableInclReportConflict*(t: var TStrTable, n: PSym; replaceSlot = h h = nextTry(h, high(t.data)) if replaceSlot >= 0: + result = t.data[replaceSlot] # found it if not onConflictKeepOld: t.data[replaceSlot] = n # overwrite it with newer definition! - return t.data[replaceSlot] # found it + return result elif mustRehash(t.data.len, t.counter): strTableEnlarge(t) strTableRawInsert(t.data, n) diff --git a/compiler/lookups.nim b/compiler/lookups.nim index a21e35ddc..9dac106e4 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -293,29 +293,28 @@ proc wrongRedefinition*(c: PContext; info: TLineInfo, s: string; "redefinition of '$1'; previous declaration here: $2" % [s, c.config $ conflictsWith]) -proc addDecl*(c: PContext, sym: PSym, info: TLineInfo) = - let conflict = c.currentScope.addUniqueSym(sym) +proc addDeclAt*(c: PContext; scope: PScope, sym: PSym, info: TLineInfo) = + let conflict = scope.addUniqueSym(sym) if conflict != nil: wrongRedefinition(c, info, sym.name.s, conflict.info) -proc addDecl*(c: PContext, sym: PSym) = - let conflict = strTableInclReportConflict(c.currentScope.symbols, sym, true) - if conflict != nil: - wrongRedefinition(c, sym.info, sym.name.s, conflict.info) +proc addDeclAt*(c: PContext; scope: PScope, sym: PSym) {.inline.} = + addDeclAt(c, scope, sym, sym.info) + +proc addDecl*(c: PContext, sym: PSym, info: TLineInfo) {.inline.} = + addDeclAt(c, c.currentScope, sym, info) + +proc addDecl*(c: PContext, sym: PSym) {.inline.} = + addDeclAt(c, c.currentScope, sym) proc addPrelimDecl*(c: PContext, sym: PSym) = discard c.currentScope.addUniqueSym(sym) -proc addDeclAt*(c: PContext; scope: PScope, sym: PSym) = - let conflict = scope.addUniqueSym(sym) - if conflict != nil: - wrongRedefinition(c, sym.info, sym.name.s, conflict.info) - from ic / ic import addHidden -proc addInterfaceDeclAux*(c: PContext, sym: PSym, forceExport = false) = +proc addInterfaceDeclAux(c: PContext, sym: PSym) = ## adds symbol to the module for either private or public access. - if sfExported in sym.flags or forceExport: + if sfExported in sym.flags: # add to interface: if c.module != nil: exportSym(c, sym) else: internalError(c.config, sym.info, "addInterfaceDeclAux") @@ -331,6 +330,10 @@ proc addInterfaceDeclAt*(c: PContext, scope: PScope, sym: PSym) = # adding into a non-shadow scope, we need to handle exports, etc addInterfaceDeclAux(c, sym) +proc addInterfaceDecl*(c: PContext, sym: PSym) {.inline.} = + ## adds a decl and the interface if appropriate + addInterfaceDeclAt(c, c.currentScope, sym) + proc addOverloadableSymAt*(c: PContext; scope: PScope, fn: PSym) = ## adds an symbol to the given scope, will check for and raise errors if it's ## a redefinition as opposed to an overload. @@ -343,16 +346,11 @@ proc addOverloadableSymAt*(c: PContext; scope: PScope, fn: PSym) = else: scope.addSym(fn) -proc addInterfaceDecl*(c: PContext, sym: PSym) = - ## adds a decl and the interface if appropriate - addDecl(c, sym) - if not c.currentScope.isShadowScope: - addInterfaceDeclAux(c, sym) - proc addInterfaceOverloadableSymAt*(c: PContext, scope: PScope, sym: PSym) = ## adds an overloadable symbol on the scope and the interface if appropriate addOverloadableSymAt(c, scope, sym) if not scope.isShadowScope: + # adding into a non-shadow scope, we need to handle exports, etc addInterfaceDeclAux(c, sym) proc openShadowScope*(c: PContext) = diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 84556ca1a..38307a9ac 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -138,15 +138,13 @@ proc semEnum(c: PContext, n: PNode, prev: PType): PType = identToReplace[] = symNode if e.position == 0: hasNull = true if result.sym != nil and sfExported in result.sym.flags: - incl(e.flags, {sfUsed, sfExported}) - if result.sym != nil and not isPure: - addInterfaceDeclAux(c, e, forceExport = sfExported in result.sym.flags) + e.flags.incl {sfUsed, sfExported} result.n.add symNode styleCheckDef(c.config, e) onDef(e.info, e) if sfGenSym notin e.flags: - if not isPure: addDecl(c, e) + if not isPure: addInterfaceDecl(c, e) else: declarePureEnumField(c, e) if isPure and (let conflict = strTableInclReportConflict(symbols, e); conflict != nil): wrongRedefinition(c, e.info, e.name.s, conflict.info) diff --git a/tests/errmsgs/t10251.nim b/tests/errmsgs/t10251.nim new file mode 100644 index 000000000..5ad373ba3 --- /dev/null +++ b/tests/errmsgs/t10251.nim @@ -0,0 +1,12 @@ +discard """ +errormsg: "redefinition of 'foo'; previous declaration here: t10251.nim(9, 9)" +line: 11 +column: 9 +""" + +type + Enum1 = enum + foo, bar, baz + Enum2 = enum + foo, bar, baz + From e80d7ff0f2716fd1f892a936a78986531b107fb6 Mon Sep 17 00:00:00 2001 From: Mark Pointing Date: Tue, 15 Jun 2021 03:29:23 +1000 Subject: [PATCH 0925/2472] httpclient.nim Fixes #14794 and an issue where content-header is not set on postContent (#18208) * Fixed missing newline after bound marker in mulipart post (#14794) and a problem where calling postContent with multipart data does not set content-length header. * Update lib/pure/httpclient.nim * Added comment outlining the reason for changes to httpclient.nim and added tests to ensure that multipart post has a newline at the end of the body, and that the content-length header is present. * Fixed typo in comments. * Removed redundant blank lines in thttpclient_standalone.nim. Co-authored-by: Mark Pointing Co-authored-by: Andreas Rumpf --- lib/pure/httpclient.nim | 8 ++++-- tests/stdlib/thttpclient_standalone.nim | 33 ++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 2f9f1913f..33da6ef6c 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -975,7 +975,7 @@ proc format(client: HttpClient | AsyncHttpClient, if entry.isFile: length += entry.fileSize + httpNewLine.len - result.add "--" & bound & "--" + result.add "--" & bound & "--" & httpNewLine for s in result: length += s.len client.headers["Content-Length"] = $length @@ -1010,12 +1010,16 @@ proc requestAux(client: HttpClient | AsyncHttpClient, url: Uri, await newConnection(client, url) - let newHeaders = client.headers.override(headers) + var newHeaders: HttpHeaders var data: seq[string] if multipart != nil and multipart.content.len > 0: + # `format` modifies `client.headers`, see + # https://github.com/nim-lang/Nim/pull/18208#discussion_r647036979 data = await client.format(multipart) + newHeaders = client.headers.override(headers) else: + newHeaders = client.headers.override(headers) # Only change headers if they have not been specified already if not newHeaders.hasKey("Content-Length"): if body.len != 0: diff --git a/tests/stdlib/thttpclient_standalone.nim b/tests/stdlib/thttpclient_standalone.nim index 44a88e91e..5fa1ea41a 100644 --- a/tests/stdlib/thttpclient_standalone.nim +++ b/tests/stdlib/thttpclient_standalone.nim @@ -2,7 +2,7 @@ discard """ cmd: "nim c --threads:on $file" """ -import asynchttpserver, httpclient, asyncdispatch +import asynchttpserver, httpclient, asyncdispatch, strutils block: # bug #16436 proc startServer() {.async.} = @@ -21,3 +21,34 @@ block: # bug #16436 asyncCheck startServer() doAssertRaises(ProtocolError): waitFor runClient() + +block: # bug #14794 (And test for presence of content-length header when using postContent) + proc startServer() {.async.} = + var killServer = false + proc cb(req: Request) {.async.} = + doAssert(req.body.endsWith(httpNewLine), "Multipart body does not end with a newline.") + # this next line is probably not required because asynchttpserver does not call + # the callback when there is no content-length header. It instead errors with + # Error: unhandled exception: 411 Length Required + # Added for good measure in case the server becomes more permissive. + doAssert(req.headers.hasKey("content-length"), "Content-Length header is not present.") + killServer = true + asyncCheck req.respond(Http200, "OK") + + var server = newAsyncHttpServer() + server.listen(Port(5556)) + while not killServer: + if server.shouldAcceptRequest(): + await server.acceptRequest(cb) + else: + poll() + + proc runClient() {.async.} = + let c = newAsyncHttpClient() + var data = newMultipartData() + data.add("file.txt", "This is intended to be an example text file.\r\nThis would be the second line.\r\n") + let r = await c.postContent("http://127.0.0.1:5556", multipart = data) + c.close() + + asyncCheck startServer() + waitFor runClient() From 8ef6073543b14bd48665ae075df66c2c36f564dd Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 14 Jun 2021 23:56:10 -0700 Subject: [PATCH 0926/2472] add test case for pure enum redefinition error within enum (fixed in recent PR) (#18266) * add test case for pure enum redefinition error within enum (fixed in recent PR) * remove code duplication * Revert "remove code duplication" (would require bootstrap >= 1.4) This reverts commit 3f793874c231f847ef015e37a5fd6851f85d9675. * fixup --- compiler/astalgo.nim | 8 ++++---- compiler/lookups.nim | 2 ++ tests/errmsgs/t10251.nim | 21 ++++++++++++++------- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/compiler/astalgo.nim b/compiler/astalgo.nim index 5af8caad8..3c97665f0 100644 --- a/compiler/astalgo.nim +++ b/compiler/astalgo.nim @@ -759,9 +759,9 @@ proc strTableAdd*(t: var TStrTable, n: PSym) = proc strTableInclReportConflict*(t: var TStrTable, n: PSym; onConflictKeepOld = false): PSym = - # returns true if n is already in the string table: - # It is essential that `n` is written nevertheless! - # This way the newest redefinition is picked by the semantic analyses! + # if `t` has a conflicting symbol (same identifier as `n`), return it + # otherwise return `nil`. Incl `n` to `t` unless `onConflictKeepOld = true` + # and a conflict was found. assert n.name != nil var h: Hash = n.name.h and high(t.data) var replaceSlot = -1 @@ -780,7 +780,7 @@ proc strTableInclReportConflict*(t: var TStrTable, n: PSym; result = t.data[replaceSlot] # found it if not onConflictKeepOld: t.data[replaceSlot] = n # overwrite it with newer definition! - return result + return result # but return the old one elif mustRehash(t.data.len, t.counter): strTableEnlarge(t) strTableRawInsert(t.data, n) diff --git a/compiler/lookups.nim b/compiler/lookups.nim index 9dac106e4..f72b58ac6 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -293,6 +293,8 @@ proc wrongRedefinition*(c: PContext; info: TLineInfo, s: string; "redefinition of '$1'; previous declaration here: $2" % [s, c.config $ conflictsWith]) +# xxx pending bootstrap >= 1.4, replace all those overloads with a single one: +# proc addDecl*(c: PContext, sym: PSym, info = sym.info, scope = c.currentScope) {.inline.} = proc addDeclAt*(c: PContext; scope: PScope, sym: PSym, info: TLineInfo) = let conflict = scope.addUniqueSym(sym) if conflict != nil: diff --git a/tests/errmsgs/t10251.nim b/tests/errmsgs/t10251.nim index 5ad373ba3..0c7fe0b3d 100644 --- a/tests/errmsgs/t10251.nim +++ b/tests/errmsgs/t10251.nim @@ -1,12 +1,19 @@ discard """ -errormsg: "redefinition of 'foo'; previous declaration here: t10251.nim(9, 9)" -line: 11 -column: 9 + action:reject + cmd: "nim check $options $file" + nimout: ''' +t10251.nim(15, 5) Error: redefinition of 'foo'; previous declaration here: t10251.nim(13, 5) +t10251.nim(19, 23) Error: redefinition of 'goo1'; previous declaration here: t10251.nim(19, 11) +''' """ +# line 10 type - Enum1 = enum - foo, bar, baz - Enum2 = enum - foo, bar, baz + Enum1 = enum + foo, bar, baz + Enum2 = enum + foo, bar, baz +type + Enum3 {.pure.} = enum # fixed (by accident?) in https://github.com/nim-lang/Nim/pull/18263 + goo0, goo1, goo2, goo1 From d3b27eb63e18fbe36a7579a3322db316115a3d81 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 15 Jun 2021 14:56:49 -0700 Subject: [PATCH 0927/2472] link stable and devel docs in nim docs (#18272) [backport:1.2] --- config/nimdoc.cfg | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index cdf49197d..15f887023 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -127,21 +127,12 @@ doc.body_toc_group = """
    From c5cf21c0c4f314e4128658d35000339d67e4824b Mon Sep 17 00:00:00 2001 From: Clyybber Date: Wed, 16 Jun 2021 02:45:05 +0200 Subject: [PATCH 0928/2472] Don't report unused hints for consumed AST (#18270) * Fix #18203 * Add testcase * Fix testcase * Fix test --- compiler/lookups.nim | 4 +++- tests/macros/t18203.nim | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 tests/macros/t18203.nim diff --git a/compiler/lookups.nim b/compiler/lookups.nim index f72b58ac6..4f413f472 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -364,7 +364,9 @@ proc openShadowScope*(c: PContext) = proc closeShadowScope*(c: PContext) = ## closes the shadow scope, but doesn't merge any of the symbols - c.closeScope + ## Does not check for unused symbols or missing forward decls since a macro + ## or template consumes this AST + rawCloseScope(c) proc mergeShadowScope*(c: PContext) = ## close the existing scope and merge in all defined symbols, this will also diff --git a/tests/macros/t18203.nim b/tests/macros/t18203.nim new file mode 100644 index 000000000..aae0a2690 --- /dev/null +++ b/tests/macros/t18203.nim @@ -0,0 +1,15 @@ +discard """ + matrix: "--hint:SuccessX:off --hint:Link:off --hint:Conf:off --hint:CC:off --hint:XDeclaredButNotUsed:on" + nimout: ''' +''' +nimoutFull: true +action: compile +""" + +# bug #18203 +import std/macros + +macro foo(x: typed) = newProc ident"bar" +proc bar() {.foo.} = raise +bar() + From c51680e7012bff32156623fad0996d62f7918222 Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 16 Jun 2021 17:31:20 +0800 Subject: [PATCH 0929/2472] fixes #17696 (#18276) --- lib/pure/collections/sharedlist.nim | 7 ++-- tests/stdlib/tsharedlist.nim | 53 +++++++++++++++++++++++------ 2 files changed, 47 insertions(+), 13 deletions(-) diff --git a/lib/pure/collections/sharedlist.nim b/lib/pure/collections/sharedlist.nim index c72477675..79f2391f6 100644 --- a/lib/pure/collections/sharedlist.nim +++ b/lib/pure/collections/sharedlist.nim @@ -35,8 +35,10 @@ template withLock(t, x: untyped) = release(t.lock) proc iterAndMutate*[A](x: var SharedList[A]; action: proc(x: A): bool) = - ## iterates over the list. If 'action' returns true, the + ## Iterates over the list. If 'action' returns true, the ## current item is removed from the list. + ## + ## .. warning:: It may not preserve the element order after some modifications. withLock(x): var n = x.head while n != nil: @@ -47,8 +49,9 @@ proc iterAndMutate*[A](x: var SharedList[A]; action: proc(x: A): bool) = if action(n.d[i]): acquire(x.lock) let t = x.tail + dec t.dataLen # TODO considering t.dataLen == 0, + # probably the module should be refactored using doubly linked lists n.d[i] = t.d[t.dataLen] - dec t.dataLen else: acquire(x.lock) inc i diff --git a/tests/stdlib/tsharedlist.nim b/tests/stdlib/tsharedlist.nim index a795be0f3..693f1018e 100644 --- a/tests/stdlib/tsharedlist.nim +++ b/tests/stdlib/tsharedlist.nim @@ -1,17 +1,48 @@ -import sharedlist +discard """ + matrix: "--threads:on" +""" -var - list: SharedList[int] - count: int +import std/sharedlist -init(list) +block: + var + list: SharedList[int] + count: int -for i in 1 .. 250: - list.add i + init(list) -for i in list: - inc count + for i in 1 .. 250: + list.add i -doAssert count == 250 + for i in list: + inc count -deinitSharedList(list) + doAssert count == 250 + + deinitSharedList(list) + + +block: # bug #17696 + var keysList = SharedList[string]() + init(keysList) + + keysList.add("a") + keysList.add("b") + keysList.add("c") + keysList.add("d") + keysList.add("e") + keysList.add("f") + + + # Remove element "b" and "d" from the list. + keysList.iterAndMutate(proc (key: string): bool = + if key == "b" or key == "d": # remove only "b" and "d" + return true + return false + ) + + var results: seq[string] + for key in keysList.items: + results.add key + + doAssert results == @["a", "f", "c", "e"] From 13b94c0297806911cf317ff6169dba8595a0333d Mon Sep 17 00:00:00 2001 From: Clyybber Date: Wed, 16 Jun 2021 16:40:22 +0200 Subject: [PATCH 0930/2472] Fix doubly typed forward declarations (#18279) * Add testcase * Fix testcase * Fix doubly typed forward decls * Better fix --- compiler/semstmts.nim | 7 ++++++- tests/macros/tmacros_issues.nim | 22 +++++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index d6ad7ab54..f29dc24ef 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -1890,11 +1890,16 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, var (proto, comesFromShadowScope) = if isAnon: (nil, false) else: searchForProc(c, declarationScope, s) - if proto == nil and sfForward in s.flags: + if proto == nil and sfForward in s.flags and n[bodyPos].kind != nkEmpty: ## In cases such as a macro generating a proc with a gensymmed name we ## know `searchForProc` will not find it and sfForward will be set. In ## such scenarios the sym is shared between forward declaration and we ## can treat the `s` as the proto. + ## To differentiate between that happening and a macro just returning a + ## forward declaration that has been typed before we check if the body + ## is not empty. This has the sideeffect of allowing multiple forward + ## declarations if they share the same sym. + ## See the "doubly-typed forward decls" case in tmacros_issues.nim proto = s let hasProto = proto != nil diff --git a/tests/macros/tmacros_issues.nim b/tests/macros/tmacros_issues.nim index a964f46ba..a81c51658 100644 --- a/tests/macros/tmacros_issues.nim +++ b/tests/macros/tmacros_issues.nim @@ -484,6 +484,26 @@ func expMin: float {.aadMin.} = 1 echo expMin() +# doubly-typed forward decls +macro noop(x: typed) = x +noop: + proc cally() = discard + +cally() + +noop: + proc barry() + +proc barry() = discard + +# some more: +proc barry2() {.noop.} +proc barry2() = discard + +proc barry3() {.noop.} +proc barry3() {.noop.} = discard + + # issue #15389 block double_sem_for_procs: @@ -498,4 +518,4 @@ block double_sem_for_procs: return x1 + 1.0 result = 10.0 - discard exp(5.0) \ No newline at end of file + discard exp(5.0) From 3c854e8c24b394900dc65df680b7679f09cee97c Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 16 Jun 2021 23:04:12 -0700 Subject: [PATCH 0931/2472] followup #16400; use use -d:nimCompilerStackraceHints in more places (#18282) --- compiler/vmgen.nim | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index c6fd75259..bb203e0ab 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -35,6 +35,9 @@ import from modulegraphs import getBody +when defined(nimCompilerStackraceHints): + import std/stackframes + const debugEchoCode* = defined(nimVMDebug) @@ -2000,6 +2003,8 @@ proc procIsCallback(c: PCtx; s: PSym): bool = dec i proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) = + when defined(nimCompilerStackraceHints): + setFrameMsg c.config$n.info & " " & $n.kind & " " & $flags case n.kind of nkSym: let s = n.sym From 49e945ed082bacd5cd5ecff305ec47ff3a00f1c5 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 16 Jun 2021 23:07:40 -0700 Subject: [PATCH 0932/2472] increase backoffDuration to avoid timeouts (#18281) --- testament/categories.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/categories.nim b/testament/categories.nim index 7f05797ff..bf7855b2c 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -429,7 +429,7 @@ proc testNimblePackages(r: var TResults; cat: Category; packageFilter: string) = let buildPath = packagesDir / pkg.name template tryCommand(cmd: string, workingDir2 = buildPath, reFailed = reInstallFailed, maxRetries = 1): string = var outp: string - let ok = retryCall(maxRetry = maxRetries, backoffDuration = 1.0): + let ok = retryCall(maxRetry = maxRetries, backoffDuration = 10.0): var status: int (outp, status) = execCmdEx(cmd, workingDir = workingDir2) status == QuitSuccess From 969cb97c0a9f045a2fb31866fc804cdf1b2698dd Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Thu, 17 Jun 2021 09:19:52 +0300 Subject: [PATCH 0933/2472] PCRE, nimgrep: add limit for buffer size (#18280) --- lib/impure/re.nim | 13 ++++++++++--- tools/nimgrep.nim | 5 ++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/lib/impure/re.nim b/lib/impure/re.nim index 78ea7b002..504d8f22e 100644 --- a/lib/impure/re.nim +++ b/lib/impure/re.nim @@ -141,6 +141,10 @@ proc matchOrFind(buf: cstring, pattern: Regex, matches: var openArray[string], else: matches[i-1] = "" return rawMatches[1] - rawMatches[0] +const MaxReBufSize* = high(cint) + ## Maximum PCRE (API 1) buffer start/size equal to `high(cint)`, which even + ## for 64-bit systems can be either 2`31`:sup:-1 or 2`63`:sup:-1. + proc findBounds*(buf: cstring, pattern: Regex, matches: var openArray[string], start = 0, bufSize: int): tuple[first, last: int] = ## returns the starting position and end position of `pattern` in `buf` @@ -167,7 +171,8 @@ proc findBounds*(s: string, pattern: Regex, matches: var openArray[string], ## and the captured substrings in the array `matches`. ## If it does not match, nothing ## is written into `matches` and `(-1,0)` is returned. - result = findBounds(cstring(s), pattern, matches, start, s.len) + result = findBounds(cstring(s), pattern, matches, + min(start, MaxReBufSize), min(s.len, MaxReBufSize)) proc findBounds*(buf: cstring, pattern: Regex, matches: var openArray[tuple[first, last: int]], @@ -197,7 +202,8 @@ proc findBounds*(s: string, pattern: Regex, ## and the captured substrings in the array `matches`. ## If it does not match, nothing is written into `matches` and ## `(-1,0)` is returned. - result = findBounds(cstring(s), pattern, matches, start, s.len) + result = findBounds(cstring(s), pattern, matches, + min(start, MaxReBufSize), min(s.len, MaxReBufSize)) proc findBoundsImpl(buf: cstring, pattern: Regex, start = 0, bufSize = 0, flags = 0): tuple[first, last: int] = @@ -232,7 +238,8 @@ proc findBounds*(s: string, pattern: Regex, ## Note: there is a speed improvement if the matches do not need to be captured. runnableExamples: assert findBounds("01234abc89", re"abc") == (5,7) - result = findBounds(cstring(s), pattern, start, s.len) + result = findBounds(cstring(s), pattern, + min(start, MaxReBufSize), min(s.len, MaxReBufSize)) proc matchOrFind(buf: cstring, pattern: Regex, start, bufSize: int, flags: cint): cint = var diff --git a/tools/nimgrep.nim b/tools/nimgrep.nim index a368c40ef..a589cfb14 100644 --- a/tools/nimgrep.nim +++ b/tools/nimgrep.nim @@ -649,7 +649,7 @@ template updateCounters(output: Output) = proc printInfo(filename:string, output: Output) = case output.kind of openError: - printError("can not open path " & filename & " " & output.msg) + printError("cannot open path '" & filename & "': " & output.msg) of rejected: if optVerbose in options: echo "(rejected: ", output.reason, ")" @@ -719,6 +719,9 @@ iterator searchFile(pattern: Pattern; buffer: string): Output = pre: pre, match: move(curMi)) i = t.last+1 + when typeof(pattern) is Regex: + if buffer.len > MaxReBufSize: + yield Output(kind: openError, msg: "PCRE size limit is " & $MaxReBufSize) func detectBin(buffer: string): bool = for i in 0 ..< min(1024, buffer.len): From fd8b79707c1b936ed1ba42892f5e06f846e2816b Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 17 Jun 2021 08:20:10 +0200 Subject: [PATCH 0934/2472] Revert "system/excpt: check if the exception is not nil before pop (#18247)" (#18265) This reverts commit 0adb47aa15e242983c8251d85367c0fe45fc5f12. --- lib/system/excpt.nim | 5 ++--- tests/exception/tsetexceptions.nim | 7 ------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index 72225cf96..5b7d4d49f 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -135,9 +135,8 @@ proc pushCurrentException(e: sink(ref Exception)) {.compilerRtl, inl.} = #showErrorMessage2 "A" proc popCurrentException {.compilerRtl, inl.} = - if currException != nil: - currException = currException.up - #showErrorMessage2 "B" + currException = currException.up + #showErrorMessage2 "B" proc popCurrentExceptionEx(id: uint) {.compilerRtl.} = discard "only for bootstrapping compatbility" diff --git a/tests/exception/tsetexceptions.nim b/tests/exception/tsetexceptions.nim index 386a6ae4c..557fc1898 100644 --- a/tests/exception/tsetexceptions.nim +++ b/tests/exception/tsetexceptions.nim @@ -6,10 +6,3 @@ let ex = newException(CatchableError, "test") setCurrentException(ex) doAssert getCurrentException().msg == ex.msg doAssert getCurrentExceptionMsg() == ex.msg -setCurrentException(nil) - -try: - raise newException(CatchableError, "test2") -except: - setCurrentException(nil) -doAssert getCurrentException() == nil From 99411674a6da72f608bfa870f51b3a296cd85eee Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 16 Jun 2021 23:20:33 -0700 Subject: [PATCH 0935/2472] tests/exception/tsetexceptions.nim not joinable (#18264) --- tests/exception/tsetexceptions.nim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/exception/tsetexceptions.nim b/tests/exception/tsetexceptions.nim index 557fc1898..0e353f43e 100644 --- a/tests/exception/tsetexceptions.nim +++ b/tests/exception/tsetexceptions.nim @@ -1,7 +1,10 @@ discard """ targets: "c cpp js" + joinable: false """ +# refs https://github.com/nim-lang/Nim/pull/18247#issuecomment-860877161 + let ex = newException(CatchableError, "test") setCurrentException(ex) doAssert getCurrentException().msg == ex.msg From c83ac16671170e513fa1ca40419892ccc618ec85 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 17 Jun 2021 00:17:46 -0700 Subject: [PATCH 0936/2472] Renamed `-d:nimCompilerStackraceHints` to `-d:nimCompilerStacktraceHints`. (#18283) --- changelog.md | 1 + compiler/ccgexprs.nim | 4 ++-- compiler/semexprs.nim | 4 ++-- compiler/vmgen.nim | 4 ++-- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/changelog.md b/changelog.md index cc6f826ee..26048eedd 100644 --- a/changelog.md +++ b/changelog.md @@ -88,6 +88,7 @@ The downside is that these defines now have custom logic that doesn't apply for other defines. +- Renamed `-d:nimCompilerStackraceHints` to `-d:nimCompilerStacktraceHints`. ## Standard library additions and changes diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 7ed80145b..c07ddf1db 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -9,7 +9,7 @@ # included from cgen.nim -when defined(nimCompilerStackraceHints): +when defined(nimCompilerStacktraceHints): import std/stackframes proc getNullValueAuxT(p: BProc; orig, t: PType; obj, constOrNil: PNode, @@ -2694,7 +2694,7 @@ proc genConstStmt(p: BProc, n: PNode) = genConstDefinition(m, p, sym) proc expr(p: BProc, n: PNode, d: var TLoc) = - when defined(nimCompilerStackraceHints): + when defined(nimCompilerStacktraceHints): setFrameMsg p.config$n.info & " " & $n.kind p.currLineInfo = n.info diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index ba8b93ae6..53a5f149e 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -10,7 +10,7 @@ # this module does the semantic checking for expressions # included from sem.nim -when defined(nimCompilerStackraceHints): +when defined(nimCompilerStacktraceHints): import std/stackframes const @@ -2714,7 +2714,7 @@ proc getNilType(c: PContext): PType = c.nilTypeCache = result proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = - when defined(nimCompilerStackraceHints): + when defined(nimCompilerStacktraceHints): setFrameMsg c.config$n.info & " " & $n.kind result = n if c.config.cmd == cmdIdeTools: suggestExpr(c, n) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index bb203e0ab..14f9e0b30 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -35,7 +35,7 @@ import from modulegraphs import getBody -when defined(nimCompilerStackraceHints): +when defined(nimCompilerStacktraceHints): import std/stackframes const @@ -2003,7 +2003,7 @@ proc procIsCallback(c: PCtx; s: PSym): bool = dec i proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) = - when defined(nimCompilerStackraceHints): + when defined(nimCompilerStacktraceHints): setFrameMsg c.config$n.info & " " & $n.kind & " " & $flags case n.kind of nkSym: From 5a3456d220d0c2a4022969b98ab33baaf1fd4d00 Mon Sep 17 00:00:00 2001 From: Adrian Veith Date: Thu, 17 Jun 2021 11:23:38 +0200 Subject: [PATCH 0937/2472] fix for #18284 int32 should be int (#18285) the var exp was typed as int32 - it should be int since frep expects an int --- lib/pure/math.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/math.nim b/lib/pure/math.nim index cff946581..410660d4a 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -997,7 +997,7 @@ when not defined(js): # taken from Go-lang Math.Log2 const ln2 = 0.693147180559945309417232121458176568075500134360255254120680009 template log2Impl[T](x: T): T = - var exp: int32 + var exp: int var frac = frexp(x, exp) # Make sure exact powers of two give an exact answer. # Don't depend on Log(0.5)*(1/Ln2)+exp being exactly exp-1. From ef121f3b99c4d020d3df7cb369f5060c76c73d57 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 17 Jun 2021 16:19:16 -0700 Subject: [PATCH 0938/2472] followup #17876: remove annoying enum name clashes in tests/enum/tenum.nim (#18291) --- tests/enum/tenum.nim | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/tests/enum/tenum.nim b/tests/enum/tenum.nim index 97fd4c68c..37383890c 100644 --- a/tests/enum/tenum.nim +++ b/tests/enum/tenum.nim @@ -128,22 +128,21 @@ block tnamedfields: doAssert $x == "abc", $x -block toptions: +block tfakeOptions: type - # please make sure we have under 32 options (improves code efficiency!) - TOption = enum - optNone, optForceFullMake, optBoehmGC, optRefcGC, optRangeCheck, - optBoundsCheck, optOverflowCheck, optNilCheck, optAssert, optLineDir, - optWarns, optHints, optListCmd, optCompileOnly, - optSafeCode, # only allow safe code - optStyleCheck, optOptimizeSpeed, optOptimizeSize, optGenDynLib, - optGenGuiApp, optStackTrace + TFakeOption = enum + fakeNone, fakeForceFullMake, fakeBoehmGC, fakeRefcGC, fakeRangeCheck, + fakeBoundsCheck, fakeOverflowCheck, fakeNilCheck, fakeAssert, fakeLineDir, + fakeWarns, fakeHints, fakeListCmd, fakeCompileOnly, + fakeSafeCode, # only allow safe code + fakeStyleCheck, fakeOptimizeSpeed, fakeOptimizeSize, fakeGenDynLib, + fakeGenGuiApp, fakeStackTrace - TOptionset = set[TOption] + TFakeOptionset = set[TFakeOption] var - gOptions: TOptionset = {optRefcGC, optRangeCheck, optBoundsCheck, - optOverflowCheck, optAssert, optWarns, optHints, optLineDir, optStackTrace} + gFakeOptions: TFakeOptionset = {fakeRefcGC, fakeRangeCheck, fakeBoundsCheck, + fakeOverflowCheck, fakeAssert, fakeWarns, fakeHints, fakeLineDir, fakeStackTrace} compilerArgs: int gExitcode: int8 From fc76565574b86566e4a642b9aac541e025cf5de3 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 17 Jun 2021 16:38:15 -0700 Subject: [PATCH 0939/2472] disable pkg manu (#18292) --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 3d3c340eb..efee361da 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -82,7 +82,7 @@ pkg "karax", "nim c -r tests/tester.nim" pkg "kdtree", "nimble test -d:nimLegacyRandomInitRand", "https://github.com/jblindsay/kdtree" pkg "loopfusion" pkg "macroutils" -pkg "manu" +pkg "manu", allowFailure=true # pending https://github.com/planetis-m/manu/issues/5 pkg "markdown" pkg "memo" pkg "msgpack4nim", "nim c -r tests/test_spec.nim" From a250481dcde2167d568cc1b61624658419c18e69 Mon Sep 17 00:00:00 2001 From: j-james <35242550+j-james@users.noreply.github.com> Date: Fri, 18 Jun 2021 03:10:45 -0700 Subject: [PATCH 0940/2472] Make the existence of high(type) clearer, especially near subranges (#18286) * Make the existence of high(type) clearer, especially around subranges * Use a better example and dry up the wording --- doc/manual.rst | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/doc/manual.rst b/doc/manual.rst index 867db245b..46a40a690 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -989,10 +989,12 @@ Ordinal types have the following characteristics: - Ordinal types are countable and ordered. This property allows the operation of functions such as `inc`, `ord`, and `dec` on ordinal types to be defined. -- Ordinal values have the smallest possible value. Trying to count further - down than the smallest value produces a panic or a static error. -- Ordinal values have the largest possible value. Trying to count further - than the largest value produces a panic or a static error. +- Ordinal types have a smallest possible value, accessible with `low(type)`. + Trying to count further down than the smallest value produces a panic or + a static error. +- Ordinal types have a largest possible value, accessible with `high(type)`. + Trying to count further up than the largest value produces a panic or + a static error. Integers, bool, characters, and enumeration types (and subranges of these types) belong to ordinal types. @@ -1095,6 +1097,7 @@ lowest and highest value of the type. For example: type Subrange = range[0..5] PositiveFloat = range[0.0..Inf] + Positive* = range[1..high(int)] # as defined in `system` `Subrange` is a subrange of an integer which can only hold the values 0 From 87cd9b24a3df66dcde96bbdec2bd7620015ddeba Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 18 Jun 2021 06:52:08 -0700 Subject: [PATCH 0941/2472] add more APIs to compiler/debugutils; re-export it (#18243) --- compiler/debugutils.nim | 37 +++++++++++++++++++++++++++++++++ compiler/options.nim | 3 +++ compiler/semexprs.nim | 7 +++++++ tests/compiler/tdebugutils.nim | 38 ++++++++++++++++++++++++++++++++++ 4 files changed, 85 insertions(+) create mode 100644 tests/compiler/tdebugutils.nim diff --git a/compiler/debugutils.nim b/compiler/debugutils.nim index 235c100b3..d109d2121 100644 --- a/compiler/debugutils.nim +++ b/compiler/debugutils.nim @@ -4,7 +4,26 @@ Utilities to help with debugging nim compiler. Experimental API, subject to change. ]## +#[ +## example +useful debugging flags: +--stacktrace -d:debug -d:nimDebugUtils + nim c -o:bin/nim_temp --stacktrace -d:debug -d:nimDebugUtils compiler/nim + +## future work +* expose and improve astalgo.debug, replacing it by std/prettyprints, + refs https://github.com/nim-lang/RFCs/issues/385 +]# + import options +import std/wrapnils +export wrapnils + # allows using things like: `?.n.sym.typ.len` + +import std/stackframes +export stackframes + # allows using things like: `setFrameMsg c.config$n.info & " " & $n.kind` + # which doesn't log, but augments stacktrace with side channel information var conf0: ConfigRef @@ -17,3 +36,21 @@ proc onNewConfigRef*(conf: ConfigRef) {.inline.} = proc getConfigRef*(): ConfigRef = ## nil, if -d:nimDebugUtils wasn't specified result = conf0 + +proc isCompilerDebug*(): bool = + ##[ + Provides a simple way for user code to enable/disable logging in the compiler + in a granular way. This can then be used in the compiler as follows: + ```nim + if isCompilerDebug(): + echo ?.n.sym.typ.len + ``` + ]## + runnableExamples: + proc main = + echo 2 + {.define(nimCompilerDebug).} + echo 3.5 # code section in which `isCompilerDebug` will be true + {.undef(nimCompilerDebug).} + echo 'x' + conf0.isDefined("nimCompilerDebug") diff --git a/compiler/options.nim b/compiler/options.nim index 1851770d9..05d8ee5b7 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -472,7 +472,10 @@ const foreignPackageNotesDefault* = { proc isDefined*(conf: ConfigRef; symbol: string): bool when defined(nimDebugUtils): + # this allows inserting debugging utilties in all modules that import `options` + # with a single switch, which is useful when debugging compiler. import debugutils + export debugutils proc initConfigRefCommon(conf: ConfigRef) = conf.selectedGC = gcRefc diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 53a5f149e..c885f4963 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -2716,6 +2716,13 @@ proc getNilType(c: PContext): PType = proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = when defined(nimCompilerStacktraceHints): setFrameMsg c.config$n.info & " " & $n.kind + when false: # see `tdebugutils` + if isCompilerDebug(): + echo (">", c.config$n.info, n, flags, n.kind) + defer: + if isCompilerDebug(): + echo ("<", c.config$n.info, n, ?.result.typ) + result = n if c.config.cmd == cmdIdeTools: suggestExpr(c, n) if nfSem in n.flags: return diff --git a/tests/compiler/tdebugutils.nim b/tests/compiler/tdebugutils.nim new file mode 100644 index 000000000..50b15cb78 --- /dev/null +++ b/tests/compiler/tdebugutils.nim @@ -0,0 +1,38 @@ +discard """ + joinable: false + disabled: true +""" + +#[ +This test illustrates some features of `debugutils` to debug the compiler. + +## example +this shows how to enable compiler logging just for a section of user code, +without generating tons of unrelated log messages for code you're not interested +in debugging. + +```sh +# enable some debugging code, e.g. the `when false:` block in `semExpr` +nim c -o:bin/nim_temp --stacktrace -d:debug -d:nimDebugUtils compiler/nim +bin/nim_temp c tests/compiler/tdebugutils.nim +``` + +(use --filenames:abs for abs files) + +## result +("<", "tdebugutils.nim(16, 3)", {.define(nimCompilerDebug).}, nil) +(">", "tdebugutils.nim(17, 3)", let a = 2.5 * 3, {}, nkLetSection) +(">", "tdebugutils.nim(17, 15)", 2.5 * 3, {efAllowDestructor, efWantValue}, nkInfix) +(">", "tdebugutils.nim(17, 11)", 2.5, {efAllowStmt, efDetermineType, efOperand}, nkFloatLit) +("<", "tdebugutils.nim(17, 11)", 2.5, float64) +(">", "tdebugutils.nim(17, 17)", 3, {efAllowStmt, efDetermineType, efOperand}, nkIntLit) +("<", "tdebugutils.nim(17, 17)", 3, int literal(3)) +("<", "tdebugutils.nim(17, 15)", 2.5 * 3, float) +("<", "tdebugutils.nim(17, 3)", let a = 2.5 * 3, nil) +(">", "tdebugutils.nim(18, 3)", {.undef(nimCompilerDebug).}, {}, nkPragma) +]# + +proc main = + {.define(nimCompilerDebug).} + let a = 2.5 * 3 + {.undef(nimCompilerDebug).} From 5600a622297a9b06b2f47039e0397cded1bd425c Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 18 Jun 2021 08:57:51 -0700 Subject: [PATCH 0942/2472] strformat.fmt now supports non-literal const strings (#18274) Co-authored-by: Andreas Rumpf --- changelog.md | 4 ++- lib/core/macros.nim | 3 +- lib/pure/strformat.nim | 63 +++++++++++++++++++++++++----------------- 3 files changed, 43 insertions(+), 27 deletions(-) diff --git a/changelog.md b/changelog.md index 26048eedd..4fd1ef355 100644 --- a/changelog.md +++ b/changelog.md @@ -93,7 +93,9 @@ ## Standard library additions and changes -- Added support for parenthesized expressions in `strformat` +- `strformat`: + added support for parenthesized expressions. + added support for const string's instead of just string literals - Fixed buffer overflow bugs in `net` diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 37f855d94..c09fae6b3 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -476,8 +476,9 @@ proc genSym*(kind: NimSymKind = nskLet; ident = ""): NimNode {. ## needs to occur in a declaration context. proc callsite*(): NimNode {.magic: "NCallSite", benign, deprecated: - "Deprecated since v0.18.1; use varargs[untyped] in the macro prototype instead".} + "Deprecated since v0.18.1; use `varargs[untyped]` in the macro prototype instead".} ## Returns the AST of the invocation expression that invoked this macro. + # see https://github.com/nim-lang/RFCs/issues/387 as candidate replacement. proc toStrLit*(n: NimNode): NimNode = ## Converts the AST `n` to the concrete Nim code and wraps that diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim index 52731e970..872141383 100644 --- a/lib/pure/strformat.nim +++ b/lib/pure/strformat.nim @@ -573,15 +573,12 @@ template formatValue(result: var string; value: char; specifier: string) = template formatValue(result: var string; value: cstring; specifier: string) = result.add value -proc strformatImpl(pattern: NimNode; openChar, closeChar: char): NimNode = - if pattern.kind notin {nnkStrLit..nnkTripleStrLit}: - error "string formatting (fmt(), &) only works with string literals", pattern +proc strformatImpl(f: string; openChar, closeChar: char): NimNode = if openChar == ':' or closeChar == ':': error "openChar and closeChar must not be ':'" - let f = pattern.strVal var i = 0 let res = genSym(nskVar, "fmtRes") - result = newNimNode(nnkStmtListExpr, lineInfoFrom = pattern) + result = newNimNode(nnkStmtListExpr) # XXX: https://github.com/nim-lang/Nim/issues/8405 # When compiling with -d:useNimRtl, certain procs such as `count` from the strutils # module are not accessible at compile-time: @@ -633,12 +630,8 @@ proc strformatImpl(pattern: NimNode; openChar, closeChar: char): NimNode = var x: NimNode try: x = parseExpr(subexpr) - except ValueError: - when declared(getCurrentExceptionMsg): - let msg = getCurrentExceptionMsg() - error("could not parse `" & subexpr & "`.\n" & msg, pattern) - else: - error("could not parse `" & subexpr & "`.\n", pattern) + except ValueError as e: + error("could not parse `$#` in `$#`.\n$#" % [subexpr, f, e.msg]) let formatSym = bindSym("formatValue", brOpen) var options = "" if f[i] == ':': @@ -667,19 +660,39 @@ proc strformatImpl(pattern: NimNode; openChar, closeChar: char): NimNode = when defined(debugFmtDsl): echo repr result -macro `&`*(pattern: string): untyped = strformatImpl(pattern, '{', '}') - ## For a specification of the `&` macro, see the module level documentation. - -macro fmt*(pattern: string): untyped = strformatImpl(pattern, '{', '}') - ## An alias for `& <#&.m,string>`_. - -macro fmt*(pattern: string; openChar, closeChar: char): untyped = - ## The same as `fmt <#fmt.m,string>`_, but uses `openChar` instead of `'{'` - ## and `closeChar` instead of `'}'`. +macro fmt*(pattern: static string; openChar: static char, closeChar: static char): string = + ## Interpolates `pattern` using symbols in scope. runnableExamples: - let testInt = 123 - assert "".fmt('<', '>') == "123" - assert """(()"foo" & "bar"())""".fmt(')', '(') == "(foobar)" - assert """ ""{"123+123"}"" """.fmt('"', '"') == " \"{246}\" " + let x = 7 + assert "var is {x * 2}".fmt == "var is 14" + assert "var is {{x}}".fmt == "var is {x}" # escape via doubling + const s = "foo: {x}" + assert s.fmt == "foo: 7" # also works with const strings - strformatImpl(pattern, openChar.intVal.char, closeChar.intVal.char) + assert fmt"\n" == r"\n" # raw string literal + assert "\n".fmt == "\n" # regular literal (likewise with `fmt("\n")` or `fmt "\n"`) + runnableExamples: + # custom `openChar`, `closeChar` + let x = 7 + assert "".fmt('<', '>') == "7" + assert "<<>>".fmt('<', '>') == "<7>" + assert "`x`".fmt('`', '`') == "7" + strformatImpl(pattern, openChar, closeChar) + +template fmt*(pattern: static string): untyped = + ## Alias for `fmt(pattern, '{', '}')`. + fmt(pattern, '{', '}') + +macro `&`*(pattern: string{lit}): string = + ## `&pattern` is the same as `pattern.fmt`. + ## For a specification of the `&` macro, see the module level documentation. + # pending bug #18275, bug #18278, use `pattern: static string` + # consider deprecating this, it's redundant with `fmt` and `fmt` is strictly + # more flexible, readable (no confusion with the binary `&`), self-documenting, + # not to mention #18275, bug #18278. + runnableExamples: + let x = 7 + assert &"{x}\n" == "7\n" # regular string literal + assert &"{x}\n" == "7\n".fmt # `fmt` can be used instead + assert &"{x}\n" != fmt"7\n" # see `fmt` docs, this would use a raw string literal + strformatImpl(pattern.strVal, '{', '}') From 5d15bd7b618cda06e3ae88883914467f89d38112 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 18 Jun 2021 10:23:27 -0700 Subject: [PATCH 0943/2472] refs #18278: do not gag fatal msgs (#18290) --- compiler/lineinfos.nim | 13 ++++++++----- compiler/modules.nim | 3 +-- compiler/msgs.nim | 15 +++++++++++---- compiler/pragmas.nim | 2 +- 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index da0a8d623..052dae272 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -27,7 +27,10 @@ proc createDocLink*(urlSuffix: string): string = type TMsgKind* = enum - errUnknown, errInternal, errIllFormedAstX, errCannotOpenFile, + # fatal errors + errUnknown, errFatal, errInternal, + # non-fatal errors + errIllFormedAstX, errCannotOpenFile, errXExpected, errGridTableNotImplemented, errMarkdownIllformedTable, @@ -38,7 +41,7 @@ type errProveInit, # deadcode errGenerated, errUser, - + # warnings warnCannotOpenFile = "CannotOpenFile", warnOctalEscape = "OctalEscape", warnXIsNeverRead = "XIsNeverRead", warnXmightNotBeenInit = "XmightNotBeenInit", warnDeprecated = "Deprecated", warnConfigDeprecated = "ConfigDeprecated", @@ -64,7 +67,7 @@ type warnCannotOpen = "CannotOpen", warnFileChanged = "FileChanged", warnUser = "User", - + # hints hintSuccess = "Success", hintSuccessX = "SuccessX", hintCC = "CC", hintLineTooLong = "LineTooLong", hintXDeclaredButNotUsed = "XDeclaredButNotUsed", @@ -83,6 +86,7 @@ type const MsgKindToStr*: array[TMsgKind, string] = [ errUnknown: "unknown error", + errFatal: "fatal error: $1", errInternal: "internal error: $1", errIllFormedAstX: "illformed AST: $1", errCannotOpenFile: "cannot open '$1'", @@ -180,8 +184,7 @@ const ] const - fatalMin* = errUnknown - fatalMax* = errInternal + fatalMsgs* = {errUnknown..errInternal} errMin* = errUnknown errMax* = errUser warnMin* = warnCannotOpenFile diff --git a/compiler/modules.nim b/compiler/modules.nim index 27b93e8df..dcfcfd508 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -153,8 +153,7 @@ proc compileSystemModule*(graph: ModuleGraph) = proc wantMainModule*(conf: ConfigRef) = if conf.projectFull.isEmpty: - fatal(conf, newLineInfo(conf, AbsoluteFile(commandLineDesc), 1, 1), errGenerated, - "command expects a filename") + fatal(conf, gCmdLineInfo, "command expects a filename") conf.projectMainIdx = fileInfoIdx(conf, addFileExt(conf.projectFull, NimExt)) proc compileProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) = diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 6b005be3d..8b4365ec4 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -408,7 +408,7 @@ To create a stacktrace, rerun compilation with './koch temp $1 ', see $2 f quit 1 proc handleError(conf: ConfigRef; msg: TMsgKind, eh: TErrorHandling, s: string) = - if msg >= fatalMin and msg <= fatalMax: + if msg in fatalMsgs: if conf.cmd == cmdIdeTools: log(s) quit(conf, msg) if msg >= errMin and msg <= errMax or @@ -498,6 +498,12 @@ proc liMessage*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string, color: ForegroundColor ignoreMsg = false sev: Severity + let errorOutputsOld = conf.m.errorOutputs + if msg in fatalMsgs: + # don't gag, refs bug #7080, bug #18278; this can happen with `{.fatal.}` + # or inside a `tryConstExpr`. + conf.m.errorOutputs = {eStdOut, eStdErr} + let kind = if msg in warnMin..hintMax and msg != hintUserRaw: $msg else: "" # xxx not sure why hintUserRaw is special case msg of errMin..errMax: @@ -553,6 +559,9 @@ proc liMessage*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string, KindFormat % $hintMsgOrigin, resetStyle, conf.unitSep) handleError(conf, msg, eh, s) + if msg in fatalMsgs: + # most likely would have died here but just in case, we restore state + conf.m.errorOutputs = errorOutputsOld template rawMessage*(conf: ConfigRef; msg: TMsgKind, args: openArray[string]) = let arg = msgKindToString(msg) % args @@ -561,9 +570,7 @@ template rawMessage*(conf: ConfigRef; msg: TMsgKind, args: openArray[string]) = template rawMessage*(conf: ConfigRef; msg: TMsgKind, arg: string) = liMessage(conf, unknownLineInfo, msg, arg, eh = doAbort, instLoc()) -template fatal*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") = - # this fixes bug #7080 so that it is at least obvious 'fatal' was executed. - conf.m.errorOutputs = {eStdOut, eStdErr} +template fatal*(conf: ConfigRef; info: TLineInfo, arg = "", msg = errFatal) = liMessage(conf, info, msg, arg, doAbort, instLoc()) template globalAssert*(conf: ConfigRef; cond: untyped, info: TLineInfo = unknownLineInfo, arg = "") = diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 3a2d9cede..695e595c3 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -1047,7 +1047,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, let s = expectStrLit(c, it) recordPragma(c, it, "error", s) localError(c.config, it.info, errUser, s) - of wFatal: fatal(c.config, it.info, errUser, expectStrLit(c, it)) + of wFatal: fatal(c.config, it.info, expectStrLit(c, it)) of wDefine: processDefine(c, it) of wUndef: processUndef(c, it) of wCompile: processCompile(c, it) From 7714ab468a665e772c6d37d39038974cbcea75dc Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 19 Jun 2021 11:24:46 -0700 Subject: [PATCH 0944/2472] make privateAccess work with generic types and generic instantiations; fix a SIGSEGV (#18260) Co-authored-by: Andreas Rumpf --- compiler/ast.nim | 20 +++++++++++++ compiler/semmagic.nim | 10 ++++--- lib/std/importutils.nim | 32 +++++++++++++------- tests/stdlib/mimportutils.nim | 15 ++++++++++ tests/stdlib/timportutils.nim | 56 +++++++++++++++++++++++++++++++++++ 5 files changed, 118 insertions(+), 15 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 96adb8c1f..8bb8de1d6 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -1887,6 +1887,26 @@ proc toObject*(typ: PType): PType = if t.kind == tyRef: t.lastSon else: typ +proc toObjectFromRefPtrGeneric*(typ: PType): PType = + #[ + See also `toObject`. + Finds the underlying `object`, even in cases like these: + type + B[T] = object f0: int + A1[T] = ref B[T] + A2[T] = ref object f1: int + A3 = ref object f2: int + A4 = object f3: int + ]# + result = typ + while true: + case result.kind + of tyGenericBody: result = result.lastSon + of tyRef, tyPtr, tyGenericInst, tyGenericInvocation, tyAlias: result = result[0] + # automatic dereferencing is deep, refs #18298. + else: break + assert result.sym != nil + proc isImportedException*(t: PType; conf: ConfigRef): bool = assert t != nil diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim index e4e007678..d3f26e630 100644 --- a/compiler/semmagic.nim +++ b/compiler/semmagic.nim @@ -457,6 +457,11 @@ proc semOld(c: PContext; n: PNode): PNode = localError(c.config, n[1].info, n[1].sym.name.s & " does not belong to " & getCurrOwner(c).name.s) result = n +proc semPrivateAccess(c: PContext, n: PNode): PNode = + let t = n[1].typ[0].toObjectFromRefPtrGeneric + c.currentScope.allowPrivateAccess.add t.sym + result = newNodeIT(nkEmpty, n.info, getSysType(c.graph, n.info, tyVoid)) + proc magicsAfterOverloadResolution(c: PContext, n: PNode, flags: TExprFlags): PNode = ## This is the preferred code point to implement magics. @@ -574,9 +579,6 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode, n[0].sym.magic = mSubU result = n of mPrivateAccess: - var t = n[1].typ[0] - if t.kind in {tyRef, tyPtr}: t = t[0] - c.currentScope.allowPrivateAccess.add t.sym - result = newNodeIT(nkEmpty, n.info, getSysType(c.graph, n.info, tyVoid)) + result = semPrivateAccess(c, n) else: result = n diff --git a/lib/std/importutils.nim b/lib/std/importutils.nim index 0c0f546b9..d2da76ea8 100644 --- a/lib/std/importutils.nim +++ b/lib/std/importutils.nim @@ -13,22 +13,32 @@ Possible future APIs: ]# when defined(nimImportutilsExample): - type Foo = object - x1: int # private + type + Foo = object + f0: int # private + Goo*[T] = object + g0: int # private proc initFoo*(): auto = Foo() -proc privateAccess*(t: typedesc[object|(ref object)|(ptr object)]) {.magic: "PrivateAccess".} = +proc privateAccess*(t: typedesc) {.magic: "PrivateAccess".} = ## Enables access to private fields of `t` in current scope. runnableExamples("-d:nimImportutilsExample"): # here we're importing a module containing: - # type Foo = object - # x1: int # private + # type + # Foo = object + # f0: int # private + # Goo*[T] = object + # g0: int # private # proc initFoo*(): auto = Foo() - var a = initFoo() + var f = initFoo() block: - assert not compiles(a.x1) - privateAccess(a.type) - a.x1 = 1 # accessible in this scope + assert not compiles(f.f0) + privateAccess(f.type) + f.f0 = 1 # accessible in this scope block: - assert a.x1 == 1 # still in scope - assert not compiles(a.x1) + assert f.f0 == 1 # still in scope + assert not compiles(f.f0) + + # this also works with generics + privateAccess(Goo) + assert Goo[float](g0: 1).g0 == 1 diff --git a/tests/stdlib/mimportutils.nim b/tests/stdlib/mimportutils.nim index d2b185cd3..e89d58d27 100644 --- a/tests/stdlib/mimportutils.nim +++ b/tests/stdlib/mimportutils.nim @@ -13,5 +13,20 @@ type hd1: float PA* = ref A PtA* = ptr A + E*[T] = object + he1: int + FSub[T1, T2] = object + h3: T1 + h4: T2 + F*[T1, T2] = ref FSub[T1, T2] + G*[T] = ref E[T] + H3*[T] = object + h5: T + H2*[T] = H3[T] + H1*[T] = ref H2[T] + H*[T] = H1[T] + +type BAalias* = typeof(B.default) + # typeof is not a transparent abstraction, creates a `tyAlias` proc initB*(): B = B() diff --git a/tests/stdlib/timportutils.nim b/tests/stdlib/timportutils.nim index 37e2b7102..be912e702 100644 --- a/tests/stdlib/timportutils.nim +++ b/tests/stdlib/timportutils.nim @@ -36,6 +36,55 @@ template main = let c = C(c0: 1, hc1: 2) c.hc1 == 2 + block: + assertAll: + not compiles(E[int](he1: 1)) + privateAccess E[int] + var e = E[int](he1: 1) + e.he1 == 1 + e.he1 = 2 + e.he1 == 2 + e.he1 += 3 + e.he1 == 5 + # xxx caveat: this currently compiles but in future, we may want + # to make `privateAccess E[int]` only affect a specific instantiation; + # note that `privateAccess E` does work to cover all instantiations. + var e2 = E[float](he1: 1) + + block: + assertAll: + not compiles(E[int](he1: 1)) + privateAccess E + var e = E[int](he1: 1) + e.he1 == 1 + + block: + assertAll: + not compiles(F[int, int](h3: 1)) + privateAccess F[int, int] + var e = F[int, int](h3: 1) + e.h3 == 1 + + block: + assertAll: + not compiles(F[int, int](h3: 1)) + privateAccess F[int, int].default[].typeof + var e = F[int, int](h3: 1) + e.h3 == 1 + + block: + assertAll: + var a = G[int]() + var b = a.addr + privateAccess b.type + discard b.he1 + discard b[][].he1 + + block: + assertAll: + privateAccess H[int] + var a = H[int](h5: 2) + block: assertAll: privateAccess PA @@ -44,6 +93,13 @@ template main = pa.ha1 = 3 pa.ha1 == 3 + block: + assertAll: + var b = BAalias() + not compiles(b.hb1) + privateAccess BAalias + discard b.hb1 + block: assertAll: var a = A(a0: 1) From ad5063aed101f0102fb867f0b4e518ede7fa188b Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 19 Jun 2021 14:41:43 -0700 Subject: [PATCH 0945/2472] testutils.assertAll: show correct location on assert failure (#18299) --- testament/lib/stdtest/testutils.nim | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/testament/lib/stdtest/testutils.nim b/testament/lib/stdtest/testutils.nim index 6d8620dc7..a5476b8d7 100644 --- a/testament/lib/stdtest/testutils.nim +++ b/testament/lib/stdtest/testutils.nim @@ -107,10 +107,12 @@ macro assertAll*(body) = # remove this once these support VM, pending #10129 (closed but not yet fixed) result = newStmtList() for a in body: - result.add genAst(a) do: + result.add genAst(a, a2 = a.repr, info = lineInfo(a)) do: # D20210421T014713:here # xxx pending https://github.com/nim-lang/Nim/issues/12030, # `typeof` should introduce its own scope, so that this # is sufficient: `typeof(a)` instead of `typeof(block: a)` when typeof(block: a) is void: a - else: doAssert a + else: + if not a: + raise newException(AssertionDefect, info & " " & a2) From 128d21be1cdf28d6aac6a63910fabf02f186e548 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 20 Jun 2021 00:49:18 -0700 Subject: [PATCH 0946/2472] add `typetraits.pointerBase` to return `T` in `ref T|ptr T` (#18293) * add typetraits.deref to return T in ref T|ptr T * deref => refBase * refBase=>pointerBase * [skip ci] address comment --- changelog.md | 15 ++++++--------- lib/pure/typetraits.nim | 10 ++++++++++ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/changelog.md b/changelog.md index 4fd1ef355..80e38f420 100644 --- a/changelog.md +++ b/changelog.md @@ -113,8 +113,12 @@ - Make `{.requiresInit.}` pragma to work for `distinct` types. -- Added a macros `enumLen` for returning the number of items in an enum to the - `typetraits.nim` module. +- `typetraits`: + `distinctBase` now is identity instead of error for non distinct types. + Added `enumLen` to return the number of elements in an enum. + Added `HoleyEnum` for enums with holes, `OrdinalEnum` for enums without holes. + Added `hasClosure`. + Added `pointerBase` to return `T` for `ref T | ptr T`. - `prelude` now works with the JavaScript target. Added `sequtils` import to `prelude`. @@ -162,8 +166,6 @@ Added `symbolName` to return the enum symbol name ignoring the human readable name. Added `symbolRank` to return the index in which an enum member is listed in an enum. -- Added `typetraits.HoleyEnum` for enums with holes, `OrdinalEnum` for enums without holes. - - Removed deprecated `iup` module from stdlib, it has already moved to [nimble](https://github.com/nim-lang/iup). @@ -293,7 +295,6 @@ - Added `algorithm.merge`. - - Added `std/jsfetch` module [Fetch](https://developer.mozilla.org/docs/Web/API/Fetch_API) wrapper for JavaScript target. - Added `std/jsheaders` module [Headers](https://developer.mozilla.org/en-US/docs/Web/API/Headers) wrapper for JavaScript target. @@ -322,8 +323,6 @@ - Added `hasDataBuffered` to `asyncnet`. -- Added `hasClosure` to `std/typetraits`. - - Added `std/tempfiles`. - Added `genasts.genAst` that avoids the problems inherent with `quote do` and can @@ -345,8 +344,6 @@ - nil dereference is not allowed at compile time. `cast[ptr int](nil)[]` is rejected at compile time. -- `typetraits.distinctBase` now is identity instead of error for non distinct types. - - `os.copyFile` is now 2.5x faster on OSX, by using `copyfile` from `copyfile.h`; use `-d:nimLegacyCopyFile` for OSX < 10.5. diff --git a/lib/pure/typetraits.nim b/lib/pure/typetraits.nim index 3e215a0ea..e63585f63 100644 --- a/lib/pure/typetraits.nim +++ b/lib/pure/typetraits.nim @@ -100,6 +100,16 @@ proc isNamedTuple*(T: typedesc): bool {.magic: "TypeTrait".} = doAssert not isNamedTuple((string, int)) doAssert isNamedTuple(tuple[name: string, age: int]) +template pointerBase*[T](_: typedesc[ptr T | ref T]): typedesc = + ## Returns `T` for `ref T | ptr T`. + runnableExamples: + assert (ref int).pointerBase is int + type A = ptr seq[float] + assert A.pointerBase is seq[float] + assert (ref A).pointerBase is A # not seq[float] + assert (var s = "abc"; s[0].addr).typeof.pointerBase is char + T + proc distinctBase*(T: typedesc): typedesc {.magic: "TypeTrait".} = ## Returns the base type for distinct types, or the type itself otherwise. ## From 590d457631290beacf97d945c5d3a625cc671645 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Sun, 20 Jun 2021 10:50:03 +0300 Subject: [PATCH 0947/2472] docgen: move to shared RST state (fix #16990) (#18256) * docgen: move to shared RST state (fix #16990) * Update lib/packages/docutils/rst.nim Co-authored-by: Andreas Rumpf * Update lib/packages/docutils/rst.nim Co-authored-by: Andreas Rumpf * Update lib/packages/docutils/rst.nim Co-authored-by: Andreas Rumpf * Update compiler/docgen.nim Co-authored-by: Timothee Cour * Update compiler/docgen.nim Co-authored-by: Timothee Cour * Update compiler/docgen.nim Co-authored-by: Timothee Cour * Update lib/packages/docutils/rst.nim Co-authored-by: Timothee Cour * rename `cmdDoc2` to `cmdDoc` * fix (P)RstSharedState convention * new style of initialization * misc suggestions * 1 more rename * fix a regression Co-authored-by: Andreas Rumpf Co-authored-by: Timothee Cour --- compiler/commands.nim | 2 +- compiler/docgen.nim | 241 ++++++++++++------ compiler/docgen2.nim | 1 + compiler/main.nim | 2 +- compiler/options.nim | 6 +- lib/packages/docutils/rst.nim | 215 +++++++++------- .../expected/subdir/subdir_b/utils.idx | 10 +- tests/stdlib/trstgen.nim | 7 +- 8 files changed, 292 insertions(+), 192 deletions(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index 994b224f1..8f8df71a7 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -443,7 +443,7 @@ proc parseCommand*(command: string): Command = of "check": cmdCheck of "e": cmdNimscript of "doc0": cmdDoc0 - of "doc2", "doc": cmdDoc2 + of "doc2", "doc": cmdDoc of "doc2tex": cmdDoc2tex of "rst2html": cmdRst2html of "rst2tex": cmdRst2tex diff --git a/compiler/docgen.nim b/compiler/docgen.nim index dc2a63f48..2c99adba3 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -28,6 +28,24 @@ const DocColOffset = "## ".len # assuming that a space was added after ## type + ItemFragment = object ## A fragment from each item will be eventually + ## constructed by converting `rst` fields to strings. + case isRst: bool + of true: + rst: PRstNode + of false: ## contains ready markup e.g. from runnableExamples + str: string + ItemPre = seq[ItemFragment] ## A pre-processed item. + Item = object ## Any item in documentation, e.g. symbol + ## entry. Configuration variable ``doc.item`` + ## is used for its HTML rendering. + descRst: ItemPre ## Description of the item (may contain + ## runnableExamples). + substitutions: seq[string] ## Variable names in `doc.item`... + ModSection = object ## Section like Procs, Types, etc. + secItems: seq[Item] ## Pre-processed items. + finalMarkup: string ## The items, after RST pass 2 and rendering. + ModSections = array[TSymKind, ModSection] TSections = array[TSymKind, string] ExampleGroup = ref object ## a group of runnableExamples with same rdoccmd @@ -35,17 +53,25 @@ type docCmd: string ## from user config, e.g. --doccmd:-d:foo code: string ## contains imports; each import contains `body` index: int ## group index + JsonItem = object # pre-processed item: `rst` should be finalized + json: JsonNode + rst: PRstNode + rstField: string TDocumentor = object of rstgen.RstGenerator - modDesc: string # module description + modDescPre: ItemPre # module description, not finalized + modDescFinal: string # module description, after RST pass 2 and rendering module: PSym modDeprecationMsg: string - toc, toc2, section: TSections + section: ModSections # entries of ``.nim`` file (for `proc`s, etc) + toc, toc2: TSections # toc2 - grouped TOC tocTable: array[TSymKind, Table[string, string]] indexValFilename: string analytics: string # Google Analytics javascript, "" if doesn't exist seenSymbols: StringTableRef # avoids duplicate symbol generation for HTML. - jArray: JsonNode + jEntriesPre: seq[JsonItem] # pre-processed RST + JSON content + jEntriesFinal: JsonNode # final JSON after RST pass 2 and rendering types: TStrTable + sharedState: PRstSharedState isPureRst: bool conf*: ConfigRef cache*: IdentCache @@ -58,6 +84,9 @@ type PDoc* = ref TDocumentor ## Alias to type less. +proc add(dest: var ItemPre, rst: PRstNode) = dest.add ItemFragment(isRst: true, rst: rst) +proc add(dest: var ItemPre, str: string) = dest.add ItemFragment(isRst: false, str: str) + proc prettyString(a: object): string = # xxx pending std/prettyprint refs https://github.com/nim-lang/RFCs/issues/203#issuecomment-602534906 for k, v in fieldPairs(a): @@ -151,12 +180,12 @@ template declareClosures = if not fileExists(result): result = "" proc parseRst(text, filename: string, - line, column: int, hasToc: var bool, + line, column: int, rstOptions: RstParseOptions; - conf: ConfigRef): PRstNode = + conf: ConfigRef, sharedState: PRstSharedState): PRstNode = declareClosures() - result = rstParse(text, filename, line, column, hasToc, rstOptions, - docgenFindFile, compilerMsgHandler) + result = rstParsePass1(text, filename, line, column, rstOptions, + sharedState) proc getOutFile2(conf: ConfigRef; filename: RelativeFile, ext: string, guessTarget: bool): AbsoluteFile = @@ -179,10 +208,13 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, result.conf = conf result.cache = cache result.outDir = conf.outDir.string + const options = {roSupportRawDirective, roSupportMarkdown, + roPreferMarkdown, roNimFile} + result.sharedState = newRstSharedState( + options, filename.string, + docgenFindFile, compilerMsgHandler) initRstGenerator(result[], (if conf.isLatexCmd: outLatex else: outHtml), - conf.configVars, filename.string, - {roSupportRawDirective, roSupportMarkdown, - roPreferMarkdown, roNimFile}, + conf.configVars, filename.string, options, docgenFindFile, compilerMsgHandler) if conf.configVars.hasKey("doc.googleAnalytics"): @@ -203,7 +235,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, result.seenSymbols = newStringTable(modeCaseInsensitive) result.id = 100 - result.jArray = newJArray() + result.jEntriesFinal = newJArray() initStrTable result.types result.onTestSnippet = proc (gen: var RstGenerator; filename, cmd: string; status: int; content: string) = @@ -261,37 +293,31 @@ proc getVarIdx(varnames: openArray[string], id: string): int = return i result = -1 -proc genComment(d: PDoc, n: PNode): string = - result = "" +proc genComment(d: PDoc, n: PNode): PRstNode = if n.comment.len > 0: - let comment = n.comment - when false: - # RFC: to preseve newlines in comments, this would work: - comment = comment.replace("\n", "\n\n") - renderRstToOut(d[], - parseRst(comment, toFullPath(d.conf, n.info), - toLinenumber(n.info), - toColumn(n.info) + DocColOffset, - (var dummy: bool; dummy), d.options, d.conf), - result) + result = parseRst(n.comment, toFullPath(d.conf, n.info), + toLinenumber(n.info), + toColumn(n.info) + DocColOffset, + d.options, d.conf, + d.sharedState) -proc genRecCommentAux(d: PDoc, n: PNode): string = - if n == nil: return "" +proc genRecCommentAux(d: PDoc, n: PNode): PRstNode = + if n == nil: return nil result = genComment(d, n) - if result == "": + if result == nil: if n.kind in {nkStmtList, nkStmtListExpr, nkTypeDef, nkConstDef, nkObjectTy, nkRefTy, nkPtrTy, nkAsgn, nkFastAsgn, nkHiddenStdConv}: # notin {nkEmpty..nkNilLit, nkEnumTy, nkTupleTy}: for i in 0.. 0: msg.add " cmd: " & rdoccmd - dispA(d.conf, dest, "\n

    $1

    \n", + var s: string + dispA(d.conf, s, "\n

    $1

    \n", "\n\n\\textbf{$1}\n", [msg]) + dest.add s inc d.listingCounter let id = $d.listingCounter dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim", ""]) @@ -597,7 +625,7 @@ proc getRoutineBody(n: PNode): PNode = doAssert result.len == 2 result = result[1] -proc getAllRunnableExamples(d: PDoc, n: PNode, dest: var string) = +proc getAllRunnableExamples(d: PDoc, n: PNode, dest: var ItemPre) = var n = n var state = rsStart template fn(n2, topLevel) = @@ -706,7 +734,7 @@ proc complexName(k: TSymKind, n: PNode, baseName: string): string = ## types will be added with a preceding dash. Return types won't be added. ## ## If you modify the output of this proc, please update the anchor generation - ## section of ``doc/docgen.txt``. + ## section of ``doc/docgen.rst``. result = baseName case k of skProc, skFunc: discard @@ -804,7 +832,7 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) = var result = "" var literal, plainName = "" var kind = tkEof - var comm = "" + var comm: ItemPre if n.kind in routineDefs: getAllRunnableExamples(d, n, comm) else: @@ -840,9 +868,9 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) = let seeSrc = genSeeSrc(d, toFullPath(d.conf, n.info), n.info.line.int) - d.section[k].add(getConfigVar(d.conf, "doc.item") % - ["name", name, "uniqueName", uniqueName, - "header", result, "desc", comm, "itemID", $d.id, + d.section[k].secItems.add Item(descRst: comm, substitutions: @[ + "name", name, "uniqueName", uniqueName, + "header", result, "itemID", $d.id, "header_plain", plainNameEsc, "itemSym", cleanPlainSymbol, "itemSymOrID", symbolOrId, "itemSymEnc", plainSymbolEnc, "itemSymOrIDEnc", symbolOrIdEnc, "seeSrc", seeSrc, @@ -886,25 +914,26 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) = if k == skType and nameNode.kind == nkSym: d.types.strTableAdd nameNode.sym -proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonNode = +proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonItem = if not isVisible(d, nameNode): return var name = getName(d, nameNode) comm = genRecComment(d, n) r: TSrcGen initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments}) - result = %{ "name": %name, "type": %($k), "line": %n.info.line.int, - "col": %n.info.col} - if comm.len > 0: - result["description"] = %comm + result.json = %{ "name": %name, "type": %($k), "line": %n.info.line.int, + "col": %n.info.col} + if comm != nil: + result.rst = comm + result.rstField = "description" if r.buf.len > 0: - result["code"] = %r.buf + result.json["code"] = %r.buf if k in routineKinds: - result["signature"] = newJObject() + result.json["signature"] = newJObject() if n[paramsPos][0].kind != nkEmpty: - result["signature"]["return"] = %($n[paramsPos][0]) + result.json["signature"]["return"] = %($n[paramsPos][0]) if n[paramsPos].len > 1: - result["signature"]["arguments"] = newJArray() + result.json["signature"]["arguments"] = newJArray() for paramIdx in 1 ..< n[paramsPos].len: for identIdx in 0 ..< n[paramsPos][paramIdx].len - 2: let @@ -912,22 +941,22 @@ proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonNode = paramType = $n[paramsPos][paramIdx][^2] if n[paramsPos][paramIdx][^1].kind != nkEmpty: let paramDefault = $n[paramsPos][paramIdx][^1] - result["signature"]["arguments"].add %{"name": %paramName, "type": %paramType, "default": %paramDefault} + result.json["signature"]["arguments"].add %{"name": %paramName, "type": %paramType, "default": %paramDefault} else: - result["signature"]["arguments"].add %{"name": %paramName, "type": %paramType} + result.json["signature"]["arguments"].add %{"name": %paramName, "type": %paramType} if n[pragmasPos].kind != nkEmpty: - result["signature"]["pragmas"] = newJArray() + result.json["signature"]["pragmas"] = newJArray() for pragma in n[pragmasPos]: - result["signature"]["pragmas"].add %($pragma) + result.json["signature"]["pragmas"].add %($pragma) if n[genericParamsPos].kind != nkEmpty: - result["signature"]["genericParams"] = newJArray() + result.json["signature"]["genericParams"] = newJArray() for genericParam in n[genericParamsPos]: var param = %{"name": %($genericParam)} if genericParam.sym.typ.sons.len > 0: param["types"] = newJArray() for kind in genericParam.sym.typ.sons: param["types"].add %($kind) - result["signature"]["genericParams"].add param + result.json["signature"]["genericParams"].add param proc checkForFalse(n: PNode): bool = result = n.kind == nkIdent and cmpIgnoreStyle(n.ident.s, "false") == 0 @@ -946,8 +975,8 @@ proc traceDeps(d: PDoc, it: PNode) = traceDeps(d, a) elif it.kind == nkSym and belongsToPackage(d.conf, it.sym): let external = externalDep(d, it.sym) - if d.section[k] != "": d.section[k].add(", ") - dispA(d.conf, d.section[k], + if d.section[k].finalMarkup != "": d.section[k].finalMarkup.add(", ") + dispA(d.conf, d.section[k].finalMarkup, "$1", "$1", [esc(d.target, external.prettyLink), changeFileExt(external, "html")]) @@ -956,8 +985,8 @@ proc exportSym(d: PDoc; s: PSym) = const k = exportSection if s.kind == skModule and belongsToPackage(d.conf, s): let external = externalDep(d, s) - if d.section[k] != "": d.section[k].add(", ") - dispA(d.conf, d.section[k], + if d.section[k].finalMarkup != "": d.section[k].finalMarkup.add(", ") + dispA(d.conf, d.section[k].finalMarkup, "$1", "$1", [esc(d.target, external.prettyLink), changeFileExt(external, "html")]) @@ -968,9 +997,9 @@ proc exportSym(d: PDoc; s: PSym) = complexSymbol = complexName(s.kind, s.ast, s.name.s) symbolOrId = d.newUniquePlainSymbol(complexSymbol) external = externalDep(d, module) - if d.section[k] != "": d.section[k].add(", ") + if d.section[k].finalMarkup != "": d.section[k].finalMarkup.add(", ") # XXX proper anchor generation here - dispA(d.conf, d.section[k], + dispA(d.conf, d.section[k].finalMarkup, "$1", "$1", [esc(d.target, s.name.s), changeFileExt(external, "html"), @@ -1034,13 +1063,16 @@ proc documentRaises*(cache: IdentCache; n: PNode) = if p5 != nil: n[pragmasPos].add p5 proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) = + ## Goes through nim nodes recursively and collects doc comments. + ## Main function for `doc`:option: command, + ## which is implemented in ``docgen2.nim``. template genItemAux(skind) = genItem(d, n, n[namePos], skind, docFlags) case n.kind of nkPragma: let pragmaNode = findPragma(n, wDeprecated) d.modDeprecationMsg.add(genDeprecationMsg(d, pragmaNode)) - of nkCommentStmt: d.modDesc.add(genComment(d, n)) + of nkCommentStmt: d.modDescPre.add(genComment(d, n)) of nkProcDef, nkFuncDef: when useEffectSystem: documentRaises(d.cache, n) genItemAux(skProc) @@ -1079,21 +1111,63 @@ proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) = of nkExportExceptStmt: discard "transformed into nkExportStmt by semExportExcept" of nkFromStmt, nkImportExceptStmt: traceDeps(d, n[0]) of nkCallKinds: - var comm = "" + var comm: ItemPre getAllRunnableExamples(d, n, comm) - if comm != "": d.modDesc.add(comm) + if comm.len != 0: d.modDescPre.add(comm) else: discard -proc add(d: PDoc; j: JsonNode) = - if j != nil: d.jArray.add j +proc finishGenerateDoc*(d: var PDoc) = + ## Perform 2nd RST pass for resolution of links/footnotes/headings... + # Main title/subtitle are allowed only in the first RST fragment of document + var firstRst = PRstNode(nil) + for fragment in d.modDescPre: + if fragment.isRst: + firstRst = fragment.rst + break + preparePass2(d.sharedState, firstRst) + + # Finalize fragments of ``.nim`` or ``.rst`` file + proc renderItemPre(d: PDoc, fragments: ItemPre, result: var string) = + for f in fragments: + case f.isRst: + of true: + var resolved = resolveSubs(d.sharedState, f.rst) + renderRstToOut(d[], resolved, result) + of false: result &= f.str + for k in TSymKind: + for item in d.section[k].secItems: + var itemDesc: string + renderItemPre(d, item.descRst, itemDesc) + d.section[k].finalMarkup.add( + getConfigVar(d.conf, "doc.item") % ( + item.substitutions & @["desc", itemDesc])) + itemDesc = "" + d.section[k].secItems.setLen 0 + renderItemPre(d, d.modDescPre, d.modDescFinal) + d.modDescPre.setLen 0 + d.hasToc = d.hasToc or d.sharedState.hasToc + + # Finalize fragments of ``.json`` file + for i, entry in d.jEntriesPre: + if entry.rst != nil: + let resolved = resolveSubs(d.sharedState, entry.rst) + var str: string + renderRstToOut(d[], resolved, str) + entry.json[entry.rstField] = %str + d.jEntriesFinal.add entry.json + d.jEntriesPre[i].rst = nil + +proc add(d: PDoc; j: JsonItem) = + if j.json != nil or j.rst != nil: d.jEntriesPre.add j proc generateJson*(d: PDoc, n: PNode, includeComments: bool = true) = case n.kind of nkCommentStmt: if includeComments: - d.add %*{"comment": genComment(d, n)} + d.add JsonItem(rst: genComment(d, n), rstField: "comment", + json: %Table[string, string]()) else: - d.modDesc.add(genComment(d, n)) + d.modDescPre.add(genComment(d, n)) of nkProcDef, nkFuncDef: when useEffectSystem: documentRaises(d.cache, n) d.add genJsonItem(d, n, n[namePos], skProc) @@ -1173,11 +1247,11 @@ proc genSection(d: PDoc, kind: TSymKind, groupedToc = false) = "Imports", "Types", "Vars", "Lets", "Consts", "Vars", "Procs", "Funcs", "Methods", "Iterators", "Converters", "Macros", "Templates", "Exports" ] - if d.section[kind] == "": return + if d.section[kind].finalMarkup == "": return var title = sectionNames[kind] - d.section[kind] = getConfigVar(d.conf, "doc.section") % [ + d.section[kind].finalMarkup = getConfigVar(d.conf, "doc.section") % [ "sectionid", $ord(kind), "sectionTitle", title, - "sectionTitleID", $(ord(kind) + 50), "content", d.section[kind]] + "sectionTitleID", $(ord(kind) + 50), "content", d.section[kind].finalMarkup] var tocSource = d.toc if groupedToc: @@ -1209,7 +1283,7 @@ proc genOutFile(d: PDoc, groupedToc = false): string = if toc != "" or d.target == outLatex: # for Latex $doc.toc will automatically generate TOC if `d.hasToc` is set toc = getConfigVar(d.conf, "doc.toc") % ["content", toc] - for i in TSymKind: code.add(d.section[i]) + for i in TSymKind: code.add(d.section[i].finalMarkup) # Extract the title. Non API modules generate an entry in the index table. if d.meta[metaTitle].len != 0: @@ -1234,7 +1308,7 @@ proc genOutFile(d: PDoc, groupedToc = false): string = let seeSrc = genSeeSrc(d, d.filename, 1) content = getConfigVar(d.conf, bodyname) % [ "title", title, "subtitle", subtitle, - "tableofcontents", toc, "moduledesc", d.modDesc, "date", getDateStr(), + "tableofcontents", toc, "moduledesc", d.modDescFinal, "date", getDateStr(), "time", getClockStr(), "content", code, "deprecationMsg", d.modDeprecationMsg, "theindexhref", relLink(d.conf.outDir, d.destFile.AbsoluteFile, @@ -1248,7 +1322,7 @@ proc genOutFile(d: PDoc, groupedToc = false): string = "dochackjs", relLink(d.conf.outDir, d.destFile.AbsoluteFile, docHackJsFname.RelativeFile), "title", title, "subtitle", subtitle, "tableofcontents", toc, - "moduledesc", d.modDesc, "date", getDateStr(), "time", getClockStr(), + "moduledesc", d.modDescFinal, "date", getDateStr(), "time", getClockStr(), "content", content, "author", d.meta[metaAuthor], "version", esc(d.target, d.meta[metaVersion]), "analytics", d.analytics, "deprecationMsg", d.modDeprecationMsg] @@ -1297,12 +1371,12 @@ proc writeOutput*(d: PDoc, useWarning = false, groupedToc = false) = proc writeOutputJson*(d: PDoc, useWarning = false) = runAllExamples(d) var modDesc: string - for desc in d.modDesc: + for desc in d.modDescFinal: modDesc &= desc let content = %*{"orig": d.filename, "nimble": getPackageName(d.conf, d.filename), "moduleDescription": modDesc, - "entries": d.jArray} + "entries": d.jEntriesFinal} if optStdout in d.conf.globalOptions: write(stdout, $content) else: @@ -1324,12 +1398,14 @@ proc handleDocOutputOptions*(conf: ConfigRef) = conf.outDir = AbsoluteDir(conf.outDir / conf.outFile) proc commandDoc*(cache: IdentCache, conf: ConfigRef) = + ## implementation of deprecated ``doc0`` command (without semantic checking) handleDocOutputOptions conf var ast = parseFile(conf.projectMainIdx, cache, conf) if ast == nil: return var d = newDocumentor(conf.projectFull, cache, conf) d.hasToc = true generateDoc(d, ast, ast) + finishGenerateDoc(d) writeOutput(d) generateIndex(d) @@ -1339,14 +1415,13 @@ proc commandRstAux(cache: IdentCache, conf: ConfigRef; var d = newDocumentor(filen, cache, conf, outExt) d.isPureRst = true - var rst = parseRst(readFile(filen.string), filen.string, + let rst = parseRst(readFile(filen.string), filen.string, line=LineRstInit, column=ColRstInit, - d.hasToc, - {roSupportRawDirective, roSupportMarkdown, roPreferMarkdown}, - conf) - var modDesc = newStringOfCap(30_000) - renderRstToOut(d[], rst, modDesc) - d.modDesc = modDesc + {roSupportRawDirective, roSupportMarkdown, + roPreferMarkdown}, conf, + d.sharedState) + d.modDescPre = @[ItemFragment(isRst: true, rst: rst)] + finishGenerateDoc(d) writeOutput(d) generateIndex(d) @@ -1357,6 +1432,7 @@ proc commandRst2TeX*(cache: IdentCache, conf: ConfigRef) = commandRstAux(cache, conf, conf.projectFull, TexExt) proc commandJson*(cache: IdentCache, conf: ConfigRef) = + ## implementation of a deprecated jsondoc0 command var ast = parseFile(conf.projectMainIdx, cache, conf) if ast == nil: return var d = newDocumentor(conf.projectFull, cache, conf) @@ -1366,7 +1442,8 @@ proc commandJson*(cache: IdentCache, conf: ConfigRef) = warnUser, "the ':test:' attribute is not supported by this backend") d.hasToc = true generateJson(d, ast) - let json = d.jArray + finishGenerateDoc(d) + let json = d.jEntriesFinal let content = pretty(json) if optStdout in d.conf.globalOptions: diff --git a/compiler/docgen2.nim b/compiler/docgen2.nim index d077ddc67..bfdb4568c 100644 --- a/compiler/docgen2.nim +++ b/compiler/docgen2.nim @@ -31,6 +31,7 @@ template closeImpl(body: untyped) {.dirty.} = let useWarning = sfMainModule notin g.module.flags let groupedToc = true if shouldProcess(g): + finishGenerateDoc(g.doc) body try: generateIndex(g.doc) diff --git a/compiler/main.nim b/compiler/main.nim index 7824e8cb2..1c8034c6c 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -282,7 +282,7 @@ proc mainCommand*(graph: ModuleGraph) = else: rawMessage(conf, errGenerated, "'run' command not available; rebuild with -d:tinyc") of cmdDoc0: docLikeCmd commandDoc(cache, conf) - of cmdDoc2: + of cmdDoc: docLikeCmd(): conf.setNoteDefaults(warnLockLevel, false) # issue #13218 conf.setNoteDefaults(warnRedefinitionOfLabel, false) # issue #13218 diff --git a/compiler/options.nim b/compiler/options.nim index 05d8ee5b7..ffb0f4501 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -143,8 +143,8 @@ type cmdIdeTools # ide tools (e.g. nimsuggest) cmdNimscript # evaluate nimscript cmdDoc0 - cmdDoc2 - cmdDoc2tex + cmdDoc # convert .nim doc comments to HTML + cmdDoc2tex # convert .nim doc comments to LaTeX cmdRst2html # convert a reStructuredText file to HTML cmdRst2tex # convert a reStructuredText file to TeX cmdJsondoc0 @@ -161,7 +161,7 @@ type const cmdBackends* = {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC, cmdCompileToJS, cmdCrun} - cmdDocLike* = {cmdDoc0, cmdDoc2, cmdDoc2tex, cmdJsondoc0, cmdJsondoc, + cmdDocLike* = {cmdDoc0, cmdDoc, cmdDoc2tex, cmdJsondoc0, cmdJsondoc, cmdCtags, cmdBuildindex} type diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 336caa2df..031a86d98 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -441,7 +441,7 @@ proc rawGetTok(L: var Lexer, tok: var Token) = inc L.col tok.col = max(tok.col - L.baseIndent, 0) -proc getTokens(buffer: string, tokens: var TokenSeq): int = +proc getTokens(buffer: string, tokens: var TokenSeq) = var L: Lexer var length = tokens.len L.buf = cstring(buffer) @@ -489,7 +489,7 @@ type autoSymIdx: int # order of occurence: fnAutoSymbol label: string # valid for fnAutoNumberLabel - SharedState = object + RstSharedState = object options: RstParseOptions # parsing options hLevels: LevelMap # hierarchy of heading styles hTitleCnt: int # =0 if no title, =1 if only main title, @@ -498,8 +498,8 @@ type currRole: string # current interpreted text role currRoleKind: RstNodeKind # ... and its node kind subs: seq[Substitution] # substitutions - refs: seq[Substitution] # references - anchors: seq[AnchorSubst] # internal target substitutions + refs*: seq[Substitution] # references + anchors*: seq[AnchorSubst] # internal target substitutions lineFootnoteNum: seq[int] # footnote line, auto numbers .. [#] lineFootnoteNumRef: seq[int] # footnote line, their reference [#]_ lineFootnoteSym: seq[int] # footnote line, auto symbols .. [*] @@ -508,19 +508,19 @@ type # number, order of occurrence msgHandler: MsgHandler # How to handle errors. findFile: FindFileHandler # How to find files. + filename: string + hasToc*: bool - PSharedState = ref SharedState + PRstSharedState* = ref RstSharedState RstParser = object of RootObj idx*: int tok*: TokenSeq - s*: PSharedState + s*: PRstSharedState indentStack*: seq[int] - filename*: string line*, col*: int ## initial line/column of whole text or ## documenation fragment that will be added ## in case of error/warning reporting to ## (relative) line/column of the token. - hasToc*: bool curAnchor*: string # variable to track latest anchor in s.anchors EParseError* = object of ValueError @@ -578,35 +578,42 @@ proc whichRoleAux(sym: string): RstNodeKind = else: # unknown role result = rnUnknownRole -proc newSharedState(options: RstParseOptions, - findFile: FindFileHandler, - msgHandler: MsgHandler): PSharedState = - new(result) - result.currRole = defaultRole(options) - result.currRoleKind = whichRoleAux(result.currRole) - result.subs = @[] - result.refs = @[] - result.options = options - result.msgHandler = if not isNil(msgHandler): msgHandler else: defaultMsgHandler - result.findFile = if not isNil(findFile): findFile else: defaultFindFile +proc newRstSharedState*(options: RstParseOptions, + filename: string, + findFile: FindFileHandler, + msgHandler: MsgHandler): PRstSharedState = + let r = defaultRole(options) + result = PRstSharedState( + currRole: r, + currRoleKind: whichRoleAux(r), + options: options, + msgHandler: if not isNil(msgHandler): msgHandler else: defaultMsgHandler, + filename: filename, + findFile: if not isNil(findFile): findFile else: defaultFindFile + ) proc curLine(p: RstParser): int = p.line + currentTok(p).line proc findRelativeFile(p: RstParser; filename: string): string = - result = p.filename.splitFile.dir / filename + result = p.s.filename.splitFile.dir / filename if not fileExists(result): result = p.s.findFile(filename) proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string) = - p.s.msgHandler(p.filename, curLine(p), + p.s.msgHandler(p.s.filename, curLine(p), p.col + currentTok(p).col, msgKind, arg) +proc rstMessage(s: PRstSharedState, msgKind: MsgKind, arg: string) = + ## Print warnings for footnotes/substitutions. + ## TODO: their line/column info is not known, to fix it. + s.msgHandler(s.filename, LineRstInit, ColRstInit, msgKind, arg) + proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string, line, col: int) = - p.s.msgHandler(p.filename, p.line + line, + p.s.msgHandler(p.s.filename, p.line + line, p.col + col, msgKind, arg) proc rstMessage(p: RstParser, msgKind: MsgKind) = - p.s.msgHandler(p.filename, curLine(p), + p.s.msgHandler(p.s.filename, curLine(p), p.col + currentTok(p).col, msgKind, currentTok(p).symbol) @@ -680,12 +687,10 @@ proc popInd(p: var RstParser) = # whether it should continue its processing or not, and decided not to, # then this B.E. handler should step back (e.g. do `dec p.idx`). -proc initParser(p: var RstParser, sharedState: PSharedState) = +proc initParser(p: var RstParser, sharedState: PRstSharedState) = p.indentStack = @[0] p.tok = @[] p.idx = 0 - p.filename = "" - p.hasToc = false p.col = ColRstInit p.line = LineRstInit p.s = sharedState @@ -754,14 +759,14 @@ proc rstnodeToRefname(n: PRstNode): string = var b = false rstnodeToRefnameAux(n, result, b) -proc findSub(p: var RstParser, n: PRstNode): int = +proc findSub(s: PRstSharedState, n: PRstNode): int = var key = addNodes(n) # the spec says: if no exact match, try one without case distinction: - for i in countup(0, high(p.s.subs)): - if key == p.s.subs[i].key: + for i in countup(0, high(s.subs)): + if key == s.subs[i].key: return i - for i in countup(0, high(p.s.subs)): - if cmpIgnoreStyle(key, p.s.subs[i].key) == 0: + for i in countup(0, high(s.subs)): + if cmpIgnoreStyle(key, s.subs[i].key) == 0: return i result = -1 @@ -783,10 +788,10 @@ proc setRef(p: var RstParser, key: string, value: PRstNode) = return p.s.refs.add(Substitution(key: key, value: value)) -proc findRef(p: var RstParser, key: string): PRstNode = - for i in countup(0, high(p.s.refs)): - if key == p.s.refs[i].key: - return p.s.refs[i].value +proc findRef(s: PRstSharedState, key: string): PRstNode = + for i in countup(0, high(s.refs)): + if key == s.refs[i].key: + return s.refs[i].value proc addAnchor(p: var RstParser, refn: string, reset: bool) = ## add anchor `refn` to anchor aliases and update last anchor ``curAnchor`` @@ -800,8 +805,8 @@ proc addAnchor(p: var RstParser, refn: string, reset: bool) = else: p.curAnchor = refn -proc findMainAnchor(p: RstParser, refn: string): string = - for subst in p.s.anchors: +proc findMainAnchor(s: PRstSharedState, refn: string): string = + for subst in s.anchors: if subst.mainAnchor == refn: # no need to rename result = subst.mainAnchor break @@ -838,28 +843,28 @@ proc addFootnoteSymAuto(p: var RstParser) = p.s.lineFootnoteSym.add curLine(p) p.s.footnotes.add((fnAutoSymbol, -1, -1, p.s.lineFootnoteSym.len, "")) -proc orderFootnotes(p: var RstParser) = +proc orderFootnotes(s: PRstSharedState) = ## numerate auto-numbered footnotes taking into account that all ## manually numbered ones always have preference. - ## Save the result back to p.s.footnotes. + ## Save the result back to `s.footnotes`. # Report an error if found any mismatch in number of automatic footnotes proc listFootnotes(lines: seq[int]): string = result.add $lines.len & " (lines " & join(lines, ", ") & ")" - if p.s.lineFootnoteNum.len != p.s.lineFootnoteNumRef.len: - rstMessage(p, meFootnoteMismatch, - "$1 != $2" % [listFootnotes(p.s.lineFootnoteNum), - listFootnotes(p.s.lineFootnoteNumRef)] & + if s.lineFootnoteNum.len != s.lineFootnoteNumRef.len: + rstMessage(s, meFootnoteMismatch, + "$1 != $2" % [listFootnotes(s.lineFootnoteNum), + listFootnotes(s.lineFootnoteNumRef)] & " for auto-numbered footnotes") - if p.s.lineFootnoteSym.len != p.s.lineFootnoteSymRef.len: - rstMessage(p, meFootnoteMismatch, - "$1 != $2" % [listFootnotes(p.s.lineFootnoteSym), - listFootnotes(p.s.lineFootnoteSymRef)] & + if s.lineFootnoteSym.len != s.lineFootnoteSymRef.len: + rstMessage(s, meFootnoteMismatch, + "$1 != $2" % [listFootnotes(s.lineFootnoteSym), + listFootnotes(s.lineFootnoteSymRef)] & " for auto-symbol footnotes") var result: seq[FootnoteSubst] var manuallyN, autoN, autoSymbol: seq[FootnoteSubst] - for fs in p.s.footnotes: + for fs in s.footnotes: if fs.kind == fnManualNumber: manuallyN.add fs elif fs.kind in {fnAutoNumber, fnAutoNumberLabel}: autoN.add fs else: autoSymbol.add fs @@ -906,26 +911,26 @@ proc orderFootnotes(p: var RstParser) = let label = footnoteAutoSymbols[symbolNum].repeat(nSymbols) result.add((fs.kind, -1, -1, fs.autoSymIdx, label)) - p.s.footnotes = result + s.footnotes = result -proc getFootnoteNum(p: var RstParser, label: string): int = +proc getFootnoteNum(s: PRstSharedState, label: string): int = ## get number from label. Must be called after `orderFootnotes`. result = -1 - for fnote in p.s.footnotes: + for fnote in s.footnotes: if fnote.label == label: return fnote.number -proc getFootnoteNum(p: var RstParser, order: int): int = +proc getFootnoteNum(s: PRstSharedState, order: int): int = ## get number from occurrence. Must be called after `orderFootnotes`. result = -1 - for fnote in p.s.footnotes: + for fnote in s.footnotes: if fnote.autoNumIdx == order: return fnote.number -proc getAutoSymbol(p: var RstParser, order: int): string = +proc getAutoSymbol(s: PRstSharedState, order: int): string = ## get symbol from occurrence of auto-symbol footnote. result = "???" - for fnote in p.s.footnotes: + for fnote in s.footnotes: if fnote.autoSymIdx == order: return fnote.label @@ -1765,17 +1770,18 @@ proc getLevel(p: var RstParser, c: char, hasOverline: bool): int = line: curLine(p), hasPeers: false) result = p.s.hLevels.len - 1 -proc countTitles(p: var RstParser, n: PRstNode) = - ## Fill `p.s.hTitleCnt` +proc countTitles(s: PRstSharedState, n: PRstNode) = + ## Fill `s.hTitleCnt` + if n == nil: return for node in n.sons: if node != nil: if node.kind notin {rnOverline, rnSubstitutionDef, rnDefaultRole}: break if node.kind == rnOverline: - if p.s.hLevels[p.s.hTitleCnt].hasPeers: + if s.hLevels[s.hTitleCnt].hasPeers: break - inc p.s.hTitleCnt - if p.s.hTitleCnt >= 2: + inc s.hTitleCnt + if s.hTitleCnt >= 2: break proc isAdornmentHeadline(p: RstParser, adornmentIdx: int): bool = @@ -2103,8 +2109,7 @@ proc parseSimpleTable(p: var RstParser): PRstNode = initParser(q, p.s) q.col = cols[j] q.line = line - 1 - q.filename = p.filename - q.col += getTokens(row[j], q.tok) + getTokens(row[j], q.tok) b = newRstNode(rnTableDataCell) b.add(parseDoc(q)) a.add(b) @@ -2156,8 +2161,7 @@ proc parseMarkdownTable(p: var RstParser): PRstNode = initParser(q, p.s) q.col = p.col q.line = currentTok(p).line - 1 - q.filename = p.filename - q.col += getTokens(getColContents(p, row[j]), q.tok) + getTokens(getColContents(p, row[j]), q.tok) b.add(parseDoc(q)) a.add(b) result.add(a) @@ -2562,8 +2566,8 @@ proc dirInclude(p: var RstParser): PRstNode = var q: RstParser initParser(q, p.s) - q.filename = path - q.col += getTokens( + q.s.filename = path + getTokens( inputString[startPosition..endPosition].strip(), q.tok) # workaround a GCC bug; more like the interior pointer bug? @@ -2806,7 +2810,27 @@ proc parseDotDot(p: var RstParser): PRstNode = else: result = parseComment(p) -proc resolveSubs(p: var RstParser, n: PRstNode): PRstNode = +proc rstParsePass1*(fragment, filename: string, + line, column: int, + options: RstParseOptions, + sharedState: PRstSharedState): PRstNode = + ## Parses an RST `fragment`. + ## The result should be further processed by + ## `preparePass2` and `resolveSubs` (which is pass 2). + var p: RstParser + initParser(p, sharedState) + p.line = line + p.col = column + getTokens(fragment, p.tok) + result = parseDoc(p) + +proc preparePass2*(s: PRstSharedState, mainNode: PRstNode) = + ## Records titles in node `mainNode` and orders footnotes. + countTitles(s, mainNode) + orderFootnotes(s) + +proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode = + ## Makes pass 2 of RST parsing. ## Resolves substitutions and anchor aliases, groups footnotes. ## Takes input node `n` and returns the same node with recursive ## substitutions in `n.sons` to `result`. @@ -2814,32 +2838,32 @@ proc resolveSubs(p: var RstParser, n: PRstNode): PRstNode = if n == nil: return case n.kind of rnSubstitutionReferences: - var x = findSub(p, n) + var x = findSub(s, n) if x >= 0: - result = p.s.subs[x].value + result = s.subs[x].value else: var key = addNodes(n) var e = getEnv(key) if e != "": result = newLeaf(e) - else: rstMessage(p, mwUnknownSubstitution, key) + else: rstMessage(s, mwUnknownSubstitution, key) of rnHeadline, rnOverline: # fix up section levels depending on presence of a title and subtitle - if p.s.hTitleCnt == 2: + if s.hTitleCnt == 2: if n.level == 1: # it's the subtitle n.level = 0 elif n.level >= 2: # normal sections n.level -= 1 - elif p.s.hTitleCnt == 0: + elif s.hTitleCnt == 0: n.level += 1 of rnRef: let refn = rstnodeToRefname(n) - var y = findRef(p, refn) + var y = findRef(s, refn) if y != nil: result = newRstNode(rnHyperlink) let text = newRstNode(rnInner, n.sons) result.sons = @[text, y] else: - let s = findMainAnchor(p, refn) + let s = findMainAnchor(s, refn) if s != "": result = newRstNode(rnInternalRef) let text = newRstNode(rnInner, n.sons) @@ -2853,16 +2877,16 @@ proc resolveSubs(p: var RstParser, n: PRstNode): PRstNode = of fnAutoNumberLabel, fnAutoNumber: if fnType == fnAutoNumberLabel: let labelR = rstnodeToRefname(n.sons[0]) - num = getFootnoteNum(p, labelR) + num = getFootnoteNum(s, labelR) else: - num = getFootnoteNum(p, n.order) + num = getFootnoteNum(s, n.order) var nn = newRstNode(rnInner) nn.add newLeaf($num) result.sons[0] = nn of fnAutoSymbol: - let sym = getAutoSymbol(p, n.order) + let sym = getAutoSymbol(s, n.order) n.sons[0].sons[0].text = sym - n.sons[1] = resolveSubs(p, n.sons[1]) + n.sons[1] = resolveSubs(s, n.sons[1]) of rnFootnoteRef: var (fnType, num) = getFootnoteType(n.sons[0]) template addLabel(number: int | string) = @@ -2877,31 +2901,31 @@ proc resolveSubs(p: var RstParser, n: PRstNode): PRstNode = addLabel num refn.add $num of fnAutoNumber: - addLabel getFootnoteNum(p, n.order) + addLabel getFootnoteNum(s, n.order) refn.add $n.order of fnAutoNumberLabel: - addLabel getFootnoteNum(p, rstnodeToRefname(n)) + addLabel getFootnoteNum(s, rstnodeToRefname(n)) refn.add rstnodeToRefname(n) of fnAutoSymbol: - addLabel getAutoSymbol(p, n.order) + addLabel getAutoSymbol(s, n.order) refn.add $n.order of fnCitation: result.add n.sons[0] refn.add rstnodeToRefname(n) - let s = findMainAnchor(p, refn) - if s != "": - result.add newLeaf(s) # add link + let anch = findMainAnchor(s, refn) + if anch != "": + result.add newLeaf(anch) # add link else: - rstMessage(p, mwUnknownSubstitution, refn) + rstMessage(s, mwUnknownSubstitution, refn) result.add newLeaf(refn) # add link of rnLeaf: discard of rnContents: - p.hasToc = true + s.hasToc = true else: var regroup = false for i in 0 ..< n.len: - n.sons[i] = resolveSubs(p, n.sons[i]) + n.sons[i] = resolveSubs(s, n.sons[i]) if n.sons[i] != nil and n.sons[i].kind == rnFootnote: regroup = true if regroup: # group footnotes together into rnFootnoteGroup @@ -2924,13 +2948,10 @@ proc rstParse*(text, filename: string, options: RstParseOptions, findFile: FindFileHandler = nil, msgHandler: MsgHandler = nil): PRstNode = - var p: RstParser - initParser(p, newSharedState(options, findFile, msgHandler)) - p.filename = filename - p.line = line - p.col = column + getTokens(text, p.tok) - let unresolved = parseDoc(p) - countTitles(p, unresolved) - orderFootnotes(p) - result = resolveSubs(p, unresolved) - hasToc = p.hasToc + ## Parses the whole `text`. The result is ready for `rstgen.renderRstToOut`. + var sharedState = newRstSharedState(options, filename, findFile, msgHandler) + let unresolved = rstParsePass1(text, filename, line, column, + options, sharedState) + preparePass2(sharedState, unresolved) + result = resolveSubs(sharedState, unresolved) + hasToc = sharedState.hasToc diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.idx b/nimdoc/testproject/expected/subdir/subdir_b/utils.idx index b49a777c8..6dc3953af 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.idx +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.idx @@ -1,8 +1,3 @@ -This is now a header subdir/subdir_b/utils.html#this-is-now-a-header This is now a header -Next header subdir/subdir_b/utils.html#this-is-now-a-header-next-header Next header -And so on subdir/subdir_b/utils.html#next-header-and-so-on And so on -More headers subdir/subdir_b/utils.html#more-headers More headers -Up to level 6 subdir/subdir_b/utils.html#more-headers-up-to-level-6 Up to level 6 enumValueA subdir/subdir_b/utils.html#enumValueA SomeType.enumValueA enumValueB subdir/subdir_b/utils.html#enumValueB SomeType.enumValueB enumValueC subdir/subdir_b/utils.html#enumValueC SomeType.enumValueC @@ -11,3 +6,8 @@ someType subdir/subdir_b/utils.html#someType_2 utils: someType(): SomeType aEnum subdir/subdir_b/utils.html#aEnum.t utils: aEnum(): untyped bEnum subdir/subdir_b/utils.html#bEnum.t utils: bEnum(): untyped fromUtilsGen subdir/subdir_b/utils.html#fromUtilsGen.t utils: fromUtilsGen(): untyped +This is now a header subdir/subdir_b/utils.html#this-is-now-a-header This is now a header +Next header subdir/subdir_b/utils.html#this-is-now-a-header-next-header Next header +And so on subdir/subdir_b/utils.html#next-header-and-so-on And so on +More headers subdir/subdir_b/utils.html#more-headers More headers +Up to level 6 subdir/subdir_b/utils.html#more-headers-up-to-level-6 Up to level 6 diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index 34b172935..300229648 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -997,7 +997,7 @@ Test1 """ var error5 = new string let output5 = input5.toHtml(error=error5) - check(error5[] == "input(6, 1) Error: mismatch in number of footnotes " & + check(error5[] == "input(1, 1) Error: mismatch in number of footnotes " & "and their refs: 1 (lines 2) != 0 (lines ) for auto-numbered " & "footnotes") @@ -1011,7 +1011,7 @@ Test1 """ var error6 = new string let output6 = input6.toHtml(error=error6) - check(error6[] == "input(6, 1) Error: mismatch in number of footnotes " & + check(error6[] == "input(1, 1) Error: mismatch in number of footnotes " & "and their refs: 1 (lines 3) != 2 (lines 2, 6) for auto-symbol " & "footnotes") @@ -1034,7 +1034,8 @@ Test1 """ var warnings8 = new seq[string] let output8 = input8.toHtml(warnings=warnings8) - check(warnings8[] == @["input(4, 1) Warning: unknown substitution " & + # TODO: the line 1 is arbitrary because reference lines are not preserved + check(warnings8[] == @["input(1, 1) Warning: unknown substitution " & "\'citation-som\'"]) # check that footnote group does not break parsing of other directives: From 6030e139b5c630b5508dbe12d7b58c56264212b1 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 20 Jun 2021 00:51:07 -0700 Subject: [PATCH 0948/2472] move {.injectStmt.} to experimental; add a test (#18300) * move {.injectStmt.} to experimental; add a test * undocument and deprecat `.injectStmt` but keep its implementation until we have a replacement --- compiler/pragmas.nim | 5 ++-- doc/manual.rst | 11 -------- doc/manual_experimental.rst | 1 + lib/pure/asyncdispatch.nim | 2 -- lib/system/gc_ms.nim | 1 + tests/pragmas/tinjectstmt.nim | 48 +++++++++++++++++++++++++++++++++++ 6 files changed, 53 insertions(+), 15 deletions(-) create mode 100644 tests/pragmas/tinjectstmt.nim diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 695e595c3..8ae931d84 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -1183,6 +1183,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, if sym == nil: invalidPragma(c, it) else: magicsys.registerNimScriptSymbol(c.graph, sym) of wInjectStmt: + warningDeprecated(c.config, it.info, "'.injectStmt' pragma is deprecated") if it.kind notin nkPragmaCallKinds or it.len != 2: localError(c.config, it.info, "expression expected") else: @@ -1194,10 +1195,10 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, of wThis: if it.kind in nkPragmaCallKinds and it.len == 2: c.selfName = considerQuotedIdent(c, it[1]) - message(c.config, n.info, warnDeprecated, "the '.this' pragma is deprecated") + message(c.config, n.info, warnDeprecated, "'.this' pragma is deprecated") elif it.kind == nkIdent or it.len == 1: c.selfName = getIdent(c.cache, "self") - message(c.config, n.info, warnDeprecated, "the '.this' pragma is deprecated") + message(c.config, n.info, warnDeprecated, "'.this' pragma is deprecated") else: localError(c.config, it.info, "'this' pragma is allowed to have zero or one arguments") of wNoRewrite: diff --git a/doc/manual.rst b/doc/manual.rst index 46a40a690..e8eec9a57 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -7432,17 +7432,6 @@ work properly (in particular regarding constructor and destructor) for proc main()= var a {.threadvar.}: Foo -InjectStmt pragma ------------------ - -The `injectStmt` pragma can be used to inject a statement before every -other statement in the current module. It is only supposed to be used for -debugging: - -.. code-block:: nim - {.injectStmt: gcInvariants().} - - # ... complex code here that produces crashes ... compile-time define pragmas --------------------------- diff --git a/doc/manual_experimental.rst b/doc/manual_experimental.rst index 3157708da..d10ef8d1b 100644 --- a/doc/manual_experimental.rst +++ b/doc/manual_experimental.rst @@ -380,6 +380,7 @@ pass multiple blocks to a macro: # code to undo it + Special Operators ================= diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 3866ebd24..b88d5ede8 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -176,8 +176,6 @@ export Port, SocketFlag export asyncfutures except callSoon export asyncstreams -#{.injectStmt: newGcInvariant().} - # TODO: Check if yielded future is nil and throw a more meaningful exception type diff --git a/lib/system/gc_ms.nim b/lib/system/gc_ms.nim index c20e93699..9e306c497 100644 --- a/lib/system/gc_ms.nim +++ b/lib/system/gc_ms.nim @@ -429,6 +429,7 @@ proc sweep(gch: var GcHeap) = else: freeCyclicCell(gch, c) when false: + # meant to be used with the now-deprected `.injectStmt`: {.injectStmt: newGcInvariant().} proc newGcInvariant*() = for x in allObjects(gch.region): if isCell(x): diff --git a/tests/pragmas/tinjectstmt.nim b/tests/pragmas/tinjectstmt.nim new file mode 100644 index 000000000..bca041e46 --- /dev/null +++ b/tests/pragmas/tinjectstmt.nim @@ -0,0 +1,48 @@ +discard """ + joinable: false + output:''' +onInject: 1 +onInject: 2 +ok0 +ok1 +onInject: 3 +onInject: 4 +0 +onInject: 5 +onInject: 6 +1 +onInject: 7 +onInject: 8 +2 +ok2 +onInject: 9 +''' +""" + +# test {.injectStmt.} + +#[ +{.injectStmt.} pragma can be used to inject a statement before every +other statement in the current module. It's now undocumented and may be removed +in the future and replaced with something more general and without its limitations. +e.g. (e.g. doesn't work in VM or js backends). +]# + +from system/ansi_c import c_printf + +var count = 0 +proc onInject*() = + count.inc + # echo count # xxx would fail, probably infinite recursion + c_printf("onInject: %d\n", cast[int](count)) + +{.injectStmt: onInject().} +echo "ok0" +proc main()= + echo "ok1" + for a in 0..<3: + echo a + echo "ok2" + +static: main() # xxx injectStmt not honred in VM +main() From e45b858f0402bbae428bdfa86f72613b69c2671a Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 20 Jun 2021 15:57:59 +0800 Subject: [PATCH 0949/2472] [std/terminal] improve docs a bit (#18296) * Revert "add missing import to asynchttpserver's example" This reverts commit 7ef364a402d3d827f10c893280f8dc7b9ef056f5. * alternative to #18185 * add std/mutexes * cvlose #17696 * Revert "add std/mutexes" This reverts commit 69abc8b64954206da6ffe5fc40a1142b39777762. * tiny * test * improve terminal docs * follow advice --- lib/pure/terminal.nim | 115 +++++++++++++++++------------------------- 1 file changed, 45 insertions(+), 70 deletions(-) diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index cada72196..283652bb3 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -16,6 +16,34 @@ ## Similarly, if you hide the cursor, make sure to unhide it with ## `showCursor` before quitting. +##[ +## Playing with colorful and styled text +]## + +## Procs like `styledWriteLine`, `styledEcho` etc. have a temporary effect on +## text parameters. Style parameters only affect the text parameter right after them. +## After being called, these procs will reset the default style of the terminal. +## While `setBackGroundColor`, `setForeGroundColor` etc. have a lasting +## influence on the terminal, you can use `resetAttributes` to +## reset the default style of the terminal. +runnableExamples("-r:off"): + stdout.styledWriteLine({styleBright, styleBlink, styleUnderscore}, "styled text ") + stdout.styledWriteLine(fgRed, "red text ") + stdout.styledWriteLine(fgWhite, bgRed, "white text in red background") + stdout.styledWriteLine(" ordinary text without style ") + + stdout.setBackGroundColor(bgCyan, true) + stdout.setForeGroundColor(fgBlue) + stdout.write("blue text in cyan background") + stdout.resetAttributes() + + # You can specify multiple text parameters. Style parameters + # only affect the text parameter right after them. + styledEcho styleBright, fgGreen, "[PASS]", resetStyle, fgGreen, " Yay!" + + stdout.styledWriteLine(fgRed, "red text ", styleBright, "bold red", fgDefault, " bold text") + + import macros import strformat from strutils import toLowerAscii, `%` @@ -418,6 +446,9 @@ else: proc eraseLine*(f: File) = ## Erases the entire current line. + runnableExamples("-r:off"): + write(stdout, "never mind") + stdout.eraseLine() # nothing will be printed on the screen when defined(windows): let h = conHandle(f) var scrbuf: CONSOLE_SCREEN_BUFFER_INFO @@ -480,7 +511,7 @@ proc resetAttributes*(f: File) = gBG = 0 type - Style* = enum ## different styles for text output + Style* = enum ## Different styles for text output. styleBright = 1, ## bright text styleDim, ## dim text styleItalic, ## italic (or reverse on terminals not supporting) @@ -534,7 +565,7 @@ proc writeStyled*(txt: string, style: set[Style] = {styleBright}) = stdout.write(ansiStyleCode(gBG)) type - ForegroundColor* = enum ## terminal's foreground colors + ForegroundColor* = enum ## Terminal's foreground colors. fgBlack = 30, ## black fgRed, ## red fgGreen, ## green @@ -546,7 +577,7 @@ type fg8Bit, ## 256-color (not supported, see `enableTrueColors` instead.) fgDefault ## default terminal foreground color - BackgroundColor* = enum ## terminal's background colors + BackgroundColor* = enum ## Terminal's background colors. bgBlack = 40, ## black bgRed, ## red bgGreen, ## green @@ -701,14 +732,10 @@ macro styledWrite*(f: File, m: varargs[typed]): untyped = ## When some argument is `Style`, `set[Style]`, `ForegroundColor`, ## `BackgroundColor` or `TerminalCmd` then it is not sent directly to ## `f`, but instead corresponding terminal style proc is called. - ## - ## Example: - ## - ## .. code-block:: nim - ## - ## stdout.styledWrite(fgRed, "red text ") - ## stdout.styledWrite(fgGreen, "green text") - ## + runnableExamples("-r:off"): + stdout.styledWrite(fgRed, "red text ") + stdout.styledWrite(fgGreen, "green text") + var reset = false result = newNimNode(nnkStmtList) @@ -731,14 +758,10 @@ macro styledWrite*(f: File, m: varargs[typed]): untyped = template styledWriteLine*(f: File, args: varargs[untyped]) = ## Calls `styledWrite` and appends a newline at the end. - ## - ## Example: - ## - ## .. code-block:: nim - ## - ## proc error(msg: string) = - ## styledWriteLine(stderr, fgRed, "Error: ", resetStyle, msg) - ## + runnableExamples: + proc error(msg: string) = + styledWriteLine(stderr, fgRed, "Error: ", resetStyle, msg) + styledWrite(f, args) write(f, "\n") @@ -747,7 +770,7 @@ template styledEcho*(args: varargs[untyped]) = stdout.styledWriteLine(args) proc getch*(): char = - ## Read a single character from the terminal, blocking until it is entered. + ## Reads a single character from the terminal, blocking until it is entered. ## The character is not printed to the terminal. when defined(windows): let fd = getStdHandle(STD_INPUT_HANDLE) @@ -852,7 +875,7 @@ when defined(windows): import os proc enableTrueColors*() = - ## Enable true color. + ## Enables true color. var term = getTerminal() when defined(windows): var @@ -885,7 +908,7 @@ proc enableTrueColors*() = term.trueColorIsEnabled = term.trueColorIsSupported proc disableTrueColors*() = - ## Disable true color. + ## Disables true color. var term = getTerminal() when defined(windows): if term.trueColorIsSupported: @@ -902,51 +925,3 @@ proc newTerminal(): owned(PTerminal) = new result when defined(windows): initTerminal(result) - -when not defined(testing) and isMainModule: - assert ansiStyleCode(styleBright) == "\e[1m" - assert ansiStyleCode(styleStrikethrough) == "\e[9m" - # exitprocs.addExitProc(resetAttributes) - write(stdout, "never mind") - stdout.eraseLine() - stdout.styledWriteLine({styleBright, styleBlink, styleUnderscore}, "styled text ") - stdout.styledWriteLine("italic text ", {styleItalic}) - stdout.setBackGroundColor(bgCyan, true) - stdout.setForeGroundColor(fgBlue) - stdout.write("blue text in cyan background") - stdout.resetAttributes() - echo "" - stdout.writeLine("ordinary text") - echo "more ordinary text" - styledEcho styleBright, fgGreen, "[PASS]", resetStyle, fgGreen, " Yay!" - echo "ordinary text again" - styledEcho styleBright, fgRed, "[FAIL]", resetStyle, fgRed, " Nay :(" - echo "ordinary text again" - setForeGroundColor(fgGreen) - echo "green text" - echo "more green text" - setForeGroundColor(fgBlue) - echo "blue text" - resetAttributes() - echo "ordinary text" - - stdout.styledWriteLine(fgRed, "red text ") - stdout.styledWriteLine(fgWhite, bgRed, "white text in red background") - stdout.styledWriteLine(" ordinary text ") - stdout.styledWriteLine(fgGreen, "green text") - - writeStyled("underscored text", {styleUnderscore}) - stdout.styledWrite(fgRed, " red text ") - writeStyled("bright text ", {styleBright}) - echo "ordinary text" - - stdout.styledWrite(fgRed, "red text ") - stdout.styledWrite(fgWhite, bgRed, "white text in red background") - stdout.styledWrite(" ordinary text ") - stdout.styledWrite(fgGreen, "green text") - echo "" - echo "ordinary text" - stdout.styledWriteLine(fgRed, "red text ", styleBright, "bold red", fgDefault, " bold text") - stdout.styledWriteLine(bgYellow, "text in yellow bg", styleBright, - " bold text in yellow bg", bgDefault, " bold text") - echo "ordinary text" From eb0b323f452f5f06d74a4f747c50ae5115e44eae Mon Sep 17 00:00:00 2001 From: Antonis Geralis <43617260+planetis-m@users.noreply.github.com> Date: Sun, 20 Jun 2021 15:51:13 +0300 Subject: [PATCH 0950/2472] Revert "disable pkg manu (#18292)" (#18307) This reverts commit fc76565574b86566e4a642b9aac541e025cf5de3. --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index efee361da..3d3c340eb 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -82,7 +82,7 @@ pkg "karax", "nim c -r tests/tester.nim" pkg "kdtree", "nimble test -d:nimLegacyRandomInitRand", "https://github.com/jblindsay/kdtree" pkg "loopfusion" pkg "macroutils" -pkg "manu", allowFailure=true # pending https://github.com/planetis-m/manu/issues/5 +pkg "manu" pkg "markdown" pkg "memo" pkg "msgpack4nim", "nim c -r tests/test_spec.nim" From 92cb765714325703d1a8e1b0409bfcd1b17af103 Mon Sep 17 00:00:00 2001 From: Heiko Nickerl Date: Sun, 20 Jun 2021 17:56:33 +0200 Subject: [PATCH 0951/2472] Raise IndexDefect when deleting element at out of bounds index (#17821) Co-authored-by: Timothee Cour Co-authored-by: Heiko Nickerl Co-authored-by: Heiko Nickerl --- changelog.md | 3 ++ lib/system.nim | 58 ++++++++++++++++++++--------------- tests/gc/gcleak3.nim | 8 ++--- tests/stdlib/tsystem.nim | 32 ++++++++++++++++++- tests/system/tsystem_misc.nim | 5 --- 5 files changed, 70 insertions(+), 36 deletions(-) diff --git a/changelog.md b/changelog.md index 80e38f420..5af195e59 100644 --- a/changelog.md +++ b/changelog.md @@ -28,6 +28,9 @@ - `math.round` now is rounded "away from zero" in JS backend which is consistent with other backends. See #9125. Use `-d:nimLegacyJsRound` for previous behavior. +- Instead of deleting the element at the last index, + `system.delete()` now raises `IndexDefect` when given index is out of bounds. + - Changed the behavior of `uri.decodeQuery` when there are unencoded `=` characters in the decoded values. Prior versions would raise an error. This is no longer the case to comply with the HTML spec and other languages diff --git a/lib/system.nim b/lib/system.nim index cae0ec3e2..a4608997a 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1325,30 +1325,6 @@ proc del*[T](x: var seq[T], i: Natural) {.noSideEffect.} = movingCopy(x[i], x[xl]) setLen(x, xl) -proc delete*[T](x: var seq[T], i: Natural) {.noSideEffect.} = - ## Deletes the item at index `i` by moving all `x[i+1..]` items by one position. - ## - ## This is an `O(n)` operation. - ## - ## See also: - ## * `del <#del,seq[T],Natural>`_ for O(1) operation - ## - ## .. code-block:: Nim - ## var i = @[1, 2, 3, 4, 5] - ## i.delete(2) # => @[1, 2, 4, 5] - template defaultImpl = - let xl = x.len - for j in i.int..xl-2: movingCopy(x[j], x[j+1]) - setLen(x, xl-1) - - when nimvm: - defaultImpl() - else: - when defined(js): - {.emit: "`x`.splice(`i`, 1);".} - else: - defaultImpl() - proc insert*[T](x: var seq[T], item: sink T, i = 0.Natural) {.noSideEffect.} = ## Inserts `item` into `x` at position `i`. ## @@ -2154,6 +2130,40 @@ const import system/dollars export dollars +proc delete*[T](x: var seq[T], i: Natural) {.noSideEffect.} = + ## Deletes the item at index `i` by moving all `x[i+1..^1]` items by one position. + ## + ## This is an `O(n)` operation. + ## + ## See also: + ## * `del <#del,seq[T],Natural>`_ for O(1) operation + ## + runnableExamples: + var s = @[1, 2, 3, 4, 5] + s.delete(2) + doAssert s == @[1, 2, 4, 5] + + doAssertRaises(IndexDefect): + s.delete(4) + + if i > high(x): + # xxx this should call `raiseIndexError2(i, high(x))` after some refactoring + raise (ref IndexDefect)(msg: "index out of bounds: '" & $i & "' < '" & $x.len & "' failed") + + template defaultImpl = + let xl = x.len + for j in i.int..xl-2: movingCopy(x[j], x[j+1]) + setLen(x, xl-1) + + when nimvm: + defaultImpl() + else: + when defined(js): + {.emit: "`x`.splice(`i`, 1);".} + else: + defaultImpl() + + const NimVersion*: string = $NimMajor & "." & $NimMinor & "." & $NimPatch ## is the version of Nim as a string. diff --git a/tests/gc/gcleak3.nim b/tests/gc/gcleak3.nim index 588e238e9..5e146d69f 100644 --- a/tests/gc/gcleak3.nim +++ b/tests/gc/gcleak3.nim @@ -17,14 +17,10 @@ for i in 0..1024: s.add(obj) proc limit*[t](a: var seq[t]) = - var loop = s.len() - 512 - for i in 0..loop: - #echo i - #GC_fullCollect() + while s.len > 0: if getOccupiedMem() > 3000_000: quit("still a leak!") - s.delete(i) + s.delete(0) s.limit() - echo "no leak: ", getOccupiedMem() diff --git a/tests/stdlib/tsystem.nim b/tests/stdlib/tsystem.nim index f8b172b44..b48034043 100644 --- a/tests/stdlib/tsystem.nim +++ b/tests/stdlib/tsystem.nim @@ -2,9 +2,10 @@ discard """ targets: "c cpp js" """ +import stdtest/testutils + # TODO: in future work move existing `system` tests here, where they belong -import stdtest/testutils template main = block: # closure @@ -42,5 +43,34 @@ template main = # doAssert rawEnv(inner3) != rawEnv(inner1) # because `a` vs `b` # this doesn't hold outer() + block: # system.delete + block: + var s = @[1] + s.delete(0) + doAssert s == @[] + + block: + var s = @["foo", "bar"] + s.delete(1) + doAssert s == @["foo"] + + block: + var s: seq[string] + doAssertRaises(IndexDefect): + s.delete(0) + + block: + doAssert not compiles(@["foo"].delete(-1)) + + block: # bug #6710 + var s = @["foo"] + s.delete(0) + doAssert s == @[] + + block: # bug #16544: deleting out of bounds index should raise + var s = @["foo"] + doAssertRaises(IndexDefect): + s.delete(1) + static: main() main() diff --git a/tests/system/tsystem_misc.nim b/tests/system/tsystem_misc.nim index cb879d3b3..13b8c18a9 100644 --- a/tests/system/tsystem_misc.nim +++ b/tests/system/tsystem_misc.nim @@ -59,11 +59,6 @@ doAssert high(float) > low(float) doAssert high(float32) > low(float32) doAssert high(float64) > low(float64) -# bug #6710 -var s = @[1] -s.delete(0) - - proc foo(a: openArray[int]) = for x in a: echo x From 7d5e6b0169fb83c7682bfad5d675c21db785c324 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 20 Jun 2021 10:52:22 -0700 Subject: [PATCH 0952/2472] support `--hint:all:off --hint:x` (ditto with `--warnings` + friends) (#17852) * select all hints via `--hint:all:on|off`, etc * simplify code with setutils * address comment --- changelog.md | 4 +++ compiler/commands.nim | 59 +++++++++++++++++++++--------------------- compiler/condsyms.nim | 2 +- doc/advopt.txt | 15 ++++++----- tests/misc/trunner.nim | 9 +++---- 5 files changed, 45 insertions(+), 44 deletions(-) diff --git a/changelog.md b/changelog.md index 5af195e59..570aff542 100644 --- a/changelog.md +++ b/changelog.md @@ -429,6 +429,10 @@ - `--hint:CC` now goes to stderr (like all other hints) instead of stdout. +- `--hint:all:on|off` is now supported to select or deselect all hints; it + differs from `--hints:on|off` which acts as a (reversible) gate. + Likewise with `--warning:all:on|off`. + - json build instructions are now generated in `$nimcache/outFileBasename.json` instead of `$nimcache/projectName.json`. This allows avoiding recompiling a given project compiled with different options if the output file differs. diff --git a/compiler/commands.nim b/compiler/commands.nim index 8f8df71a7..4c1db8ead 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -25,10 +25,10 @@ bootSwitch(usedMarkAndSweep, defined(gcmarkandsweep), "--gc:markAndSweep") bootSwitch(usedGoGC, defined(gogc), "--gc:go") bootSwitch(usedNoGC, defined(nogc), "--gc:none") +import std/[setutils, os, strutils, parseutils, parseopt, sequtils, strtabs] import - os, msgs, options, nversion, condsyms, strutils, extccomp, platform, - wordrecg, parseutils, nimblecmd, parseopt, sequtils, lineinfos, - pathutils, strtabs, pathnorm + msgs, options, nversion, condsyms, extccomp, platform, + wordrecg, nimblecmd, lineinfos, pathutils, pathnorm from ast import eqTypeFlags, tfGcSafe, tfNoSideEffect @@ -182,7 +182,7 @@ proc processSpecificNote*(arg: string, state: TSpecialWord, pass: TCmdLinePass, info: TLineInfo; orig: string; conf: ConfigRef) = var id = "" # arg = key or [key] or key:val or [key]:val; with val=on|off var i = 0 - var n = hintMin + var notes: set[TMsgKind] var isBracket = false if i < arg.len and arg[i] == '[': isBracket = true @@ -197,37 +197,36 @@ proc processSpecificNote*(arg: string, state: TSpecialWord, pass: TCmdLinePass, if i == arg.len: discard elif i < arg.len and (arg[i] in {':', '='}): inc(i) else: invalidCmdLineOption(conf, pass, orig, info) - # unfortunately, hintUser and warningUser clash - if state in {wHint, wHintAsError}: - let x = findStr(hintMin, hintMax, id, errUnknown) - if x != errUnknown: n = TNoteKind(x) - else: localError(conf, info, "unknown hint: " & id) - else: - let x = findStr(warnMin, warnMax, id, errUnknown) - if x != errUnknown: n = TNoteKind(x) - else: localError(conf, info, "unknown warning: " & id) + let isSomeHint = state in {wHint, wHintAsError} + template findNote(noteMin, noteMax, name) = + # unfortunately, hintUser and warningUser clash, otherwise implementation would simplify a bit + let x = findStr(noteMin, noteMax, id, errUnknown) + if x != errUnknown: notes = {TNoteKind(x)} + else: localError(conf, info, "unknown $#: $#" % [name, id]) + case id.normalize + of "all": # other note groups would be easy to support via additional cases + notes = if isSomeHint: {hintMin..hintMax} else: {warnMin..warnMax} + elif isSomeHint: findNote(hintMin, hintMax, "hint") + else: findNote(warnMin, warnMax, "warning") var val = substr(arg, i).normalize if val == "": val = "on" if val notin ["on", "off"]: + # xxx in future work we should also allow users to have control over `foreignPackageNotes` + # so that they can enable `hints|warnings|warningAsErrors` for all the code they depend on. localError(conf, info, errOnOrOffExpectedButXFound % arg) - elif n notin conf.cmdlineNotes or pass == passCmd1: - if pass == passCmd1: incl(conf.cmdlineNotes, n) - incl(conf.modifiedyNotes, n) - case val - of "on": - if state in {wWarningAsError, wHintAsError}: - incl(conf.warningAsErrors, n) # xxx rename warningAsErrors to noteAsErrors - else: - incl(conf.notes, n) - incl(conf.mainPackageNotes, n) - of "off": - if state in {wWarningAsError, wHintAsError}: - excl(conf.warningAsErrors, n) - else: - excl(conf.notes, n) - excl(conf.mainPackageNotes, n) - excl(conf.foreignPackageNotes, n) + else: + let isOn = val == "on" + for n in notes: + if n notin conf.cmdlineNotes or pass == passCmd1: + if pass == passCmd1: incl(conf.cmdlineNotes, n) + incl(conf.modifiedyNotes, n) + if state in {wWarningAsError, wHintAsError}: + conf.warningAsErrors[n] = isOn # xxx rename warningAsErrors to noteAsErrors + else: + conf.notes[n] = isOn + conf.mainPackageNotes[n] = isOn + if not isOn: excl(conf.foreignPackageNotes, n) proc processCompile(conf: ConfigRef; filename: string) = var found = findFile(conf, filename) diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index ff0c603cb..b3f8465f0 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -134,5 +134,5 @@ proc initDefines*(symbols: StringTableRef) = defineSymbol("nimHasUnifiedTuple") defineSymbol("nimHasIterable") defineSymbol("nimHasTypeofVoid") - defineSymbol("nimHasDragonBox") + defineSymbol("nimHasHintAll") diff --git a/doc/advopt.txt b/doc/advopt.txt index e5b4c9568..b56649a86 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -48,14 +48,15 @@ Advanced options: --spellSuggest|:num show at most `num >= 0` spelling suggestions on typos. if `num` is not specified (or `auto`), return an implementation defined set of suggestions. - -w:on|off|list, --warnings:on|off|list - turn all warnings on|off or list all available - --warning:X:on|off turn specific warning X on|off. - `warning:X` means `warning:X:on`, as with similar flags. - --hints:on|off|list turn all hints on|off or list all available - --hint:X:on|off turn specific hint X on|off. - --warningAsError:X:on|off turn specific warning X into an error on|off + --hints:on|off|list. `on|off` enables or disables hints. + `list` reports which hints are selected. + --hint:X:on|off turn specific hint X on|off. `hint:X` means `hint:X:on`, + as with similar flags. `all` can be used for "all hints". --hintAsError:X:on|off turn specific hint X into an error on|off + -w:on|off|list, --warnings:on|off|list + same as `--hints` but for warnings. + --warning:X:on|off ditto + --warningAsError:X:on|off ditto --styleCheck:off|hint|error produce hints or errors for Nim identifiers that do not adhere to Nim's official style guide diff --git a/tests/misc/trunner.nim b/tests/misc/trunner.nim index f2ea1d040..77ae10f3d 100644 --- a/tests/misc/trunner.nim +++ b/tests/misc/trunner.nim @@ -23,9 +23,6 @@ proc isDots(a: string): bool = a.startsWith(".") and a.strip(chars = {'.'}) == "" const - defaultHintsOff = "--hint:successx:off --hint:exec:off --hint:link:off --hint:cc:off --hint:conf:off --hint:processing:off --hint:QuitCalled:off" - # useful when you want to turn only some hints on, and some common ones off. - # pending https://github.com/timotheecour/Nim/issues/453, simplify to: `--hints:off` nim = getCurrentCompilerExe() mode = querySetting(backend) nimcache = buildDir / "nimcacheTrunner" @@ -236,7 +233,7 @@ sub/mmain.idx""", context check execCmdEx(cmd) == ("witness\n", 0) block: # config.nims, nim.cfg, hintConf, bug #16557 - let cmd = fmt"{nim} r {defaultHintsOff} --hint:conf tests/newconfig/bar/mfoo.nim" + let cmd = fmt"{nim} r --hint:all:off --hint:conf tests/newconfig/bar/mfoo.nim" let (outp, exitCode) = execCmdEx(cmd, options = {poStdErrToStdOut}) doAssert exitCode == 0 let dir = getCurrentDir() @@ -268,7 +265,7 @@ tests/newconfig/bar/mfoo.nims""".splitLines check fmt"""{nim} r -b:js {opt} --eval:"echo defined(js)"""".execCmdEx == ("true\n", 0) block: # `hintProcessing` dots should not interfere with `static: echo` + friends - let cmd = fmt"""{nim} r {defaultHintsOff} --hint:processing -f --eval:"static: echo 1+1"""" + let cmd = fmt"""{nim} r --hint:all:off --hint:processing -f --eval:"static: echo 1+1"""" let (outp, exitCode) = execCmdEx(cmd, options = {poStdErrToStdOut}) template check3(cond) = doAssert cond, $(outp,) doAssert exitCode == 0 @@ -282,7 +279,7 @@ tests/newconfig/bar/mfoo.nims""".splitLines check3 "2" in outp block: # nim secret - let opt = fmt"{defaultHintsOff} --hint:processing" + let opt = "--hint:all:off --hint:processing" template check3(cond) = doAssert cond, $(outp,) for extra in ["", "--stdout"]: let cmd = fmt"""{nim} secret {opt} {extra}""" From 1b3c0f142dd5abc3b3509feb194f54e6acf046c0 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Sun, 20 Jun 2021 21:00:42 +0300 Subject: [PATCH 0953/2472] validate rst field for :number-lines:, :status: (#18304) --- compiler/docgen.nim | 1 + compiler/lineinfos.nim | 2 ++ lib/packages/docutils/rst.nim | 1 + lib/packages/docutils/rstgen.nim | 30 ++++++++++++++++++++++-------- tests/stdlib/trstgen.nim | 12 ++++++++++++ 5 files changed, 38 insertions(+), 8 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 2c99adba3..08311eade 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -164,6 +164,7 @@ template declareClosures = of meNewSectionExpected: k = errNewSectionExpected of meGeneralParseError: k = errGeneralParseError of meInvalidDirective: k = errInvalidDirectiveX + of meInvalidRstField: k = errInvalidRstField of meFootnoteMismatch: k = errFootnoteMismatch of mwRedefinitionOfLabel: k = warnRedefinitionOfLabel of mwUnknownSubstitution: k = warnUnknownSubstitutionX diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index 052dae272..abe6bce7c 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -37,6 +37,7 @@ type errGeneralParseError, errNewSectionExpected, errInvalidDirectiveX, + errInvalidRstField, errFootnoteMismatch, errProveInit, # deadcode errGenerated, @@ -96,6 +97,7 @@ const errGeneralParseError: "general parse error", errNewSectionExpected: "new section expected $1", errInvalidDirectiveX: "invalid directive: '$1'", + errInvalidRstField: "invalid field: $1", errFootnoteMismatch: "number of footnotes and their references don't match: $1", errProveInit: "Cannot prove that '$1' is initialized.", # deadcode errGenerated: "$1", diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 031a86d98..64910aa19 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -225,6 +225,7 @@ type meNewSectionExpected = "new section expected $1", meGeneralParseError = "general parse error", meInvalidDirective = "invalid directive: '$1'", + meInvalidRstField = "invalid field: $1", meFootnoteMismatch = "mismatch in number of footnotes and their refs: $1", mwRedefinitionOfLabel = "redefinition of label '$1'", mwUnknownSubstitution = "unknown substitution '$1'", diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index e3f889283..829177263 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -907,6 +907,25 @@ proc renderSmiley(d: PDoc, n: PRstNode, result: var string) = "\\includegraphics{$1}", [d.config.getOrDefault"doc.smiley_format" % n.text]) +proc getField1Int(d: PDoc, n: PRstNode, fieldName: string): int = + # TODO: proper column/line info + template err(msg: string) = + d.msgHandler(d.filename, 1, 0, meInvalidRstField, msg) + let value = n.getFieldValue + var number: int + let nChars = parseInt(value, number) + if nChars == 0: + if value.len == 0: + err("field $1 requires an argument" % [fieldName]) + else: + err("field $1 requires an integer, but '$2' was given" % + [fieldName, value]) + elif nChars < value.len: + err("extra arguments were given to $1: '$2'" % + [fieldName, value[nChars..^1]]) + else: + result = number + proc parseCodeBlockField(d: PDoc, n: PRstNode, params: var CodeBlockParams) = ## Parses useful fields which can appear before a code block. ## @@ -916,9 +935,7 @@ proc parseCodeBlockField(d: PDoc, n: PRstNode, params: var CodeBlockParams) = of "number-lines": params.numberLines = true # See if the field has a parameter specifying a different line than 1. - var number: int - if parseInt(n.getFieldValue, number) > 0: - params.startLine = number + params.startLine = getField1Int(d, n, "number-lines") of "file", "filename": # The ``file`` option is a Nim extension to the official spec, it acts # like it would for other directives like ``raw`` or ``cvs-table``. This @@ -936,9 +953,7 @@ proc parseCodeBlockField(d: PDoc, n: PRstNode, params: var CodeBlockParams) = # consider whether `$docCmd` should be appended here too params.testCmd = unescape(params.testCmd) of "status", "exitcode": - var status: int - if parseInt(n.getFieldValue, status) > 0: - params.status = status + params.status = getField1Int(d, n, n.getArgument) of "default-language": params.langStr = n.getFieldValue.strip params.lang = params.langStr.getSourceLanguage @@ -954,7 +969,6 @@ proc parseCodeBlockParams(d: PDoc, n: PRstNode): CodeBlockParams = if n.isNil: return assert n.kind in {rnCodeBlock, rnInlineCode} - assert(not n.sons[2].isNil) # Parse the field list for rendering parameters if there are any. if not n.sons[1].isNil: @@ -1028,8 +1042,8 @@ proc renderCode(d: PDoc, n: PRstNode, result: var string) = ## option to differentiate between a plain code block and Nim's code block ## extension. assert n.kind in {rnCodeBlock, rnInlineCode} - if n.sons[2] == nil: return var params = d.parseCodeBlockParams(n) + if n.sons[2] == nil: return var m = n.sons[2].sons[0] assert m.kind == rnLeaf diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index 300229648..d5d902e1d 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -1136,6 +1136,18 @@ Test1 check "
    55\n
    " in output check "x" in output + test "Nim code-block indentation": + let input = dedent """ + .. code-block:: nim + :number-lines: 55 + let a = 1 + """ + var error = new string + let output = input.toHtml(error=error) + check(error[] == "input(1, 1) Error: invalid field: " & + "extra arguments were given to number-lines: ' let a = 1'") + check "" == output + test "RST admonitions": # check that all admonitions are implemented let input0 = dedent """ From 40ec8184ad51ccdaa6042132dc9b37b6f242c362 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 20 Jun 2021 13:49:14 -0700 Subject: [PATCH 0954/2472] followup #17852, disallow all:on for now (#18311) --- compiler/commands.nim | 2 ++ doc/advopt.txt | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index 4c1db8ead..1c3de29ba 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -217,6 +217,8 @@ proc processSpecificNote*(arg: string, state: TSpecialWord, pass: TCmdLinePass, localError(conf, info, errOnOrOffExpectedButXFound % arg) else: let isOn = val == "on" + if isOn and id.normalize == "all": + localError(conf, info, "only 'all:off' is supported") for n in notes: if n notin conf.cmdlineNotes or pass == passCmd1: if pass == passCmd1: incl(conf.cmdlineNotes, n) diff --git a/doc/advopt.txt b/doc/advopt.txt index b56649a86..063d018d1 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -51,7 +51,8 @@ Advanced options: --hints:on|off|list. `on|off` enables or disables hints. `list` reports which hints are selected. --hint:X:on|off turn specific hint X on|off. `hint:X` means `hint:X:on`, - as with similar flags. `all` can be used for "all hints". + as with similar flags. `all` is the set of all hints + (only `all:off` is supported). --hintAsError:X:on|off turn specific hint X into an error on|off -w:on|off|list, --warnings:on|off|list same as `--hints` but for warnings. From abd21ef213b7368253050a18e5c4907d22992c8c Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 21 Jun 2021 00:45:26 -0700 Subject: [PATCH 0955/2472] close #17403; improve docs for tuple (#18312) --- doc/manual.rst | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/doc/manual.rst b/doc/manual.rst index e8eec9a57..f80b03369 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -1616,7 +1616,7 @@ heterogeneous storage types with few abstractions. The `()` syntax can be used to construct tuples. The order of the fields in the constructor must match the order of the tuple's definition. Different tuple-types are *equivalent* if they specify the same fields of the same type in the same -order. The *names* of the fields also have to be identical. +order. The *names* of the fields also have to be the same. The assignment operator for tuples copies each component. The default assignment operator for objects copies each component. Overloading @@ -1627,15 +1627,16 @@ of the assignment operator is described `here type Person = tuple[name: string, age: int] # type representing a person: - # a person consists of a name - # and an age - var - person: Person + # it consists of a name and an age. + var person: Person person = (name: "Peter", age: 30) - echo person.name + assert person.name == "Peter" # the same, but less readable: person = ("Peter", 30) - echo person[0] + assert person[0] == "Peter" + assert Person is (string, int) + assert (string, int) is Person + assert Person isnot tuple[other: string, age: int] # `other` is a different identifier A tuple with one unnamed field can be constructed with the parentheses and a trailing comma: From ad70a65e0e3eda39a3ca074af9feadb68f10598f Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 21 Jun 2021 00:46:31 -0700 Subject: [PATCH 0956/2472] fix #18310 system.== in vm for NimNode (#18313) * fix #18310 == in vm * fixup * fixup --- compiler/vm.nim | 11 ++++++----- tests/vm/tvmmisc.nim | 11 +++++++++++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/compiler/vm.nim b/compiler/vm.nim index b540ac071..837a1c366 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -10,14 +10,15 @@ ## This file implements the new evaluation engine for Nim code. ## An instruction is 1-3 int32s in memory, it is a register based VM. -import ast except getstr import - strutils, msgs, vmdef, vmgen, nimsets, types, passes, - parser, vmdeps, idents, trees, renderer, options, transf, parseutils, - vmmarshal, gorgeimpl, lineinfos, tables, btrees, macrocacheimpl, + std/[strutils, tables, parseutils], + msgs, vmdef, vmgen, nimsets, types, passes, + parser, vmdeps, idents, trees, renderer, options, transf, + vmmarshal, gorgeimpl, lineinfos, btrees, macrocacheimpl, modulegraphs, sighashes, int128, vmprofiler +import ast except getstr from semfold import leValueConv, ordinalValToString from evaltempl import evalTemplate from magicsys import getSysType @@ -1026,7 +1027,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = let nc = regs[rc].node if nb.kind != nc.kind: discard elif (nb == nc) or (nb.kind == nkNilLit): ret = true # intentional - elif (nb.kind in {nkSym, nkTupleConstr, nkClosure} and nb.typ.kind == tyProc) and sameConstant(nb, nc): + elif nb.kind in {nkSym, nkTupleConstr, nkClosure} and nb.typ != nil and nb.typ.kind == tyProc and sameConstant(nb, nc): ret = true # this also takes care of procvar's, represented as nkTupleConstr, e.g. (nil, nil) elif nb.kind == nkIntLit and nc.kind == nkIntLit and nb.intVal == nc.intVal: # TODO: nkPtrLit diff --git a/tests/vm/tvmmisc.nim b/tests/vm/tvmmisc.nim index f0d2c2dbf..a4f6391cc 100644 --- a/tests/vm/tvmmisc.nim +++ b/tests/vm/tvmmisc.nim @@ -256,6 +256,17 @@ block: doAssert f == @[] +block: # bug #18310 + macro t() : untyped = + let + x = nnkTupleConstr.newTree(newLit(1)) + y = nnkTupleConstr.newTree(newLit(2)) + doAssert not (x == y) # not using != intentionally + doAssert not(cast[int](x) == cast[int](y)) + doAssert not(system.`==`(x, y)) + doAssert system.`==`(x, x) + t() + block: # bug #10815 type Opcode = enum From 2deb7011f5388559a70617c398360639102724c4 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 21 Jun 2021 18:50:47 +0800 Subject: [PATCH 0957/2472] fixes #17768 [backport:1.4] (#18317) * fixes #17768 [backport:1.4] * tiny --- lib/pure/unicode.nim | 10 ++++------ tests/stdlib/tunicode.nim | 9 ++++++++- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/lib/pure/unicode.nim b/lib/pure/unicode.nim index 903f01fb4..dda631633 100644 --- a/lib/pure/unicode.nim +++ b/lib/pure/unicode.nim @@ -333,7 +333,7 @@ proc runeReverseOffset*(s: string, rev: Positive): (int, int) = ## from the end (starting with 1) and the total ## number of runes in the string. ## - ## Returns a negative value for offset if there are to few runes in + ## Returns a negative value for offset if there are too few runes in ## the string to satisfy the request. ## ## **Beware:** This can lead to unoptimized code and slow execution! @@ -346,16 +346,14 @@ proc runeReverseOffset*(s: string, rev: Positive): (int, int) = a = rev.int o = 0 x = 0 + let times = 2*rev.int-s.runeLen # transformed from rev.int - a < s.runeLen - rev.int while o < s.len: let r = runeLenAt(s, o) o += r - if a < 0: + if a > times: x += r dec a - - if a > 0: - return (-a, rev.int-a) - return (x, -a+rev.int) + result = if a > 0: (-a, rev.int-a) else: (x, -a+rev.int) proc runeAtPos*(s: string, pos: int): Rune = ## Returns the rune at position ``pos``. diff --git a/tests/stdlib/tunicode.nim b/tests/stdlib/tunicode.nim index a346106f9..33405cf38 100644 --- a/tests/stdlib/tunicode.nim +++ b/tests/stdlib/tunicode.nim @@ -1,4 +1,4 @@ -import unicode +import std/unicode proc asRune(s: static[string]): Rune = @@ -213,3 +213,10 @@ block differentSizes: doAssert swapCase("ⱥbCd") == "ȺBcD" doAssert swapCase("XyꟆaB") == "xYᶎAb" doAssert swapCase("aᵹcᲈd") == "AꝽCꙊD" + +block: # bug #17768 + let s1 = "abcdef" + let s2 = "abcdéf" + + doAssert s1.runeSubstr(0, -1) == "abcde" + doAssert s2.runeSubstr(0, -1) == "abcdé" From d398c558a44ad346c2cfadf266a8cf808585de6d Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 21 Jun 2021 03:51:00 -0700 Subject: [PATCH 0958/2472] close #13196 now that we have dragonbox (#18316) --- tests/stdlib/tjson.nim | 11 +++++++++++ tests/stdlib/tjsonutils.nim | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/tests/stdlib/tjson.nim b/tests/stdlib/tjson.nim index e757e6c7e..289ef9d05 100644 --- a/tests/stdlib/tjson.nim +++ b/tests/stdlib/tjson.nim @@ -11,6 +11,8 @@ import std/[json,parsejson,strutils] from std/math import isNaN when not defined(js): import std/streams +import stdtest/testutils +from std/fenv import epsilon proc testRoundtrip[T](t: T, expected: string) = # checks that `T => json => T2 => json2` is such that json2 = json @@ -325,6 +327,15 @@ block: # bug #18007 let a = parseJson($(%NaN)).to(float) doAssert a.isNaN + whenRuntimeJs: discard # refs bug #18009 + do: + testRoundtripVal(0.0): "0.0" + testRoundtripVal(-0.0): "-0.0" + +block: # bug #15397, bug #13196 + testRoundtripVal(1.0 + epsilon(float64)): "1.0000000000000002" + testRoundtripVal(0.12345678901234567890123456789): "0.12345678901234568" + block: let a = "18446744073709551615" let b = a.parseJson diff --git a/tests/stdlib/tjsonutils.nim b/tests/stdlib/tjsonutils.nim index 88c05facf..31ec4316f 100644 --- a/tests/stdlib/tjsonutils.nim +++ b/tests/stdlib/tjsonutils.nim @@ -5,6 +5,7 @@ discard """ import std/jsonutils import std/json from std/math import isNaN, signbit +from std/fenv import epsilon from stdtest/testutils import whenRuntimeJs proc testRoundtrip[T](t: T, expected: string) = @@ -160,6 +161,16 @@ template fn() = doAssert b[2].signbit doAssert not b[3].signbit + block: # bug #15397, bug #13196 + let a = 0.1 + let x = 0.12345678901234567890123456789 + let b = (a + 0.2, 0.3, x) + testRoundtripVal(b): "[0.30000000000000004,0.3,0.12345678901234568]" + + testRoundtripVal(0.12345678901234567890123456789): "0.12345678901234568" + testRoundtripVal(epsilon(float64)): "2.220446049250313e-16" + testRoundtripVal(1.0 + epsilon(float64)): "1.0000000000000002" + block: # case object type Foo = object x0: float From 16461a88101a398fd7349eb56392b27cdb285c17 Mon Sep 17 00:00:00 2001 From: xioren <40043405+xioren@users.noreply.github.com> Date: Mon, 21 Jun 2021 18:53:35 -0700 Subject: [PATCH 0959/2472] add multi type exception catching to manual (#18258) (#18323) --- doc/manual.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/doc/manual.rst b/doc/manual.rst index f80b03369..a99c4b5af 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -4462,10 +4462,8 @@ Example: echo "sum: " & $(parseInt(a) + parseInt(b)) except OverflowDefect: echo "overflow!" - except ValueError: - echo "could not convert string to integer" - except IOError: - echo "IO error!" + except ValueError, IOError: + echo "catch multiple exceptions!" except: echo "Unknown exception!" finally: From 9d3a813fb96950631c69858e8e26b6503abd02fe Mon Sep 17 00:00:00 2001 From: xioren <40043405+xioren@users.noreply.github.com> Date: Mon, 21 Jun 2021 18:56:16 -0700 Subject: [PATCH 0960/2472] convert code-blocks to runnableExamples in io (#18315) Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- lib/system/io.nim | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/lib/system/io.nim b/lib/system/io.nim index 594c78209..62a9d1e18 100644 --- a/lib/system/io.nim +++ b/lib/system/io.nim @@ -912,14 +912,14 @@ iterator lines*(filename: string): string {.tags: [ReadIOEffect].} = ## If the file does not exist `IOError` is raised. The trailing newline ## character(s) are removed from the iterated lines. Example: ## - ## .. code-block:: nim - ## import std/strutils - ## - ## proc transformLetters(filename: string) = - ## var buffer = "" - ## for line in filename.lines: - ## buffer.add(line.replace("a", "0") & '\n') - ## writeFile(filename, buffer) + runnableExamples: + import std/strutils + + proc transformLetters(filename: string) = + var buffer = "" + for line in filename.lines: + buffer.add(line.replace("a", "0") & '\n') + writeFile(filename, buffer) var f = open(filename, bufSize=8000) try: var res = newStringOfCap(80) @@ -931,14 +931,13 @@ iterator lines*(f: File): string {.tags: [ReadIOEffect].} = ## Iterate over any line in the file `f`. ## ## The trailing newline character(s) are removed from the iterated lines. - ## Example: ## - ## .. code-block:: nim - ## proc countZeros(filename: File): tuple[lines, zeros: int] = - ## for line in filename.lines: - ## for letter in line: - ## if letter == '0': - ## result.zeros += 1 - ## result.lines += 1 + runnableExamples: + proc countZeros(filename: File): tuple[lines, zeros: int] = + for line in filename.lines: + for letter in line: + if letter == '0': + result.zeros += 1 + result.lines += 1 var res = newStringOfCap(80) while f.readLine(res): yield res From d8488e41e85c84a6b5fe687940b7fee10314d1f1 Mon Sep 17 00:00:00 2001 From: mantielero Date: Tue, 22 Jun 2021 06:19:11 +0200 Subject: [PATCH 0961/2472] readAsText supports both Blob and File (fixes #18187) (#18189) --- lib/js/dom.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/js/dom.nim b/lib/js/dom.nim index 2b2332020..ca325fdd2 100644 --- a/lib/js/dom.nim +++ b/lib/js/dom.nim @@ -1767,5 +1767,5 @@ since (1, 3): ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readAsBinaryString proc readAsDataURL*(f: FileReader, b: Blob) {.importcpp: "#.readAsDataURL(#)".} ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readAsDataURL - proc readAsText*(f: FileReader, b: Blob, encoding = cstring"UTF-8") {.importcpp: "#.readAsText(#, #)".} - ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readAsText + proc readAsText*(f: FileReader, b: Blob|File, encoding = cstring"UTF-8") {.importcpp: "#.readAsText(#, #)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readAsText \ No newline at end of file From 037715285c088af46c94939f36abef63c95406e8 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 22 Jun 2021 18:44:56 +0800 Subject: [PATCH 0962/2472] fix #18327 (#18328) --- compiler/semtypes.nim | 2 +- tests/errmsgs/t18327.nim | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 tests/errmsgs/t18327.nim diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 38307a9ac..484997e3a 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -1295,7 +1295,7 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode, let param = strTableGet(c.signatures, arg.name) if param != nil: typ = param.typ else: - localError(c.config, a.info, "parameter '$1' requires a type" % param.name.s) + localError(c.config, a.info, "parameter '$1' requires a type" % arg.name.s) typ = errorType(c) let lifted = liftParamType(c, kind, genericParams, typ, arg.name.s, arg.info) diff --git a/tests/errmsgs/t18327.nim b/tests/errmsgs/t18327.nim new file mode 100644 index 000000000..686a1bd0c --- /dev/null +++ b/tests/errmsgs/t18327.nim @@ -0,0 +1,5 @@ +discard """ + errormsg: "parameter 'n' requires a type" +""" + +proc fn3(n) = discard \ No newline at end of file From 9a81e91fa5d01cd6d60a408d0888dce4a9f49898 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 22 Jun 2021 23:02:32 +0800 Subject: [PATCH 0963/2472] merge similar procs regarding digits (#18318) --- lib/std/private/digitsutils.nim | 88 +++++++++++++++++++++++++++++++++ lib/system/dollars.nim | 23 ++++----- lib/system/dragonbox.nim | 32 ++---------- lib/system/schubfach.nim | 30 ++--------- lib/system/strmantle.nim | 65 +----------------------- 5 files changed, 105 insertions(+), 133 deletions(-) create mode 100644 lib/std/private/digitsutils.nim diff --git a/lib/std/private/digitsutils.nim b/lib/std/private/digitsutils.nim new file mode 100644 index 000000000..cc985c7bd --- /dev/null +++ b/lib/std/private/digitsutils.nim @@ -0,0 +1,88 @@ +const + trailingZeros100: array[100, int8] = [2'i8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0] + + digits100: array[200, char] = ['0', '0', '0', '1', '0', '2', '0', '3', '0', '4', '0', '5', + '0', '6', '0', '7', '0', '8', '0', '9', '1', '0', '1', '1', '1', '2', '1', '3', '1', '4', + '1', '5', '1', '6', '1', '7', '1', '8', '1', '9', '2', '0', '2', '1', '2', '2', '2', '3', + '2', '4', '2', '5', '2', '6', '2', '7', '2', '8', '2', '9', '3', '0', '3', '1', '3', '2', + '3', '3', '3', '4', '3', '5', '3', '6', '3', '7', '3', '8', '3', '9', '4', '0', '4', '1', + '4', '2', '4', '3', '4', '4', '4', '5', '4', '6', '4', '7', '4', '8', '4', '9', '5', '0', + '5', '1', '5', '2', '5', '3', '5', '4', '5', '5', '5', '6', '5', '7', '5', '8', '5', '9', + '6', '0', '6', '1', '6', '2', '6', '3', '6', '4', '6', '5', '6', '6', '6', '7', '6', '8', + '6', '9', '7', '0', '7', '1', '7', '2', '7', '3', '7', '4', '7', '5', '7', '6', '7', '7', + '7', '8', '7', '9', '8', '0', '8', '1', '8', '2', '8', '3', '8', '4', '8', '5', '8', '6', + '8', '7', '8', '8', '8', '9', '9', '0', '9', '1', '9', '2', '9', '3', '9', '4', '9', '5', + '9', '6', '9', '7', '9', '8', '9', '9'] + +# Inspired by https://engineering.fb.com/2013/03/15/developer-tools/three-optimization-tips-for-c +# Generates: +# .. code-block:: nim +# var res = "" +# for i in 0 .. 99: +# if i < 10: +# res.add "0" & $i +# else: +# res.add $i +# doAssert res == digits100 + +proc utoa2Digits*(buf: var openArray[char]; pos: int; digits: uint32) {.inline.} = + assert(digits <= 99) + buf[pos] = digits100[2 * digits] + buf[pos+1] = digits100[2 * digits + 1] + #copyMem(buf, unsafeAddr(digits100[2 * digits]), 2 * sizeof((char))) + +proc trailingZeros2Digits*(digits: uint32): int32 {.inline.} = + assert(digits <= 99) + return trailingZeros100[digits] + +func digits10*(num: uint64): int {.noinline.} = + if num < 10'u64: + result = 1 + elif num < 100'u64: + result = 2 + elif num < 1_000'u64: + result = 3 + elif num < 10_000'u64: + result = 4 + elif num < 100_000'u64: + result = 5 + elif num < 1_000_000'u64: + result = 6 + elif num < 10_000_000'u64: + result = 7 + elif num < 100_000_000'u64: + result = 8 + elif num < 1_000_000_000'u64: + result = 9 + elif num < 10_000_000_000'u64: + result = 10 + elif num < 100_000_000_000'u64: + result = 11 + elif num < 1_000_000_000_000'u64: + result = 12 + else: + result = 12 + digits10(num div 1_000_000_000_000'u64) + +template numToString*(result: var string, origin: uint64, length: int) = + var num = origin + var next = length - 1 + const nbatch = 100 + + while num >= nbatch: + let originNum = num + num = num div nbatch + let index = (originNum - num * nbatch) shl 1 + result[next] = digits100[index + 1] + result[next - 1] = digits100[index] + dec(next, 2) + + # process last 1-2 digits + if num < 10: + result[next] = chr(ord('0') + num) + else: + let index = num * 2 + result[next] = digits100[index + 1] + result[next - 1] = digits100[index] diff --git a/lib/system/dollars.nim b/lib/system/dollars.nim index 5a7a77e56..75390b0e1 100644 --- a/lib/system/dollars.nim +++ b/lib/system/dollars.nim @@ -1,3 +1,6 @@ +import std/private/digitsutils + + proc `$`*(x: int): string {.magic: "IntToStr", noSideEffect.} ## The stringify operator for an integer argument. Returns `x` ## converted to a decimal string. `$` is Nim's general way of @@ -6,22 +9,14 @@ proc `$`*(x: int): string {.magic: "IntToStr", noSideEffect.} template dollarImpl(x: uint | uint64, result: var string) = type destTyp = typeof(x) if x == 0: - result = "0" - else: - result = newString(60) - var i = 0 - var n = x - while n != 0: - let nn = n div destTyp(10) - result[i] = char(n - destTyp(10) * nn + ord('0')) - inc i - n = nn - result.setLen i + return "0" + elif x == 1: + return "1" - let half = i div 2 - # Reverse - for t in 0 .. half-1: swap(result[t], result[i-t-1]) + let length = digits10(x) + setLen(result, length) + numToString(result, x, length) when defined(js): import std/private/since diff --git a/lib/system/dragonbox.nim b/lib/system/dragonbox.nim index eec8749da..336c982c1 100644 --- a/lib/system/dragonbox.nim +++ b/lib/system/dragonbox.nim @@ -21,6 +21,10 @@ ## Note: ## This function may temporarily write up to DtoaMinBufferLength characters into the buffer. + +import std/private/digitsutils + + const dtoaMinBufferLength*: cint = 64 @@ -1038,34 +1042,6 @@ proc toDecimal64*(ieeeSignificand: uint64; ieeeExponent: uint64): FloatingDecima ## ToChars ## ================================================================================================== -proc utoa2Digits*(buf: var openArray[char]; pos: int; digits: uint32) {.inline.} = - const - digits100: array[200, char] = ['0', '0', '0', '1', '0', '2', '0', '3', '0', '4', '0', '5', - '0', '6', '0', '7', '0', '8', '0', '9', '1', '0', '1', '1', '1', '2', '1', '3', '1', '4', - '1', '5', '1', '6', '1', '7', '1', '8', '1', '9', '2', '0', '2', '1', '2', '2', '2', '3', - '2', '4', '2', '5', '2', '6', '2', '7', '2', '8', '2', '9', '3', '0', '3', '1', '3', '2', - '3', '3', '3', '4', '3', '5', '3', '6', '3', '7', '3', '8', '3', '9', '4', '0', '4', '1', - '4', '2', '4', '3', '4', '4', '4', '5', '4', '6', '4', '7', '4', '8', '4', '9', '5', '0', - '5', '1', '5', '2', '5', '3', '5', '4', '5', '5', '5', '6', '5', '7', '5', '8', '5', '9', - '6', '0', '6', '1', '6', '2', '6', '3', '6', '4', '6', '5', '6', '6', '6', '7', '6', '8', - '6', '9', '7', '0', '7', '1', '7', '2', '7', '3', '7', '4', '7', '5', '7', '6', '7', '7', - '7', '8', '7', '9', '8', '0', '8', '1', '8', '2', '8', '3', '8', '4', '8', '5', '8', '6', - '8', '7', '8', '8', '8', '9', '9', '0', '9', '1', '9', '2', '9', '3', '9', '4', '9', '5', - '9', '6', '9', '7', '9', '8', '9', '9'] - dragonbox_Assert(digits <= 99) - buf[pos] = digits100[2 * digits] - buf[pos+1] = digits100[2 * digits + 1] - #copyMem(buf, unsafeAddr(digits100[2 * digits]), 2 * sizeof((char))) - -proc trailingZeros2Digits*(digits: uint32): int32 {.inline.} = - const - trailingZeros100: array[100, int8] = [2'i8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0] - dragonbox_Assert(digits <= 99) - return trailingZeros100[digits] - when false: template `+!`(x: cstring; offset: int): cstring = cast[cstring](cast[uint](x) + uint(offset)) diff --git a/lib/system/schubfach.nim b/lib/system/schubfach.nim index 94d3d3c25..5f974a79e 100644 --- a/lib/system/schubfach.nim +++ b/lib/system/schubfach.nim @@ -10,6 +10,9 @@ ## https://drive.google.com/open?id=1luHhyQF9zKlM8yJ1nebU0OgVYhfC6CBN ## -------------------------------------------------------------------------------------------------- +import std/private/digitsutils + + template sf_Assert(x: untyped): untyped = assert(x) @@ -233,33 +236,6 @@ proc toDecimal32(ieeeSignificand: uint32; ieeeExponent: uint32): FloatingDecimal ## ToChars ## ================================================================================================== -proc utoa2Digits(buf: var openArray[char]; pos: int; digits: uint32) {.inline.} = - const - digits100: array[200, char] = ['0', '0', '0', '1', '0', '2', '0', '3', '0', '4', '0', '5', - '0', '6', '0', '7', '0', '8', '0', '9', '1', '0', '1', '1', '1', '2', '1', '3', '1', '4', - '1', '5', '1', '6', '1', '7', '1', '8', '1', '9', '2', '0', '2', '1', '2', '2', '2', '3', - '2', '4', '2', '5', '2', '6', '2', '7', '2', '8', '2', '9', '3', '0', '3', '1', '3', '2', - '3', '3', '3', '4', '3', '5', '3', '6', '3', '7', '3', '8', '3', '9', '4', '0', '4', '1', - '4', '2', '4', '3', '4', '4', '4', '5', '4', '6', '4', '7', '4', '8', '4', '9', '5', '0', - '5', '1', '5', '2', '5', '3', '5', '4', '5', '5', '5', '6', '5', '7', '5', '8', '5', '9', - '6', '0', '6', '1', '6', '2', '6', '3', '6', '4', '6', '5', '6', '6', '6', '7', '6', '8', - '6', '9', '7', '0', '7', '1', '7', '2', '7', '3', '7', '4', '7', '5', '7', '6', '7', '7', - '7', '8', '7', '9', '8', '0', '8', '1', '8', '2', '8', '3', '8', '4', '8', '5', '8', '6', - '8', '7', '8', '8', '8', '9', '9', '0', '9', '1', '9', '2', '9', '3', '9', '4', '9', '5', - '9', '6', '9', '7', '9', '8', '9', '9'] - sf_Assert(digits <= 99) - buf[pos] = digits100[2 * digits] - buf[pos+1] = digits100[2 * digits + 1] - -proc trailingZeros2Digits(digits: uint32): int32 {.inline.} = - const - trailingZeros100: array[100, int8] = [2'i8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0] - sf_Assert(digits <= 99) - return trailingZeros100[digits] - proc printDecimalDigitsBackwards(buf: var openArray[char]; pos: int; output: uint32): int32 {.inline.} = var output = output var pos = pos diff --git a/lib/system/strmantle.nim b/lib/system/strmantle.nim index 1a32a4afe..b60bc4b80 100644 --- a/lib/system/strmantle.nim +++ b/lib/system/strmantle.nim @@ -9,72 +9,9 @@ # Compilerprocs for strings that do not depend on the string implementation. -const digitsTable = "0001020304050607080910111213141516171819" & - "2021222324252627282930313233343536373839" & - "4041424344454647484950515253545556575859" & - "6061626364656667686970717273747576777879" & - "8081828384858687888990919293949596979899" - # Inspired by https://engineering.fb.com/2013/03/15/developer-tools/three-optimization-tips-for-c - # Generates: - # .. code-block:: nim - # var res = "" - # for i in 0 .. 99: - # if i < 10: - # res.add "0" & $i - # else: - # res.add $i - # doAssert res == digitsTable +import std/private/digitsutils -func digits10(num: uint64): int {.noinline.} = - if num < 10'u64: - result = 1 - elif num < 100'u64: - result = 2 - elif num < 1_000'u64: - result = 3 - elif num < 10_000'u64: - result = 4 - elif num < 100_000'u64: - result = 5 - elif num < 1_000_000'u64: - result = 6 - elif num < 10_000_000'u64: - result = 7 - elif num < 100_000_000'u64: - result = 8 - elif num < 1_000_000_000'u64: - result = 9 - elif num < 10_000_000_000'u64: - result = 10 - elif num < 100_000_000_000'u64: - result = 11 - elif num < 1_000_000_000_000'u64: - result = 12 - else: - result = 12 + digits10(num div 1_000_000_000_000'u64) - -template numToString(result: var string, origin: uint64, length: int) = - var num = origin - var next = length - 1 - const nbatch = 100 - - while num >= nbatch: - let originNum = num - num = num div nbatch - let index = (originNum - num * nbatch) shl 1 - result[next] = digitsTable[index + 1] - result[next - 1] = digitsTable[index] - dec(next, 2) - - # process last 1-2 digits - if num < 10: - result[next] = chr(ord('0') + num) - else: - let index = num * 2 - result[next] = digitsTable[index + 1] - result[next - 1] = digitsTable[index] - proc cmpStrings(a, b: string): int {.inline, compilerproc.} = let alen = a.len let blen = b.len From 5badeea17055c85ea8177baeee62c8d1d06ac13e Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 22 Jun 2021 21:42:39 -0700 Subject: [PATCH 0964/2472] followup #18318: simplify `dollarImpl` and add a test (#18330) --- lib/system/dollars.nim | 7 ------- tests/stdlib/tdigitsutils.nim | 23 +++++++++++++++++++++++ 2 files changed, 23 insertions(+), 7 deletions(-) create mode 100644 tests/stdlib/tdigitsutils.nim diff --git a/lib/system/dollars.nim b/lib/system/dollars.nim index 75390b0e1..3d824a965 100644 --- a/lib/system/dollars.nim +++ b/lib/system/dollars.nim @@ -7,15 +7,8 @@ proc `$`*(x: int): string {.magic: "IntToStr", noSideEffect.} ## spelling `toString`:idx:. template dollarImpl(x: uint | uint64, result: var string) = - type destTyp = typeof(x) - if x == 0: - return "0" - elif x == 1: - return "1" - let length = digits10(x) setLen(result, length) - numToString(result, x, length) when defined(js): diff --git a/tests/stdlib/tdigitsutils.nim b/tests/stdlib/tdigitsutils.nim new file mode 100644 index 000000000..7cf63411f --- /dev/null +++ b/tests/stdlib/tdigitsutils.nim @@ -0,0 +1,23 @@ +import std/private/digitsutils + +template main = + block: # digits10 + doAssert digits10(0'u64) == 1 + # checks correctness on all powers of 10 + [0,-1,1] + var x = 1'u64 + var num = 1 + while true: + # echo (x, num) + doAssert digits10(x) == num + doAssert digits10(x+1) == num + if x > 1: + doAssert digits10(x-1) == num - 1 + num += 1 + let xOld = x + x *= 10 + if x < xOld: + # wrap-around + break + +static: main() +main() From 9c43f05099aaddae236df8b86da0cee342a785b8 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Wed, 23 Jun 2021 08:50:05 +0300 Subject: [PATCH 0965/2472] Markdown: allow to end URL with balanced parenthesis (#18321) * Markdown: allow to end URL with balanced parenthesis * Update lib/packages/docutils/rst.nim Co-authored-by: Timothee Cour * apply suggestion * remove unnecessary if Co-authored-by: Timothee Cour --- lib/packages/docutils/rst.nim | 36 ++++++++++++++++++++++++++++-- tests/stdlib/trst.nim | 41 +++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 64910aa19..7833e6e9a 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -1246,15 +1246,43 @@ proc isUrl(p: RstParser, i: int): bool = p.tok[i+3].kind == tkWord and p.tok[i].symbol in ["http", "https", "ftp", "telnet", "file"] +proc checkParen(token: Token, parensStack: var seq[char]): bool {.inline.} = + ## Returns `true` iff `token` is a closing parenthesis for some + ## previous opening parenthesis saved in `parensStack`. + ## This is according Markdown balanced parentheses rule + ## (https://spec.commonmark.org/0.29/#link-destination) + ## to allow links like + ## https://en.wikipedia.org/wiki/APL_(programming_language), + ## we use it for RST also. + result = false + if token.kind == tkPunct: + let c = token.symbol[0] + if c in {'(', '[', '{'}: # push + parensStack.add c + elif c in {')', ']', '}'}: # try pop + # a case like ([) inside a link is allowed and [ is also `pop`ed: + for i in countdown(parensStack.len - 1, 0): + if (parensStack[i] == '(' and c == ')' or + parensStack[i] == '[' and c == ']' or + parensStack[i] == '{' and c == '}'): + parensStack.setLen i + result = true + break + proc parseUrl(p: var RstParser): PRstNode = ## https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#standalone-hyperlinks result = newRstNode(rnStandaloneHyperlink) var lastIdx = p.idx + var closedParenIdx = p.idx - 1 # for balanced parens rule + var parensStack: seq[char] while p.tok[lastIdx].kind in {tkWord, tkPunct, tkOther}: + let isClosing = checkParen(p.tok[lastIdx], parensStack) + if isClosing: + closedParenIdx = lastIdx inc lastIdx dec lastIdx # standalone URL can not end with punctuation in RST - while lastIdx >= p.idx and p.tok[lastIdx].kind == tkPunct and + while lastIdx > closedParenIdx and p.tok[lastIdx].kind == tkPunct and p.tok[lastIdx].symbol != "/": dec lastIdx var s = "" @@ -1393,11 +1421,15 @@ proc parseMarkdownLink(p: var RstParser; father: PRstNode): bool = var desc, link = "" var i = p.idx + var parensStack: seq[char] template parse(endToken, dest) = + parensStack.setLen 0 inc i # skip begin token while true: if p.tok[i].kind in {tkEof, tkIndent}: return false - if p.tok[i].symbol == endToken: break + let isClosing = checkParen(p.tok[i], parensStack) + if p.tok[i].symbol == endToken and not isClosing: + break dest.add p.tok[i].symbol inc i inc i # skip end token diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index ec34edc91..c69450cd9 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -580,3 +580,44 @@ suite "RST inline markup": rnLeaf ' ' rnLeaf 'end' """) + + test "URL with balanced parentheses (Markdown rule)": + # 2 balanced parens, 1 unbalanced: + check(dedent""" + https://en.wikipedia.org/wiki/APL_((programming_language)))""".toAst == + dedent""" + rnInner + rnStandaloneHyperlink + rnLeaf 'https://en.wikipedia.org/wiki/APL_((programming_language))' + rnLeaf ')' + """) + + # the same for Markdown-style link: + check(dedent""" + [foo [bar]](https://en.wikipedia.org/wiki/APL_((programming_language))))""".toAst == + dedent""" + rnInner + rnHyperlink + rnLeaf 'foo [bar]' + rnLeaf 'https://en.wikipedia.org/wiki/APL_((programming_language))' + rnLeaf ')' + """) + + # unbalanced (here behavior is more RST-like actually): + check(dedent""" + https://en.wikipedia.org/wiki/APL_(programming_language(""".toAst == + dedent""" + rnInner + rnStandaloneHyperlink + rnLeaf 'https://en.wikipedia.org/wiki/APL_(programming_language' + rnLeaf '(' + """) + + # unbalanced [, but still acceptable: + check(dedent""" + [my {link example](http://example.com/bracket_(symbol_[))""".toAst == + dedent""" + rnHyperlink + rnLeaf 'my {link example' + rnLeaf 'http://example.com/bracket_(symbol_[)' + """) From 0eee7f572796d3626db1d94d3eb344b399780db1 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 23 Jun 2021 14:39:10 +0200 Subject: [PATCH 0966/2472] OS.nim: style changes (#18331) --- lib/pure/os.nim | 58 ++++++++++++++++++++++++------------------------- 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 4dd5c8d8b..213da2aed 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -320,17 +320,17 @@ else: template `!=?`(a, b: char): bool = toLowerAscii(a) != toLowerAscii(b) when doslikeFileSystem: - proc isAbsFromCurrentDrive(path: string): bool {.noSideEffect, raises: []} = + proc isAbsFromCurrentDrive(path: string): bool {.noSideEffect, raises: [].} = ## An absolute path from the root of the current drive (e.g. "\foo") path.len > 0 and (path[0] == AltSep or (path[0] == DirSep and (path.len == 1 or path[1] notin {DirSep, AltSep, ':'}))) - proc isUNCPrefix(path: string): bool {.noSideEffect, raises: []} = + proc isUNCPrefix(path: string): bool {.noSideEffect, raises: [].} = path[0] == DirSep and path[1] == DirSep - proc sameRoot(path1, path2: string): bool {.noSideEffect, raises: []} = + proc sameRoot(path1, path2: string): bool {.noSideEffect, raises: [].} = ## Return true if path1 and path2 have a same root. ## ## Detail of windows path formats: @@ -935,7 +935,7 @@ proc getConfigDir*(): string {.rtl, extern: "nos$1", proc getCacheDir*(): string = ## Returns the cache directory of the current user for applications. - ## + ## ## This makes use of the following environment variables: ## ## * On Windows: `getEnv("LOCALAPPDATA")` @@ -1038,7 +1038,7 @@ proc expandTilde*(path: string): string {. ## Expands ``~`` or a path starting with ``~/`` to a full path, replacing ## ``~`` with `getHomeDir() <#getHomeDir>`_ (otherwise returns ``path`` unmodified). ## - ## Windows: this is still supported despite Windows platform not having this + ## Windows: this is still supported despite the Windows platform not having this ## convention; also, both ``~/`` and ``~\`` are handled. ## ## .. warning:: `~bob` and `~bob/` are not yet handled correctly. @@ -1101,18 +1101,14 @@ proc quoteShellWindows*(s: string): string {.noSideEffect, rtl, extern: "nosp$1" proc quoteShellPosix*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = ## Quote ``s``, so it can be safely passed to POSIX shell. - ## Based on Python's `pipes.quote`. const safeUnixChars = {'%', '+', '-', '.', '/', '_', ':', '=', '@', '0'..'9', 'A'..'Z', 'a'..'z'} if s.len == 0: - return "''" - - let safe = s.allCharsInSet(safeUnixChars) - - if safe: - return s + result = "''" + elif s.allCharsInSet(safeUnixChars): + result = s else: - return "'" & s.replace("'", "'\"'\"'") & "'" + result = "'" & s.replace("'", "'\"'\"'") & "'" when defined(windows) or defined(posix) or defined(nintendoswitch): proc quoteShell*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = @@ -1122,9 +1118,9 @@ when defined(windows) or defined(posix) or defined(nintendoswitch): ## <#quoteShellWindows,string>`_. Otherwise, calls `quoteShellPosix proc ## <#quoteShellPosix,string>`_. when defined(windows): - return quoteShellWindows(s) + result = quoteShellWindows(s) else: - return quoteShellPosix(s) + result = quoteShellPosix(s) proc quoteShellCommand*(args: openArray[string]): string = ## Concatenates and quotes shell arguments `args`. @@ -1217,7 +1213,7 @@ proc dirExists*(dir: string): bool {.rtl, extern: "nos$1", tags: [ReadDirEffect] result = (a and FILE_ATTRIBUTE_DIRECTORY) != 0'i32 else: var res: Stat - return stat(dir, res) >= 0'i32 and S_ISDIR(res.st_mode) + result = stat(dir, res) >= 0'i32 and S_ISDIR(res.st_mode) proc symlinkExists*(link: string): bool {.rtl, extern: "nos$1", tags: [ReadDirEffect], @@ -1239,7 +1235,7 @@ proc symlinkExists*(link: string): bool {.rtl, extern: "nos$1", result = (a and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32 else: var res: Stat - return lstat(link, res) >= 0'i32 and S_ISLNK(res.st_mode) + result = lstat(link, res) >= 0'i32 and S_ISLNK(res.st_mode) when not defined(windows): @@ -1752,17 +1748,18 @@ proc isAdmin*: bool {.noWeirdTarget.} = addr administratorsGroup)): raiseOSError(osLastError(), "could not get SID for Administrators group") - defer: + try: + var b: WINBOOL + if not isSuccess(checkTokenMembership(0, administratorsGroup, addr b)): + raiseOSError(osLastError(), "could not check access token membership") + + result = isSuccess(b) + finally: if freeSid(administratorsGroup) != nil: raiseOSError(osLastError(), "failed to free SID for Administrators group") - var b: WINBOOL - if not isSuccess(checkTokenMembership(0, administratorsGroup, addr b)): - raiseOSError(osLastError(), "could not check access token membership") - - return isSuccess(b) else: - return geteuid() == 0 + result = geteuid() == 0 proc createSymlink*(src, dest: string) {.noWeirdTarget.} = ## Create a symbolic link at `dest` which points to the item specified @@ -1828,10 +1825,11 @@ when hasCCopyfile: COPYFILE_XATTR {.nodecl.}: copyfile_flags_t {.pop.} -type CopyFlag* = enum ## Copy options. - cfSymlinkAsIs, ## Copy symlinks as symlinks - cfSymlinkFollow, ## Copy the files symlinks point to - cfSymlinkIgnore ## Ignore symlinks +type + CopyFlag* = enum ## Copy options. + cfSymlinkAsIs, ## Copy symlinks as symlinks + cfSymlinkFollow, ## Copy the files symlinks point to + cfSymlinkIgnore ## Ignore symlinks const copyFlagSymlink = {cfSymlinkAsIs, cfSymlinkFollow, cfSymlinkIgnore} @@ -2027,7 +2025,7 @@ proc tryMoveFSObject(source, dest: string, isDir: bool): bool {.noWeirdTarget.} if not result: let err = osLastError() - let isAccessDeniedError = + let isAccessDeniedError = when defined(windows): const AccessDeniedError = OSErrorCode(5) isDir and err == AccessDeniedError @@ -2066,7 +2064,7 @@ proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", except: discard tryRemoveFile(dest) raise - + proc exitStatusLikeShell*(status: cint): cint = ## Converts exit code from `c_system` into a shell exit code. From 63456c6d7f84f29da38703fb3e59acf00acec1b8 Mon Sep 17 00:00:00 2001 From: Antonis Geralis <43617260+planetis-m@users.noreply.github.com> Date: Wed, 23 Jun 2021 18:56:20 +0300 Subject: [PATCH 0967/2472] Add some tests (#18333) --- tests/destructor/tv2_cast.nim | 72 ++++++++++++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 2 deletions(-) diff --git a/tests/destructor/tv2_cast.nim b/tests/destructor/tv2_cast.nim index ef0b3a936..5502cb38d 100644 --- a/tests/destructor/tv2_cast.nim +++ b/tests/destructor/tv2_cast.nim @@ -1,10 +1,78 @@ discard """ - cmd: '''nim c --gc:arc $file''' output: '''@[1] @[116, 101, 115, 116] @[1953719668, 875770417]''' + cmd: '''nim c --gc:arc --expandArc:main --expandArc:main1 --expandArc:main2 --expandArc:main3 --hints:off --assertions:off $file''' + nimout: '''--expandArc: main + +var + data + :tmpD +`=copy`(data, cast[string](encode(cast[seq[byte]]( + :tmpD = newString(100) + :tmpD)))) +`=destroy`(:tmpD) +`=destroy`(data) +-- end of expandArc ------------------------ +--expandArc: main1 + +var + s + data +s = newString(100) +`=copy`(data, cast[string](encode(toOpenArrayByte(s, 0, len(s) - 1)))) +`=destroy`(data) +`=destroy`(s) +-- end of expandArc ------------------------ +--expandArc: main2 + +var + s + data +s = newSeq(100) +`=copy`(data, cast[string](encode(s))) +`=destroy`(data) +`=destroy_1`(s) +-- end of expandArc ------------------------ +--expandArc: main3 + +var + data + :tmpD +`=copy`(data, cast[string](encode do: + :tmpD = newSeq(100) + :tmpD)) +`=destroy`(:tmpD) +`=destroy_1`(data) +-- end of expandArc ------------------------''' """ +func encode*(src: openArray[byte]): seq[byte] = + result = newSeq[byte](src.len) + +template compress*(src: string): string = + cast[string](encode(cast[seq[byte]](src))) + +proc main = + let data = compress(newString(100)) +main() + +proc main1 = + var + s = newString(100) + let data = cast[string](encode(s.toOpenArrayByte(0, s.len-1))) +main1() + +proc main2 = + var + s = newSeq[byte](100) + let data = cast[string](encode(s)) +main2() + +proc main3 = + let data = cast[string](encode(newSeq[byte](100))) +main3() + # bug #11018 discard cast[seq[uint8]](@[1]) discard cast[seq[uint8]]("test") @@ -20,4 +88,4 @@ echo a #issue 11204 var ac {.compileTime.} = @["a", "b"] -const bc = ac.len \ No newline at end of file +const bc = ac.len From 496bd790e1b641a7c025bde2c6a1c6dca433ed5d Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 24 Jun 2021 03:33:19 +0800 Subject: [PATCH 0968/2472] [std/times]getTime now uses high resolution API on windows (#17901) --- lib/pure/times.nim | 5 +++-- lib/windows/winlean.nim | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index a92eccc8c..7bb393aa0 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -880,6 +880,7 @@ since((1, 1)): export fromUnixFloat export toUnixFloat + proc fromWinTime*(win: int64): Time = ## Convert a Windows file time (100-nanosecond intervals since ## `1601-01-01T00:00:00Z`) to a `Time`. @@ -912,7 +913,7 @@ proc getTime*(): Time {.tags: [TimeEffect], benign.} = result = initTime(ts.tv_sec.int64, ts.tv_nsec.int) elif defined(windows): var f {.noinit.}: FILETIME - getSystemTimeAsFileTime(f) + getSystemTimePreciseAsFileTime(f) result = fromWinTime(rdFileTime(f)) proc `-`*(a, b: Time): Duration {.operator, extern: "ntDiffTime".} = @@ -2595,7 +2596,7 @@ proc epochTime*(): float {.tags: [TimeEffect].} = toBiggestFloat(ts.tv_nsec.int64) / 1_000_000_000 elif defined(windows): var f {.noinit.}: winlean.FILETIME - getSystemTimeAsFileTime(f) + getSystemTimePreciseAsFileTime(f) var i64 = rdFileTime(f) - epochDiff var secs = i64 div rateDiff var subsecs = i64 mod rateDiff diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index d2d7d26cc..a71dd97d8 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -928,6 +928,9 @@ proc getProcessTimes*(hProcess: Handle; lpCreationTime, lpExitTime, lpKernelTime, lpUserTime: var FILETIME): WINBOOL {.stdcall, dynlib: "kernel32", importc: "GetProcessTimes".} +proc getSystemTimePreciseAsFileTime*(lpSystemTimeAsFileTime: var FILETIME) {. + importc: "GetSystemTimePreciseAsFileTime", dynlib: "kernel32", stdcall, sideEffect.} + type inet_ntop_proc = proc(family: cint, paddr: pointer, pStringBuffer: cstring, stringBufSize: int32): cstring {.gcsafe, stdcall, tags: [].} From 0f91b67f5c15328330f74a8769aed9961940aab2 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 23 Jun 2021 23:31:17 +0200 Subject: [PATCH 0969/2472] fixes #18326 (#18335) * fixes #18326 * make tests green again --- compiler/isolation_check.nim | 4 ++-- tests/isolate/tisolate.nim | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/compiler/isolation_check.nim b/compiler/isolation_check.nim index 43db5d59b..777e7f6ce 100644 --- a/compiler/isolation_check.nim +++ b/compiler/isolation_check.nim @@ -64,7 +64,7 @@ proc canAlias(arg, ret: PType; marker: var IntSet): bool = else: result = false -proc isValueOnlyType(t: PType): bool = +proc isValueOnlyType(t: PType): bool = # t doesn't contain pointers and references proc wrap(t: PType): bool {.nimcall.} = t.kind in {tyRef, tyPtr, tyVar, tyLent} result = not types.searchTypeFor(t, wrap) @@ -88,7 +88,7 @@ proc checkIsolate*(n: PNode): bool = # XXX: as long as we don't update the analysis while examining arguments # we can do an early check of the return type, otherwise this is a # bug and needs to be moved below - if n[0].typ.flags * {tfGcSafe, tfNoSideEffect} == {}: + if tfNoSideEffect notin n[0].typ.flags: return false for i in 1.. Date: Wed, 23 Jun 2021 23:31:55 -0700 Subject: [PATCH 0970/2472] fix #18332: XDeclaredButNotUsed hints now in deterministic order (#18336) --- compiler/lookups.nim | 10 ++++++---- tests/errmsgs/treportunused.nim | 24 +++++++++++++----------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/compiler/lookups.nim b/compiler/lookups.nim index 4f413f472..ee79673d0 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -8,11 +8,10 @@ # # This module implements lookup helpers. - +import std/[algorithm, strutils] import intsets, ast, astalgo, idents, semdata, types, msgs, options, - renderer, nimfix/prettybase, lineinfos, strutils, - modulegraphs, astmsgs + renderer, nimfix/prettybase, lineinfos, modulegraphs, astmsgs proc ensureNoMissingOrUnusedSymbols(c: PContext; scope: PScope) @@ -268,6 +267,7 @@ proc ensureNoMissingOrUnusedSymbols(c: PContext; scope: PScope) = var it: TTabIter var s = initTabIter(it, scope.symbols) var missingImpls = 0 + var unusedSyms: seq[tuple[sym: PSym, key: string]] while s != nil: if sfForward in s.flags and s.kind notin {skType, skModule}: # too many 'implementation of X' errors are annoying @@ -282,8 +282,10 @@ proc ensureNoMissingOrUnusedSymbols(c: PContext; scope: PScope) = # maybe they can be made skGenericParam as well. if s.typ != nil and tfImplicitTypeParam notin s.typ.flags and s.typ.kind != tyGenericParam: - message(c.config, s.info, hintXDeclaredButNotUsed, s.name.s) + unusedSyms.add (s, toFileLineCol(c.config, s.info)) s = nextIter(it, scope.symbols) + for (s, _) in sortedByIt(unusedSyms, it.key): + message(c.config, s.info, hintXDeclaredButNotUsed, s.name.s) proc wrongRedefinition*(c: PContext; info: TLineInfo, s: string; conflictsWith: TLineInfo) = diff --git a/tests/errmsgs/treportunused.nim b/tests/errmsgs/treportunused.nim index b339e06bf..3e945013e 100644 --- a/tests/errmsgs/treportunused.nim +++ b/tests/errmsgs/treportunused.nim @@ -1,16 +1,18 @@ discard """ + matrix: "--hint:all:off --hint:XDeclaredButNotUsed" + nimoutFull: true nimout: ''' -treportunused.nim(19, 10) Hint: 's1' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(26, 5) Hint: 's8' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(22, 6) Hint: 's4' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(25, 7) Hint: 's7' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(24, 7) Hint: 's6' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(23, 6) Hint: 's5' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(20, 10) Hint: 's2' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(29, 6) Hint: 's11' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(27, 5) Hint: 's9' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(21, 10) Hint: 's3' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(28, 6) Hint: 's10' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(21, 10) Hint: 's1' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(22, 10) Hint: 's2' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(23, 10) Hint: 's3' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(24, 6) Hint: 's4' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(25, 6) Hint: 's5' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(26, 7) Hint: 's6' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(27, 7) Hint: 's7' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(28, 5) Hint: 's8' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(29, 5) Hint: 's9' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(30, 6) Hint: 's10' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(31, 6) Hint: 's11' is declared but not used [XDeclaredButNotUsed] ''' action: compile """ From 55c1953f636ff2d2e2b9e9599194fa6c3fb49050 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 24 Jun 2021 00:58:11 -0700 Subject: [PATCH 0971/2472] fix #18334: handle path with spaces on windows during bootstrap (#18337) --- compiler/nim.nim | 27 +++++++++++++++------------ koch.nim | 2 +- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/compiler/nim.nim b/compiler/nim.nim index f18238f60..91b1bc2db 100644 --- a/compiler/nim.nim +++ b/compiler/nim.nim @@ -7,21 +7,24 @@ # distribution, for details about the copyright. # -when defined(gcc) and defined(windows): - when defined(x86): - {.link: "../icons/nim.res".} - else: - {.link: "../icons/nim_icon.o".} +import std/[os, strutils, parseopt] +when defined(windows) and not defined(nimKochBootstrap): + # remove workaround pending bootstrap >= 1.5.1 + # refs https://github.com/nim-lang/Nim/issues/18334#issuecomment-867114536 + # alternative would be to prepend `currentSourcePath.parentDir.quoteShell` + when defined(gcc): + when defined(x86): + {.link: "../icons/nim.res".} + else: + {.link: "../icons/nim_icon.o".} -when defined(amd64) and defined(windows) and defined(vcc): - {.link: "../icons/nim-amd64-windows-vcc.res".} -when defined(i386) and defined(windows) and defined(vcc): - {.link: "../icons/nim-i386-windows-vcc.res".} + when defined(amd64) and defined(vcc): + {.link: "../icons/nim-amd64-windows-vcc.res".} + when defined(i386) and defined(vcc): + {.link: "../icons/nim-i386-windows-vcc.res".} import - commands, options, msgs, - extccomp, strutils, os, main, parseopt, - idents, lineinfos, cmdlinehelper, + commands, options, msgs, extccomp, main, idents, lineinfos, cmdlinehelper, pathutils, modulegraphs from browsers import openDefaultBrowser diff --git a/koch.nim b/koch.nim index 665b5e042..51be53cc7 100644 --- a/koch.nim +++ b/koch.nim @@ -308,7 +308,7 @@ proc boot(args: string) = var nimi = i.thVersion if i == 0: nimi = nimStart - extraOption.add " --skipUserCfg --skipParentCfg" + extraOption.add " --skipUserCfg --skipParentCfg -d:nimKochBootstrap" # The configs are skipped for bootstrap # (1st iteration) to prevent newer flags from breaking bootstrap phase. let ret = execCmdEx(nimStart & " --version") From 0c8d3ae9852721147ddf878e204c14e949ed676a Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Thu, 24 Jun 2021 11:28:28 +0300 Subject: [PATCH 0972/2472] rst: allow comment to continue on second line (#18338) --- lib/packages/docutils/rst.nim | 28 ++++++--------- tests/stdlib/trst.nim | 66 +++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 18 deletions(-) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 7833e6e9a..8ffbbd4d3 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -1615,24 +1615,16 @@ proc getDirective(p: var RstParser): string = "too many colons for a directive (should be ::)", p.tok[afterIdx].line, p.tok[afterIdx].col) -proc parseComment(p: var RstParser): PRstNode = - case currentTok(p).kind - of tkIndent, tkEof: - if currentTok(p).kind != tkEof and nextTok(p).kind == tkIndent: - inc p.idx # empty comment - else: - var indent = currentTok(p).ival - while true: - case currentTok(p).kind - of tkEof: - break - of tkIndent: - if currentTok(p).ival < indent: break - else: - discard - inc p.idx +proc parseComment(p: var RstParser, col: int): PRstNode = + if currentTok(p).kind != tkEof and nextTok(p).kind == tkIndent: + inc p.idx # empty comment else: - while currentTok(p).kind notin {tkIndent, tkEof}: inc p.idx + while currentTok(p).kind != tkEof: + if currentTok(p).kind == tkIndent and currentTok(p).ival > col or + currentTok(p).kind != tkIndent and currentTok(p).col > col: + inc p.idx + else: + break result = nil proc parseLine(p: var RstParser, father: PRstNode) = @@ -2841,7 +2833,7 @@ proc parseDotDot(p: var RstParser): PRstNode = (n = parseFootnote(p); n != nil): result = n else: - result = parseComment(p) + result = parseComment(p, col) proc rstParsePass1*(fragment, filename: string, line, column: int, diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index c69450cd9..d53092d3b 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -83,6 +83,72 @@ suite "RST parsing": rnLeaf 'set' """) + test "RST comment": + check(dedent""" + .. comment1 + comment2 + someParagraph""".toAst == + dedent""" + rnLeaf 'someParagraph' + """) + + check(dedent""" + .. + comment1 + comment2 + someParagraph""".toAst == + dedent""" + rnLeaf 'someParagraph' + """) + + test "check that additional line right after .. ends comment": + check(dedent""" + .. + + notAcomment1 + notAcomment2 + someParagraph""".toAst == + dedent""" + rnInner + rnBlockQuote + rnInner + rnLeaf 'notAcomment1' + rnLeaf ' ' + rnLeaf 'notAcomment2' + rnParagraph + rnLeaf 'someParagraph' + """) + + test "but blank lines after 2nd non-empty line don't end the comment": + check(dedent""" + .. + comment1 + + + comment2 + someParagraph""".toAst == + dedent""" + rnLeaf 'someParagraph' + """) + + test "using .. as separator b/w directives and block quotes": + check(dedent""" + .. note:: someNote + + .. + + someBlockQuote""".toAst == + dedent""" + rnInner + rnAdmonition adType=note + [nil] + [nil] + rnLeaf 'someNote' + rnBlockQuote + rnInner + rnLeaf 'someBlockQuote' + """) + suite "RST indentation": test "nested bullet lists": let input = dedent """ From 565e07a9936657fa356777eec5f0f52b2d5d7e02 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 24 Jun 2021 02:55:31 -0700 Subject: [PATCH 0973/2472] enable VM tracing in user code via `{.define(nimVmTrace).}` (#18244) * enable VM tracing in user code via `{.define(nimVmTrace).}` * add vmutils.vmTrace * add vmTrace --- changelog.md | 2 ++ compiler/options.nim | 1 + compiler/vm.nim | 7 +++++-- compiler/vmops.nim | 3 +++ lib/std/vmutils.nim | 11 +++++++++++ tests/stdlib/tvmutils.nim | 31 +++++++++++++++++++++++++++++++ 6 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 lib/std/vmutils.nim create mode 100644 tests/stdlib/tvmutils.nim diff --git a/changelog.md b/changelog.md index 570aff542..6672f7538 100644 --- a/changelog.md +++ b/changelog.md @@ -387,6 +387,8 @@ - Added `--declaredlocs` to show symbol declaration location in messages. +- You can now enable/disable VM tracing in user code via `vmutils.vmTrace`. + - Deprecated `TaintedString` and `--taintmode`. - Deprecated `--nilseqs` which is now a noop. diff --git a/compiler/options.nim b/compiler/options.nim index ffb0f4501..135ecf1de 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -330,6 +330,7 @@ type warnCounter*: int errorMax*: int maxLoopIterationsVM*: int ## VM: max iterations of all loops + isVmTrace*: bool configVars*: StringTableRef symbols*: StringTableRef ## We need to use a StringTableRef here as defined ## symbols are always guaranteed to be style diff --git a/compiler/vm.nim b/compiler/vm.nim index 837a1c366..9541af701 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -548,9 +548,12 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = "pc", $pc, "opcode", alignLeft($c.code[pc].opcode, 15), "ra", regDescr("ra", ra), "rb", regDescr("rb", instr.regB), "rc", regDescr("rc", instr.regC)] - + if c.config.isVmTrace: + # unlike nimVMDebug, this doesn't require re-compiling nim and is controlled by user code + let info = c.debug[pc] + # other useful variables: c.loopIterations + echo "$# [$#] $#" % [c.config$info, $instr.opcode, c.config.sourceLine(info)] c.profiler.enter(c, tos) - case instr.opcode of opcEof: return regs[ra] of opcRet: diff --git a/compiler/vmops.nim b/compiler/vmops.nim index 85729fd59..67a196b8f 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -249,6 +249,9 @@ proc registerAdditionalOps*(c: PCtx) = "isExported() requires a symbol. '" & $n & "' is of kind '" & $n.kind & "'", n.info) setResult(a, sfExported in n.sym.flags) + registerCallback c, "stdlib.vmutils.vmTrace", proc (a: VmArgs) = + c.config.isVmTrace = getBool(a, 0) + proc hashVmImpl(a: VmArgs) = var res = hashes.hash(a.getString(0), a.getInt(1).int, a.getInt(2).int) if c.config.backend == backendJs: diff --git a/lib/std/vmutils.nim b/lib/std/vmutils.nim new file mode 100644 index 000000000..e16912a3c --- /dev/null +++ b/lib/std/vmutils.nim @@ -0,0 +1,11 @@ +##[ +Experimental API, subject to change. +]## + +proc vmTrace*(on: bool) {.compileTime.} = + runnableExamples: + static: vmTrace(true) + proc fn = + var a = 1 + vmTrace(false) + static: fn() diff --git a/tests/stdlib/tvmutils.nim b/tests/stdlib/tvmutils.nim new file mode 100644 index 000000000..f43557ad8 --- /dev/null +++ b/tests/stdlib/tvmutils.nim @@ -0,0 +1,31 @@ +discard """ + joinable: false + nimout: ''' +0 +1 +2 +tvmutils.nim(28, 13) [opcLdImmInt] if i == 4: +tvmutils.nim(28, 10) [opcEqInt] if i == 4: +tvmutils.nim(28, 10) [opcFJmp] if i == 4: +tvmutils.nim(28, 13) [opcLdImmInt] if i == 4: +tvmutils.nim(28, 10) [opcEqInt] if i == 4: +tvmutils.nim(28, 10) [opcFJmp] if i == 4: +tvmutils.nim(29, 7) [opcLdConst] vmTrace(false) +tvmutils.nim(29, 15) [opcLdImmInt] vmTrace(false) +tvmutils.nim(29, 14) [opcIndCall] vmTrace(false) +5 +6 +''' +""" +# line 20 (only showing a subset of nimout to avoid making the test rigid) +import std/vmutils + +proc main() = + for i in 0..<7: + echo i + if i == 2: + vmTrace(true) + if i == 4: + vmTrace(false) + +static: main() From 16038d44f6b246c5f39d1a8609a8c45f2cd005a4 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 24 Jun 2021 18:37:04 +0200 Subject: [PATCH 0974/2472] fixes #18320 (#18343) * TSymFlag has 47 flags already * fixes #18320 --- compiler/ast.nim | 2 +- compiler/varpartitions.nim | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 8bb8de1d6..56043692f 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -229,7 +229,7 @@ type TNodeKinds* = set[TNodeKind] type - TSymFlag* = enum # 46 flags! + TSymFlag* = enum # 47 flags! sfUsed, # read access of sym (for warnings) or simply used sfExported, # symbol is exported from module sfFromGeneric, # symbol is instantiation of a generic; this is needed diff --git a/compiler/varpartitions.nim b/compiler/varpartitions.nim index 697dd2b1d..213c3b80e 100644 --- a/compiler/varpartitions.nim +++ b/compiler/varpartitions.nim @@ -51,6 +51,7 @@ type SubgraphFlag = enum isMutated, # graph might be mutated isMutatedDirectly, # graph is mutated directly by a non-var parameter. + isMutatedByVarParam, # graph is mutated by a var parameter. connectsConstParam # graph is connected to a non-var parameter. VarFlag = enum @@ -184,7 +185,7 @@ proc potentialMutation(v: var Partitions; s: PSym; level: int; info: TLineInfo) {isMutated, isMutatedDirectly} elif s.typ.kind == tyVar and level <= 1: # varParam[i] = v is different from varParam[i][] = v - {} + {isMutatedByVarParam} else: {isMutated} else: @@ -878,7 +879,7 @@ proc computeGraphPartitions*(s: PSym; n: PNode; g: ModuleGraph; goals: set[Goal] proc dangerousMutation(g: MutationInfo; v: VarIndex): bool = #echo "range ", v.aliveStart, " .. ", v.aliveEnd, " ", v.sym - if isMutated in g.flags: + if {isMutated, isMutatedByVarParam} * g.flags != {}: for m in g.mutations: #echo "mutation ", m if m in v.aliveStart..v.aliveEnd: @@ -941,4 +942,5 @@ proc computeCursors*(s: PSym; n: PNode; g: ModuleGraph) = discard "cannot cursor into a graph that is mutated" else: v.sym.flags.incl sfCursor - #echo "this is now a cursor ", v.sym, " ", par.s[rid].flags, " ", g.config $ v.sym.info + when false: + echo "this is now a cursor ", v.sym, " ", par.s[rid].flags, " ", g.config $ v.sym.info From f128f7971f6eda5dc9b21af3dbaeccc5dc71da5a Mon Sep 17 00:00:00 2001 From: Kaushal Modi Date: Thu, 24 Jun 2021 13:19:50 -0400 Subject: [PATCH 0975/2472] float parsing: Add test for a fixed issue (#18232) Fixes https://github.com/nim-lang/Nim/issues/14407 . This issue was fixed by https://github.com/nim-lang/Nim/pull/18139. --- tests/errmsgs/treportunused.nim | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/tests/errmsgs/treportunused.nim b/tests/errmsgs/treportunused.nim index 3e945013e..65e00163b 100644 --- a/tests/errmsgs/treportunused.nim +++ b/tests/errmsgs/treportunused.nim @@ -2,17 +2,19 @@ discard """ matrix: "--hint:all:off --hint:XDeclaredButNotUsed" nimoutFull: true nimout: ''' -treportunused.nim(21, 10) Hint: 's1' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(22, 10) Hint: 's2' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(23, 10) Hint: 's3' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(24, 6) Hint: 's4' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(25, 6) Hint: 's5' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(26, 7) Hint: 's6' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(27, 7) Hint: 's7' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(28, 5) Hint: 's8' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(29, 5) Hint: 's9' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(30, 6) Hint: 's10' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(31, 6) Hint: 's11' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(23, 10) Hint: 's1' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(24, 10) Hint: 's2' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(25, 10) Hint: 's3' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(26, 6) Hint: 's4' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(27, 6) Hint: 's5' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(28, 7) Hint: 's6' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(29, 7) Hint: 's7' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(30, 5) Hint: 's8' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(31, 5) Hint: 's9' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(32, 6) Hint: 's10' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(33, 6) Hint: 's11' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(37, 3) Hint: 'v0.99' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(38, 3) Hint: 'v0.99.99' is declared but not used [XDeclaredButNotUsed] ''' action: compile """ @@ -29,3 +31,8 @@ let s8 = 0 var s9: int type s10 = object type s11 = type(1.2) + +# https://github.com/nim-lang/Nim/issues/14407 +let + `v0.99` = "0.99" + `v0.99.99` = "0.99.99" From 6be8a66833ee25ad183fa3e4a667e3c0948413d8 Mon Sep 17 00:00:00 2001 From: Smooth Operator Date: Thu, 24 Jun 2021 13:48:57 -0400 Subject: [PATCH 0976/2472] couple tiny typo fixes (#18344) --- lib/pure/times.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 7bb393aa0..fb48a199c 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -359,7 +359,7 @@ type Timezone* = ref object ## \ ## Timezone interface for supporting `DateTime <#DateTime>`_\s of arbitrary ## timezones. The `times` module only supplies implementations for the - ## systems local time and UTC. + ## system's local time and UTC. zonedTimeFromTimeImpl: proc (x: Time): ZonedTime {.tags: [], raises: [], benign.} zonedTimeFromAdjTimeImpl: proc (x: Time): ZonedTime @@ -1145,7 +1145,7 @@ proc name*(zone: Timezone): string = ## If the timezone doesn't exist in the tz database, or if the timezone ## name is unknown, then any string that describes the timezone ## unambiguously might be used. For example, the string "LOCAL" is used - ## for the systems local timezone. + ## for the system's local timezone. ## ## See also: https://en.wikipedia.org/wiki/Tz_database zone.name From 0d194cdbf90953f28450c4bf1db744d2c1332996 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 25 Jun 2021 06:22:52 +0200 Subject: [PATCH 0977/2472] fixes #18287 (#18346) --- compiler/varpartitions.nim | 48 ++++++++++++++++----------------- tests/arc/topt_no_cursor.nim | 51 +++++++++++++++++++++++++++++++++++- 2 files changed, 74 insertions(+), 25 deletions(-) diff --git a/compiler/varpartitions.nim b/compiler/varpartitions.nim index 213c3b80e..8f422face 100644 --- a/compiler/varpartitions.nim +++ b/compiler/varpartitions.nim @@ -416,7 +416,6 @@ proc allRoots(n: PNode; result: var seq[(PSym, int)]; level: int) = proc destMightOwn(c: var Partitions; dest: var VarIndex; n: PNode) = ## Analyse if 'n' is an expression that owns the data, if so mark 'dest' ## with 'ownsData'. - if n.typ == nil: return case n.kind of nkEmpty, nkCharLit..nkNilLit: # primitive literals including the empty are harmless: @@ -475,30 +474,31 @@ proc destMightOwn(c: var Partitions; dest: var VarIndex; n: PNode) = destMightOwn(c, dest, n[0]) of nkCallKinds: - if hasDestructor(n.typ): - # calls do construct, what we construct must be destroyed, - # so dest cannot be a cursor: - dest.flags.incl ownsData - elif n.typ.kind in {tyLent, tyVar}: - # we know the result is derived from the first argument: - var roots: seq[(PSym, int)] - allRoots(n[1], roots, RootEscapes) - for r in roots: - connect(c, dest.sym, r[0], n[1].info) + if n.typ != nil: + if hasDestructor(n.typ): + # calls do construct, what we construct must be destroyed, + # so dest cannot be a cursor: + dest.flags.incl ownsData + elif n.typ.kind in {tyLent, tyVar}: + # we know the result is derived from the first argument: + var roots: seq[(PSym, int)] + allRoots(n[1], roots, RootEscapes) + for r in roots: + connect(c, dest.sym, r[0], n[1].info) - else: - let magic = if n[0].kind == nkSym: n[0].sym.magic else: mNone - # this list is subtle, we try to answer the question if after 'dest = f(src)' - # there is a connection betwen 'src' and 'dest' so that mutations to 'src' - # also reflect 'dest': - if magic in {mNone, mMove, mSlice, mAppendStrCh, mAppendStrStr, mAppendSeqElem, mArrToSeq}: - for i in 1.. Date: Fri, 25 Jun 2021 14:12:23 +0200 Subject: [PATCH 0978/2472] fixes #18240 (#18354) * ORC: track escaping parameters properly * fixes #18240 --- compiler/sempass2.nim | 37 +++++++++++++++++++++++++------------ tests/arc/torcmisc.nim | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 12 deletions(-) diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index e66c9596a..e124baf26 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -77,6 +77,7 @@ type config: ConfigRef graph: ModuleGraph c: PContext + escapingParams: IntSet PEffects = var TEffects proc `<`(a, b: TLockLevel): bool {.borrow.} @@ -900,6 +901,25 @@ proc castBlock(tracked: PEffects, pragma: PNode, bc: var PragmaBlockContext) = localError(tracked.config, pragma.info, "invalid pragma block: " & $pragma) +proc trackInnerProc(tracked: PEffects, n: PNode) = + case n.kind + of nkSym: + let s = n.sym + if s.kind == skParam and s.owner == tracked.owner: + tracked.escapingParams.incl s.id + of nkNone..pred(nkSym), succ(nkSym)..nkNilLit: + discard + of nkProcDef, nkConverterDef, nkMethodDef, nkIteratorDef, nkLambda, nkFuncDef, nkDo: + if n[0].kind == nkSym and n[0].sym.ast != nil: + trackInnerProc(tracked, getBody(tracked.graph, n[0].sym)) + of nkTypeSection, nkMacroDef, nkTemplateDef, nkError, + nkConstSection, nkConstDef, nkIncludeStmt, nkImportStmt, + nkExportStmt, nkPragma, nkCommentStmt, nkBreakState, + nkTypeOfExpr, nkMixinStmt, nkBindStmt: + discard + else: + for ch in n: trackInnerProc(tracked, ch) + proc track(tracked: PEffects, n: PNode) = case n.kind of nkSym: @@ -1095,8 +1115,10 @@ proc track(tracked: PEffects, n: PNode) = track(tracked, n.lastSon) unapplyBlockContext(tracked, bc) - of nkTypeSection, nkProcDef, nkConverterDef, nkMethodDef, nkIteratorDef, - nkMacroDef, nkTemplateDef, nkLambda, nkDo, nkFuncDef: + of nkProcDef, nkConverterDef, nkMethodDef, nkIteratorDef, nkLambda, nkFuncDef, nkDo: + if n[0].kind == nkSym and n[0].sym.ast != nil: + trackInnerProc(tracked, getBody(tracked.graph, n[0].sym)) + of nkTypeSection, nkMacroDef, nkTemplateDef: discard of nkCast: if n.len == 2: @@ -1259,15 +1281,6 @@ proc hasRealBody(s: PSym): bool = ## which is not a real implementation, refs #14314 result = {sfForward, sfImportc} * s.flags == {} -proc maybeWrappedInClosure(tracked: PEffects; t: PType): bool {.inline.} = - ## The spec does say when to produce destructors. However, the spec - ## was written in mind with the idea that "lambda lifting" already - ## happened. Not true in our implementation, so we need to workaround - ## here: - result = tracked.isInnerProc and - sfSystemModule notin tracked.c.module.flags and - tfCheckedForDestructor notin t.flags and containsGarbageCollectedRef(t) - proc trackProc*(c: PContext; s: PSym, body: PNode) = let g = c.graph var effects = s.typ.n[0] @@ -1287,7 +1300,7 @@ proc trackProc*(c: PContext; s: PSym, body: PNode) = let typ = param.typ if isSinkTypeForParam(typ) or (t.config.selectedGC in {gcArc, gcOrc} and - (isClosure(typ.skipTypes(abstractInst)) or maybeWrappedInClosure(t, typ))): + (isClosure(typ.skipTypes(abstractInst)) or param.id in t.escapingParams)): createTypeBoundOps(t, typ, param.info) when false: if typ.kind == tyOut and param.id notin t.init: diff --git a/tests/arc/torcmisc.nim b/tests/arc/torcmisc.nim index 20dd18fb3..e41ad7c77 100644 --- a/tests/arc/torcmisc.nim +++ b/tests/arc/torcmisc.nim @@ -32,3 +32,35 @@ when true: waitFor hello(Flags()) echo "success" +# bug #18240 +import tables + +type + TopicHandler* = proc(topic: string, + data: seq[byte]): Future[void] {.gcsafe, raises: [Defect].} + + PeerID* = object + data*: seq[byte] + + PeerInfo* = ref object of RootObj + peerId*: PeerID + + Connection* = ref object of RootObj + peerInfo*: PeerInfo + + PubSubPeer* = ref object of RootObj + codec*: string + + PubSub* = ref object of RootObj + topics*: Table[string, seq[TopicHandler]] + peers*: Table[PeerID, PubSubPeer] + +proc getOrCreatePeer*(myParam: PubSub, peerId: PeerID, protos: seq[string]): PubSubPeer = + myParam.peers.withValue(peerId, peer): + return peer[] + +method handleConn*(myParam: PubSub, + conn: Connection, + proto: string) {.base, async.} = + myParam.peers.withValue(conn.peerInfo.peerId, peer): + let peerB = peer[] From 8535b26a754b2daea46627019e3145d7c12d826e Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 25 Jun 2021 07:47:10 -0700 Subject: [PATCH 0979/2472] docs now show nimExperimentalX APIs (#18345) * docs now show nimExperimentalX APIs * fix for windows --- lib/std/jsfetch.nim | 2 +- lib/wrappers/linenoise/linenoise.nim | 12 ++++++------ tests/config.nims | 3 +++ tools/kochdocs.nim | 3 ++- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/lib/std/jsfetch.nim b/lib/std/jsfetch.nim index 36bc772d5..1e7bb4d29 100644 --- a/lib/std/jsfetch.nim +++ b/lib/std/jsfetch.nim @@ -3,7 +3,7 @@ when not defined(js): {.fatal: "Module jsfetch is designed to be used with the JavaScript backend.".} -when defined(nimExperimentalJsfetch) or defined(nimdoc): +when defined(nimExperimentalJsfetch): import std/[asyncjs, jsheaders, jsformdata] from std/httpcore import HttpMethod from std/jsffi import JsObject diff --git a/lib/wrappers/linenoise/linenoise.nim b/lib/wrappers/linenoise/linenoise.nim index 5b1fa7190..186b3b252 100644 --- a/lib/wrappers/linenoise/linenoise.nim +++ b/lib/wrappers/linenoise/linenoise.nim @@ -32,17 +32,17 @@ proc printKeyCodes*() {.importc: "linenoisePrintKeyCodes".} proc free*(s: cstring) {.importc: "free", header: "".} -when defined nimExperimentalLinenoiseExtra: +when defined(nimExperimentalLinenoiseExtra) and not defined(windows): # C interface - type linenoiseStatus = enum + type LinenoiseStatus = enum linenoiseStatus_ctrl_unknown linenoiseStatus_ctrl_C linenoiseStatus_ctrl_D - type linenoiseData* = object - status: linenoiseStatus + type LinenoiseData* = object + status: LinenoiseStatus - proc linenoiseExtra(prompt: cstring, data: ptr linenoiseData): cstring {.importc.} + proc linenoiseExtra(prompt: cstring, data: ptr LinenoiseData): cstring {.importc.} # stable nim interface type Status* = enum @@ -65,7 +65,7 @@ when defined nimExperimentalLinenoiseExtra: if ret.line.len > 0: echo ret.line if ret.status == lnCtrlD: break echo "exiting" - var data: linenoiseData + var data: LinenoiseData let buf = linenoiseExtra(prompt, data.addr) result.line = $buf free(buf) diff --git a/tests/config.nims b/tests/config.nims index 578b69e76..539de5e8d 100644 --- a/tests/config.nims +++ b/tests/config.nims @@ -30,4 +30,7 @@ hint("Processing", off) # switch("hint", "ConvFromXtoItselfNotNeeded") # experimental API's are enabled in testament, refs https://github.com/timotheecour/Nim/issues/575 +# sync with `kochdocs.docDefines` or refactor. switch("define", "nimExperimentalAsyncjsThen") +switch("define", "nimExperimentalJsfetch") +switch("define", "nimExperimentalLinenoiseExtra") diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index f25564fad..469059348 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -8,7 +8,8 @@ const gaCode* = " --doc.googleAnalytics:UA-48159761-1" # errormax: subsequent errors are probably consequences of 1st one; a simple # bug could cause unlimited number of errors otherwise, hard to debug in CI. - nimArgs = "--errormax:3 --hint:Conf:off --hint:Path:off --hint:Processing:off --hint:XDeclaredButNotUsed:off --warning:UnusedImport:off -d:boot --putenv:nimversion=$#" % system.NimVersion + docDefines = "-d:nimExperimentalAsyncjsThen -d:nimExperimentalJsfetch -d:nimExperimentalLinenoiseExtra" + nimArgs = "--errormax:3 --hint:Conf:off --hint:Path:off --hint:Processing:off --hint:XDeclaredButNotUsed:off --warning:UnusedImport:off -d:boot --putenv:nimversion=$# $#" % [system.NimVersion, docDefines] gitUrl = "https://github.com/nim-lang/Nim" docHtmlOutput = "doc/html" webUploadOutput = "web/upload" From f6bea08eac7938e460ca9088693f0cf3f274e75b Mon Sep 17 00:00:00 2001 From: tomc1998 Date: Fri, 25 Jun 2021 21:06:24 +0100 Subject: [PATCH 0980/2472] Add sink annotation to option some() (#18358) --- lib/pure/options.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/options.nim b/lib/pure/options.nim index bd5aa389e..850bfa555 100644 --- a/lib/pure/options.nim +++ b/lib/pure/options.nim @@ -92,7 +92,7 @@ type UnpackDefect* = object of Defect UnpackError* {.deprecated: "See corresponding Defect".} = UnpackDefect -proc option*[T](val: T): Option[T] {.inline.} = +proc option*[T](val: sink T): Option[T] {.inline.} = ## Can be used to convert a pointer type (`ptr`, `pointer`, `ref` or `proc`) to an option type. ## It converts `nil` to `none(T)`. When `T` is no pointer type, this is equivalent to `some(val)`. ## @@ -112,7 +112,7 @@ proc option*[T](val: T): Option[T] {.inline.} = when T isnot SomePointer: result.has = true -proc some*[T](val: T): Option[T] {.inline.} = +proc some*[T](val: sink T): Option[T] {.inline.} = ## Returns an `Option` that has the value `val`. ## ## **See also:** From 39fbf3c84bd83613407e22b3de215d9a221b9422 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 26 Jun 2021 03:39:52 +0200 Subject: [PATCH 0981/2472] ensure 'koch boot --gc:orc' stays green (#18353) * ensure 'koch boot --gc:orc' stays green * disable for C++ code generator for now --- koch.nim | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/koch.nim b/koch.nim index 51be53cc7..6a0b7a0d3 100644 --- a/koch.nim +++ b/koch.nim @@ -582,6 +582,11 @@ proc runCI(cmd: string) = when defined(posix): execFold("Run nimsuggest tests", "nim r nimsuggest/tester") + when not defined(bsd): + if not doUseCpp: + # the BSDs are overwhelmed already, so only run this test on the other machines: + kochExecFold("Boot Nim ORC", "boot -d:release --gc:orc --lib:lib") + proc testUnixInstall(cmdLineRest: string) = csource("-d:danger" & cmdLineRest) xz(false, cmdLineRest) From b8f761b7e2cb3f28abd6486d28ea19228887cdf5 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 26 Jun 2021 06:21:46 -0700 Subject: [PATCH 0982/2472] even lighter version of #17938: fix most issues with UnusedImport, XDeclaredButNotUsed, etc; fix #17511, #17510, #14246 (without realModule) (#18362) * {.used: symbol} * add tests * fix tests with --import * --import works without giving spurious unused warnings * new warning warnDuplicateModuleImport for `import foo; import foo` * fix test, add resolveModuleAlias, use proper line info for module aliases * fix spurious warnings * fix deprecation msg for deprecated modules even with `import foo as bar` * disable a test for i386 pending sorting XDeclaredButNotUsed errors * UnusedImport now works with re-exported symbols * fix typo [skip ci] * ic support * add genPNode to allow writing PNode-based compiler code similarly to `genAst` * fix DuplicateModuleImport warning * adjust test * fixup * fixup * fixup * fix after rebase * fix for IC * keep the proc inline, move the const out * [skip ci] fix changelog * experiment: remove calls to resolveModuleAlias * followup * fixup * fix tests/modules/tselfimport.nim * workaround tests/deprecated/tmodule1.nim * fix properly * simplify --- compiler/importer.nim | 34 +++++++++-------- compiler/lineinfos.nim | 2 + compiler/lookups.nim | 18 +++++---- compiler/passaux.nim | 2 +- lib/system/schubfach.nim | 59 +++++++++++++++--------------- testament/categories.nim | 1 + tests/deprecated/tmodule1.nim | 22 ++++++++--- tests/modules/tselfimport.nim | 2 +- tests/pragmas/mused2a.nim | 23 ++++++++++++ tests/pragmas/mused2b.nim | 3 ++ tests/pragmas/mused2c.nim | 1 + tests/pragmas/tused2.nim | 46 +++++++++++++++++++++++ tests/statictypes/tstatictypes.nim | 2 +- 13 files changed, 155 insertions(+), 60 deletions(-) create mode 100644 tests/pragmas/mused2a.nim create mode 100644 tests/pragmas/mused2b.nim create mode 100644 tests/pragmas/mused2c.nim create mode 100644 tests/pragmas/tused2.nim diff --git a/compiler/importer.nim b/compiler/importer.nim index 692cdb604..7a5c4de4a 100644 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -13,6 +13,7 @@ import intsets, ast, astalgo, msgs, options, idents, lookups, semdata, modulepaths, sigmatch, lineinfos, sets, modulegraphs, wordrecg +from strutils import `%` proc readExceptSet*(c: PContext, n: PNode): IntSet = assert n.kind in {nkImportExceptStmt, nkExportExceptStmt} @@ -224,19 +225,20 @@ proc importForwarded(c: PContext, n: PNode, exceptSet: IntSet; fromMod: PSym; im proc importModuleAs(c: PContext; n: PNode, realModule: PSym, importHidden: bool): PSym = result = realModule - c.unusedImports.add((realModule, n.info)) template createModuleAliasImpl(ident): untyped = - createModuleAlias(realModule, nextSymId c.idgen, ident, realModule.info, c.config.options) + createModuleAlias(realModule, nextSymId c.idgen, ident, n.info, c.config.options) if n.kind != nkImportAs: discard elif n.len != 2 or n[1].kind != nkIdent: localError(c.config, n.info, "module alias must be an identifier") elif n[1].ident.id != realModule.name.id: # some misguided guy will write 'import abc.foo as foo' ... result = createModuleAliasImpl(n[1].ident) + if result == realModule: + # avoids modifying `realModule`, see D20201209T194412 for `import {.all.}` + result = createModuleAliasImpl(realModule.name) if importHidden: - if result == realModule: # avoids modifying `realModule`, see D20201209T194412. - result = createModuleAliasImpl(realModule.name) result.options.incl optImportHidden + c.unusedImports.add((result, n.info)) proc transformImportAs(c: PContext; n: PNode): tuple[node: PNode, importHidden: bool] = var ret: typeof(result) @@ -274,23 +276,22 @@ proc myImportModule(c: PContext, n: var PNode, importStmtResult: PNode): PSym = toFullPath(c.config, c.graph.importStack[i+1]) c.recursiveDep = err + var realModule: PSym discard pushOptionEntry(c) - result = importModuleAs(c, n, c.graph.importModuleCallback(c.graph, c.module, f), transf.importHidden) + realModule = c.graph.importModuleCallback(c.graph, c.module, f) + result = importModuleAs(c, n, realModule, transf.importHidden) popOptionEntry(c) #echo "set back to ", L c.graph.importStack.setLen(L) # we cannot perform this check reliably because of - # test: modules/import_in_config) - when true: - if result.info.fileIndex == c.module.info.fileIndex and - result.info.fileIndex == n.info.fileIndex: - localError(c.config, n.info, "A module cannot import itself") - if sfDeprecated in result.flags: - if result.constraint != nil: - message(c.config, n.info, warnDeprecated, result.constraint.strVal & "; " & result.name.s & " is deprecated") - else: - message(c.config, n.info, warnDeprecated, result.name.s & " is deprecated") + # test: modules/import_in_config) # xxx is that still true? + if realModule == c.module: + localError(c.config, n.info, "module '$1' cannot import itself" % realModule.name.s) + if sfDeprecated in realModule.flags: + var prefix = "" + if realModule.constraint != nil: prefix = realModule.constraint.strVal & "; " + message(c.config, n.info, warnDeprecated, prefix & realModule.name.s & " is deprecated") suggestSym(c.graph, n.info, result, c.graph.usageSym, false) importStmtResult.add newSymNode(result, n.info) #newStrNode(toFullPath(c.config, f), n.info) @@ -303,6 +304,9 @@ proc impMod(c: PContext; it: PNode; importStmtResult: PNode) = addDecl(c, m, it.info) # add symbol to symbol table of module importAllSymbols(c, m) #importForwarded(c, m.ast, emptySet, m) + for s in allSyms(c.graph, m): # fixes bug #17510, for re-exported symbols + if s.owner != m: + c.exportIndirections.incl((m.id, s.id)) proc evalImport*(c: PContext, n: PNode): PNode = result = newNodeI(nkImportStmt, n.info) diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index abe6bce7c..11fa45480 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -67,6 +67,7 @@ type warnResultUsed = "ResultUsed", warnCannotOpen = "CannotOpen", warnFileChanged = "FileChanged", + warnDuplicateModuleImport = "DuplicateModuleImport", warnUser = "User", # hints hintSuccess = "Success", hintSuccessX = "SuccessX", @@ -148,6 +149,7 @@ const warnResultUsed: "used 'result' variable", warnCannotOpen: "cannot open: $1", warnFileChanged: "file changed: $1", + warnDuplicateModuleImport: "$1", warnUser: "$1", hintSuccess: "operation successful: $#", # keep in sync with `testament.isSuccess` diff --git a/compiler/lookups.nim b/compiler/lookups.nim index ee79673d0..0e057a79a 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -58,8 +58,8 @@ proc considerQuotedIdent*(c: PContext; n: PNode, origin: PNode = nil): PIdent = template addSym*(scope: PScope, s: PSym) = strTableAdd(scope.symbols, s) -proc addUniqueSym*(scope: PScope, s: PSym): PSym = - result = strTableInclReportConflict(scope.symbols, s) +proc addUniqueSym*(scope: PScope, s: PSym, onConflictKeepOld: bool): PSym = + result = strTableInclReportConflict(scope.symbols, s, onConflictKeepOld) proc openScope*(c: PContext): PScope {.discardable.} = result = PScope(parent: c.currentScope, @@ -288,19 +288,23 @@ proc ensureNoMissingOrUnusedSymbols(c: PContext; scope: PScope) = message(c.config, s.info, hintXDeclaredButNotUsed, s.name.s) proc wrongRedefinition*(c: PContext; info: TLineInfo, s: string; - conflictsWith: TLineInfo) = + conflictsWith: TLineInfo, note = errGenerated) = ## Emit a redefinition error if in non-interactive mode if c.config.cmd != cmdInteractive: - localError(c.config, info, + localError(c.config, info, note, "redefinition of '$1'; previous declaration here: $2" % [s, c.config $ conflictsWith]) # xxx pending bootstrap >= 1.4, replace all those overloads with a single one: # proc addDecl*(c: PContext, sym: PSym, info = sym.info, scope = c.currentScope) {.inline.} = proc addDeclAt*(c: PContext; scope: PScope, sym: PSym, info: TLineInfo) = - let conflict = scope.addUniqueSym(sym) + let conflict = scope.addUniqueSym(sym, onConflictKeepOld = true) if conflict != nil: - wrongRedefinition(c, info, sym.name.s, conflict.info) + var note = errGenerated + if sym.kind == skModule and conflict.kind == skModule and sym.owner == conflict.owner: + # import foo; import foo + note = warnDuplicateModuleImport + wrongRedefinition(c, info, sym.name.s, conflict.info, note) proc addDeclAt*(c: PContext; scope: PScope, sym: PSym) {.inline.} = addDeclAt(c, scope, sym, sym.info) @@ -312,7 +316,7 @@ proc addDecl*(c: PContext, sym: PSym) {.inline.} = addDeclAt(c, c.currentScope, sym) proc addPrelimDecl*(c: PContext, sym: PSym) = - discard c.currentScope.addUniqueSym(sym) + discard c.currentScope.addUniqueSym(sym, onConflictKeepOld = false) from ic / ic import addHidden diff --git a/compiler/passaux.nim b/compiler/passaux.nim index 9abc97be4..87cb3a730 100644 --- a/compiler/passaux.nim +++ b/compiler/passaux.nim @@ -10,7 +10,7 @@ ## implements some little helper passes import - ast, passes, idents, msgs, options, lineinfos + ast, passes, msgs, options, lineinfos from modulegraphs import ModuleGraph, PPassContext diff --git a/lib/system/schubfach.nim b/lib/system/schubfach.nim index 5f974a79e..d072cde99 100644 --- a/lib/system/schubfach.nim +++ b/lib/system/schubfach.nim @@ -95,6 +95,36 @@ proc floorLog2Pow10(e: int32): int32 {.inline.} = sf_Assert(e <= 1233) return floorDivPow2(e * 1741647, 19) +const + kMin: int32 = -31 + kMax: int32 = 45 + g: array[kMax - kMin + 1, uint64] = [0x81CEB32C4B43FCF5'u64, 0xA2425FF75E14FC32'u64, + 0xCAD2F7F5359A3B3F'u64, 0xFD87B5F28300CA0E'u64, 0x9E74D1B791E07E49'u64, + 0xC612062576589DDB'u64, 0xF79687AED3EEC552'u64, 0x9ABE14CD44753B53'u64, + 0xC16D9A0095928A28'u64, 0xF1C90080BAF72CB2'u64, 0x971DA05074DA7BEF'u64, + 0xBCE5086492111AEB'u64, 0xEC1E4A7DB69561A6'u64, 0x9392EE8E921D5D08'u64, + 0xB877AA3236A4B44A'u64, 0xE69594BEC44DE15C'u64, 0x901D7CF73AB0ACDA'u64, + 0xB424DC35095CD810'u64, 0xE12E13424BB40E14'u64, 0x8CBCCC096F5088CC'u64, + 0xAFEBFF0BCB24AAFF'u64, 0xDBE6FECEBDEDD5BF'u64, 0x89705F4136B4A598'u64, + 0xABCC77118461CEFD'u64, 0xD6BF94D5E57A42BD'u64, 0x8637BD05AF6C69B6'u64, + 0xA7C5AC471B478424'u64, 0xD1B71758E219652C'u64, 0x83126E978D4FDF3C'u64, + 0xA3D70A3D70A3D70B'u64, 0xCCCCCCCCCCCCCCCD'u64, 0x8000000000000000'u64, + 0xA000000000000000'u64, 0xC800000000000000'u64, 0xFA00000000000000'u64, + 0x9C40000000000000'u64, 0xC350000000000000'u64, 0xF424000000000000'u64, + 0x9896800000000000'u64, 0xBEBC200000000000'u64, 0xEE6B280000000000'u64, + 0x9502F90000000000'u64, 0xBA43B74000000000'u64, 0xE8D4A51000000000'u64, + 0x9184E72A00000000'u64, 0xB5E620F480000000'u64, 0xE35FA931A0000000'u64, + 0x8E1BC9BF04000000'u64, 0xB1A2BC2EC5000000'u64, 0xDE0B6B3A76400000'u64, + 0x8AC7230489E80000'u64, 0xAD78EBC5AC620000'u64, 0xD8D726B7177A8000'u64, + 0x878678326EAC9000'u64, 0xA968163F0A57B400'u64, 0xD3C21BCECCEDA100'u64, + 0x84595161401484A0'u64, 0xA56FA5B99019A5C8'u64, 0xCECB8F27F4200F3A'u64, + 0x813F3978F8940985'u64, 0xA18F07D736B90BE6'u64, 0xC9F2C9CD04674EDF'u64, + 0xFC6F7C4045812297'u64, 0x9DC5ADA82B70B59E'u64, 0xC5371912364CE306'u64, + 0xF684DF56C3E01BC7'u64, 0x9A130B963A6C115D'u64, 0xC097CE7BC90715B4'u64, + 0xF0BDC21ABB48DB21'u64, 0x96769950B50D88F5'u64, 0xBC143FA4E250EB32'u64, + 0xEB194F8E1AE525FE'u64, 0x92EFD1B8D0CF37BF'u64, 0xB7ABC627050305AE'u64, + 0xE596B7B0C643C71A'u64, 0x8F7E32CE7BEA5C70'u64, 0xB35DBF821AE4F38C'u64] + proc computePow10Single(k: int32): uint64 {.inline.} = ## There are unique beta and r such that 10^k = beta 2^r and ## 2^63 <= beta < 2^64, namely r = floor(log_2 10^k) - 63 and @@ -103,35 +133,6 @@ proc computePow10Single(k: int32): uint64 {.inline.} = ## value being a pretty good overestimate for 10^k. ## NB: Since for all the required exponents k, we have g < 2^64, ## all constants can be stored in 128-bit integers. - const - kMin: int32 = -31 - kMax: int32 = 45 - g: array[kMax - kMin + 1, uint64] = [0x81CEB32C4B43FCF5'u64, 0xA2425FF75E14FC32'u64, - 0xCAD2F7F5359A3B3F'u64, 0xFD87B5F28300CA0E'u64, 0x9E74D1B791E07E49'u64, - 0xC612062576589DDB'u64, 0xF79687AED3EEC552'u64, 0x9ABE14CD44753B53'u64, - 0xC16D9A0095928A28'u64, 0xF1C90080BAF72CB2'u64, 0x971DA05074DA7BEF'u64, - 0xBCE5086492111AEB'u64, 0xEC1E4A7DB69561A6'u64, 0x9392EE8E921D5D08'u64, - 0xB877AA3236A4B44A'u64, 0xE69594BEC44DE15C'u64, 0x901D7CF73AB0ACDA'u64, - 0xB424DC35095CD810'u64, 0xE12E13424BB40E14'u64, 0x8CBCCC096F5088CC'u64, - 0xAFEBFF0BCB24AAFF'u64, 0xDBE6FECEBDEDD5BF'u64, 0x89705F4136B4A598'u64, - 0xABCC77118461CEFD'u64, 0xD6BF94D5E57A42BD'u64, 0x8637BD05AF6C69B6'u64, - 0xA7C5AC471B478424'u64, 0xD1B71758E219652C'u64, 0x83126E978D4FDF3C'u64, - 0xA3D70A3D70A3D70B'u64, 0xCCCCCCCCCCCCCCCD'u64, 0x8000000000000000'u64, - 0xA000000000000000'u64, 0xC800000000000000'u64, 0xFA00000000000000'u64, - 0x9C40000000000000'u64, 0xC350000000000000'u64, 0xF424000000000000'u64, - 0x9896800000000000'u64, 0xBEBC200000000000'u64, 0xEE6B280000000000'u64, - 0x9502F90000000000'u64, 0xBA43B74000000000'u64, 0xE8D4A51000000000'u64, - 0x9184E72A00000000'u64, 0xB5E620F480000000'u64, 0xE35FA931A0000000'u64, - 0x8E1BC9BF04000000'u64, 0xB1A2BC2EC5000000'u64, 0xDE0B6B3A76400000'u64, - 0x8AC7230489E80000'u64, 0xAD78EBC5AC620000'u64, 0xD8D726B7177A8000'u64, - 0x878678326EAC9000'u64, 0xA968163F0A57B400'u64, 0xD3C21BCECCEDA100'u64, - 0x84595161401484A0'u64, 0xA56FA5B99019A5C8'u64, 0xCECB8F27F4200F3A'u64, - 0x813F3978F8940985'u64, 0xA18F07D736B90BE6'u64, 0xC9F2C9CD04674EDF'u64, - 0xFC6F7C4045812297'u64, 0x9DC5ADA82B70B59E'u64, 0xC5371912364CE306'u64, - 0xF684DF56C3E01BC7'u64, 0x9A130B963A6C115D'u64, 0xC097CE7BC90715B4'u64, - 0xF0BDC21ABB48DB21'u64, 0x96769950B50D88F5'u64, 0xBC143FA4E250EB32'u64, - 0xEB194F8E1AE525FE'u64, 0x92EFD1B8D0CF37BF'u64, 0xB7ABC627050305AE'u64, - 0xE596B7B0C643C71A'u64, 0x8F7E32CE7BEA5C70'u64, 0xB35DBF821AE4F38C'u64] sf_Assert(k >= kMin) sf_Assert(k <= kMax) return g[k - kMin] diff --git a/testament/categories.nim b/testament/categories.nim index bf7855b2c..b5e8a5454 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -508,6 +508,7 @@ proc icTests(r: var TResults; testsDir: string, cat: Category, options: string; const tempExt = "_temp.nim" for it in walkDirRec(testsDir): + # for it in ["tests/ic/timports.nim"]: # debugging: to try a specific test if isTestFile(it) and not it.endsWith(tempExt): let nimcache = nimcacheDir(it, options, getTestSpecTarget()) removeDir(nimcache) diff --git a/tests/deprecated/tmodule1.nim b/tests/deprecated/tmodule1.nim index 954836889..f26e4ce3f 100644 --- a/tests/deprecated/tmodule1.nim +++ b/tests/deprecated/tmodule1.nim @@ -1,13 +1,23 @@ discard """ - nimout: '''tmodule1.nim(11, 8) Warning: goodbye; importme is deprecated [Deprecated] -tmodule1.nim(14, 10) Warning: Ty is deprecated [Deprecated] -tmodule1.nim(17, 10) Warning: hello; Ty1 is deprecated [Deprecated] -tmodule1.nim(20, 8) Warning: aVar is deprecated [Deprecated] -tmodule1.nim(22, 3) Warning: aProc is deprecated [Deprecated] -tmodule1.nim(23, 3) Warning: hello; aProc1 is deprecated [Deprecated] + matrix: "--hint:all:off" + nimoutFull: true + nimout: ''' +tmodule1.nim(21, 8) Warning: goodbye; importme is deprecated [Deprecated] +tmodule1.nim(24, 10) Warning: Ty is deprecated [Deprecated] +tmodule1.nim(27, 10) Warning: hello; Ty1 is deprecated [Deprecated] +tmodule1.nim(30, 8) Warning: aVar is deprecated [Deprecated] +tmodule1.nim(32, 3) Warning: aProc is deprecated [Deprecated] +tmodule1.nim(33, 3) Warning: hello; aProc1 is deprecated [Deprecated] ''' """ + + + + + + +# line 20 import importme block: diff --git a/tests/modules/tselfimport.nim b/tests/modules/tselfimport.nim index 7e50bef7c..ba5d9b4cf 100644 --- a/tests/modules/tselfimport.nim +++ b/tests/modules/tselfimport.nim @@ -1,5 +1,5 @@ discard """ - errormsg: "A module cannot import itself" + errormsg: "module 'tselfimport' cannot import itself" file: "tselfimport.nim" line: 7 """ diff --git a/tests/pragmas/mused2a.nim b/tests/pragmas/mused2a.nim new file mode 100644 index 000000000..d9b2bb9bf --- /dev/null +++ b/tests/pragmas/mused2a.nim @@ -0,0 +1,23 @@ +import std/strutils + +from std/os import fileExists + +import std/typetraits as typetraits2 +from std/setutils import complement + + + + + +proc fn1() = discard +proc fn2*() = discard + + +let fn4 = 0 +let fn5* = 0 + + +const fn7 = 0 +const fn8* = 0 + +type T1 = object diff --git a/tests/pragmas/mused2b.nim b/tests/pragmas/mused2b.nim new file mode 100644 index 000000000..39c92b964 --- /dev/null +++ b/tests/pragmas/mused2b.nim @@ -0,0 +1,3 @@ +import mused2c +export mused2c + diff --git a/tests/pragmas/mused2c.nim b/tests/pragmas/mused2c.nim new file mode 100644 index 000000000..a374e634e --- /dev/null +++ b/tests/pragmas/mused2c.nim @@ -0,0 +1 @@ +proc baz*() = discard \ No newline at end of file diff --git a/tests/pragmas/tused2.nim b/tests/pragmas/tused2.nim new file mode 100644 index 000000000..f80c198d8 --- /dev/null +++ b/tests/pragmas/tused2.nim @@ -0,0 +1,46 @@ +discard """ + matrix: "--hint:all:off --hint:XDeclaredButNotUsed --path:." + joinable: false + nimoutFull: true + nimout: ''' +mused2a.nim(12, 6) Hint: 'fn1' is declared but not used [XDeclaredButNotUsed] +mused2a.nim(16, 5) Hint: 'fn4' is declared but not used [XDeclaredButNotUsed] +mused2a.nim(20, 7) Hint: 'fn7' is declared but not used [XDeclaredButNotUsed] +mused2a.nim(23, 6) Hint: 'T1' is declared but not used [XDeclaredButNotUsed] +mused2a.nim(1, 11) Warning: imported and not used: 'strutils' [UnusedImport] +mused2a.nim(3, 9) Warning: imported and not used: 'os' [UnusedImport] +mused2a.nim(5, 23) Warning: imported and not used: 'typetraits2' [UnusedImport] +mused2a.nim(6, 9) Warning: imported and not used: 'setutils' [UnusedImport] +tused2.nim(42, 8) Warning: imported and not used: 'mused2a' [UnusedImport] +tused2.nim(45, 11) Warning: imported and not used: 'strutils' [UnusedImport] +''' +""" + + + + + + + + + + + + + + + + + + + + + + +# line 40 + +import mused2a +import mused2b + +import std/strutils +baz() diff --git a/tests/statictypes/tstatictypes.nim b/tests/statictypes/tstatictypes.nim index 16cb55c28..5df3f35fd 100644 --- a/tests/statictypes/tstatictypes.nim +++ b/tests/statictypes/tstatictypes.nim @@ -271,7 +271,7 @@ block: fails(foo) -import macros, tables +import tables var foo{.compileTime.} = [ "Foo", From 1b9b8060075efab82912dc33ba64e671d102b999 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Sat, 26 Jun 2021 19:10:46 +0300 Subject: [PATCH 0983/2472] rst: fix bug 20 from #17340 (#18360) and a leftover bug: priority of option list inside definition list --- lib/packages/docutils/rst.nim | 4 ++- tests/stdlib/trst.nim | 47 +++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 8ffbbd4d3..419cf50e7 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -2245,7 +2245,7 @@ proc parseOptionList(p: var RstParser): PRstNode = popInd(p) else: parseLine(p, b) - if currentTok(p).kind == tkIndent: inc p.idx + while currentTok(p).kind == tkIndent: inc p.idx c.add(a) c.add(b) c.order = order; inc order @@ -2262,6 +2262,8 @@ proc parseDefinitionList(p: var RstParser): PRstNode = var col = currentTok(p).col result = newRstNodeA(p, rnDefList) while true: + if isOptionList(p): + break # option list has priority over def.list j = p.idx var a = newRstNode(rnDefName) parseLine(p, a) diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index d53092d3b..fb9552479 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -83,6 +83,53 @@ suite "RST parsing": rnLeaf 'set' """) + test "items of 1 option list can be separated by blank lines": + check(dedent""" + -a desc1 + + -b desc2 + """.toAst == + dedent""" + rnOptionList + rnOptionListItem order=1 + rnOptionGroup + rnLeaf '-' + rnLeaf 'a' + rnDescription + rnLeaf 'desc1' + rnOptionListItem order=2 + rnOptionGroup + rnLeaf '-' + rnLeaf 'b' + rnDescription + rnLeaf 'desc2' + """) + + test "option list has priority over definition list": + check(dedent""" + defName + defBody + + -b desc2 + """.toAst == + dedent""" + rnInner + rnDefList + rnDefItem + rnDefName + rnLeaf 'defName' + rnDefBody + rnInner + rnLeaf 'defBody' + rnOptionList + rnOptionListItem order=1 + rnOptionGroup + rnLeaf '-' + rnLeaf 'b' + rnDescription + rnLeaf 'desc2' + """) + test "RST comment": check(dedent""" .. comment1 From 0b7361e938134305d6893ee2876b5fc8f9ba419b Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 27 Jun 2021 11:39:16 -0700 Subject: [PATCH 0984/2472] followup #18362: make `UnusedImport` work robustly (#18366) * warnDuplicateModuleImport => hintDuplicateModuleImport * improve DuplicateModuleImport msg, add test --- compiler/cgen.nim | 2 - compiler/dfa.nim | 2 +- compiler/docgen.nim | 2 +- compiler/importer.nim | 16 +++++-- compiler/injectdestructors.nim | 6 +-- compiler/jsgen.nim | 2 - compiler/lineinfos.nim | 6 +-- compiler/lookups.nim | 12 ++++-- compiler/semdata.nim | 3 +- compiler/suggest.nim | 5 ++- tests/misc/trunner.nim | 26 +++++++++++- tests/pragmas/mused3.nim | 76 ++++++++++++++++++++++++++++++++++ tests/pragmas/mused3a.nim | 41 ++++++++++++++++++ tests/pragmas/mused3b.nim | 12 ++++++ 14 files changed, 186 insertions(+), 25 deletions(-) create mode 100644 tests/pragmas/mused3.nim create mode 100644 tests/pragmas/mused3a.nim create mode 100644 tests/pragmas/mused3b.nim diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 5d6de2a03..3b004d399 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -24,8 +24,6 @@ import strutils except `%` # collides with ropes.`%` from ic / ic import ModuleBackendFlag from modulegraphs import ModuleGraph, PPassContext -from lineinfos import - warnGcMem, errXMustBeCompileTime, hintDependency, errGenerated, errCannotOpenFile import dynlib when not declared(dynlib.libCandidates): diff --git a/compiler/dfa.nim b/compiler/dfa.nim index ea60dbc59..402945042 100644 --- a/compiler/dfa.nim +++ b/compiler/dfa.nim @@ -29,7 +29,7 @@ ## "A Graph–Free Approach to Data–Flow Analysis" by Markus Mohnen. ## https://link.springer.com/content/pdf/10.1007/3-540-45937-5_6.pdf -import ast, types, intsets, lineinfos, renderer +import ast, intsets, lineinfos, renderer import std/private/asciitables type diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 08311eade..e6bb7cef8 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -16,7 +16,7 @@ import packages/docutils/rst, packages/docutils/rstgen, json, xmltree, trees, types, typesrenderer, astalgo, lineinfos, intsets, - pathutils, trees, tables, nimpaths, renderverbatim, osproc + pathutils, tables, nimpaths, renderverbatim, osproc from uri import encodeUrl from std/private/globs import nativeToUnixPath diff --git a/compiler/importer.nim b/compiler/importer.nim index 7a5c4de4a..af392f849 100644 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -12,7 +12,7 @@ import intsets, ast, astalgo, msgs, options, idents, lookups, semdata, modulepaths, sigmatch, lineinfos, sets, - modulegraphs, wordrecg + modulegraphs, wordrecg, tables from strutils import `%` proc readExceptSet*(c: PContext, n: PNode): IntSet = @@ -239,6 +239,7 @@ proc importModuleAs(c: PContext; n: PNode, realModule: PSym, importHidden: bool) if importHidden: result.options.incl optImportHidden c.unusedImports.add((result, n.info)) + c.importModuleMap[result.id] = realModule.id proc transformImportAs(c: PContext; n: PNode): tuple[node: PNode, importHidden: bool] = var ret: typeof(result) @@ -296,6 +297,13 @@ proc myImportModule(c: PContext, n: var PNode, importStmtResult: PNode): PSym = importStmtResult.add newSymNode(result, n.info) #newStrNode(toFullPath(c.config, f), n.info) +proc afterImport(c: PContext, m: PSym) = + # fixes bug #17510, for re-exported symbols + let realModuleId = c.importModuleMap[m.id] + for s in allSyms(c.graph, m): + if s.owner.id != realModuleId: + c.exportIndirections.incl((m.id, s.id)) + proc impMod(c: PContext; it: PNode; importStmtResult: PNode) = var it = it let m = myImportModule(c, it, importStmtResult) @@ -304,9 +312,7 @@ proc impMod(c: PContext; it: PNode; importStmtResult: PNode) = addDecl(c, m, it.info) # add symbol to symbol table of module importAllSymbols(c, m) #importForwarded(c, m.ast, emptySet, m) - for s in allSyms(c.graph, m): # fixes bug #17510, for re-exported symbols - if s.owner != m: - c.exportIndirections.incl((m.id, s.id)) + afterImport(c, m) proc evalImport*(c: PContext, n: PNode): PNode = result = newNodeI(nkImportStmt, n.info) @@ -345,6 +351,7 @@ proc evalFrom*(c: PContext, n: PNode): PNode = if n[i].kind != nkNilLit: importSymbol(c, n[i], m, im.imported) c.addImport im + afterImport(c, m) proc evalImportExcept*(c: PContext, n: PNode): PNode = result = newNodeI(nkImportStmt, n.info) @@ -355,3 +362,4 @@ proc evalImportExcept*(c: PContext, n: PNode): PNode = addDecl(c, m, n.info) # add symbol to symbol table of module importAllSymbolsExcept(c, m, readExceptSet(c, n)) #importForwarded(c, m.ast, exceptSet, m) + afterImport(c, m) diff --git a/compiler/injectdestructors.nim b/compiler/injectdestructors.nim index 8ef08adeb..6447557c5 100644 --- a/compiler/injectdestructors.nim +++ b/compiler/injectdestructors.nim @@ -15,7 +15,7 @@ import intsets, strtabs, ast, astalgo, msgs, renderer, magicsys, types, idents, - strutils, options, dfa, lowerings, tables, modulegraphs, msgs, + strutils, options, dfa, lowerings, tables, modulegraphs, lineinfos, parampatterns, sighashes, liftdestructors, optimizer, varpartitions @@ -73,7 +73,7 @@ proc nestedScope(parent: var Scope): Scope = proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode): PNode proc moveOrCopy(dest, ri: PNode; c: var Con; s: var Scope; isDecl = false): PNode -import sets, hashes, tables +import sets, hashes proc hash(n: PNode): Hash = hash(cast[pointer](n)) @@ -245,8 +245,6 @@ template isUnpackedTuple(n: PNode): bool = ## hence unpacked tuples themselves don't need to be destroyed (n.kind == nkSym and n.sym.kind == skTemp and n.sym.typ.kind == tyTuple) -from strutils import parseInt - proc checkForErrorPragma(c: Con; t: PType; ri: PNode; opname: string) = var m = "'" & opname & "' is not available for type <" & typeToString(t) & ">" if (opname == "=" or opname == "=copy") and ri != nil: diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 3b183b9d6..393ac83c2 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -37,8 +37,6 @@ import import json, sets, math, tables, intsets, strutils -from modulegraphs import ModuleGraph, PPassContext - type TJSGen = object of PPassContext module: PSym diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index 11fa45480..131fe480b 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -67,12 +67,12 @@ type warnResultUsed = "ResultUsed", warnCannotOpen = "CannotOpen", warnFileChanged = "FileChanged", - warnDuplicateModuleImport = "DuplicateModuleImport", warnUser = "User", # hints hintSuccess = "Success", hintSuccessX = "SuccessX", hintCC = "CC", - hintLineTooLong = "LineTooLong", hintXDeclaredButNotUsed = "XDeclaredButNotUsed", + hintLineTooLong = "LineTooLong", + hintXDeclaredButNotUsed = "XDeclaredButNotUsed", hintDuplicateModuleImport = "DuplicateModuleImport", hintXCannotRaiseY = "XCannotRaiseY", hintConvToBaseNotNeeded = "ConvToBaseNotNeeded", hintConvFromXtoItselfNotNeeded = "ConvFromXtoItselfNotNeeded", hintExprAlwaysX = "ExprAlwaysX", hintQuitCalled = "QuitCalled", hintProcessing = "Processing", hintCodeBegin = "CodeBegin", @@ -149,7 +149,6 @@ const warnResultUsed: "used 'result' variable", warnCannotOpen: "cannot open: $1", warnFileChanged: "file changed: $1", - warnDuplicateModuleImport: "$1", warnUser: "$1", hintSuccess: "operation successful: $#", # keep in sync with `testament.isSuccess` @@ -157,6 +156,7 @@ const hintCC: "CC: $1", hintLineTooLong: "line too long", hintXDeclaredButNotUsed: "'$1' is declared but not used", + hintDuplicateModuleImport: "$1", hintXCannotRaiseY: "$1", hintConvToBaseNotNeeded: "conversion to base object is not needed", hintConvFromXtoItselfNotNeeded: "conversion from $1 to itself is pointless", diff --git a/compiler/lookups.nim b/compiler/lookups.nim index 0e057a79a..9b878dd4b 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -300,11 +300,15 @@ proc wrongRedefinition*(c: PContext; info: TLineInfo, s: string; proc addDeclAt*(c: PContext; scope: PScope, sym: PSym, info: TLineInfo) = let conflict = scope.addUniqueSym(sym, onConflictKeepOld = true) if conflict != nil: - var note = errGenerated if sym.kind == skModule and conflict.kind == skModule and sym.owner == conflict.owner: - # import foo; import foo - note = warnDuplicateModuleImport - wrongRedefinition(c, info, sym.name.s, conflict.info, note) + # e.g.: import foo; import foo + # xxx we could refine this by issuing a different hint for the case + # where a duplicate import happens inside an include. + localError(c.config, info, hintDuplicateModuleImport, + "duplicate import of '$1'; previous import here: $2" % + [sym.name.s, c.config $ conflict.info]) + else: + wrongRedefinition(c, info, sym.name.s, conflict.info, errGenerated) proc addDeclAt*(c: PContext; scope: PScope, sym: PSym) {.inline.} = addDeclAt(c, scope, sym, sym.info) diff --git a/compiler/semdata.nim b/compiler/semdata.nim index f763a00e4..bd863505c 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -156,7 +156,8 @@ type features*: set[Feature] inTypeContext*, inConceptDecl*: int unusedImports*: seq[(PSym, TLineInfo)] - exportIndirections*: HashSet[(int, int)] + exportIndirections*: HashSet[(int, int)] # (module.id, symbol.id) + importModuleMap*: Table[int, int] # (module.id, module.id) lastTLineInfo*: TLineInfo template config*(c: PContext): ConfigRef = c.graph.config diff --git a/compiler/suggest.nim b/compiler/suggest.nim index a5b4ac87d..eaa30040a 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -32,7 +32,7 @@ # included from sigmatch.nim -import algorithm, sets, prefixmatches, lineinfos, parseutils, linter +import algorithm, sets, prefixmatches, lineinfos, parseutils, linter, tables from wordrecg import wDeprecated, wError, wAddr, wYield when defined(nimsuggest): @@ -572,7 +572,8 @@ proc markOwnerModuleAsUsed(c: PContext; s: PSym) = var i = 0 while i <= high(c.unusedImports): let candidate = c.unusedImports[i][0] - if candidate == module or c.exportIndirections.contains((candidate.id, s.id)): + if candidate == module or c.importModuleMap.getOrDefault(candidate.id, int.low) == module.id or + c.exportIndirections.contains((candidate.id, s.id)): # mark it as used: c.unusedImports.del(i) else: diff --git a/tests/misc/trunner.nim b/tests/misc/trunner.nim index 77ae10f3d..8426e9aee 100644 --- a/tests/misc/trunner.nim +++ b/tests/misc/trunner.nim @@ -31,7 +31,7 @@ const proc runNimCmd(file, options = "", rtarg = ""): auto = let fileabs = testsDir / file.unixToNativePath # doAssert fileabs.fileExists, fileabs # disabled because this allows passing `nim r --eval:code fakefile` - let cmd = fmt"{nim} {mode} {options} --hints:off {fileabs} {rtarg}" + let cmd = fmt"{nim} {mode} --hint:all:off {options} {fileabs} {rtarg}" result = execCmdEx(cmd) when false: # for debugging echo cmd @@ -352,5 +352,29 @@ running: v2 doAssert outp2 == "12345\n", outp2 doAssert status2 == 0 + block: # UnusedImport + proc fn(opt: string, expected: string) = + let output = runNimCmdChk("pragmas/mused3.nim", fmt"--warning:all:off --warning:UnusedImport --hint:DuplicateModuleImport {opt}") + doAssert output == expected, opt & "\noutput:\n" & output & "expected:\n" & expected + fn("-d:case1"): """ +mused3.nim(13, 8) Warning: imported and not used: 'mused3b' [UnusedImport] +""" + fn("-d:case2"): "" + fn("-d:case3"): "" + fn("-d:case4"): "" + fn("-d:case5"): "" + fn("-d:case6"): "" + fn("-d:case7"): "" + fn("-d:case8"): "" + fn("-d:case9"): "" + fn("-d:case10"): "" + when false: + fn("-d:case11"): """ + Warning: imported and not used: 'm2' [UnusedImport] + """ + fn("-d:case12"): """ +mused3.nim(75, 10) Hint: duplicate import of 'mused3a'; previous import here: mused3.nim(74, 10) [DuplicateModuleImport] +""" + else: discard # only during debugging, tests added here will run with `-d:nimTestsTrunnerDebugging` enabled diff --git a/tests/pragmas/mused3.nim b/tests/pragmas/mused3.nim new file mode 100644 index 000000000..0beec1d44 --- /dev/null +++ b/tests/pragmas/mused3.nim @@ -0,0 +1,76 @@ +#[ +ran from trunner +]# + + + + + + +# line 10 +when defined case1: + from mused3a import nil + from mused3b import nil + mused3a.fn1() + +when defined case2: + from mused3a as m1 import nil + m1.fn1() + +when defined case3: + from mused3a import fn1 + fn1() + +when defined case4: + from mused3a as m1 import fn1 + m1.fn1() + +when defined case5: + import mused3a as m1 + fn1() + +when defined case6: + import mused3a except nonexistent + fn1() + +when defined case7: + import mused3a + mused3a.fn1() + +when defined case8: + # re-export test + import mused3a except nonexistent + gn1() + +when defined case9: + # re-export test + import mused3a + gn1() + +when defined case10: + #[ + edge case which happens a lot in compiler code: + don't report UnusedImport for mused3b here even though it works without `import mused3b`, + because `a.b0.f0` is accessible from both mused3a and mused3b (fields are given implicit access) + ]# + import mused3a + import mused3b + var a: Bar + discard a.b0.f0 + +when false: + when defined case11: + #[ + xxx minor bug: this should give: + Warning: imported and not used: 'm2' [UnusedImport] + but doesn't, because currently implementation in `markOwnerModuleAsUsed` + only looks at `fn1`, not fully qualified call `m1.fn1() + ]# + from mused3a as m1 import nil + from mused3a as m2 import nil + m1.fn1() + +when defined case12: + import mused3a + import mused3a + fn1() diff --git a/tests/pragmas/mused3a.nim b/tests/pragmas/mused3a.nim new file mode 100644 index 000000000..c33d1ecb3 --- /dev/null +++ b/tests/pragmas/mused3a.nim @@ -0,0 +1,41 @@ +when defined case1: + proc fn1*() = discard + +when defined case2: + proc fn1*() = discard + +when defined case3: + proc fn1*() = discard + proc fn2*() = discard + +when defined case4: + proc fn1*() = discard + proc fn2*() = discard + +when defined case5: + proc fn1*() = discard + +when defined case6: + proc fn1*() = discard + +when defined case7: + proc fn1*() = discard + +when defined case8: + import mused3b + export mused3b + +when defined case9: + import mused3b + export mused3b + +when defined case10: + import mused3b + type Bar* = object + b0*: Foo + +when defined case11: + proc fn1*() = discard + +when defined case12: + proc fn1*() = discard diff --git a/tests/pragmas/mused3b.nim b/tests/pragmas/mused3b.nim new file mode 100644 index 000000000..de288bb08 --- /dev/null +++ b/tests/pragmas/mused3b.nim @@ -0,0 +1,12 @@ +when defined case1: + proc gn1*()=discard + +when defined case8: + proc gn1*()=discard + +when defined case9: + proc gn1*()=discard + +when defined case10: + type Foo* = object + f0*: int From 0804b4b37d5fcc43d975f56a4c2c88dd58a0b882 Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Sun, 27 Jun 2021 19:51:28 +0100 Subject: [PATCH 0985/2472] Switch packaging.rst to use makefile Minor cleanup --- doc/packaging.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/packaging.rst b/doc/packaging.rst index ecb17e482..fcdc4ae04 100644 --- a/doc/packaging.rst +++ b/doc/packaging.rst @@ -48,14 +48,15 @@ Hints on the build process: .. code:: cmd # build from C sources and then using koch - ./build.sh --os $os_type --cpu $cpu_arch - ./bin/nim c koch + make -j # supports parallel build + # alternatively: ./build.sh --os $os_type --cpu $cpu_arch + ./bin/nim c -d:release koch ./koch boot -d:release # optionally generate docs into doc/html ./koch docs - ./koch tools -d:release + ./koch tools # extract files to be really installed ./install.sh From 908b2cc2e44bededdcb943778f44b908a4f3c1df Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Sun, 27 Jun 2021 21:12:12 +0100 Subject: [PATCH 0986/2472] Switch IRC links to Libera Chat (#18370) --- readme.md | 4 ++-- tools/website.nimf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/readme.md b/readme.md index 37babf711..7c6e28be3 100644 --- a/readme.md +++ b/readme.md @@ -16,7 +16,7 @@ the latest release, check out [Nim's website][nim-site] or [bleeding edge docs]( [![Follow @nim_lang on Twitter][badge-nim-twitter]][nim-twitter] * The [forum][nim-forum] - the best place to ask questions and to discuss Nim. -* [#nim IRC Channel (Freenode)][nim-irc] - a place to discuss Nim in real-time. +* [#nim IRC Channel (Libera Chat)][nim-irc] - a place to discuss Nim in real-time. Also where most development decisions get made. * [Discord][nim-discord] - an additional place to discuss Nim in real-time. Most channels there are bridged to IRC. @@ -206,7 +206,7 @@ Copyright © 2006-2021 Andreas Rumpf, all rights reserved. [nim-forum]: https://forum.nim-lang.org [nim-issues]: https://github.com/nim-lang/Nim/issues [nim-issues-easy]: https://github.com/nim-lang/Nim/labels/Easy -[nim-irc]: https://webchat.freenode.net/?channels=nim +[nim-irc]: https://web.libera.chat/#nim [nim-twitter]: https://twitter.com/nim_lang [nim-stackoverflow]: https://stackoverflow.com/questions/tagged/nim-lang [nim-stackoverflow-newest]: https://stackoverflow.com/questions/tagged/nim-lang?sort=newest&pageSize=15 diff --git a/tools/website.nimf b/tools/website.nimf index bde84b32f..cea30f74f 100644 --- a/tools/website.nimf +++ b/tools/website.nimf @@ -188,7 +188,7 @@ runForever()
    From e720bbdd76303e2cc0a3a169e870859470a1da84 Mon Sep 17 00:00:00 2001 From: gemath <33119724+gemath@users.noreply.github.com> Date: Mon, 28 Jun 2021 12:33:20 +0200 Subject: [PATCH 0987/2472] Peg captures get stack-like behavior (#18369) * Implements reverse capture indexing. * Now works for modified backrefs too. * Changed reverse indexing syntax prefix for back-references to '$^'. --- doc/pegdocs.txt | 10 +++-- lib/pure/pegs.nim | 90 +++++++++++++++++++++++++++--------------- tests/stdlib/tpegs.nim | 28 +++++++++++++ 3 files changed, 94 insertions(+), 34 deletions(-) diff --git a/doc/pegdocs.txt b/doc/pegdocs.txt index 4c557aed8..0363d4874 100644 --- a/doc/pegdocs.txt +++ b/doc/pegdocs.txt @@ -27,7 +27,10 @@ notation meaning ``{E}`` Capture: Apply expression `E` and store the substring that matched `E` into a *capture* that can be accessed after the matching process. -``$i`` Back reference to the ``i``th capture. ``i`` counts from 1. +``{}`` Empty capture: Delete the last capture. No character + is consumed. +``$i`` Back reference to the ``i``th capture. ``i`` counts forwards + from 1 or backwards (last capture to first) from ^1. ``$`` Anchor: Matches at the end of the input. No character is consumed. Same as ``!.``. ``^`` Anchor: Matches at the start of the input. No character @@ -149,14 +152,15 @@ The PEG parser implements this grammar (written in PEG syntax):: rule <- identifier \s* "<-" expr ig identNoArrow <- identifier !(\s* "<-") prefixOpr <- ig '&' / ig '!' / ig '@' / ig '{@}' / ig '@@' - literal <- ig identifier? '$' [0-9]+ / '$' / '^' / + literal <- ig identifier? '$' '^'? [0-9]+ / '$' / '^' / ig identNoArrow / ig charset / ig stringlit / ig builtin / ig '.' / ig '_' / - (ig "(" expr ig ")") + (ig "(" expr ig ")") / + (ig "{" expr? ig "}") postfixOpr <- ig '?' / ig '*' / ig '+' primary <- prefixOpr* (literal postfixOpr*) diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index b5d9b1d07..b6b05cdc1 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -83,7 +83,7 @@ type of pkChar, pkGreedyRepChar: ch: char of pkCharChoice, pkGreedyRepSet: charChoice: ref set[char] of pkNonTerminal: nt: NonTerminal - of pkBackRef..pkBackRefIgnoreStyle: index: range[0..MaxSubpatterns] + of pkBackRef..pkBackRefIgnoreStyle: index: range[-MaxSubpatterns..MaxSubpatterns-1] else: sons: seq[Peg] NonTerminal* = ref NonTerminalObj @@ -106,7 +106,7 @@ proc nt*(p: Peg): NonTerminal = p.nt ## Returns the *NonTerminal* object of a given *Peg* variant object ## where present. -proc index*(p: Peg): range[0..MaxSubpatterns] = p.index +proc index*(p: Peg): range[-MaxSubpatterns..MaxSubpatterns-1] = p.index ## Returns the back-reference index of a captured sub-pattern in the ## *Captures* object for a given *Peg* variant object where present. @@ -304,34 +304,37 @@ proc endAnchor*: Peg {.inline.} = ## constructs the PEG ``$`` which matches the end of the input. result = !any() -proc capture*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsCapture".} = +proc capture*(a: Peg = Peg(kind: pkEmpty)): Peg {.noSideEffect, rtl, extern: "npegsCapture".} = ## constructs a capture with the PEG `a` result = Peg(kind: pkCapture, sons: @[a]) -proc backref*(index: range[1..MaxSubpatterns]): Peg {. +proc backref*(index: range[1..MaxSubpatterns], reverse: bool = false): Peg {. noSideEffect, rtl, extern: "npegs$1".} = ## constructs a back reference of the given `index`. `index` starts counting - ## from 1. - result = Peg(kind: pkBackRef, index: index-1) + ## from 1. `reverse` specifies wether indexing starts from the end of the + ## capture list. + result = Peg(kind: pkBackRef, index: (if reverse: -index else: index - 1)) -proc backrefIgnoreCase*(index: range[1..MaxSubpatterns]): Peg {. +proc backrefIgnoreCase*(index: range[1..MaxSubpatterns], reverse: bool = false): Peg {. noSideEffect, rtl, extern: "npegs$1".} = ## constructs a back reference of the given `index`. `index` starts counting - ## from 1. Ignores case for matching. - result = Peg(kind: pkBackRefIgnoreCase, index: index-1) + ## from 1. `reverse` specifies wether indexing starts from the end of the + ## capture list. Ignores case for matching. + result = Peg(kind: pkBackRefIgnoreCase, index: (if reverse: -index else: index - 1)) -proc backrefIgnoreStyle*(index: range[1..MaxSubpatterns]): Peg {. +proc backrefIgnoreStyle*(index: range[1..MaxSubpatterns], reverse: bool = false): Peg {. noSideEffect, rtl, extern: "npegs$1".} = ## constructs a back reference of the given `index`. `index` starts counting - ## from 1. Ignores style for matching. - result = Peg(kind: pkBackRefIgnoreStyle, index: index-1) + ## from 1. `reverse` specifies wether indexing starts from the end of the + ## capture list. Ignores style for matching. + result = Peg(kind: pkBackRefIgnoreStyle, index: (if reverse: -index else: index - 1)) proc spaceCost(n: Peg): int = case n.kind of pkEmpty: discard of pkTerminal, pkTerminalIgnoreCase, pkTerminalIgnoreStyle, pkChar, pkGreedyRepChar, pkCharChoice, pkGreedyRepSet, - pkAny..pkWhitespace, pkGreedyAny: + pkAny..pkWhitespace, pkGreedyAny, pkBackRef..pkBackRefIgnoreStyle: result = 1 of pkNonTerminal: # we cannot inline a rule with a non-terminal @@ -561,8 +564,10 @@ template matchOrParse(mopProc: untyped) = # Parse handler code must run in an *of* clause of its own for each # *PegKind*, so we encapsulate the identical clause body for # *pkBackRef..pkBackRefIgnoreStyle* here. - if p.index >= c.ml: return -1 - var (a, b) = c.matches[p.index] + var index = p.index + if index < 0: index.inc(c.ml) + if index < 0 or index >= c.ml: return -1 + var (a, b) = c.matches[index] var n: Peg case p.kind of pkBackRef: @@ -822,15 +827,19 @@ template matchOrParse(mopProc: untyped) = leave(pkNotPredicate, s, p, start, result) of pkCapture: enter(pkCapture, s, p, start) - var idx = c.ml # reserve a slot for the subpattern - inc(c.ml) - result = mopProc(s, p.sons[0], start, c) - if result >= 0: - if idx < MaxSubpatterns: - c.matches[idx] = (start, start+result-1) - #else: silently ignore the capture + if p.sons.len == 0 or p.sons[0].kind == pkEmpty: + # empty capture removes last match + dec(c.ml) + c.matches[c.ml] = (0, 0) + result = 0 # match of length 0 else: - c.ml = idx + var idx = c.ml # reserve a slot for the subpattern + result = mopProc(s, p.sons[0], start, c) + if result >= 0: + inc(c.ml) + if idx < MaxSubpatterns: + c.matches[idx] = (start, start+result-1) + #else: silently ignore the capture leave(pkCapture, s, p, start, result) of pkBackRef: enter(pkBackRef, s, p, start) @@ -1395,6 +1404,7 @@ type tkCurlyLe, ## '{' tkCurlyRi, ## '}' tkCurlyAt, ## '{@}' + tkEmptyCurl, ## '{}' tkArrow, ## '<-' tkBar, ## '/' tkStar, ## '*' @@ -1427,7 +1437,7 @@ type const tokKindToStr: array[TokKind, string] = [ "invalid", "[EOF]", ".", "_", "identifier", "string literal", - "character set", "(", ")", "{", "}", "{@}", + "character set", "(", ")", "{", "}", "{@}", "{}", "<-", "/", "*", "+", "&", "!", "?", "@", "built-in", "escaped", "$", "$", "^" ] @@ -1564,13 +1574,21 @@ proc getString(c: var PegLexer, tok: var Token) = proc getDollar(c: var PegLexer, tok: var Token) = var pos = c.bufpos + 1 + var neg = false + if pos < c.buf.len and c.buf[pos] == '^': + neg = true + inc(pos) if pos < c.buf.len and c.buf[pos] in {'0'..'9'}: tok.kind = tkBackref tok.index = 0 while pos < c.buf.len and c.buf[pos] in {'0'..'9'}: tok.index = tok.index * 10 + ord(c.buf[pos]) - ord('0') inc(pos) + if neg: + tok.index = -tok.index else: + if neg: + dec(pos) tok.kind = tkDollar c.bufpos = pos @@ -1670,6 +1688,10 @@ proc getTok(c: var PegLexer, tok: var Token) = tok.kind = tkCurlyAt inc(c.bufpos, 2) add(tok.literal, "{@}") + elif c.buf[c.bufpos] == '}' and c.bufpos < c.buf.len: + tok.kind = tkEmptyCurl + inc(c.bufpos) + add(tok.literal, "{}") else: tok.kind = tkCurlyLe add(tok.literal, '{') @@ -1705,7 +1727,7 @@ proc getTok(c: var PegLexer, tok: var Token) = return if c.buf[c.bufpos] in {'\'', '"'} or c.buf[c.bufpos] == '$' and c.bufpos+1 < c.buf.len and - c.buf[c.bufpos+1] in {'0'..'9'}: + c.buf[c.bufpos+1] in {'^', '0'..'9'}: case tok.literal of "i": tok.modifier = modIgnoreCase of "y": tok.modifier = modIgnoreStyle @@ -1819,10 +1841,13 @@ proc modifiedTerm(s: string, m: Modifier): Peg = of modIgnoreStyle: result = termIgnoreStyle(s) proc modifiedBackref(s: int, m: Modifier): Peg = + var + reverse = s < 0 + index = if reverse: -s else: s case m - of modNone, modVerbatim: result = backref(s) - of modIgnoreCase: result = backrefIgnoreCase(s) - of modIgnoreStyle: result = backrefIgnoreStyle(s) + of modNone, modVerbatim: result = backref(index, reverse) + of modIgnoreCase: result = backrefIgnoreCase(index, reverse) + of modIgnoreStyle: result = backrefIgnoreStyle(index, reverse) proc builtin(p: var PegParser): Peg = # do not use "y", "skip" or "i" as these would be ambiguous @@ -1896,6 +1921,9 @@ proc primary(p: var PegParser): Peg = result = capture(parseExpr(p)).token(p) eat(p, tkCurlyRi) inc(p.captures) + of tkEmptyCurl: + result = capture() + getTok(p) of tkAny: result = any().token(p) getTok(p) @@ -1915,11 +1943,11 @@ proc primary(p: var PegParser): Peg = result = startAnchor() getTok(p) of tkBackref: + if abs(p.tok.index) > p.captures or p.tok.index == 0: + pegError(p, "invalid back reference index: " & $p.tok.index) var m = p.tok.modifier if m == modNone: m = p.modifier result = modifiedBackref(p.tok.index, m).token(p) - if p.tok.index < 0 or p.tok.index > p.captures: - pegError(p, "invalid back reference index: " & $p.tok.index) getTok(p) else: pegError(p, "expression expected, but found: " & p.tok.literal) @@ -1943,7 +1971,7 @@ proc seqExpr(p: var PegParser): Peg = case p.tok.kind of tkAmp, tkNot, tkAt, tkStringLit, tkCharSet, tkParLe, tkCurlyLe, tkAny, tkAnyRune, tkBuiltin, tkEscaped, tkDollar, tkBackref, - tkHat, tkCurlyAt: + tkHat, tkCurlyAt, tkEmptyCurl: result = sequence(result, primary(p)) of tkIdentifier: if not arrowIsNextTok(p): diff --git a/tests/stdlib/tpegs.nim b/tests/stdlib/tpegs.nim index f2c93c2e5..1261d55b8 100644 --- a/tests/stdlib/tpegs.nim +++ b/tests/stdlib/tpegs.nim @@ -293,6 +293,34 @@ block: doAssert "test1".match(peg"""{@}$""") doAssert "test2".match(peg"""{(!$ .)*} $""") + + doAssert "abbb".match(peg"{a} {b} $2 $^1") + doAssert "abBA".match(peg"{a} {b} i$2 i$^2") + + doAssert "abba".match(peg"{a} {b} $^1 {} $^1") + + block: + let grammar = peg""" +program <- {''} stmt* $ +stmt <- call / block +call <- 'call()' EOL +EOL <- \n / $ +block <- 'block:' \n indBody +indBody <- {$^1 ' '+} stmt ($^1 stmt)* {} +""" + let program = """ +call() +block: + block: + call() + call() + call() +call() +""" + var c: Captures + doAssert program.len == program.rawMatch(grammar, 0, c) + doAssert c.ml == 1 + pegsTest() static: pegsTest() From 6e74fecb4275272e5224e2a0ba45df5ca2c40b82 Mon Sep 17 00:00:00 2001 From: Clyybber Date: Mon, 28 Jun 2021 15:00:14 +0200 Subject: [PATCH 0988/2472] Simplify addInt, remove digits10 (#18356) * Simplify addInt, remove digits10 Co-authored-by: Charles Blake * Fix bootstrapping * Add noInit to tmp array * noInit -> noinit Co-authored-by: Charles Blake --- lib/std/private/digitsutils.nim | 58 +++++++++++++-------------------- lib/system/dollars.nim | 4 +-- lib/system/strmantle.nim | 10 ++---- tests/stdlib/tdigitsutils.nim | 23 ------------- 4 files changed, 27 insertions(+), 68 deletions(-) delete mode 100644 tests/stdlib/tdigitsutils.nim diff --git a/lib/std/private/digitsutils.nim b/lib/std/private/digitsutils.nim index cc985c7bd..7aefc36bc 100644 --- a/lib/std/private/digitsutils.nim +++ b/lib/std/private/digitsutils.nim @@ -38,51 +38,39 @@ proc trailingZeros2Digits*(digits: uint32): int32 {.inline.} = assert(digits <= 99) return trailingZeros100[digits] -func digits10*(num: uint64): int {.noinline.} = - if num < 10'u64: - result = 1 - elif num < 100'u64: - result = 2 - elif num < 1_000'u64: - result = 3 - elif num < 10_000'u64: - result = 4 - elif num < 100_000'u64: - result = 5 - elif num < 1_000_000'u64: - result = 6 - elif num < 10_000_000'u64: - result = 7 - elif num < 100_000_000'u64: - result = 8 - elif num < 1_000_000_000'u64: - result = 9 - elif num < 10_000_000_000'u64: - result = 10 - elif num < 100_000_000_000'u64: - result = 11 - elif num < 1_000_000_000_000'u64: - result = 12 - else: - result = 12 + digits10(num div 1_000_000_000_000'u64) - -template numToString*(result: var string, origin: uint64, length: int) = +func addIntImpl*(result: var string, origin: uint64) = + var tmp {.noinit.}: array[24, char] var num = origin - var next = length - 1 + var next = tmp.len - 1 const nbatch = 100 while num >= nbatch: let originNum = num num = num div nbatch let index = (originNum - num * nbatch) shl 1 - result[next] = digits100[index + 1] - result[next - 1] = digits100[index] + tmp[next] = digits100[index + 1] + tmp[next - 1] = digits100[index] dec(next, 2) # process last 1-2 digits if num < 10: - result[next] = chr(ord('0') + num) + tmp[next] = chr(ord('0') + num) else: let index = num * 2 - result[next] = digits100[index + 1] - result[next - 1] = digits100[index] + tmp[next] = digits100[index + 1] + tmp[next - 1] = digits100[index] + dec next + let n = result.len + let length = tmp.len - next + result.setLen n + length + when nimvm: + for i in 0.. 1: - doAssert digits10(x-1) == num - 1 - num += 1 - let xOld = x - x *= 10 - if x < xOld: - # wrap-around - break - -static: main() -main() From 0be17f5d9c6b66a8f8dda83c9a3c77c61be3844d Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 28 Jun 2021 17:08:45 +0200 Subject: [PATCH 0989/2472] fixes #18319 (#18375) --- compiler/jsgen.nim | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 393ac83c2..a9339c7fd 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -29,7 +29,7 @@ implements the required case distinction. import - ast, trees, magicsys, options, + ast, astalgo, trees, magicsys, options, nversion, msgs, idents, types, ropes, passes, ccgutils, wordrecg, renderer, cgmeth, lowerings, sighashes, modulegraphs, lineinfos, rodutils, @@ -2234,6 +2234,16 @@ proc genTupleConstr(p: PProc, n: PNode, r: var TCompRes) = r.res.addf("Field$#: $#", [i.rope, a.res]) r.res.add("}") +proc lookupFieldAgain(ty: PType; field: PSym): PSym = + var ty = ty + while ty != nil: + ty = ty.skipTypes(skipPtrs) + assert(ty.kind in {tyTuple, tyObject}) + result = lookupInRecord(ty.n, field.name) + if result != nil: break + ty = ty[0] + if result == nil: result = field + proc genObjConstr(p: PProc, n: PNode, r: var TCompRes) = var a: TCompRes r.kind = resExpr @@ -2247,7 +2257,7 @@ proc genObjConstr(p: PProc, n: PNode, r: var TCompRes) = gen(p, val, a) var f = it[0].sym if f.loc.r == nil: f.loc.r = mangleName(p.module, f) - fieldIDs.incl(f.id) + fieldIDs.incl(lookupFieldAgain(n.typ, f).id) let typ = val.typ.skipTypes(abstractInst) if a.typ == etyBaseIndex: From cb9496693a718cdb9ad8b764f82f524efe943bd2 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 28 Jun 2021 08:15:20 -0700 Subject: [PATCH 0990/2472] `./koch tools` now builds bin/nim_dbg, a debug version of nim (#18289) --- koch.nim | 10 ++++++++-- tools/kochdocs.nim | 5 +++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/koch.nim b/koch.nim index 6a0b7a0d3..d3957069b 100644 --- a/koch.nim +++ b/koch.nim @@ -215,8 +215,14 @@ proc buildTools(args: string = "") = options = "-d:release " & args) when defined(windows): buildVccTool(args) bundleNimpretty(args) - nimCompileFold("Compile testament", "testament/testament.nim", - options = "-d:release " & args) + nimCompileFold("Compile testament", "testament/testament.nim", options = "-d:release" & args) + + # pre-packages a debug version of nim which can help in many cases investigate issuses + # withouth having to rebuild compiler. + # `-d:nimDebugUtils` only makes sense when temporarily editing/debugging compiler + # `-d:debug` should be changed to a flag that doesn't require re-compiling nim + # `--opt:speed` is a sensible default even for a debug build, it doesn't affect nim stacktraces + nimCompileFold("Compile nim_dbg", "compiler/nim.nim", options = "--opt:speed --stacktrace -d:debug --stacktraceMsgs -d:nimCompilerStacktraceHints" & args, outputName = "nim_dbg") proc nsis(latest: bool; args: string) = bundleNimbleExe(latest, args) diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 469059348..8e8085f73 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -98,8 +98,9 @@ proc nimCompile*(input: string, outputDir = "bin", mode = "c", options = "") = let cmd = findNim().quoteShell() & " " & mode & " -o:" & output & " " & options & " " & input exec cmd -proc nimCompileFold*(desc, input: string, outputDir = "bin", mode = "c", options = "") = - let output = outputDir / input.splitFile.name.exe +proc nimCompileFold*(desc, input: string, outputDir = "bin", mode = "c", options = "", outputName = "") = + let outputName2 = if outputName.len == 0: input.splitFile.name.exe else: outputName.exe + let output = outputDir / outputName2 let cmd = findNim().quoteShell() & " " & mode & " -o:" & output & " " & options & " " & input execFold(desc, cmd) From 72d6a9c885504a1752ee0639aa110d4bf42d95c8 Mon Sep 17 00:00:00 2001 From: Clyybber Date: Mon, 28 Jun 2021 17:41:37 +0200 Subject: [PATCH 0991/2472] Fix #16426 (#18377) --- compiler/parser.nim | 13 ++++++++----- tests/parser/tstmtlists.nim | 15 +++++++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/compiler/parser.nim b/compiler/parser.nim index 9cddf8e4f..b6c01e9c6 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -537,13 +537,16 @@ proc parseGStrLit(p: var Parser, a: PNode): PNode = proc complexOrSimpleStmt(p: var Parser): PNode proc simpleExpr(p: var Parser, mode = pmNormal): PNode -proc parseIfExpr(p: var Parser, kind: TNodeKind): PNode +proc parseIfOrWhenExpr(p: var Parser, kind: TNodeKind): PNode proc semiStmtList(p: var Parser, result: PNode) = inc p.inSemiStmtList withInd(p): # Be lenient with the first stmt/expr - let a = if p.tok.tokType == tkIf: parseIfExpr(p, nkIfStmt) else: complexOrSimpleStmt(p) + let a = case p.tok.tokType + of tkIf: parseIfOrWhenExpr(p, nkIfStmt) + of tkWhen: parseIfOrWhenExpr(p, nkWhenStmt) + else: complexOrSimpleStmt(p) result.add a while p.tok.tokType != tkEof: @@ -1203,13 +1206,13 @@ proc parseExpr(p: var Parser): PNode = result = parseBlock(p) of tkIf: nimprettyDontTouch: - result = parseIfExpr(p, nkIfExpr) + result = parseIfOrWhenExpr(p, nkIfExpr) of tkFor: nimprettyDontTouch: result = parseFor(p) of tkWhen: nimprettyDontTouch: - result = parseIfExpr(p, nkWhenExpr) + result = parseIfOrWhenExpr(p, nkWhenExpr) of tkCase: # Currently we think nimpretty is good enough with case expressions, # so it is allowed to touch them: @@ -1559,7 +1562,7 @@ proc parseIfOrWhen(p: var Parser, kind: TNodeKind): PNode = branch.add(parseStmt(p)) result.add(branch) -proc parseIfExpr(p: var Parser, kind: TNodeKind): PNode = +proc parseIfOrWhenExpr(p: var Parser, kind: TNodeKind): PNode = #| condExpr = expr colcom expr optInd #| ('elif' expr colcom expr optInd)* #| 'else' colcom expr diff --git a/tests/parser/tstmtlists.nim b/tests/parser/tstmtlists.nim index b6c7e4a94..fa47ba364 100644 --- a/tests/parser/tstmtlists.nim +++ b/tests/parser/tstmtlists.nim @@ -50,6 +50,8 @@ hello 9 hello 10 +lucky +lucky ''' """ @@ -163,3 +165,16 @@ template dim: int = else: int.high) +# lenient indentation: + +echo (if 0 == 1: + "0 == 1" +else: + "lucky") + +# bug #16426 +echo (when 0 == 1: + "0 == 1" +else: + "lucky") + From dd2cbe3164e476dafda4b59d6ed518d79f69f195 Mon Sep 17 00:00:00 2001 From: Clyybber Date: Mon, 28 Jun 2021 19:25:30 +0200 Subject: [PATCH 0992/2472] Cleanup lookups.nim again.. (#18379) --- compiler/lookups.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/compiler/lookups.nim b/compiler/lookups.nim index 9b878dd4b..44a30eefa 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -58,8 +58,8 @@ proc considerQuotedIdent*(c: PContext; n: PNode, origin: PNode = nil): PIdent = template addSym*(scope: PScope, s: PSym) = strTableAdd(scope.symbols, s) -proc addUniqueSym*(scope: PScope, s: PSym, onConflictKeepOld: bool): PSym = - result = strTableInclReportConflict(scope.symbols, s, onConflictKeepOld) +proc addUniqueSym*(scope: PScope, s: PSym): PSym = + result = strTableInclReportConflict(scope.symbols, s) proc openScope*(c: PContext): PScope {.discardable.} = result = PScope(parent: c.currentScope, @@ -298,7 +298,7 @@ proc wrongRedefinition*(c: PContext; info: TLineInfo, s: string; # xxx pending bootstrap >= 1.4, replace all those overloads with a single one: # proc addDecl*(c: PContext, sym: PSym, info = sym.info, scope = c.currentScope) {.inline.} = proc addDeclAt*(c: PContext; scope: PScope, sym: PSym, info: TLineInfo) = - let conflict = scope.addUniqueSym(sym, onConflictKeepOld = true) + let conflict = scope.addUniqueSym(sym) if conflict != nil: if sym.kind == skModule and conflict.kind == skModule and sym.owner == conflict.owner: # e.g.: import foo; import foo @@ -320,7 +320,7 @@ proc addDecl*(c: PContext, sym: PSym) {.inline.} = addDeclAt(c, c.currentScope, sym) proc addPrelimDecl*(c: PContext, sym: PSym) = - discard c.currentScope.addUniqueSym(sym, onConflictKeepOld = false) + discard c.currentScope.addUniqueSym(sym) from ic / ic import addHidden From 808db3b2845dab9822bc82ae80073a68396db999 Mon Sep 17 00:00:00 2001 From: Clyybber Date: Mon, 28 Jun 2021 22:01:06 +0200 Subject: [PATCH 0993/2472] Fix unused warning for $ on empty object (#18381) --- lib/system/dollars.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/system/dollars.nim b/lib/system/dollars.nim index c23ea347d..baae8f090 100644 --- a/lib/system/dollars.nim +++ b/lib/system/dollars.nim @@ -113,7 +113,7 @@ proc `$`*[T: tuple|object](x: T): string = ## $() == "()" result = "(" const isNamed = T is object or isNamedTuple(T) - var count = 0 + var count {.used.} = 0 for name, value in fieldPairs(x): if count > 0: result.add(", ") when isNamed: From 6387e289631c22b8fa81e8e478c97cc500b90e26 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 28 Jun 2021 13:09:14 -0700 Subject: [PATCH 0994/2472] simplify rdstdin (#18382) --- lib/impure/rdstdin.nim | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/lib/impure/rdstdin.nim b/lib/impure/rdstdin.nim index c580b89d1..32048b131 100644 --- a/lib/impure/rdstdin.nim +++ b/lib/impure/rdstdin.nim @@ -52,16 +52,6 @@ elif defined(genode): else: import linenoise - proc readLineFromStdin*(prompt: string): string {. - tags: [ReadIOEffect, WriteIOEffect].} = - var buffer = linenoise.readLine(prompt) - if isNil(buffer): - raise newException(IOError, "Linenoise returned nil") - result = $buffer - if result.len > 0: - historyAdd(buffer) - linenoise.free(buffer) - proc readLineFromStdin*(prompt: string, line: var string): bool {. tags: [ReadIOEffect, WriteIOEffect].} = var buffer = linenoise.readLine(prompt) @@ -73,3 +63,7 @@ else: historyAdd(buffer) linenoise.free(buffer) result = true + + proc readLineFromStdin*(prompt: string): string {.inline.} = + if not readLineFromStdin(prompt, result): + raise newException(IOError, "Linenoise returned nil") From 97fc95012d2725b625c492b6b72336a89c501076 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 29 Jun 2021 15:34:39 +0200 Subject: [PATCH 0995/2472] fixes #16270 (#18388) --- compiler/ast.nim | 3 ++- compiler/semexprs.nim | 1 + compiler/sigmatch.nim | 9 +++++++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 56043692f..2a2a84e76 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -499,7 +499,7 @@ type nfFirstWrite# this node is a first write TNodeFlags* = set[TNodeFlag] - TTypeFlag* = enum # keep below 32 for efficiency reasons (now: ~40) + TTypeFlag* = enum # keep below 32 for efficiency reasons (now: 43) tfVarargs, # procedure has C styled varargs # tyArray type represeting a varargs list tfNoSideEffect, # procedure type does not allow side effects @@ -566,6 +566,7 @@ type # (for importc types); type is fully specified, allowing to compute # sizeof, alignof, offsetof at CT tfExplicitCallConv + tfIsConstructor TTypeFlags* = set[TTypeFlag] diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index c885f4963..acf408db0 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -2412,6 +2412,7 @@ proc semWhen(c: PContext, n: PNode, semCheck = true): PNode = proc semSetConstr(c: PContext, n: PNode): PNode = result = newNodeI(nkCurly, n.info) result.typ = newTypeS(tySet, c) + result.typ.flags.incl tfIsConstructor if n.len == 0: rawAddSon(result.typ, newTypeS(tyEmpty, c)) else: diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index b2003c0a8..071685057 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -1341,8 +1341,13 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, result = isSubtype else: result = typeRel(c, f[0], a[0], flags) - if result <= isConvertible: - result = isNone # BUGFIX! + if result < isGeneric: + if result <= isConvertible: + result = isNone + elif tfIsConstructor notin a.flags: + # set constructors are a bit special... + result = isNone + of tyPtr, tyRef: skipOwned(a) if a.kind == f.kind: From 36d24846b6685a44ff2185a823cdcb0f3bc236d3 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 29 Jun 2021 18:08:50 -0700 Subject: [PATCH 0996/2472] typo: enmRange => enumElementsAsSet (#18394) --- lib/std/setutils.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/std/setutils.nim b/lib/std/setutils.nim index aba5f348a..84cc93623 100644 --- a/lib/std/setutils.nim +++ b/lib/std/setutils.nim @@ -36,7 +36,7 @@ template toSet*(iter: untyped): untyped = incl(result, x) result -macro enmRange(enm: typed): untyped = result = newNimNode(nnkCurly).add(enm.getType[1][1..^1]) +macro enumElementsAsSet(enm: typed): untyped = result = newNimNode(nnkCurly).add(enm.getType[1][1..^1]) # func fullSet*(T: typedesc): set[T] {.inline.} = # xxx would give: Error: ordinal type expected func fullSet*[T](U: typedesc[T]): set[T] {.inline.} = @@ -49,7 +49,7 @@ func fullSet*[T](U: typedesc[T]): set[T] {.inline.} = when T is Ordinal: {T.low..T.high} else: # Hole filled enum - enmRange(T) + enumElementsAsSet(T) func complement*[T](s: set[T]): set[T] {.inline.} = ## Returns the set complement of `a`. From afe5eb569b36ed310f6f93a33ae52274a6dce02e Mon Sep 17 00:00:00 2001 From: Elliot Waite Date: Tue, 29 Jun 2021 19:21:30 -0700 Subject: [PATCH 0997/2472] Add commas to docs for clarity (#18398) --- doc/tut2.rst | 2 +- lib/pure/asyncdispatch.nim | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/tut2.rst b/doc/tut2.rst index 2624b2690..725f68dd5 100644 --- a/doc/tut2.rst +++ b/doc/tut2.rst @@ -79,7 +79,7 @@ to introduce new object roots apart from `system.RootObj`. (This is used in the GTK wrapper for instance.) Ref objects should be used whenever inheritance is used. It isn't strictly -necessary, but with non-ref objects assignments such as `let person: Person = +necessary, but with non-ref objects, assignments such as `let person: Person = Student(id: 123)` will truncate subclass fields. **Note**: Composition (*has-a* relation) is often preferable to inheritance diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index b88d5ede8..17fd5f824 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -61,8 +61,8 @@ ## callback on this future which will be called once the future completes. ## All the callback does is write the data stored in the future to `stdout`. ## The `read` function is used for this and it checks whether the future -## completes with an error for you (if it did it will simply raise the -## error), if there is no error however it returns the value of the future. +## completes with an error for you (if it did, it will simply raise the +## error), if there is no error, however, it returns the value of the future. ## ## Asynchronous procedures ## ======================= From 0483a5ffaf3284b6bfc51e2e08bbc1d5a57cd939 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 29 Jun 2021 22:16:31 -0700 Subject: [PATCH 0998/2472] improvements to hint:processing: show import stack, distinguish nims, show includes etc (#18372) * improvements to hint:processing * fix tests; do not show hintProcessing for nimscript unless given -d:nimHintProcessingNims * fix trunner and avoid need for -d:nimHintProcessingNims * fix some tests --- compiler/lineinfos.nim | 5 +++-- compiler/modulegraphs.nim | 12 ++++++++++++ compiler/modules.nim | 11 ++++++----- compiler/passaux.nim | 13 ++++--------- compiler/semstmts.nim | 1 + nimsuggest/tests/tchk1.nim | 2 +- nimsuggest/tests/tchk_compiles.nim | 2 +- nimsuggest/tests/ttempl_inst.nim | 2 +- 8 files changed, 29 insertions(+), 19 deletions(-) diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index 131fe480b..801c20d25 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -75,7 +75,7 @@ type hintXDeclaredButNotUsed = "XDeclaredButNotUsed", hintDuplicateModuleImport = "DuplicateModuleImport", hintXCannotRaiseY = "XCannotRaiseY", hintConvToBaseNotNeeded = "ConvToBaseNotNeeded", hintConvFromXtoItselfNotNeeded = "ConvFromXtoItselfNotNeeded", hintExprAlwaysX = "ExprAlwaysX", - hintQuitCalled = "QuitCalled", hintProcessing = "Processing", hintCodeBegin = "CodeBegin", + hintQuitCalled = "QuitCalled", hintProcessing = "Processing", hintProcessingStmt = "ProcessingStmt", hintCodeBegin = "CodeBegin", hintCodeEnd = "CodeEnd", hintConf = "Conf", hintPath = "Path", hintConditionAlwaysTrue = "CondTrue", hintConditionAlwaysFalse = "CondFalse", hintName = "Name", hintPattern = "Pattern", hintExecuting = "Exec", hintLinking = "Link", hintDependency = "Dependency", @@ -163,6 +163,7 @@ const hintExprAlwaysX: "expression evaluates always to '$1'", hintQuitCalled: "quit() called", hintProcessing: "$1", + hintProcessingStmt: "$1", hintCodeBegin: "generated code listing:", hintCodeEnd: "end of listing", hintConf: "used config file '$1'", @@ -202,7 +203,7 @@ type proc computeNotesVerbosity(): array[0..3, TNoteKinds] = result[3] = {low(TNoteKind)..high(TNoteKind)} - {warnObservableStores, warnResultUsed} - result[2] = result[3] - {hintStackTrace, warnUninit, hintExtendedContext, hintDeclaredLoc} + result[2] = result[3] - {hintStackTrace, warnUninit, hintExtendedContext, hintDeclaredLoc, hintProcessingStmt} result[1] = result[2] - {warnProveField, warnProveIndex, warnGcUnsafe, hintPath, hintDependency, hintCodeBegin, hintCodeEnd, hintSource, hintGlobalVar, hintGCStats, hintMsgOrigin, hintPerformance} diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index 0d8d4f867..90b130e92 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -582,3 +582,15 @@ proc moduleFromRodFile*(g: ModuleGraph; fileIdx: FileIndex; proc configComplete*(g: ModuleGraph) = rememberStartupConfig(g.startupPackedConfig, g.config) + +from std/strutils import repeat, `%` + +proc onProcessing*(graph: ModuleGraph, fileIdx: FileIndex, moduleStatus: string, fromModule: PSym, ) = + let conf = graph.config + let isNimscript = conf.isDefined("nimscript") + if (not isNimscript) or hintProcessing in conf.cmdlineNotes: + let path = toFilenameOption(conf, fileIdx, conf.filenameOption) + let indent = ">".repeat(graph.importStack.len) + let fromModule2 = if fromModule != nil: $fromModule.name.s else: "(toplevel)" + let mode = if isNimscript: "(nims) " else: "" + rawMessage(conf, hintProcessing, "$#$# $#: $#: $#" % [mode, indent, fromModule2, moduleStatus, path]) diff --git a/compiler/modules.nim b/compiler/modules.nim index dcfcfd508..6fba606b2 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -84,12 +84,13 @@ proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym = partialInitModule(result, graph, fileIdx, filename) graph.registerModule(result) -proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): PSym = +proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags, fromModule: PSym = nil): PSym = var flags = flags if fileIdx == graph.config.projectMainIdx2: flags.incl sfMainModule result = graph.getModule(fileIdx) - template processModuleAux = + template processModuleAux(moduleStatus) = + onProcessing(graph, fileIdx, moduleStatus, fromModule = fromModule) var s: PLLStream if sfMainModule in flags: if graph.config.projectIsStdin: s = stdin.llStreamOpen @@ -103,7 +104,7 @@ proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): P result = newModule(graph, fileIdx) result.flags.incl flags registerModule(graph, result) - processModuleAux() + processModuleAux("import") else: if sfSystemModule in flags: graph.systemModule = result @@ -117,13 +118,13 @@ proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): P # reset module fields: initStrTables(graph, result) result.ast = nil - processModuleAux() + processModuleAux("import(dirty)") graph.markClientsDirty(fileIdx) proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym = # this is called by the semantic checking phase assert graph.config != nil - result = compileModule(graph, fileIdx, {}) + result = compileModule(graph, fileIdx, {}, s) graph.addDep(s, fileIdx) # keep track of import relationships if graph.config.hcrOn: diff --git a/compiler/passaux.nim b/compiler/passaux.nim index 87cb3a730..68b783248 100644 --- a/compiler/passaux.nim +++ b/compiler/passaux.nim @@ -19,18 +19,13 @@ type config: ConfigRef proc verboseOpen(graph: ModuleGraph; s: PSym; idgen: IdGenerator): PPassContext = - let conf = graph.config - result = VerboseRef(config: conf, idgen: idgen) - let path = toFilenameOption(conf, s.position.FileIndex, conf.filenameOption) - rawMessage(conf, hintProcessing, path) + # xxx consider either removing this or keeping for documentation for how to add a pass + result = VerboseRef(config: graph.config, idgen: idgen) proc verboseProcess(context: PPassContext, n: PNode): PNode = + # called from `process` in `processTopLevelStmt`. result = n let v = VerboseRef(context) - if v.config.verbosity == 3: - # system.nim deactivates all hints, for verbosity:3 we want the processing - # messages nonetheless, so we activate them again (but honor cmdlineNotes) - v.config.setNote(hintProcessing) - message(v.config, n.info, hintProcessing, $v.idgen[]) + message(v.config, n.info, hintProcessingStmt, $v.idgen[]) const verbosePass* = makePass(open = verboseOpen, process = verboseProcess) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index f29dc24ef..2907d5172 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -2158,6 +2158,7 @@ proc incMod(c: PContext, n: PNode, it: PNode, includeStmtResult: PNode) = var f = checkModuleName(c.config, it) if f != InvalidFileIdx: addIncludeFileDep(c, f) + onProcessing(c.graph, f, "include", c.module) if containsOrIncl(c.includedFiles, f.int): localError(c.config, n.info, errRecursiveDependencyX % toMsgFilename(c.config, f)) else: diff --git a/nimsuggest/tests/tchk1.nim b/nimsuggest/tests/tchk1.nim index 794392c1f..f8e2989c5 100644 --- a/nimsuggest/tests/tchk1.nim +++ b/nimsuggest/tests/tchk1.nim @@ -17,7 +17,7 @@ proc main = discard """ $nimsuggest --tester $file >chk $1 -chk;;skUnknown;;;;Hint;;???;;0;;-1;;"tests/tchk1.nim [Processing]";;0 +chk;;skUnknown;;;;Hint;;???;;0;;-1;;">> (toplevel): import(dirty): tests/tchk1.nim [Processing]";;0 chk;;skUnknown;;;;Error;;$file;;12;;0;;"identifier expected, but got \'keyword template\'";;0 chk;;skUnknown;;;;Error;;$file;;14;;0;;"nestable statement requires indentation";;0 chk;;skUnknown;;;;Error;;$file;;12;;0;;"implementation of \'foo\' expected";;0 diff --git a/nimsuggest/tests/tchk_compiles.nim b/nimsuggest/tests/tchk_compiles.nim index 2afa06899..c8a3daac4 100644 --- a/nimsuggest/tests/tchk_compiles.nim +++ b/nimsuggest/tests/tchk_compiles.nim @@ -4,5 +4,5 @@ discard compiles(2 + "hello") discard """ $nimsuggest --tester $file >chk $1 -chk;;skUnknown;;;;Hint;;???;;0;;-1;;"tests/tchk_compiles.nim [Processing]";;0 +chk;;skUnknown;;;;Hint;;???;;0;;-1;;">> (toplevel): import(dirty): tests/tchk_compiles.nim [Processing]";;0 """ diff --git a/nimsuggest/tests/ttempl_inst.nim b/nimsuggest/tests/ttempl_inst.nim index da413d99a..5f5b10fe9 100644 --- a/nimsuggest/tests/ttempl_inst.nim +++ b/nimsuggest/tests/ttempl_inst.nim @@ -7,7 +7,7 @@ foo() discard """ $nimsuggest --tester $file >chk $1 -chk;;skUnknown;;;;Hint;;???;;0;;-1;;"tests/ttempl_inst.nim [Processing]";;0 +chk;;skUnknown;;;;Hint;;???;;0;;-1;;">> (toplevel): import(dirty): tests/ttempl_inst.nim [Processing]";;0 chk;;skUnknown;;;;Hint;;$file;;4;;3;;"template/generic instantiation from here";;0 chk;;skUnknown;;;;Warning;;$file;;2;;11;;"foo [User]";;0 """ From 16f0b556643a8a9cc73781de6d71d0041a194640 Mon Sep 17 00:00:00 2001 From: Bung Date: Wed, 30 Jun 2021 13:28:41 +0800 Subject: [PATCH 0999/2472] remove isKeywordIgnoreCase from highlite, not used , can't even compile with -d:useNimRtl (#18399) --- lib/packages/docutils/highlite.nim | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/packages/docutils/highlite.nim b/lib/packages/docutils/highlite.nim index 014303837..d36b2c877 100644 --- a/lib/packages/docutils/highlite.nim +++ b/lib/packages/docutils/highlite.nim @@ -188,9 +188,6 @@ const proc isKeyword(x: openArray[string], y: string): int = binarySearch(x, y) -proc isKeywordIgnoreCase(x: openArray[string], y: string): int = - binarySearch(x, y, cmpIgnoreCase) - proc nimNextToken(g: var GeneralTokenizer, keywords: openArray[string] = @[]) = const hexChars = {'0'..'9', 'A'..'F', 'a'..'f', '_'} From eaa1c3a37754725f91f54f23080980b82c2f2356 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Wed, 30 Jun 2021 07:29:10 +0200 Subject: [PATCH 1000/2472] genDepend: fix for module names that are reserved DOT keywords (#18392) (like "node" and "edge") --- compiler/depends.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/depends.nim b/compiler/depends.nim index 7225b6b47..30fc961c5 100644 --- a/compiler/depends.nim +++ b/compiler/depends.nim @@ -25,7 +25,7 @@ type dotGraph: Rope proc addDependencyAux(b: Backend; importing, imported: string) = - b.dotGraph.addf("$1 -> \"$2\";$n", [rope(importing), rope(imported)]) + b.dotGraph.addf("\"$1\" -> \"$2\";$n", [rope(importing), rope(imported)]) # s1 -> s2_4[label="[0-9]"]; proc addDotDependency(c: PPassContext, n: PNode): PNode = From 19263f27756e56026fe8d1ac05966b84010ae8d1 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 30 Jun 2021 12:36:24 +0200 Subject: [PATCH 1001/2472] fixes #18400 (#18402) --- lib/system/schubfach.nim | 2 +- tests/float/tfloat4.nim | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/system/schubfach.nim b/lib/system/schubfach.nim index d072cde99..06813f632 100644 --- a/lib/system/schubfach.nim +++ b/lib/system/schubfach.nim @@ -359,7 +359,7 @@ proc formatDigits(buffer: var openArray[char]; pos: int; digits: uint32; decimal pos = digitsEnd elif decimalPoint < numDigits: ## dig.its - for i in 0..<8: + for i in countdown(7, 0): buffer[i + decimalPoint + 1] = buffer[i + decimalPoint] buffer[pos+decimalPoint] = '.' pos = digitsEnd + 1 diff --git a/tests/float/tfloat4.nim b/tests/float/tfloat4.nim index 6a87cbe66..5bedca371 100644 --- a/tests/float/tfloat4.nim +++ b/tests/float/tfloat4.nim @@ -54,4 +54,10 @@ doAssert 9999999999999999.0 == "9999999999999999.0".parseFloat doAssert 0.999999999999999 == ".999999999999999".parseFloat doAssert 0.9999999999999999 == ".9999999999999999".parseFloat +# bug #18400 +var s = [-13.888888'f32] +assert $s[0] == "-13.888888" +var x = 1.23456789012345'f32 +assert $x == "1.2345679" + echo("passed all tests.") From 28a4814019ea5d985d323def3766fe7f44f2e3e1 Mon Sep 17 00:00:00 2001 From: Endeg Date: Wed, 30 Jun 2021 23:19:22 +0600 Subject: [PATCH 1002/2472] Missing spaces when compiling testament and nim_dbg. (#18403) --- koch.nim | 4 ++-- lib/pure/times.nim | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/koch.nim b/koch.nim index d3957069b..4ba0d0eb5 100644 --- a/koch.nim +++ b/koch.nim @@ -215,14 +215,14 @@ proc buildTools(args: string = "") = options = "-d:release " & args) when defined(windows): buildVccTool(args) bundleNimpretty(args) - nimCompileFold("Compile testament", "testament/testament.nim", options = "-d:release" & args) + nimCompileFold("Compile testament", "testament/testament.nim", options = "-d:release " & args) # pre-packages a debug version of nim which can help in many cases investigate issuses # withouth having to rebuild compiler. # `-d:nimDebugUtils` only makes sense when temporarily editing/debugging compiler # `-d:debug` should be changed to a flag that doesn't require re-compiling nim # `--opt:speed` is a sensible default even for a debug build, it doesn't affect nim stacktraces - nimCompileFold("Compile nim_dbg", "compiler/nim.nim", options = "--opt:speed --stacktrace -d:debug --stacktraceMsgs -d:nimCompilerStacktraceHints" & args, outputName = "nim_dbg") + nimCompileFold("Compile nim_dbg", "compiler/nim.nim", options = "--opt:speed --stacktrace -d:debug --stacktraceMsgs -d:nimCompilerStacktraceHints " & args, outputName = "nim_dbg") proc nsis(latest: bool; args: string) = bundleNimbleExe(latest, args) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index fb48a199c..fcdd4ec8d 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -913,7 +913,7 @@ proc getTime*(): Time {.tags: [TimeEffect], benign.} = result = initTime(ts.tv_sec.int64, ts.tv_nsec.int) elif defined(windows): var f {.noinit.}: FILETIME - getSystemTimePreciseAsFileTime(f) + getSystemTimeAsFileTime(f) result = fromWinTime(rdFileTime(f)) proc `-`*(a, b: Time): Duration {.operator, extern: "ntDiffTime".} = @@ -2596,7 +2596,7 @@ proc epochTime*(): float {.tags: [TimeEffect].} = toBiggestFloat(ts.tv_nsec.int64) / 1_000_000_000 elif defined(windows): var f {.noinit.}: winlean.FILETIME - getSystemTimePreciseAsFileTime(f) + getSystemTimeAsFileTime(f) var i64 = rdFileTime(f) - epochDiff var secs = i64 div rateDiff var subsecs = i64 mod rateDiff From 17911a116d8677228480d74ceb2b9593b992253e Mon Sep 17 00:00:00 2001 From: Mark Pointing Date: Thu, 1 Jul 2021 03:22:38 +1000 Subject: [PATCH 1003/2472] Removed hard coded ports from thttpclient_standalone.nim. (#18374) Co-authored-by: Dominik Picheta Co-authored-by: Mark Pointing Co-authored-by: Andreas Rumpf Co-authored-by: Timothee Cour --- tests/stdlib/thttpclient_standalone.nim | 49 +++++++++++++------------ 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/tests/stdlib/thttpclient_standalone.nim b/tests/stdlib/thttpclient_standalone.nim index 5fa1ea41a..362b1cb86 100644 --- a/tests/stdlib/thttpclient_standalone.nim +++ b/tests/stdlib/thttpclient_standalone.nim @@ -2,29 +2,37 @@ discard """ cmd: "nim c --threads:on $file" """ -import asynchttpserver, httpclient, asyncdispatch, strutils +import asynchttpserver, httpclient, asyncdispatch, strutils, net block: # bug #16436 - proc startServer() {.async.} = + proc startServer(): AsyncHttpServer = + result = newAsyncHttpServer() + result.listen(Port(0)) + + proc processRequest(server: AsyncHttpServer) {.async.} = proc cb(req: Request) {.async.} = let headers = { "Content-length": "15"} # Provide invalid content-length await req.respond(Http200, "Hello World", headers.newHttpHeaders()) - var server = newAsyncHttpServer() - await server.serve(Port(5555), cb) + await server.acceptRequest(cb) - proc runClient() {.async.} = + proc runClient(port: Port) {.async.} = let c = newAsyncHttpClient(headers = {"Connection": "close"}.newHttpHeaders) - let r = await c.getContent("http://127.0.0.1:5555") + discard await c.getContent("http://127.0.0.1:" & $port) doAssert false, "should fail earlier" - asyncCheck startServer() + let server = startServer() + asyncCheck processRequest(server) + let port = server.getPort() doAssertRaises(ProtocolError): - waitFor runClient() + waitFor runClient(port) block: # bug #14794 (And test for presence of content-length header when using postContent) - proc startServer() {.async.} = - var killServer = false + proc startServer(): AsyncHttpServer = + result = newAsyncHttpServer() + result.listen(Port(0)) + + proc runServer(server: AsyncHttpServer) {.async.} = proc cb(req: Request) {.async.} = doAssert(req.body.endsWith(httpNewLine), "Multipart body does not end with a newline.") # this next line is probably not required because asynchttpserver does not call @@ -32,23 +40,18 @@ block: # bug #14794 (And test for presence of content-length header when using p # Error: unhandled exception: 411 Length Required # Added for good measure in case the server becomes more permissive. doAssert(req.headers.hasKey("content-length"), "Content-Length header is not present.") - killServer = true asyncCheck req.respond(Http200, "OK") - var server = newAsyncHttpServer() - server.listen(Port(5556)) - while not killServer: - if server.shouldAcceptRequest(): - await server.acceptRequest(cb) - else: - poll() + await server.acceptRequest(cb) - proc runClient() {.async.} = + proc runClient(port: Port) {.async.} = let c = newAsyncHttpClient() - var data = newMultipartData() + let data = newMultipartData() data.add("file.txt", "This is intended to be an example text file.\r\nThis would be the second line.\r\n") - let r = await c.postContent("http://127.0.0.1:5556", multipart = data) + discard await c.postContent("http://127.0.0.1:" & $port, multipart = data) c.close() - asyncCheck startServer() - waitFor runClient() + let server = startServer() + let port = server.getPort() + asyncCheck runServer(server) + waitFor runClient(port) From 41c29cb3a1674a8b85844888e831b79050d91293 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 1 Jul 2021 06:51:08 +0200 Subject: [PATCH 1004/2472] fixes #18130 (#18407) --- compiler/injectdestructors.nim | 2 +- tests/destructor/tv2_cast.nim | 87 +++++++++++++++++++++++++++------- 2 files changed, 72 insertions(+), 17 deletions(-) diff --git a/compiler/injectdestructors.nim b/compiler/injectdestructors.nim index 6447557c5..6e4eaa817 100644 --- a/compiler/injectdestructors.nim +++ b/compiler/injectdestructors.nim @@ -1057,7 +1057,7 @@ proc moveOrCopy(dest, ri: PNode; c: var Con; s: var Scope, isDecl = false): PNod result = c.genCopy(dest, ri) result.add p(ri, c, s, consumed) c.finishCopy(result, dest, isFromSink = false) - of nkHiddenSubConv, nkHiddenStdConv, nkConv, nkObjDownConv, nkObjUpConv: + of nkHiddenSubConv, nkHiddenStdConv, nkConv, nkObjDownConv, nkObjUpConv, nkCast: result = c.genSink(dest, p(ri, c, s, sinkArg), isDecl) of nkStmtListExpr, nkBlockExpr, nkIfExpr, nkCaseStmt, nkTryStmt: template process(child, s): untyped = moveOrCopy(dest, child, c, s, isDecl) diff --git a/tests/destructor/tv2_cast.nim b/tests/destructor/tv2_cast.nim index 5502cb38d..917cf0eb3 100644 --- a/tests/destructor/tv2_cast.nim +++ b/tests/destructor/tv2_cast.nim @@ -1,48 +1,81 @@ discard """ output: '''@[1] @[116, 101, 115, 116] -@[1953719668, 875770417]''' +@[1953719668, 875770417] +destroying O1''' cmd: '''nim c --gc:arc --expandArc:main --expandArc:main1 --expandArc:main2 --expandArc:main3 --hints:off --assertions:off $file''' nimout: '''--expandArc: main var data :tmpD -`=copy`(data, cast[string](encode(cast[seq[byte]]( - :tmpD = newString(100) - :tmpD)))) -`=destroy`(:tmpD) -`=destroy`(data) + :tmpD_1 + :tmpD_2 +data = + wasMoved(:tmpD) + `=copy`(:tmpD, cast[string]( + :tmpD_2 = encode(cast[seq[byte]]( + :tmpD_1 = newString(100) + :tmpD_1)) + :tmpD_2)) + :tmpD +`=destroy`(:tmpD_2) +`=destroy_1`(:tmpD_1) +`=destroy_1`(data) -- end of expandArc ------------------------ --expandArc: main1 var s data + :tmpD + :tmpD_1 s = newString(100) -`=copy`(data, cast[string](encode(toOpenArrayByte(s, 0, len(s) - 1)))) -`=destroy`(data) -`=destroy`(s) +data = + wasMoved(:tmpD) + `=copy`(:tmpD, cast[string]( + :tmpD_1 = encode(toOpenArrayByte(s, 0, len(s) - 1)) + :tmpD_1)) + :tmpD +`=destroy`(:tmpD_1) +`=destroy_1`(data) +`=destroy_1`(s) -- end of expandArc ------------------------ --expandArc: main2 var s data + :tmpD + :tmpD_1 s = newSeq(100) -`=copy`(data, cast[string](encode(s))) -`=destroy`(data) -`=destroy_1`(s) +data = + wasMoved(:tmpD) + `=copy`(:tmpD, cast[string]( + :tmpD_1 = encode(s) + :tmpD_1)) + :tmpD +`=destroy`(:tmpD_1) +`=destroy_1`(data) +`=destroy`(s) -- end of expandArc ------------------------ --expandArc: main3 var data :tmpD -`=copy`(data, cast[string](encode do: - :tmpD = newSeq(100) - :tmpD)) -`=destroy`(:tmpD) + :tmpD_1 + :tmpD_2 +data = + wasMoved(:tmpD) + `=copy`(:tmpD, cast[string]( + :tmpD_2 = encode do: + :tmpD_1 = newSeq(100) + :tmpD_1 + :tmpD_2)) + :tmpD +`=destroy`(:tmpD_2) +`=destroy`(:tmpD_1) `=destroy_1`(data) -- end of expandArc ------------------------''' """ @@ -89,3 +122,25 @@ echo a #issue 11204 var ac {.compileTime.} = @["a", "b"] const bc = ac.len + + +type + O = object of RootRef + i: int + + O1 = object of O + O2 = object of O + +proc `=destroy`(o: var O) = + echo "destroying O" + +proc `=destroy`(o: var O1) = + echo "destroying O1" + +proc `=destroy`(o: var O2) = + echo "destroying O2" + +proc test = + let o3 = cast[ref O2]((ref O1)()) + +test() From 779b4e307bdeb018bb2884920144a8c0d1f3b377 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 1 Jul 2021 02:14:01 -0700 Subject: [PATCH 1005/2472] don't use `{.rtl.}` for generics, otherwise `-d:useNimRtl` gives `ambiguous identifier` nimrtl error (#18406) --- lib/pure/strutils.nim | 2 +- lib/system/inclrtl.nim | 13 +++++++------ tests/dll/client.nim | 8 +++++--- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 8e0a6158b..9e0837942 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -1760,7 +1760,7 @@ func join*(a: openArray[string], sep: string = ""): string {.rtl, else: result = "" -func join*[T: not string](a: openArray[T], sep: string = ""): string {.rtl.} = +func join*[T: not string](a: openArray[T], sep: string = ""): string = ## Converts all elements in the container `a` to strings using `$`, ## and concatenates them with `sep`. runnableExamples: diff --git a/lib/system/inclrtl.nim b/lib/system/inclrtl.nim index 0bc51d693..ca41f39c6 100644 --- a/lib/system/inclrtl.nim +++ b/lib/system/inclrtl.nim @@ -30,12 +30,13 @@ when defined(createNimRtl): {.pragma: inl.} {.pragma: compilerRtl, compilerproc, exportc: "nimrtl_$1", dynlib.} elif defined(useNimRtl): - when defined(windows): - const nimrtl* = "nimrtl.dll" - elif defined(macosx): - const nimrtl* = "libnimrtl.dylib" - else: - const nimrtl* = "libnimrtl.so" + #[ + `{.rtl.}` should only be used for non-generic procs. + ]# + const nimrtl* = + when defined(windows): "nimrtl.dll" + elif defined(macosx): "libnimrtl.dylib" + else: "libnimrtl.so" {.pragma: rtl, importc: "nimrtl_$1", dynlib: nimrtl, gcsafe.} {.pragma: inl.} {.pragma: compilerRtl, compilerproc, importc: "nimrtl_$1", dynlib: nimrtl.} diff --git a/tests/dll/client.nim b/tests/dll/client.nim index 150af3a17..62697569f 100644 --- a/tests/dll/client.nim +++ b/tests/dll/client.nim @@ -1,5 +1,4 @@ discard """ - output: "Done" cmd: "nim $target --debuginfo --hints:on --define:useNimRtl $options $file" """ @@ -37,5 +36,8 @@ proc eval(n: PNode): int = for i in 0..100_000: discard eval(buildTree(2)) -echo "Done" - +# bug https://forum.nim-lang.org/t/8176; Error: ambiguous identifier: 'nimrtl' +import std/strutils +doAssert join(@[1, 2]) == "12" +doAssert join(@[1.5, 2.5]) == "1.52.5" +doAssert join(@["a", "bc"]) == "abc" From 3ceaf5c1309ac8d4d68e6f39c13b021bcc1b15f4 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 1 Jul 2021 17:35:04 +0200 Subject: [PATCH 1006/2472] fixes #18030 (#18415) --- compiler/seminst.nim | 2 +- tests/arc/tarcmisc.nim | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/compiler/seminst.nim b/compiler/seminst.nim index 68ab2a310..e1fad236c 100644 --- a/compiler/seminst.nim +++ b/compiler/seminst.nim @@ -146,8 +146,8 @@ proc instantiateBody(c: PContext, n, params: PNode, result, orig: PSym) = freshGenSyms(c, b, result, orig, symMap) b = semProcBody(c, b) result.ast[bodyPos] = hloBody(c, b) - trackProc(c, result, result.ast[bodyPos]) excl(result.flags, sfForward) + trackProc(c, result, result.ast[bodyPos]) dec c.inGenericInst proc fixupInstantiatedSymbols(c: PContext, s: PSym) = diff --git a/tests/arc/tarcmisc.nim b/tests/arc/tarcmisc.nim index a52538085..5d5d8e914 100644 --- a/tests/arc/tarcmisc.nim +++ b/tests/arc/tarcmisc.nim @@ -28,6 +28,8 @@ aaaaa hello ok true +copying +123 closed destroying variable: 20 destroying variable: 10 @@ -433,3 +435,31 @@ proc t17712 = echo el != nil t17712() + +# bug #18030 + +type + Foo = object + n: int + +proc `=copy`(dst: var Foo, src: Foo) = + echo "copying" + dst.n = src.n + +proc `=sink`(dst: var Foo, src: Foo) = + echo "sinking" + dst.n = src.n + +var a: Foo + +proc putValue[T](n: T) + +proc useForward = + putValue(123) + +proc putValue[T](n: T) = + var b = Foo(n:n) + a = b + echo b.n + +useForward() From 60eac79f7f0478565a24ff423aa52ab53527d153 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 2 Jul 2021 20:38:01 +0800 Subject: [PATCH 1007/2472] [minor]remove unnecessary stringify (#18419) --- compiler/lexer.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/compiler/lexer.nim b/compiler/lexer.nim index 59b983060..c7a9aa47f 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -536,9 +536,9 @@ proc getNumber(L: var Lexer, result: var Token) = try: len = parseBiggestUInt(result.literal, iNumber) except ValueError: - raise newException(OverflowDefect, "number out of range: " & $result.literal) + raise newException(OverflowDefect, "number out of range: " & result.literal) if len != result.literal.len: - raise newException(ValueError, "invalid integer: " & $result.literal) + raise newException(ValueError, "invalid integer: " & result.literal) result.iNumber = cast[int64](iNumber) else: var iNumber: int64 @@ -546,9 +546,9 @@ proc getNumber(L: var Lexer, result: var Token) = try: len = parseBiggestInt(result.literal, iNumber) except ValueError: - raise newException(OverflowDefect, "number out of range: " & $result.literal) + raise newException(OverflowDefect, "number out of range: " & result.literal) if len != result.literal.len: - raise newException(ValueError, "invalid integer: " & $result.literal) + raise newException(ValueError, "invalid integer: " & result.literal) result.iNumber = iNumber # Explicit bounds checks. From d1d2498c7b933b6364af450df5088aaa89b4cf4b Mon Sep 17 00:00:00 2001 From: Antonis Geralis <43617260+planetis-m@users.noreply.github.com> Date: Sat, 3 Jul 2021 00:20:59 +0300 Subject: [PATCH 1008/2472] Add lent annotation to avoid extra copies in sample (#18404) * Add lent annotation to avoid extra copies in sample * Remove lent * Update random.nim --- lib/pure/random.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/random.nim b/lib/pure/random.nim index 6a5f35bde..07db0365c 100644 --- a/lib/pure/random.nim +++ b/lib/pure/random.nim @@ -410,7 +410,7 @@ proc sample*[T](r: var Rand; a: openArray[T]): T = result = a[r.rand(a.low..a.high)] -proc sample*[T](a: openArray[T]): T = +proc sample*[T](a: openArray[T]): lent T = ## Returns a random element from `a`. ## ## If `randomize <#randomize>`_ has not been called, the order of outcomes From ffa638ed5023834a7a5040a54c6cfde550d68054 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 5 Jul 2021 05:26:40 -0700 Subject: [PATCH 1009/2472] std/net: show addr+port on address already in use error; code-block => runnableExamples (#18428) * std/net: show addr+port on address already in use error; code-block => runnableExamples * var=>let --- lib/pure/net.nim | 80 ++++++++++++++++++++++-------------------------- 1 file changed, 37 insertions(+), 43 deletions(-) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index ff179fb2c..ec3e4e74d 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -44,46 +44,44 @@ ## After you create a socket with the `newSocket` procedure, you can easily ## connect it to a server running at a known hostname (or IP address) and port. ## To do so over TCP, use the example below. -## -## .. code-block:: Nim -## var socket = newSocket() -## socket.connect("google.com", Port(80)) -## -## For SSL, use the following example (and make sure to compile with `-d:ssl`): -## -## .. code-block:: Nim -## var socket = newSocket() -## var ctx = newContext() -## wrapSocket(ctx, socket) -## socket.connect("google.com", Port(443)) -## + +runnableExamples("-r:off"): + let socket = newSocket() + socket.connect("google.com", Port(80)) + +## For SSL, use the following example: + +runnableExamples("-r:off -d:ssl"): + let socket = newSocket() + let ctx = newContext() + wrapSocket(ctx, socket) + socket.connect("google.com", Port(443)) + ## UDP is a connectionless protocol, so UDP sockets don't have to explicitly ## call the `connect `_ procedure. They can ## simply start sending data immediately. -## -## .. code-block:: Nim -## var socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) -## socket.sendTo("192.168.0.1", Port(27960), "status\n") -## + +runnableExamples("-r:off"): + let socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) + socket.sendTo("192.168.0.1", Port(27960), "status\n") + ## Creating a server ## ----------------- ## ## After you create a socket with the `newSocket` procedure, you can create a ## TCP server by calling the `bindAddr` and `listen` procedures. -## -## .. code-block:: Nim -## var socket = newSocket() -## socket.bindAddr(Port(1234)) -## socket.listen() -## -## You can then begin accepting connections using the `accept` procedure. -## -## .. code-block:: Nim -## var client: Socket -## var address = "" -## while true: -## socket.acceptAddr(client, address) -## echo("Client connected from: ", address) + +runnableExamples("-r:off"): + let socket = newSocket() + socket.bindAddr(Port(1234)) + socket.listen() + + # You can then begin accepting connections using the `accept` procedure. + var client: Socket + var address = "" + while true: + socket.acceptAddr(client, address) + echo "Client connected from: ", address import std/private/since @@ -960,7 +958,7 @@ proc bindAddr*(socket: Socket, port = Port(0), address = "") {. var aiList = getAddrInfo(realaddr, port, socket.domain) if bindAddr(socket.fd, aiList.ai_addr, aiList.ai_addrlen.SockLen) < 0'i32: freeaddrinfo(aiList) - raiseOSError(osLastError()) + raiseOSError(osLastError(), "address: $# port: $#" % [address, $port]) freeaddrinfo(aiList) proc acceptAddr*(server: Socket, client: var owned(Socket), address: var string, @@ -1233,12 +1231,10 @@ proc getPeerAddr*(socket: Socket): (string, Port) = proc setSockOpt*(socket: Socket, opt: SOBool, value: bool, level = SOL_SOCKET) {.tags: [WriteIOEffect].} = ## Sets option `opt` to a boolean value specified by `value`. - ## - ## .. code-block:: Nim - ## var socket = newSocket() - ## socket.setSockOpt(OptReusePort, true) - ## socket.setSockOpt(OptNoDelay, true, level=IPPROTO_TCP.toInt) - ## + runnableExamples("-r:off"): + let socket = newSocket() + socket.setSockOpt(OptReusePort, true) + socket.setSockOpt(OptNoDelay, true, level = IPPROTO_TCP.cint) var valuei = cint(if value: 1 else: 0) setSockOptInt(socket.fd, cint(level), toCInt(opt), valuei) @@ -2025,10 +2021,8 @@ proc getPrimaryIPAddr*(dest = parseIpAddress("8.8.8.8")): IpAddress = ## ## Supports IPv4 and v6. ## Raises OSError if external networking is not set up. - ## - ## .. code-block:: Nim - ## echo $getPrimaryIPAddr() # "192.168.1.2" - + runnableExamples("-r:off"): + echo getPrimaryIPAddr() # "192.168.1.2" let socket = if dest.family == IpAddressFamily.IPv4: newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) From 1bed7773146d6acf704b0ae5cfe62455f7a127a1 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 5 Jul 2021 18:04:07 +0200 Subject: [PATCH 1010/2472] fixes #18411 (#18432) [backport:1.4] --- lib/system/excpt.nim | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index 5b7d4d49f..9f65db2fe 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -446,9 +446,7 @@ proc raiseExceptionAux(e: sink(ref Exception)) {.nodestroy.} = pushCurrentException(e) {.emit: "throw `e`;".} elif defined(nimQuirky) or gotoBasedExceptions: - # XXX This check should likely also be done in the setjmp case below. - if e != currException: - pushCurrentException(e) + pushCurrentException(e) when gotoBasedExceptions: inc nimInErrorMode else: From 927a832dde9403492d4a2838780bf59bd1a48177 Mon Sep 17 00:00:00 2001 From: pyautogui <79544976+pyautogui@users.noreply.github.com> Date: Mon, 5 Jul 2021 10:12:58 -0700 Subject: [PATCH 1011/2472] deprecate std/mersenne (#18395) Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Co-authored-by: Timothee Cour Co-authored-by: Andreas Rumpf --- changelog.md | 2 ++ lib/pure/mersenne.nim | 12 ++++-------- tests/stdlib/tmersenne.nim | 1 + 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/changelog.md b/changelog.md index 6672f7538..a4c5b4e62 100644 --- a/changelog.md +++ b/changelog.md @@ -4,6 +4,8 @@ ## Changes affecting backward compatibility +- Deprecated `std/mersenne` + - `cuchar` now aliases `uint8` instead of `char` - `repr` now doesn't insert trailing newline; previous behavior was very inconsistent, diff --git a/lib/pure/mersenne.nim b/lib/pure/mersenne.nim index 6778e2d62..6ec0d66b4 100644 --- a/lib/pure/mersenne.nim +++ b/lib/pure/mersenne.nim @@ -2,24 +2,20 @@ # # Nim's Runtime Library # (c) Copyright 2015 Nim Contributors -# # See the file "copying.txt", included in this # distribution, for details about the copyright. -# - ## The [Mersenne Twister](https://en.wikipedia.org/wiki/Mersenne_Twister) ## random number generator. -## -## **Note:** The procs in this module work at compile-time. +## .. note:: The procs in this module work at compile-time. + +{.deprecated: "use `std/random` instead.".} runnableExamples: var rand = newMersenneTwister(uint32.high) ## must be "var" doAssert rand.getNum() != rand.getNum() ## pseudorandom number - ## See also ## ======== ## * `random module`_ for Nim's standard random number generator - type MersenneTwister* = object ## The Mersenne Twister. @@ -35,6 +31,7 @@ proc newMersenneTwister*(seed: uint32): MersenneTwister = (result.mt[i-1] shr 30'u32)) + i) proc generateNumbers(m: var MersenneTwister) = + for i in 0..623: var y = (m.mt[i] and 0x80000000'u32) + (m.mt[(i+1) mod 624] and 0x7fffffff'u32) @@ -48,7 +45,6 @@ proc getNum*(m: var MersenneTwister): uint32 = generateNumbers(m) result = m.mt[m.index] m.index = (m.index + 1) mod m.mt.len - result = result xor (result shr 11'u32) result = result xor ((result shl 7'u32) and 0x9d2c5680'u32) result = result xor ((result shl 15'u32) and 0xefc60000'u32) diff --git a/tests/stdlib/tmersenne.nim b/tests/stdlib/tmersenne.nim index 2707aa2f2..54eb7b216 100644 --- a/tests/stdlib/tmersenne.nim +++ b/tests/stdlib/tmersenne.nim @@ -7,5 +7,6 @@ template main() = doAssert mt.getNum == 1071751096'u32 doAssert mt.getNum == 3805347140'u32 + static: main() main() From 36654fd577f46c49191c5e6f7e499ea31193018c Mon Sep 17 00:00:00 2001 From: Antonis Geralis <43617260+planetis-m@users.noreply.github.com> Date: Mon, 5 Jul 2021 21:55:46 +0300 Subject: [PATCH 1012/2472] Cleanup interval and timeout procs (#18431) clear/setTimeout procs were using ref ref Timeout. Added more interval overloads. Corrections. --- lib/js/dom.nim | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/js/dom.nim b/lib/js/dom.nim index ca325fdd2..3f0b9f630 100644 --- a/lib/js/dom.nim +++ b/lib/js/dom.nim @@ -1301,7 +1301,7 @@ type width*: int TimeOut* {.importc.} = ref object of RootObj - Interval* {.importc.} = object of RootObj + Interval* {.importc.} = ref object of RootObj AddEventListenerOptions* = object capture*: bool @@ -1481,6 +1481,8 @@ else: proc setTimeout*(action: proc(); ms: int): TimeOut {.importc, nodecl.} proc clearTimeout*(t: TimeOut) {.importc, nodecl.} +proc setInterval*(action: proc(); ms: int): Interval {.importc, nodecl.} +proc clearInterval*(i: Interval) {.importc, nodecl.} {.push importcpp.} @@ -1494,8 +1496,8 @@ proc removeEventListener*(et: EventTarget; ev: cstring; cb: proc(ev: Event)) proc alert*(w: Window, msg: cstring) proc back*(w: Window) proc blur*(w: Window) -proc clearInterval*(w: Window, interval: ref Interval) -proc clearTimeout*(w: Window, timeout: ref TimeOut) +proc clearInterval*(w: Window, interval: Interval) +proc clearTimeout*(w: Window, timeout: TimeOut) proc close*(w: Window) proc confirm*(w: Window, msg: cstring): bool proc disableExternalCapture*(w: Window) @@ -1520,10 +1522,10 @@ proc resizeTo*(w: Window, x, y: int) proc routeEvent*(w: Window, event: Event) proc scrollBy*(w: Window, x, y: int) proc scrollTo*(w: Window, x, y: int) -proc setInterval*(w: Window, code: cstring, pause: int): ref Interval -proc setInterval*(w: Window, function: proc (), pause: int): ref Interval -proc setTimeout*(w: Window, code: cstring, pause: int): ref TimeOut -proc setTimeout*(w: Window, function: proc (), pause: int): ref Interval +proc setInterval*(w: Window, code: cstring, pause: int): Interval +proc setInterval*(w: Window, function: proc (), pause: int): Interval +proc setTimeout*(w: Window, code: cstring, pause: int): TimeOut +proc setTimeout*(w: Window, function: proc (), pause: int): Interval proc stop*(w: Window) proc requestAnimationFrame*(w: Window, function: proc (time: float)): int proc cancelAnimationFrame*(w: Window, id: int) @@ -1768,4 +1770,4 @@ since (1, 3): proc readAsDataURL*(f: FileReader, b: Blob) {.importcpp: "#.readAsDataURL(#)".} ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readAsDataURL proc readAsText*(f: FileReader, b: Blob|File, encoding = cstring"UTF-8") {.importcpp: "#.readAsText(#, #)".} - ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readAsText \ No newline at end of file + ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readAsText From c522f7f33ca08e6a441ebbb08ea9d2d79a3c500c Mon Sep 17 00:00:00 2001 From: Antonis Geralis <43617260+planetis-m@users.noreply.github.com> Date: Tue, 6 Jul 2021 08:21:50 +0300 Subject: [PATCH 1013/2472] Add entry (#18434) --- changelog.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/changelog.md b/changelog.md index a4c5b4e62..028743bab 100644 --- a/changelog.md +++ b/changelog.md @@ -86,8 +86,8 @@ - Deprecated `proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T]` in `std/algorithm`. -- In `std/macros`, `treeRepr,lispRepr,astGenRepr` now represent SymChoice nodes in a collapsed way, - use `-d:nimLegacyMacrosCollapseSymChoice` to get previous behavior. +- In `std/macros`, `treeRepr,lispRepr,astGenRepr` now represent SymChoice nodes in a collapsed way, + use `-d:nimLegacyMacrosCollapseSymChoice` to get previous behavior. - The configuration subsystem now allows for `-d:release` and `-d:danger` to work as expected. The downside is that these defines now have custom logic that doesn't apply for @@ -95,6 +95,8 @@ - Renamed `-d:nimCompilerStackraceHints` to `-d:nimCompilerStacktraceHints`. +- In `std/dom`, `Interval` is now a `ref object`, same as `Timeout`. Definitions of `setTimeout`, + `clearTimeout`, `setInterval`, `clearInterval` were updated. ## Standard library additions and changes @@ -341,6 +343,8 @@ - Added `dom.scrollIntoView` proc with options +- Added `dom.setInterval`, `dom.clearInterval` overloads. + ## Language changes - `nimscript` now handles `except Exception as e`. From 252eea8cae33e486b179398e193aea9459954338 Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Mon, 5 Jul 2021 23:28:38 -0600 Subject: [PATCH 1014/2472] Make procedure mismatch more informative with pragma/call convention mismatches (#18384) * Added more concise calling convention/pragma mismatch messages * Now only adds callConvMsg/lock message when sensible * Fixed message formatting * Added tests, and fixed some bugs * Tests joined, and always indenting * More tests and more bug fixes * Fixed first test in tprocmismatch * Using var param for writting mismatches * Better logic for handling proc type rel and conv/pragma mismatch * Refactored getProcConvMismatch * Fixed callConv message formatting * Fixed test for proper message * Cleanup to address issues * getProcConvMismatch now returns tuple, and reformatted code --- compiler/semcall.nim | 8 ++- compiler/sigmatch.nim | 33 +----------- compiler/types.nim | 87 ++++++++++++++++++++++++++++++++ tests/errmsgs/tproc_mismatch.nim | 74 +++++++++++++++++++++++++++ 4 files changed, 170 insertions(+), 32 deletions(-) create mode 100644 tests/errmsgs/tproc_mismatch.nim diff --git a/compiler/semcall.nim b/compiler/semcall.nim index b56a1dbac..cec656fc2 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -247,7 +247,13 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors): candidates.add typeToString(got) candidates.addDeclaredLocMaybe(c.config, got) doAssert wanted != nil - if got != nil: effectProblem(wanted, got, candidates, c) + if got != nil: + if got.kind == tyProc and wanted.kind == tyProc: + # These are proc mismatches so, + # add the extra explict detail of the mismatch + candidates.addPragmaAndCallConvMismatch(wanted, got, c.config) + effectProblem(wanted, got, candidates, c) + of kUnknown: discard "do not break 'nim check'" candidates.add "\n" if err.firstMismatch.arg == 1 and nArg.kind == nkTupleConstr and diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 071685057..73ab5e9a3 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -86,21 +86,6 @@ type TTypeRelFlags* = set[TTypeRelFlag] - TTypeRelation* = enum # order is important! - isNone, isConvertible, - isIntConv, - isSubtype, - isSubrange, # subrange of the wanted type; no type conversion - # but apart from that counts as ``isSubtype`` - isBothMetaConvertible # generic proc parameter was matched against - # generic type, e.g., map(mySeq, x=>x+1), - # maybe recoverable by rerun if the parameter is - # the proc's return value - isInferred, # generic proc was matched against a concrete type - isInferredConvertible, # same as above, but requiring proc CC conversion - isGeneric, - isFromIntLit, # conversion *from* int literal; proven safe - isEqual const isNilConversion = isConvertible # maybe 'isIntConv' fits better? @@ -640,23 +625,9 @@ proc procTypeRel(c: var TCandidate, f, a: PType): TTypeRelation = return isNone elif a[0] != nil: return isNone + + result = getProcConvMismatch(c.c.config, f, a, result)[1] - if tfNoSideEffect in f.flags and tfNoSideEffect notin a.flags: - return isNone - elif tfThread in f.flags and a.flags * {tfThread, tfNoSideEffect} == {} and - optThreadAnalysis in c.c.config.globalOptions: - # noSideEffect implies ``tfThread``! - return isNone - elif f.flags * {tfIterator} != a.flags * {tfIterator}: - return isNone - elif f.callConv != a.callConv: - # valid to pass a 'nimcall' thingie to 'closure': - if f.callConv == ccClosure and a.callConv == ccNimCall: - result = if result == isInferred: isInferredConvertible - elif result == isBothMetaConvertible: isBothMetaConvertible - else: isConvertible - else: - return isNone when useEffectSystem: if compatibleEffects(f, a) != efCompat: return isNone when defined(drnim): diff --git a/compiler/types.nim b/compiler/types.nim index bc970a6d2..bdf08ffe9 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -26,6 +26,29 @@ type # most useful, shows: symbol + resolved symbols if it differs, e.g.: # tuple[a: MyInt{int}, b: float] + TTypeRelation* = enum # order is important! + isNone, isConvertible, + isIntConv, + isSubtype, + isSubrange, # subrange of the wanted type; no type conversion + # but apart from that counts as ``isSubtype`` + isBothMetaConvertible # generic proc parameter was matched against + # generic type, e.g., map(mySeq, x=>x+1), + # maybe recoverable by rerun if the parameter is + # the proc's return value + isInferred, # generic proc was matched against a concrete type + isInferredConvertible, # same as above, but requiring proc CC conversion + isGeneric, + isFromIntLit, # conversion *from* int literal; proven safe + isEqual + + ProcConvMismatch* = enum + pcmNoSideEffect + pcmNotGcSafe + pcmLockDifference + pcmNotIterator + pcmDifferentCallConv + proc typeToString*(typ: PType; prefer: TPreferedDesc = preferName): string template `$`*(typ: PType): string = typeToString(typ) @@ -1471,6 +1494,68 @@ proc skipHiddenSubConv*(n: PNode; g: ModuleGraph; idgen: IdGenerator): PNode = else: result = n +proc getProcConvMismatch*(c: ConfigRef, f, a: PType, rel = isNone): (set[ProcConvMismatch], TTypeRelation) = + ## Returns a set of the reason of mismatch, and the relation for conversion. + result[1] = rel + if tfNoSideEffect in f.flags and tfNoSideEffect notin a.flags: + # Formal is pure, but actual is not + result[0].incl pcmNoSideEffect + result[1] = isNone + + if tfThread in f.flags and a.flags * {tfThread, tfNoSideEffect} == {} and + optThreadAnalysis in c.globalOptions: + # noSideEffect implies ``tfThread``! + result[0].incl pcmNotGcSafe + result[1] = isNone + + if f.flags * {tfIterator} != a.flags * {tfIterator}: + # One of them is an iterator so not convertible + result[0].incl pcmNotIterator + result[1] = isNone + + if f.callConv != a.callConv: + # valid to pass a 'nimcall' thingie to 'closure': + if f.callConv == ccClosure and a.callConv == ccNimCall: + case result[1] + of isInferred: result[1] = isInferredConvertible + of isBothMetaConvertible: result[1] = isBothMetaConvertible + elif result[1] != isNone: result[1] = isConvertible + else: + result[1] = isNone + result[0].incl pcmDifferentCallConv + + if f.lockLevel.ord != UnspecifiedLockLevel.ord and + a.lockLevel.ord != UnspecifiedLockLevel.ord: + # proctypeRel has more logic to catch this difference, + # so dont need to do `rel = isNone` + # but it's a pragma mismatch reason which is why it's here + result[0].incl pcmLockDifference + +proc addPragmaAndCallConvMismatch*(message: var string, formal, actual: PType, conf: ConfigRef) = + assert formal.kind == tyProc and actual.kind == tyProc + let (convMismatch, _) = getProcConvMismatch(conf, formal, actual) + var + gotPragmas = "" + expectedPragmas = "" + for reason in convMismatch: + case reason + of pcmDifferentCallConv: + message.add "\n Calling convention mismatch: got '{.$1.}', but expected '{.$2.}'." % [$actual.callConv, $formal.callConv] + of pcmNoSideEffect: + expectedPragmas.add "noSideEffect, " + of pcmNotGcSafe: + expectedPragmas.add "gcsafe, " + of pcmLockDifference: + gotPragmas.add("locks: " & $actual.lockLevel & ", ") + expectedPragmas.add("locks: " & $formal.lockLevel & ", ") + of pcmNotIterator: discard + + if expectedPragmas.len > 0: + gotPragmas.setLen(max(0, gotPragmas.len - 2)) # Remove ", " + expectedPragmas.setLen(max(0, expectedPragmas.len - 2)) # Remove ", " + message.add "\n Pragma mismatch: got '{.$1.}', but expected '{.$2.}'." % [gotPragmas, expectedPragmas] + + proc typeMismatch*(conf: ConfigRef; info: TLineInfo, formal, actual: PType, n: PNode) = if formal.kind != tyError and actual.kind != tyError: let actualStr = typeToString(actual) @@ -1491,6 +1576,7 @@ proc typeMismatch*(conf: ConfigRef; info: TLineInfo, formal, actual: PType, n: P if verbose: msg.addDeclaredLoc(conf, formal) if formal.kind == tyProc and actual.kind == tyProc: + msg.addPragmaAndCallConvMismatch(formal, actual, conf) case compatibleEffects(formal, actual) of efCompat: discard of efRaisesDiffer: @@ -1503,6 +1589,7 @@ proc typeMismatch*(conf: ConfigRef; info: TLineInfo, formal, actual: PType, n: P msg.add "\n.tag effect is 'any tag allowed'" of efLockLevelsDiffer: msg.add "\nlock levels differ" + if formal.kind == tyEnum and actual.kind == tyEnum: msg.add "\nmaybe use `-d:nimLegacyConvEnumEnum` for a transition period" localError(conf, info, msg) diff --git a/tests/errmsgs/tproc_mismatch.nim b/tests/errmsgs/tproc_mismatch.nim new file mode 100644 index 000000000..400c3d441 --- /dev/null +++ b/tests/errmsgs/tproc_mismatch.nim @@ -0,0 +1,74 @@ +discard """ + action: reject + cmd: '''nim check --hints:off $options $file''' + nimoutFull: true + nimout: ''' +tproc_mismatch.nim(35, 52) Error: type mismatch: got but expected 'proc (a: int, c: float){.closure, noSideEffect.}' + Calling convention mismatch: got '{.cdecl.}', but expected '{.closure.}'. +tproc_mismatch.nim(39, 6) Error: type mismatch: got +but expected one of: +proc bar(a: proc ()) + first type mismatch at position: 1 + required type for a: proc (){.closure.} + but expression 'fn1' is of type: proc (){.inline, noSideEffect, gcsafe, locks: 0.} + Calling convention mismatch: got '{.inline.}', but expected '{.closure.}'. + +expression: bar(fn1) +tproc_mismatch.nim(43, 8) Error: type mismatch: got but expected 'proc (){.closure.}' + Calling convention mismatch: got '{.inline.}', but expected '{.closure.}'. +tproc_mismatch.nim(48, 8) Error: type mismatch: got but expected 'proc (){.closure, noSideEffect.}' + Pragma mismatch: got '{..}', but expected '{.noSideEffect.}'. +tproc_mismatch.nim(52, 8) Error: type mismatch: got but expected 'proc (a: float){.closure.}' +tproc_mismatch.nim(61, 9) Error: type mismatch: got but expected 'proc (a: int){.closure, gcsafe.}' + Pragma mismatch: got '{..}', but expected '{.gcsafe.}'. +tproc_mismatch.nim(69, 9) Error: type mismatch: got but expected 'proc (a: int): int{.cdecl.}' + Calling convention mismatch: got '{.nimcall.}', but expected '{.cdecl.}'. +tproc_mismatch.nim(70, 9) Error: type mismatch: got but expected 'proc (a: int): int{.nimcall.}' + Calling convention mismatch: got '{.cdecl.}', but expected '{.nimcall.}'. +tproc_mismatch.nim(74, 9) Error: type mismatch: got but expected 'proc (a: int){.closure, locks: 1.}' + Pragma mismatch: got '{.locks: 3.}', but expected '{.locks: 1.}'. +lock levels differ +''' +""" +block: # CallConv mismatch + func a(a: int, c: float) {.cdecl.} = discard + var b: proc(a: int, c: float) {.noSideEffect.} = a +block: # Parameter CallConv mismatch + proc fn1() {.inline.} = discard + proc bar(a: proc()) = discard + bar(fn1) +block: # CallConv mismatch + proc fn1() {.inline.} = discard + var fn: proc() + fn = fn1 +block: # Pragma mismatch + var a = "" + proc fn1() = a.add "b" + var fn: proc() {.noSideEffect.} + fn = fn1 +block: # Fail match not do to Pragma or CallConv + proc fn1(a: int) = discard + var fn: proc(a: float) + fn = fn1 +block: # Infered noSideEffect assign + type Foo = ref object + x0: int + var g0 = Foo(x0: 1) + proc fn1(a: int) = g0.x0 = a + var fn2: proc(a: int) + var fn3: proc(a: int) {.gcsafe.} + fn2 = fn1 + fn3 = fn1 +block: # Indrection through pragmas + {.pragma: inl1, inline.} + {.pragma: inl2, inline.} + {.pragma: p1, nimcall.} + {.pragma: p2, cdecl.} + var fn1: proc(a: int): int {.inl1, p1.} + var fn2: proc(a: int): int {.inl2, p2.} + fn2 = fn1 + fn1 = fn2 +block: # Lock levels differ + var fn1: proc(a: int){.locks: 3.} + var fn2: proc(a: int){.locks: 1.} + fn2 = fn1 From b88496ac67bea92c40cf1d0f3dc55b0fa91550a4 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Tue, 6 Jul 2021 21:09:56 +0300 Subject: [PATCH 1015/2472] fix compilation on Debian 7 (no git -C) (#18427) * fix compilation on Debian 7 (no git -C) * address review * allow specify branch for testing unmerged csources * temporarily change csources checkout parameters for testing * Update tools/deps.nim Co-authored-by: Timothee Cour * fix failure * Update config/build_config.txt * set proper git branch/hash Co-authored-by: Timothee Cour Co-authored-by: Andreas Rumpf --- build_all.bat | 2 +- ci/build_autogen.bat | 2 +- ci/funs.sh | 8 ++++++-- config/build_config.txt | 3 ++- tools/ci_generate.nim | 2 +- tools/deps.nim | 9 +++++++-- tools/niminst/makefile.nimf | 3 ++- 7 files changed, 20 insertions(+), 9 deletions(-) diff --git a/build_all.bat b/build_all.bat index 1bff761b3..c8ba05e19 100644 --- a/build_all.bat +++ b/build_all.bat @@ -10,7 +10,7 @@ SET nim_csources=bin\nim_csources_%nim_csourcesHash%.exe echo "building from csources: %nim_csources%" if not exist %nim_csourcesDir% ( - git clone -q --depth 1 %nim_csourcesUrl% %nim_csourcesDir% + git clone -q --depth 1 -b %nim_csourcesBranch% %nim_csourcesUrl% %nim_csourcesDir% ) if not exist %nim_csources% ( diff --git a/ci/build_autogen.bat b/ci/build_autogen.bat index 8b1113175..0b6ca566e 100644 --- a/ci/build_autogen.bat +++ b/ci/build_autogen.bat @@ -10,7 +10,7 @@ SET nim_csources=bin\nim_csources_%nim_csourcesHash%.exe echo "building from csources: %nim_csources%" if not exist %nim_csourcesDir% ( - git clone -q --depth 1 %nim_csourcesUrl% %nim_csourcesDir% + git clone -q --depth 1 -b %nim_csourcesBranch% %nim_csourcesUrl% %nim_csourcesDir% ) if not exist %nim_csources% ( diff --git a/ci/funs.sh b/ci/funs.sh index 4a8887658..e25922c8c 100644 --- a/ci/funs.sh +++ b/ci/funs.sh @@ -120,8 +120,12 @@ nimBuildCsourcesIfNeeded(){ else # Note: using git tags would allow fetching just what's needed, unlike git hashes, e.g. # via `git clone -q --depth 1 --branch $tag $nim_csourcesUrl`. - echo_run git clone -q --depth 1 $nim_csourcesUrl "$nim_csourcesDir" - echo_run git -C "$nim_csourcesDir" checkout $nim_csourcesHash + echo_run git clone -q --depth 1 -b $nim_csourcesBranch \ + $nim_csourcesUrl "$nim_csourcesDir" + # old `git` versions don't support -C option, using `cd` explicitly: + echo_run cd "$nim_csourcesDir" + echo_run git checkout $nim_csourcesHash + echo_run cd "$OLDPWD" # if needed we could also add: `git reset --hard $nim_csourcesHash` fi _nimBuildCsourcesIfNeeded "$@" diff --git a/config/build_config.txt b/config/build_config.txt index 2484d7a52..75dbb1da3 100644 --- a/config/build_config.txt +++ b/config/build_config.txt @@ -1,4 +1,5 @@ nim_comment="key-value pairs for windows/posix bootstrapping build scripts" nim_csourcesDir=csources_v1 nim_csourcesUrl=https://github.com/nim-lang/csources_v1.git -nim_csourcesHash=a8a5241f9475099c823cfe1a5e0ca4022ac201ff +nim_csourcesBranch=master +nim_csourcesHash=9a7f751d23c49c75a0b6f63a234c575dc0df7231 diff --git a/tools/ci_generate.nim b/tools/ci_generate.nim index 52b84f0d8..ecf0023d3 100644 --- a/tools/ci_generate.nim +++ b/tools/ci_generate.nim @@ -61,7 +61,7 @@ SET nim_csources=bin\nim_csources_%nim_csourcesHash%.exe echo "building from csources: %nim_csources%" if not exist %nim_csourcesDir% ( - git clone -q --depth 1 %nim_csourcesUrl% %nim_csourcesDir% + git clone -q --depth 1 -b %nim_csourcesBranch% %nim_csourcesUrl% %nim_csourcesDir% ) if not exist %nim_csources% ( diff --git a/tools/deps.nim b/tools/deps.nim index 8ed95b59a..95b2916a1 100644 --- a/tools/deps.nim +++ b/tools/deps.nim @@ -28,8 +28,13 @@ proc cloneDependency*(destDirBase: string, url: string, commit = commitHead, # from failing execRetry fmt"git clone -q {url} {destDir2}" if isGitRepo(destDir): - execRetry fmt"git -C {destDir2} fetch -q" - exec fmt"git -C {destDir2} checkout -q {commit}" + let oldDir = getCurrentDir() + setCurrentDir(destDir2) + try: + execRetry "git fetch -q" + exec fmt"git checkout -q {commit}" + finally: + setCurrentDir(oldDir) elif allowBundled: discard "this dependency was bundled with Nim, don't do anything" else: diff --git a/tools/niminst/makefile.nimf b/tools/niminst/makefile.nimf index aff0c5da0..8c40cf50f 100644 --- a/tools/niminst/makefile.nimf +++ b/tools/niminst/makefile.nimf @@ -25,7 +25,8 @@ endif ifeq ($(uos),linux) myos = linux - LDFLAGS += -ldl -lm + # add -lrt to avoid "undefined reference to `clock_gettime'" with glibc<2.17 + LDFLAGS += -ldl -lm -lrt endif ifeq ($(uos),dragonfly) myos = freebsd From b72ecaf639ca7b2edf5a762cfea3a11f7cc5da9a Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 6 Jul 2021 21:00:59 -0700 Subject: [PATCH 1016/2472] fix shebangs #! /xxx => #!/xxx (#18444) --- build_all.sh | 2 +- lib/pure/unidecode/gen.py | 2 +- tools/ci_generate.nim | 2 +- tools/niminst/buildsh.nimf | 2 +- tools/niminst/deinstall.nimf | 2 +- tools/niminst/install.nimf | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/build_all.sh b/build_all.sh index 37374aa7c..fb3d15d45 100755 --- a/build_all.sh +++ b/build_all.sh @@ -1,4 +1,4 @@ -#! /bin/sh +#!/bin/sh # DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim` # build development version of the compiler; can be rerun safely. diff --git a/lib/pure/unidecode/gen.py b/lib/pure/unidecode/gen.py index 0b180a381..2fb69f7b2 100644 --- a/lib/pure/unidecode/gen.py +++ b/lib/pure/unidecode/gen.py @@ -1,4 +1,4 @@ -#! usr/bin/env python3 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Generates the unidecode.dat module diff --git a/tools/ci_generate.nim b/tools/ci_generate.nim index ecf0023d3..a8a80e026 100644 --- a/tools/ci_generate.nim +++ b/tools/ci_generate.nim @@ -82,7 +82,7 @@ if not exist %nim_csources% ( proc genPosixScript(): string = result = fmt""" -#! /bin/sh +#!/bin/sh # {doNotEdit} # build development version of the compiler; can be rerun safely. diff --git a/tools/niminst/buildsh.nimf b/tools/niminst/buildsh.nimf index e7afd402e..f74413c09 100644 --- a/tools/niminst/buildsh.nimf +++ b/tools/niminst/buildsh.nimf @@ -1,6 +1,6 @@ #? stdtmpl(subsChar='?') | standard #proc generateBuildShellScript(c: ConfigData): string = -# result = "#! /bin/sh\n# Generated from niminst\n" & +# result = "#!/bin/sh\n# Generated from niminst\n" & # "# Template is in tools/niminst/buildsh.nimf\n" & # "# To regenerate run ``niminst csource`` or ``koch csource``\n" diff --git a/tools/niminst/deinstall.nimf b/tools/niminst/deinstall.nimf index 8b4477369..0dcfda75e 100644 --- a/tools/niminst/deinstall.nimf +++ b/tools/niminst/deinstall.nimf @@ -1,6 +1,6 @@ #? stdtmpl(subsChar='?') | standard #proc generateDeinstallScript(c: ConfigData): string = -# result = "#! /bin/sh\n# Generated by niminst\n" +# result = "#!/bin/sh\n# Generated by niminst\n" # var proj = c.name.toLowerAscii if [ $# -eq 1 ] ; then diff --git a/tools/niminst/install.nimf b/tools/niminst/install.nimf index a78914ecd..c2fc96e94 100644 --- a/tools/niminst/install.nimf +++ b/tools/niminst/install.nimf @@ -1,6 +1,6 @@ #? stdtmpl(subsChar = '?') | standard #proc generateInstallScript(c: ConfigData): string = -# result = "#! /bin/sh\n# Generated by niminst\n" +# result = "#!/bin/sh\n# Generated by niminst\n" # var proj = c.name.toLowerAscii set -e From d1447fe25d40e35d4746d570701d23333ff480a0 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 6 Jul 2021 21:04:36 -0700 Subject: [PATCH 1017/2472] major improvements to `std/wrapnils`: optimal codegen, case objects, lvalue semantics (#18435) * wrapnils now generates optimal code; also handles case objects * changelog * unsafeAddr => addr --- changelog.md | 4 + lib/std/wrapnils.nim | 185 +++++++++++++++++++-------- tests/stdlib/twrapnils.nim | 249 ++++++++++++++++++++++++++++--------- 3 files changed, 329 insertions(+), 109 deletions(-) diff --git a/changelog.md b/changelog.md index 028743bab..1deb97161 100644 --- a/changelog.md +++ b/changelog.md @@ -279,6 +279,10 @@ issues like https://github.com/nim-lang/Nim/issues/13063 (which affected error messages) for modules importing `std/wrapnils`. Added `??.` macro which returns an `Option`. + `std/wrapnils` can now be used to protect against `FieldDefect` errors in + case objects, generates optimal code (no overhead compared to manual + if-else branches), and preserves lvalue semantics which allows modifying + an expression. - Added `math.frexp` overload procs. Deprecated `c_frexp`, use `frexp` instead. diff --git a/lib/std/wrapnils.nim b/lib/std/wrapnils.nim index ed0a79d79..facba85fa 100644 --- a/lib/std/wrapnils.nim +++ b/lib/std/wrapnils.nim @@ -1,9 +1,20 @@ -## This module allows chains of field-access and indexing where the LHS can be nil. -## This simplifies code by reducing need for if-else branches around intermediate values -## that may be nil. +## This module allows evaluating expressions safely against the following conditions: +## * nil dereferences +## * field accesses with incorrect discriminant in case objects +## +## `default(T)` is returned in those cases when evaluating an expression of type `T`. +## This simplifies code by reducing need for if-else branches. ## ## Note: experimental module, unstable API. +#[ +TODO: +consider handling indexing operations, eg: +doAssert ?.default(seq[int])[3] == default(int) +]# + +import macros + runnableExamples: type Foo = ref object x1: string @@ -24,8 +35,124 @@ runnableExamples: assert (?.f2.x2.x2).x3 == nil # this terminates ?. early +runnableExamples: + # ?. also allows case object + type B = object + b0: int + case cond: bool + of false: discard + of true: + b1: float + + var b = B(cond: false, b0: 3) + doAssertRaises(FieldDefect): discard b.b1 # wrong discriminant + doAssert ?.b.b1 == 0.0 # safe + b = B(cond: true, b1: 4.5) + doAssert ?.b.b1 == 4.5 + + # lvalue semantics are preserved: + if (let p = ?.b.b1.addr; p != nil): p[] = 4.7 + doAssert b.b1 == 4.7 + +proc finalize(n: NimNode, lhs: NimNode, level: int): NimNode = + if level == 0: + result = quote: `lhs` = `n` + else: + result = quote: (let `lhs` = `n`) + +proc process(n: NimNode, lhs: NimNode, level: int): NimNode = + var n = n.copyNimTree + var it = n + let addr2 = bindSym"addr" + var old: tuple[n: NimNode, index: int] + while true: + if it.len == 0: + result = finalize(n, lhs, level) + break + elif it.kind == nnkCheckedFieldExpr: + let dot = it[0] + let obj = dot[0] + let objRef = quote do: `addr2`(`obj`) + # avoids a copy and preserves lvalue semantics, see tests + let check = it[1] + let okSet = check[1] + let kind1 = check[2] + let tmp = genSym(nskLet, "tmpCase") + let body = process(objRef, tmp, level + 1) + let tmp3 = nnkDerefExpr.newTree(tmp) + it[0][0] = tmp3 + let dot2 = nnkDotExpr.newTree(@[tmp, dot[1]]) + if old.n != nil: old.n[old.index] = dot2 + else: n = dot2 + let assgn = finalize(n, lhs, level) + result = quote do: + `body` + if `tmp3`.`kind1` notin `okSet`: break + `assgn` + break + elif it.kind in {nnkHiddenDeref, nnkDerefExpr}: + let tmp = genSym(nskLet, "tmp") + let body = process(it[0], tmp, level + 1) + it[0] = tmp + let assgn = finalize(n, lhs, level) + result = quote do: + `body` + if `tmp` == nil: break + `assgn` + break + elif it.kind == nnkCall: # consider extending to `nnkCallKinds` + # `copyNimTree` needed to avoid `typ = nil` issues + old = (it, 1) + it = it[1].copyNimTree + else: + old = (it, 0) + it = it[0] + +macro `?.`*(a: typed): auto = + ## Transforms `a` into an expression that can be safely evaluated even in + ## presence of intermediate nil pointers/references, in which case a default + ## value is produced. + let lhs = genSym(nskVar, "lhs") + let body = process(a, lhs, 0) + result = quote do: + var `lhs`: type(`a`) + block: + `body` + `lhs` + +# the code below is not needed for `?.` from options import Option, isSome, get, option, unsafeGet, UnpackDefect +macro `??.`*(a: typed): Option = + ## Same as `?.` but returns an `Option`. + runnableExamples: + import std/options + type Foo = ref object + x1: ref int + x2: int + # `?.` can't distinguish between a valid vs invalid default value, but `??.` can: + var f1 = Foo(x1: int.new, x2: 2) + doAssert (??.f1.x1[]).get == 0 # not enough to tell when the chain was valid. + doAssert (??.f1.x1[]).isSome # a nil didn't occur in the chain + doAssert (??.f1.x2).get == 2 + + var f2: Foo + doAssert not (??.f2.x1[]).isSome # f2 was nil + + doAssertRaises(UnpackDefect): discard (??.f2.x1[]).get + doAssert ?.f2.x1[] == 0 # in contrast, this returns default(int) + + let lhs = genSym(nskVar, "lhs") + let lhs2 = genSym(nskVar, "lhs") + let body = process(a, lhs2, 0) + result = quote do: + var `lhs`: Option[type(`a`)] + block: + var `lhs2`: type(`a`) + `body` + `lhs` = option(`lhs2`) + `lhs` + template fakeDot*(a: Option, b): untyped = ## See top-level example. let a1 = a # to avoid double evaluations @@ -58,51 +185,7 @@ func `[]`*[U](a: Option[U]): auto {.inline.} = if a2 != nil: result = option(a2[]) -import macros - -func replace(n: NimNode): NimNode = - if n.kind == nnkDotExpr: - result = newCall(bindSym"fakeDot", replace(n[0]), n[1]) - elif n.kind == nnkPar: - doAssert n.len == 1 - result = newCall(bindSym"option", n[0]) - elif n.kind in {nnkCall, nnkObjConstr}: - result = newCall(bindSym"option", n) - elif n.len == 0: - result = newCall(bindSym"option", n) - else: - n[0] = replace(n[0]) - result = n - -proc safeGet[T](a: Option[T]): T {.inline.} = - get(a, default(T)) - -macro `?.`*(a: untyped): auto = - ## Transforms `a` into an expression that can be safely evaluated even in - ## presence of intermediate nil pointers/references, in which case a default - ## value is produced. - result = replace(a) - result = quote do: - # `result`.val # TODO: expose a way to do this directly in std/options, e.g.: `getAsIs` - safeGet(`result`) - -macro `??.`*(a: untyped): Option = - ## Same as `?.` but returns an `Option`. - runnableExamples: - import std/options - type Foo = ref object - x1: ref int - x2: int - # `?.` can't distinguish between a valid vs invalid default value, but `??.` can: - var f1 = Foo(x1: int.new, x2: 2) - doAssert (??.f1.x1[]).get == 0 # not enough to tell when the chain was valid. - doAssert (??.f1.x1[]).isSome # a nil didn't occur in the chain - doAssert (??.f1.x2).get == 2 - - var f2: Foo - doAssert not (??.f2.x1[]).isSome # f2 was nil - from std/options import UnpackDefect - doAssertRaises(UnpackDefect): discard (??.f2.x1[]).get - doAssert ?.f2.x1[] == 0 # in contrast, this returns default(int) - - result = replace(a) +when false: + # xxx: expose a way to do this directly in std/options, e.g.: `getAsIs` + proc safeGet[T](a: Option[T]): T {.inline.} = + get(a, default(T)) diff --git a/tests/stdlib/twrapnils.nim b/tests/stdlib/twrapnils.nim index 956204355..1a66f4cd9 100644 --- a/tests/stdlib/twrapnils.nim +++ b/tests/stdlib/twrapnils.nim @@ -7,80 +7,213 @@ proc checkNotZero(x: float): float = proc main() = var witness = 0 - type Bar = object - b1: int - b2: ptr string + block: + type Bar = object + b1: int + b2: ptr string - type Foo = ref object - x1: float - x2: Foo - x3: string - x4: Bar - x5: seq[int] - x6: ptr Bar - x7: array[2, string] - x8: seq[int] - x9: ref Bar + type Foo = ref object + x1: float + x2: Foo + x3: string + x4: Bar + x5: seq[int] + x6: ptr Bar + x7: array[2, string] + x8: seq[int] + x9: ref Bar - type Gook = ref object - foo: Foo + type Gook = ref object + foo: Foo - proc fun(a: Bar): auto = a.b2 + proc fun(a: Bar): auto = a.b2 - var a: Foo + var a: Foo - var x6: ptr Bar - when nimvm: discard # pending https://github.com/timotheecour/Nim/issues/568 - else: - x6 = create(Bar) - x6.b1 = 42 - var a2 = Foo(x1: 1.0, x5: @[10, 11], x6: x6) - var a3 = Foo(x1: 1.2, x3: "abc") - a3.x2 = a3 + var x6: ptr Bar + when nimvm: discard # pending https://github.com/timotheecour/Nim/issues/568 + else: + x6 = create(Bar) + x6.b1 = 42 + var a2 = Foo(x1: 1.0, x5: @[10, 11], x6: x6) + var a3 = Foo(x1: 1.2, x3: "abc") + a3.x2 = a3 - var gook = Gook(foo: a) + var gook = Gook(foo: a) - proc initFoo(x1: float): auto = - witness.inc - result = Foo(x1: x1) + proc initFoo(x1: float): auto = + witness.inc + result = Foo(x1: x1) - doAssert ?.a.x2.x2.x1 == 0.0 - doAssert ?.a3.x2.x2.x1 == 1.2 - doAssert ?.a3.x2.x2.x3[1] == 'b' + doAssert ?.a.x2.x2.x1 == 0.0 + doAssert ?.a3.x2.x2.x1 == 1.2 + doAssert ?.a3.x2.x2.x3[1] == 'b' - doAssert ?.a3.x2.x2.x5.len == 0 - doAssert a3.x2.x2.x3.len == 3 + doAssert ?.a3.x2.x2.x5.len == 0 + doAssert a3.x2.x2.x3.len == 3 - doAssert ?.a.x2.x2.x3[1] == default(char) - # here we only apply wrapnil around gook.foo, not gook (and assume gook is not nil) - doAssert ?.(gook.foo).x2.x2.x1 == 0.0 + doAssert ?.a.x2.x2.x3[1] == default(char) + # here we only apply wrapnil around gook.foo, not gook (and assume gook is not nil) + doAssert ?.(gook.foo).x2.x2.x1 == 0.0 - when nimvm: discard - else: - doAssert ?.a2.x6[] == Bar(b1: 42) # deref for ptr Bar + when nimvm: discard + else: + doAssert ?.a2.x6[] == Bar(b1: 42) # deref for ptr Bar - doAssert ?.a2.x1.checkNotZero == 1.0 - doAssert a == nil - # shows that checkNotZero won't be called if a nil is found earlier in chain - doAssert ?.a.x1.checkNotZero == 0.0 + doAssert ?.a2.x1.checkNotZero == 1.0 + doAssert a == nil + # shows that checkNotZero won't be called if a nil is found earlier in chain + doAssert ?.a.x1.checkNotZero == 0.0 - when nimvm: discard - else: - # checks that a chain without nil but with an empty seq still raises - doAssertRaises(IndexDefect): discard ?.a2.x8[3] + when nimvm: discard + else: + # checks that a chain without nil but with an empty seq still raises + doAssertRaises(IndexDefect): discard ?.a2.x8[3] - # make sure no double evaluation bug - doAssert witness == 0 - doAssert ?.initFoo(1.3).x1 == 1.3 - doAssert witness == 1 + # make sure no double evaluation bug + doAssert witness == 0 + doAssert ?.initFoo(1.3).x1 == 1.3 + doAssert witness == 1 - # here, it's used twice, to deref `ref Bar` and then `ptr string` - doAssert ?.a.x9[].fun[] == "" + # here, it's used twice, to deref `ref Bar` and then `ptr string` + doAssert ?.a.x9[].fun[] == "" - block: # `??.` - doAssert (??.a3.x2.x2.x3.len).get == 3 - doAssert (??.a2.x4).isSome - doAssert not (??.a.x4).isSome + block: # `??.` + doAssert (??.a3.x2.x2.x3.len).get == 3 + doAssert (??.a2.x4).isSome + doAssert not (??.a.x4).isSome + + block: + type + A = object + b: B + B = object + c: C + C = object + d: D + D = ref object + e: E + e2: array[2, E] + e3: seq[E] + d3: D + i4: int + E = object + f: int + d2: D + proc identity[T](a: T): T = a + proc identity2[T](a: T, ignore: int): T = a + var a: A + doAssert ?.a.b.c.d.e.f == 0 + doAssert ?.a.b.c.d.e.d2.d3[].d3.e.d2.e.f == 0 + doAssert ?.a.b.c.d.d3[].e.f == 0 + doAssert ?.a.b.c.d.e2[0].d2.e3[0].f == 0 + doAssert ?.a == A.default + doAssert ?.a.b.c.d.e == E.default + doAssert ?.a.b.c.d.e.d2 == nil + + doAssert ?.a.identity.b.c.identity2(12).d.d3.e.f == 0 + doAssert ?.a.b.c.d.d3.e2[0].f == 0 + a.b.c.d = D() + a.b.c.d.d3 = a.b.c.d + a.b.c.d.e2[0].f = 5 + doAssert ?.a.b.c.d.d3.e2[0].f == 5 + + var d: D = nil + doAssert ?.d.identity.i4 == 0 + doAssert ?.d.i4.identity == 0 + + block: # case objects + type + Kind = enum k0, k1, k2 + V = object + case kind: Kind + of k0: + x0: int + of k1: + x1: int + of k2: + x2: int + A = object + v0: V + + block: + var a = V(kind: k0, x0: 3) + doAssert ?.a.x0 == 3 + doAssert ?.a.x1 == 0 + a = V(kind: k1, x1: 5) + doAssert ?.a.x0 == 0 + doAssert ?.a.x1 == 5 + + block: + var a = A(v0: V(kind: k0, x0: 10)) + doAssert ?.a.v0.x0 == 10 + doAssert ?.a.v0.x1 == 0 + a.v0 = V(kind: k2, x2: 8) + doAssert ?.a.v0.x0 == 0 + doAssert ?.a.v0.x1 == 0 + doAssert ?.a.v0.x2 == 8 + + block: # `nnkCall` + type + A = object + a0: int + d: D + D = ref object + i4: int + + proc identity[T](a: T): T = a + var d: D = nil + doAssert ?.d.i4.identity == 0 + doAssert ?.identity(?.d.i4) == 0 + doAssert ?.identity(d.i4) == 0 + doAssert ?.identity(d) == nil + doAssert ?.identity(d[]) == default(typeof(d[])) + doAssert ?.identity(d[]).i4 == 0 + var a: A + doAssert ?.identity(a) == default(A) + doAssert ?.identity(a.a0) == 0 + doAssert ?.identity(a.d) == nil + doAssert ?.identity(a.d.i4) == 0 + + block: # lvalue semantic propagation + type + A = ref object + a0: A + a1: seq[A] + a2: int + + B = object + b0: int + case cond: bool + of false: discard + of true: + b1: float + + block: + var a: A + doAssert ?.a.a0.a1[0].a2.addr == nil + a = A(a2: 3) + doAssert ?.a.a0.a1[0].a2.addr == nil + a.a0 = a + a.a1 = @[a] + let p = ?.a.a0.a1[0].a2.addr + doAssert p != nil + p[] = 5 + doAssert a.a2 == 5 + + block: + var b = B(cond: false, b0: 3) + let p = ?.b.b1.addr + doAssert p == nil + b = B(cond: true, b1: 4.5) + let p2 = ?.b.b1.addr + doAssert p2 != nil + p2[] = 4.6 + doAssert b.b1 == 4.6 + # useful pattern, impossible with Options + if (let p3 = ?.b.b1.addr; p3 != nil): + p3[] = 4.7 + doAssert b.b1 == 4.7 main() static: main() From 3eb3e6b9a38aed1f26014bd5dece216b0c625b1a Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 7 Jul 2021 09:39:01 +0200 Subject: [PATCH 1018/2472] ORC: use =destroy instead of =dispose (#18440) * ORC refactoring in preparation for further changes (=dispose must die) * ORC: embrace =destroy, avoid =dispose * ORC: no need for =dispose * closes #18421 --- compiler/ast.nim | 3 +- compiler/ccgtypes.nim | 5 +- compiler/liftdestructors.nim | 49 ++--------- compiler/semstmts.nim | 2 - doc/manual.rst | 2 +- lib/system.nim | 1 - lib/system/cellseqs_v2.nim | 26 +++--- lib/system/cyclebreaker.nim | 3 +- lib/system/orc.nim | 40 +++++---- tests/arc/thamming_orc.nim | 155 +++++++++++++++++++++++++++++++++++ 10 files changed, 205 insertions(+), 81 deletions(-) create mode 100644 tests/arc/thamming_orc.nim diff --git a/compiler/ast.nim b/compiler/ast.nim index 2a2a84e76..26050426a 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -911,7 +911,6 @@ type attachedAsgn, attachedSink, attachedTrace, - attachedDispose, attachedDeepCopy TType* {.acyclic.} = object of TIdObj # \ @@ -1408,7 +1407,7 @@ const MaxLockLevel* = 1000'i16 UnknownLockLevel* = TLockLevel(1001'i16) AttachedOpToStr*: array[TTypeAttachedOp, string] = [ - "=destroy", "=copy", "=sink", "=trace", "=dispose", "=deepcopy"] + "=destroy", "=copy", "=sink", "=trace", "=deepcopy"] proc `$`*(x: TLockLevel): string = if x.ord == UnspecifiedLockLevel.ord: result = "" diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 6e5ffeaa4..67e701804 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -1328,14 +1328,13 @@ proc genTypeInfoV2Impl(m: BModule, t, origType: PType, name: Rope; info: TLineIn m.s[cfsData].addf("N_LIB_PRIVATE TNimTypeV2 $1;$n", [name]) let destroyImpl = genHook(m, t, info, attachedDestructor) let traceImpl = genHook(m, t, info, attachedTrace) - let disposeImpl = genHook(m, t, info, attachedDispose) var flags = 0 if not canFormAcycle(t): flags = flags or 1 - addf(m.s[cfsTypeInit3], "$1.destructor = (void*)$2; $1.size = sizeof($3); $1.align = NIM_ALIGNOF($3); $1.name = $4;$n; $1.traceImpl = (void*)$5; $1.disposeImpl = (void*)$6; $1.flags = $7;", [ + addf(m.s[cfsTypeInit3], "$1.destructor = (void*)$2; $1.size = sizeof($3); $1.align = NIM_ALIGNOF($3); $1.name = $4;$n; $1.traceImpl = (void*)$5; $1.flags = $6;", [ name, destroyImpl, getTypeDesc(m, t), typeName, - traceImpl, disposeImpl, rope(flags)]) + traceImpl, rope(flags)]) if t.kind == tyObject and t.len > 0 and t[0] != nil and optEnableDeepCopy in m.config.globalOptions: discard genTypeInfoV1(m, t, info) diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim index 43f87e179..1824557e6 100644 --- a/compiler/liftdestructors.nim +++ b/compiler/liftdestructors.nim @@ -421,11 +421,6 @@ proc considerUserDefinedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool = result = considerAsgnOrSink(c, t, body, x, y, op) if op != nil: setAttachedOp(c.g, c.idgen.module, t, c.kind, op) - of attachedDispose: - var op = getAttachedOp(c.g, t, c.kind) - result = considerAsgnOrSink(c, t, body, x, nil, op) - if op != nil: - setAttachedOp(c.g, c.idgen.module, t, c.kind, op) of attachedDeepCopy: let op = getAttachedOp(c.g, t, attachedDeepCopy) if op != nil: @@ -516,9 +511,6 @@ proc fillSeqOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = if canFormAcycle(t.elemType): # follow all elements: forallElements(c, t, body, x, y) - of attachedDispose: - forallElements(c, t, body, x, y) - body.add genBuiltin(c, mDestroy, "destroy", x) proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = createTypeBoundOps(c.g, c.c, t, body.info, c.idgen) @@ -556,11 +548,6 @@ proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = if op == nil: return # protect from recursion body.add newHookCall(c, op, x, y) - of attachedDispose: - let op = getAttachedOp(c.g, t, c.kind) - if op == nil: - return # protect from recursion - body.add newHookCall(c, op, x, nil) proc fillStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = case c.kind @@ -572,7 +559,7 @@ proc fillStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = doAssert t.destructor != nil moveCall.add destructorCall(c, t.destructor, x) body.add moveCall - of attachedDestructor, attachedDispose: + of attachedDestructor: body.add genBuiltin(c, mDestroy, "destroy", x) of attachedTrace: discard "strings are atomic and have no inner elements that are to trace" @@ -667,16 +654,6 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = # If the ref is polymorphic we have to account for this body.add callCodegenProc(c.g, "nimTraceRefDyn", c.info, genAddrOf(x, c.idgen), y) #echo "can follow ", elemType, " static ", isFinal(elemType) - of attachedDispose: - # this is crucial! dispose is like =destroy but we don't follow refs - # as that is dealt within the cycle collector. - if not isCyclic: - body.add genIf(c, cond, actions) - when false: - let cond = copyTree(x) - cond.typ = getSysType(c.g, x.info, tyBool) - actions.add callCodegenProc(c.g, "nimRawDispose", c.info, x) - body.add genIf(c, cond, actions) proc atomicClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = ## Closures are really like refs except they always use a virtual destructor @@ -725,14 +702,6 @@ proc atomicClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = of attachedDeepCopy: assert(false, "cannot happen") of attachedTrace: body.add callCodegenProc(c.g, "nimTraceRefDyn", c.info, genAddrOf(xenv, c.idgen), y) - of attachedDispose: - # this is crucial! dispose is like =destroy but we don't follow refs - # as that is dealt within the cycle collector. - when false: - let cond = copyTree(xenv) - cond.typ = getSysType(c.g, xenv.info, tyBool) - actions.add callCodegenProc(c.g, "nimRawDispose", c.info, xenv) - body.add genIf(c, cond, actions) proc weakrefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = case c.kind @@ -756,7 +725,7 @@ proc weakrefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = else: body.sons.insert(des, 0) of attachedDeepCopy: assert(false, "cannot happen") - of attachedTrace, attachedDispose: discard + of attachedTrace: discard proc ownedRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = var actions = newNodeI(nkStmtList, c.info) @@ -781,7 +750,7 @@ proc ownedRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = of attachedDestructor: body.add genIf(c, x, actions) of attachedDeepCopy: assert(false, "cannot happen") - of attachedTrace, attachedDispose: discard + of attachedTrace: discard proc closureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = if c.kind == attachedDeepCopy: @@ -815,7 +784,7 @@ proc closureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = else: body.sons.insert(des, 0) of attachedDeepCopy: assert(false, "cannot happen") - of attachedTrace, attachedDispose: discard + of attachedTrace: discard proc ownedClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = let xx = genBuiltin(c, mAccessEnv, "accessEnv", x) @@ -830,7 +799,7 @@ proc ownedClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = of attachedDestructor: body.add genIf(c, xx, actions) of attachedDeepCopy: assert(false, "cannot happen") - of attachedTrace, attachedDispose: discard + of attachedTrace: discard proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode) = case t.kind @@ -946,7 +915,7 @@ proc symPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttachedOp result.typ = newProcType(info, nextTypeId(idgen), owner) result.typ.addParam dest - if kind notin {attachedDestructor, attachedDispose}: + if kind != attachedDestructor: result.typ.addParam src if kind == attachedAsgn and g.config.selectedGC == gcOrc and @@ -980,7 +949,7 @@ proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp; let dest = result.typ.n[1].sym let d = newDeref(newSymNode(dest)) - let src = if kind in {attachedDestructor, attachedDispose}: newNodeIT(nkSym, info, getSysType(g, info, tyPointer)) + let src = if kind == attachedDestructor: newNodeIT(nkSym, info, getSysType(g, info, tyPointer)) else: newSymNode(result.typ.n[2].sym) # register this operation already: @@ -1098,13 +1067,13 @@ proc createTypeBoundOps(g: ModuleGraph; c: PContext; orig: PType; info: TLineInf # we do not generate '=trace' nor '=dispose' procs if we # have the cycle detection disabled, saves code size. - let lastAttached = if g.config.selectedGC == gcOrc: attachedDispose + let lastAttached = if g.config.selectedGC == gcOrc: attachedTrace else: attachedSink # bug #15122: We need to produce all prototypes before entering the # mind boggling recursion. Hacks like these imply we should rewrite # this module. - var generics: array[attachedDestructor..attachedDispose, bool] + var generics: array[attachedDestructor..attachedTrace, bool] for k in attachedDestructor..lastAttached: generics[k] = getAttachedOp(g, canon, k) != nil if not generics[k]: diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 2907d5172..c3402aee3 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -1753,8 +1753,6 @@ proc semOverride(c: PContext, s: PSym, n: PNode) = "signature for '" & s.name.s & "' must be proc[T: object](x: var T; y: T)") of "=trace": bindTypeHook(c, s, n, attachedTrace) - of "=dispose": - bindTypeHook(c, s, n, attachedDispose) else: if sfOverriden in s.flags: localError(c.config, n.info, errGenerated, diff --git a/doc/manual.rst b/doc/manual.rst index a99c4b5af..9ddf02009 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -3841,7 +3841,7 @@ the operator is in scope (including if it is private). doAssert witness == 3 Type bound operators currently include: -`=destroy`, `=copy`, `=sink`, `=trace`, `=dispose`, `=deepcopy` +`=destroy`, `=copy`, `=sink`, `=trace`, `=deepcopy` (some of which are still implementation defined and not yet documented). For more details on some of those procs, see diff --git a/lib/system.nim b/lib/system.nim index a4608997a..090ab309e 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1779,7 +1779,6 @@ when not defined(js) and defined(nimV2): align: int name: cstring traceImpl: pointer - disposeImpl: pointer typeInfoV1: pointer # for backwards compat, usually nil flags: int PNimTypeV2 = ptr TNimTypeV2 diff --git a/lib/system/cellseqs_v2.nim b/lib/system/cellseqs_v2.nim index 59cf67dcd..27be48d78 100644 --- a/lib/system/cellseqs_v2.nim +++ b/lib/system/cellseqs_v2.nim @@ -10,20 +10,20 @@ # Cell seqs for cyclebreaker and cyclicrefs_v2. type - CellTuple = (PT, PNimTypeV2) - CellArray = ptr UncheckedArray[CellTuple] - CellSeq = object + CellTuple[T] = (T, PNimTypeV2) + CellArray[T] = ptr UncheckedArray[CellTuple[T]] + CellSeq[T] = object len, cap: int - d: CellArray + d: CellArray[T] -proc add(s: var CellSeq, c: PT; t: PNimTypeV2) {.inline.} = +proc add[T](s: var CellSeq[T], c: T; t: PNimTypeV2) {.inline.} = if s.len >= s.cap: s.cap = s.cap * 3 div 2 when compileOption("threads"): - var d = cast[CellArray](allocShared(uint(s.cap * sizeof(CellTuple)))) + var d = cast[CellArray[T]](allocShared(uint(s.cap * sizeof(CellTuple[T])))) else: - var d = cast[CellArray](alloc(s.cap * sizeof(CellTuple))) - copyMem(d, s.d, s.len * sizeof(CellTuple)) + var d = cast[CellArray[T]](alloc(s.cap * sizeof(CellTuple[T]))) + copyMem(d, s.d, s.len * sizeof(CellTuple[T])) when compileOption("threads"): deallocShared(s.d) else: @@ -33,15 +33,15 @@ proc add(s: var CellSeq, c: PT; t: PNimTypeV2) {.inline.} = s.d[s.len] = (c, t) inc(s.len) -proc init(s: var CellSeq, cap: int = 1024) = +proc init[T](s: var CellSeq[T], cap: int = 1024) = s.len = 0 s.cap = cap when compileOption("threads"): - s.d = cast[CellArray](allocShared(uint(s.cap * sizeof(CellTuple)))) + s.d = cast[CellArray[T]](allocShared(uint(s.cap * sizeof(CellTuple[T])))) else: - s.d = cast[CellArray](alloc(s.cap * sizeof(CellTuple))) + s.d = cast[CellArray[T]](alloc(s.cap * sizeof(CellTuple[T]))) -proc deinit(s: var CellSeq) = +proc deinit[T](s: var CellSeq[T]) = if s.d != nil: when compileOption("threads"): deallocShared(s.d) @@ -51,6 +51,6 @@ proc deinit(s: var CellSeq) = s.len = 0 s.cap = 0 -proc pop(s: var CellSeq): (PT, PNimTypeV2) = +proc pop[T](s: var CellSeq[T]): (T, PNimTypeV2) = result = s.d[s.len-1] dec s.len diff --git a/lib/system/cyclebreaker.nim b/lib/system/cyclebreaker.nim index a3159f95c..6ec0f01f7 100644 --- a/lib/system/cyclebreaker.nim +++ b/lib/system/cyclebreaker.nim @@ -53,7 +53,6 @@ depth-first traversal suffices. ]# -type PT = ptr pointer include cellseqs_v2 const @@ -78,7 +77,7 @@ proc nimMarkCyclic(p: pointer) {.compilerRtl, inl.} = discard type GcEnv = object - traceStack: CellSeq + traceStack: CellSeq[ptr pointer] proc trace(p: pointer; desc: PNimTypeV2; j: var GcEnv) {.inline.} = when false: diff --git a/lib/system/orc.nim b/lib/system/orc.nim index df81afcf1..6273e1a21 100644 --- a/lib/system/orc.nim +++ b/lib/system/orc.nim @@ -14,7 +14,6 @@ # R.D. Lins / Information Processing Letters 109 (2008) 71–78 # -type PT = Cell include cellseqs_v2 const @@ -68,10 +67,10 @@ const type GcEnv = object - traceStack: CellSeq + traceStack: CellSeq[ptr pointer] when useJumpStack: - jumpStack: CellSeq # Lins' jump stack in order to speed up traversals - toFree: CellSeq + jumpStack: CellSeq[ptr pointer] # Lins' jump stack in order to speed up traversals + toFree: CellSeq[Cell] freed, touched, edges, rcSum: int keepThreshold: bool @@ -92,13 +91,13 @@ proc free(s: Cell; desc: PNimTypeV2) {.inline.} = when logOrc: writeCell("free", s, desc) - if desc.disposeImpl != nil: - cast[DisposeProc](desc.disposeImpl)(p) + if desc.destructor != nil: + cast[DestructorProc](desc.destructor)(p) when false: cstderr.rawWrite desc.name cstderr.rawWrite " " - if desc.disposeImpl == nil: + if desc.destructor == nil: cstderr.rawWrite "lacks dispose" if desc.traceImpl != nil: cstderr.rawWrite ", but has trace\n" @@ -125,16 +124,16 @@ proc nimTraceRef(q: pointer; desc: PNimTypeV2; env: pointer) {.compilerRtl, inli orcAssert strstr(desc.name, "TType") == nil, "following a TType but it's acyclic!" var j = cast[ptr GcEnv](env) - j.traceStack.add(head p[], desc) + j.traceStack.add(p, desc) proc nimTraceRefDyn(q: pointer; env: pointer) {.compilerRtl, inline.} = let p = cast[ptr pointer](q) if p[] != nil: var j = cast[ptr GcEnv](env) - j.traceStack.add(head p[], cast[ptr PNimTypeV2](p[])[]) + j.traceStack.add(p, cast[ptr PNimTypeV2](p[])[]) var - roots {.threadvar.}: CellSeq + roots {.threadvar.}: CellSeq[Cell] proc unregisterCycle(s: Cell) = # swap with the last element. O(1) @@ -162,7 +161,8 @@ proc scanBlack(s: Cell; desc: PNimTypeV2; j: var GcEnv) = trace(s, desc, j) when logOrc: writeCell("root still alive", s, desc) while j.traceStack.len > until: - let (t, desc) = j.traceStack.pop() + let (entry, desc) = j.traceStack.pop() + let t = head entry[] inc t.rc, rcIncrement if t.color != colBlack: t.setColor colBlack @@ -187,7 +187,8 @@ proc markGray(s: Cell; desc: PNimTypeV2; j: var GcEnv) = orcAssert(j.traceStack.len == 0, "markGray: trace stack not empty") trace(s, desc, j) while j.traceStack.len > 0: - let (t, desc) = j.traceStack.pop() + let (entry, desc) = j.traceStack.pop() + let t = head entry[] dec t.rc, rcIncrement inc j.edges when useJumpStack: @@ -195,7 +196,7 @@ proc markGray(s: Cell; desc: PNimTypeV2; j: var GcEnv) = t.rc = t.rc or jumpStackFlag when traceCollector: cprintf("[Now in jumpstack] %p %ld color %ld in jumpstack %ld\n", t, t.rc shr rcShift, t.color, t.rc and jumpStackFlag) - j.jumpStack.add(t, desc) + j.jumpStack.add(entry, desc) if t.color != colGray: t.setColor colGray inc j.touched @@ -225,7 +226,8 @@ proc scan(s: Cell; desc: PNimTypeV2; j: var GcEnv) = # that are still alive; we also need to mark what they # refer to as alive: while j.jumpStack.len > 0: - let (t, desc) = j.jumpStack.pop + let (entry, desc) = j.jumpStack.pop + let t = head entry[] # not in jump stack anymore! t.rc = t.rc and not jumpStackFlag if t.color == colGray and (t.rc shr rcShift) >= 0: @@ -239,7 +241,8 @@ proc scan(s: Cell; desc: PNimTypeV2; j: var GcEnv) = s.setColor(colWhite) trace(s, desc, j) while j.traceStack.len > 0: - let (t, desc) = j.traceStack.pop() + let (entry, desc) = j.traceStack.pop() + let t = head entry[] if t.color == colGray: if (t.rc shr rcShift) >= 0: scanBlack(t, desc, j) @@ -249,7 +252,8 @@ proc scan(s: Cell; desc: PNimTypeV2; j: var GcEnv) = # that are still alive; we also need to mark what they # refer to as alive: while j.jumpStack.len > 0: - let (t, desc) = j.jumpStack.pop + let (entry, desc) = j.jumpStack.pop + let t = head entry[] # not in jump stack anymore! t.rc = t.rc and not jumpStackFlag if t.color == colGray and (t.rc shr rcShift) >= 0: @@ -285,7 +289,9 @@ proc collectColor(s: Cell; desc: PNimTypeV2; col: int; j: var GcEnv) = j.toFree.add(s, desc) trace(s, desc, j) while j.traceStack.len > 0: - let (t, desc) = j.traceStack.pop() + let (entry, desc) = j.traceStack.pop() + let t = head entry[] + entry[] = nil # ensure that the destructor does touch moribund objects! if t.color == col and t.rootIdx == 0: j.toFree.add(t, desc) t.setColor(colBlack) diff --git a/tests/arc/thamming_orc.nim b/tests/arc/thamming_orc.nim new file mode 100644 index 000000000..463f7117e --- /dev/null +++ b/tests/arc/thamming_orc.nim @@ -0,0 +1,155 @@ +discard """ + output: '''(allocCount: 1114, deallocCount: 1112) +created 439 destroyed 402''' + cmd: "nim c --gc:orc -d:nimAllocStats $file" +""" + +# bug #18421 + +# test Nim Hamming Number Lazy List algo with reference counts and not... +# compile with "-d:release -d:danger" and test with various +# memory managment GC's, allocators, threading, etc. +# it should be guaranteed to work with zero memory leaks with `--gc:orc`... + +# compile with `-d:trace20` to trace creation and destruction of first 20 values. + +from math import log2 + +# implement our own basic BigInt so the bigints library isn't necessary... +type + BigInt = object + digits: seq[uint32] +let zeroBigInt = BigInt(digits: @[ 0'u32 ]) +let oneBigInt = BigInt(digits: @[ 1'u32 ]) + +proc shladd(bi: var BigInt; n: int; a: BigInt) = + # assume that both `bi` and `a` are sized correctly with + # msuint32 for both not containing a zero + let alen = a.digits.len + let mx = max(bi.digits.len, a.digits.len) + for i in bi.digits.len ..< mx: bi.digits.add 0'u32 + var cry = 0'u64 + for i in 0 ..< alen: + cry += (bi.digits[i].uint64 shl n) + a.digits[i].uint64 + bi.digits[i] = cry.uint32; cry = cry shr 32 + for i in alen ..< mx: + cry += bi.digits[i].uint64 shl n + bi.digits[i] = cry.uint32; cry = cry shr 32 + if cry > 0'u64: + bi.digits.add cry.uint32 + +proc `$`(x: BigInt): string = + if x.digits.len == 0 or (x.digits.len == 1 and x.digits[0] == 0'u32): + return "0" + result = ""; var n = x; var msd = n.digits.high + while msd >= 0: + if n.digits[msd] == 0'u32: msd.dec; continue + var brw = 0.uint64 + for i in countdown(msd, 0): + let dvdnd = n.digits[i].uint64 + (brw shl 32) + let q = dvdnd div 10'u64; brw = dvdnd - q * 10'u64 + n.digits[i] = q.uint32 + result &= $brw + for i in 0 .. result.high shr 1: # reverse result string in place + let tmp = result[^(i + 1)] + result[^(i + 1)] = result[i] + result[i] = tmp + +type TriVal = (uint32, uint32, uint32) +type LogRep = (float64, TriVal) +type LogRepf = proc(x: LogRep): LogRep +const one: LogRep = (0.0'f64, (0'u32, 0'u32, 0'u32)) +proc `<`(me: LogRep, othr: LogRep): bool = me[0] < othr[0] + +proc convertTriVal2BigInt(tpl: TriVal): BigInt = + result = oneBigInt + let (x2, x3, x5) = tpl + for _ in 1 .. x2: result.shladd 1, zeroBigInt + for _ in 1 .. x3: result.shladd 1, result + for _ in 1 .. x5: result.shladd 2, result + +const lb2 = 1.0'f64 +const lb3 = 3.0'f64.log2 +const lb5 = 5.0'f64.log2 + +proc mul2(me: LogRep): LogRep = + let (lr, tpl) = me; let (x2, x3, x5) = tpl + (lr + lb2, (x2 + 1, x3, x5)) + +proc mul3(me: LogRep): LogRep = + let (lr, tpl) = me; let (x2, x3, x5) = tpl + (lr + lb3, (x2, x3 + 1, x5)) + +proc mul5(me: LogRep): LogRep = + let (lr, tpl) = me; let (x2, x3, x5) = tpl + (lr + lb5, (x2, x3, x5 + 1)) + +type + LazyListObj = object + hd: LogRep + tlf: proc(): LazyList {.closure.} + tl: LazyList + LazyList = ref LazyListObj + +var destroyed = 0 + +proc `=destroy`(ll: var LazyListObj) = + if ll.tlf == nil and ll.tl == nil: return + destroyed += 1 + + when defined(trace20): + echo "destroying: ", (destroyed, ll.hd[1].convertTriVal2BigInt) + if ll.tlf != nil: ll.tlf.`=destroy` + if ll.tl != nil: ll.tl.`=destroy` + #wasMoved(ll) + +proc rest(ll: LazyList): LazyList = # not thread-safe; needs lock on thunk + if ll.tlf != nil: ll.tl = ll.tlf(); ll.tlf = nil + ll.tl + +var created = 0 +iterator hammings(until: int): TriVal = + proc merge(x, y: LazyList): LazyList = + let xh = x.hd; let yh = y.hd; created += 1 + when defined(trace20): + echo "merge create: ", (created - 1, (if xh < yh: xh else: yh)[1].convertTriVal2BigInt) + if xh < yh: LazyList(hd: xh, tlf: proc(): auto = merge x.rest, y) + else: LazyList(hd: yh, tlf: proc(): auto = merge x, y.rest) + proc smult(mltf: LogRepf; s: LazyList): LazyList = + proc smults(ss: LazyList): LazyList = + when defined(trace20): + echo "mult create: ", (created, ss.hd.mltf[1].convertTriVal2BigInt) + created += 1; LazyList(hd: ss.hd.mltf, tlf: proc(): auto = ss.rest.smults) + s.smults + proc unnsm(s: LazyList, mltf: LogRepf): LazyList = + var r: LazyList = nil + when defined(trace20): + echo "first create: ", (created, one[1].convertTriVal2BigInt) + let frst = LazyList(hd: one, tlf: proc(): LazyList = r); created += 1 + r = if s == nil: smult(mltf, frst) else: s.merge smult(mltf, frst) + r + yield one[1] + var hmpll: LazyList = ((nil.unnsm mul5).unnsm mul3).unnsm mul2 + for _ in 2 .. until: + yield hmpll.hd[1]; hmpll = hmpll.rest # almost forever + +proc main = + var s = "" + for h in hammings(20): s &= $h.convertTrival2BigInt & " " + doAssert s == "1 2 3 4 5 6 8 9 10 12 15 16 18 20 24 25 27 30 32 36 ", + "Algorithmic error finding first 20 Hamming numbers!!!" + + when not defined(trace20): + var lsth: TriVal; created = 0; destroyed = 0 + for h in hammings(200): lsth = h + doAssert $lsth.convertTriVal2BigInt == "16200", + "Algorithmic error finding 200th Hamming number!!!" + +let mem = getOccupiedMem() +main() +GC_FullCollect() +let mb = getOccupiedMem() - mem +#doAssert mb == 0, "Found memory leak of " & $mb & " bytes!!!" + +echo getAllocStats() +echo "created ", created, " destroyed ", destroyed From ffce6de84c94348dd5a615c9855a14ed64807449 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 7 Jul 2021 00:43:16 -0700 Subject: [PATCH 1019/2472] improve rendering of newOSError.additionalInfo (#18443) * improve rendering of newOSError.additionalInfo * fixup --- lib/pure/includes/oserr.nim | 5 +++-- lib/pure/net.nim | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/pure/includes/oserr.nim b/lib/pure/includes/oserr.nim index b0740cbe6..e42771d52 100644 --- a/lib/pure/includes/oserr.nim +++ b/lib/pure/includes/oserr.nim @@ -80,8 +80,9 @@ proc newOSError*( e.msg = osErrorMsg(errorCode) if additionalInfo.len > 0: if e.msg.len > 0 and e.msg[^1] != '\n': e.msg.add '\n' - e.msg.add "Additional info: " - e.msg.addQuoted additionalInfo + e.msg.add "Additional info: " + e.msg.add additionalInfo + # don't add trailing `.` etc, which negatively impacts "jump to file" in IDEs. if e.msg == "": e.msg = "unknown OS error" return e diff --git a/lib/pure/net.nim b/lib/pure/net.nim index ec3e4e74d..9c3827751 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -958,7 +958,9 @@ proc bindAddr*(socket: Socket, port = Port(0), address = "") {. var aiList = getAddrInfo(realaddr, port, socket.domain) if bindAddr(socket.fd, aiList.ai_addr, aiList.ai_addrlen.SockLen) < 0'i32: freeaddrinfo(aiList) - raiseOSError(osLastError(), "address: $# port: $#" % [address, $port]) + var address2: string + address2.addQuoted address + raiseOSError(osLastError(), "address: $# port: $#" % [address2, $port]) freeaddrinfo(aiList) proc acceptAddr*(server: Socket, client: var owned(Socket), address: var string, From 2d758795f05d8edf567011eeb49f0cdd38b6dc13 Mon Sep 17 00:00:00 2001 From: Aditya Siram Date: Wed, 7 Jul 2021 21:21:30 -0500 Subject: [PATCH 1020/2472] Rename test variable due to unfortunate connotations (#18453) --- tests/stdlib/twrapnils.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/stdlib/twrapnils.nim b/tests/stdlib/twrapnils.nim index 1a66f4cd9..a0549c1bc 100644 --- a/tests/stdlib/twrapnils.nim +++ b/tests/stdlib/twrapnils.nim @@ -23,7 +23,7 @@ proc main() = x8: seq[int] x9: ref Bar - type Gook = ref object + type Goo = ref object foo: Foo proc fun(a: Bar): auto = a.b2 @@ -39,7 +39,7 @@ proc main() = var a3 = Foo(x1: 1.2, x3: "abc") a3.x2 = a3 - var gook = Gook(foo: a) + var goo = Goo(foo: a) proc initFoo(x1: float): auto = witness.inc @@ -53,8 +53,8 @@ proc main() = doAssert a3.x2.x2.x3.len == 3 doAssert ?.a.x2.x2.x3[1] == default(char) - # here we only apply wrapnil around gook.foo, not gook (and assume gook is not nil) - doAssert ?.(gook.foo).x2.x2.x1 == 0.0 + # here we only apply wrapnil around goo.foo, not goo (and assume goo is not nil) + doAssert ?.(goo.foo).x2.x2.x1 == 0.0 when nimvm: discard else: From 5a42f4a53e22051493232652cda7fb3d32c16da8 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 8 Jul 2021 03:26:57 -0700 Subject: [PATCH 1021/2472] followup #18252: show special flags -d:danger, -d:release when given (#18451) --- compiler/msgs.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 8b4365ec4..242b06587 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -677,6 +677,10 @@ proc genSuccessX*(conf: ConfigRef) = elif optOptimizeSize in conf.options: build.add "size" else: build.add "none (DEBUG BUILD, `-d:release` generates faster code)" # pending https://github.com/timotheecour/Nim/issues/752, point to optimization.html + var flags = "" + if isDefined(conf, "danger"): flags.add " -d:danger" + elif isDefined(conf, "release"): flags.add " -d:release" + if flags.len > 0: build.add "; options:" & flags let sec = formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3) let project = if conf.filenameOption == foAbs: $conf.projectFull else: $conf.projectName # xxx honor conf.filenameOption more accurately From 836b061ae30bb471dc7deef41bb454e1d047bab7 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 8 Jul 2021 03:28:06 -0700 Subject: [PATCH 1022/2472] improve `--declaredLocs` to help disambiguate types (generics, aliases etc) (#18389) * improve --declaredlocs to help disambiguate types (generics, aliases etc) * avoid a cyclic deps * fix test after rebase --- changelog.md | 2 +- compiler/astmsgs.nim | 2 +- compiler/semcall.nim | 12 ++--- compiler/semexprs.nim | 2 +- compiler/types.nim | 8 +++ tests/errmsgs/tdeclaredlocs.nim | 92 ++++++++++++++++++++++++++++++++ tests/errmsgs/tproc_mismatch.nim | 2 +- tests/errmsgs/tsigmatch.nim | 6 +-- 8 files changed, 112 insertions(+), 14 deletions(-) create mode 100644 tests/errmsgs/tdeclaredlocs.nim diff --git a/changelog.md b/changelog.md index 1deb97161..f4c31fbc9 100644 --- a/changelog.md +++ b/changelog.md @@ -395,7 +395,7 @@ ## Compiler changes -- Added `--declaredlocs` to show symbol declaration location in messages. +- Added `--declaredLocs` to show symbol declaration location in messages. - You can now enable/disable VM tracing in user code via `vmutils.vmTrace`. diff --git a/compiler/astmsgs.nim b/compiler/astmsgs.nim index 90bdce7cf..d9105b761 100644 --- a/compiler/astmsgs.nim +++ b/compiler/astmsgs.nim @@ -18,7 +18,7 @@ proc addDeclaredLoc*(result: var string, conf: ConfigRef; typ: PType) = # xxx figure out how to resolve `tyGenericParam`, e.g. for # proc fn[T](a: T, b: T) = discard # fn(1.1, "a") - let typ = typ.skipTypes(abstractInst + {tyStatic, tyUserTypeClassInst} - {tyRange}) + let typ = typ.skipTypes(abstractInst + {tyStatic, tySequence, tyArray, tySet, tyUserTypeClassInst, tyVar, tyRef, tyPtr} - {tyRange}) result.add " [$1" % typ.kind.toHumanStr if typ.sym != nil: result.add " declared in " & toFileLineCol(conf, typ.sym.info) diff --git a/compiler/semcall.nim b/compiler/semcall.nim index cec656fc2..d8dba3e6c 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -231,11 +231,10 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors): of kMissingParam: candidates.add("\n missing parameter: " & nameParam) of kTypeMismatch, kVarNeeded: doAssert nArg != nil - var wanted = err.firstMismatch.formal.typ + let wanted = err.firstMismatch.formal.typ doAssert err.firstMismatch.formal != nil candidates.add("\n required type for " & nameParam & ": ") - candidates.add typeToString(wanted) - candidates.addDeclaredLocMaybe(c.config, wanted) + candidates.addTypeDeclVerboseMaybe(c.config, wanted) candidates.add "\n but expression '" if err.firstMismatch.kind == kVarNeeded: candidates.add renderNotLValue(nArg) @@ -243,9 +242,8 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors): else: candidates.add renderTree(nArg) candidates.add "' is of type: " - var got = nArg.typ - candidates.add typeToString(got) - candidates.addDeclaredLocMaybe(c.config, got) + let got = nArg.typ + candidates.addTypeDeclVerboseMaybe(c.config, got) doAssert wanted != nil if got != nil: if got.kind == tyProc and wanted.kind == tyProc: @@ -274,7 +272,7 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors): const errTypeMismatch = "type mismatch: got <" - errButExpected = "but expected one of: " + errButExpected = "but expected one of:" errUndeclaredField = "undeclared field: '$1'" errUndeclaredRoutine = "attempting to call undeclared routine: '$1'" errBadRoutine = "attempting to call routine: '$1'$2" diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index acf408db0..f46d44bf0 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -988,7 +988,7 @@ proc semIndirectOp(c: PContext, n: PNode, flags: TExprFlags): PNode = break if not hasErrorType: let typ = n[0].typ - msg.add(">\nbut expected one of: \n" & + msg.add(">\nbut expected one of:\n" & typeToString(typ)) # prefer notin preferToResolveSymbols # t.sym != nil diff --git a/compiler/types.nim b/compiler/types.nim index bdf08ffe9..49ebb1dae 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -50,6 +50,14 @@ type pcmDifferentCallConv proc typeToString*(typ: PType; prefer: TPreferedDesc = preferName): string + +proc addTypeDeclVerboseMaybe*(result: var string, conf: ConfigRef; typ: PType) = + if optDeclaredLocs in conf.globalOptions: + result.add typeToString(typ, preferMixed) + result.addDeclaredLoc(conf, typ) + else: + result.add typeToString(typ) + template `$`*(typ: PType): string = typeToString(typ) proc base*(t: PType): PType = diff --git a/tests/errmsgs/tdeclaredlocs.nim b/tests/errmsgs/tdeclaredlocs.nim new file mode 100644 index 000000000..926ebf217 --- /dev/null +++ b/tests/errmsgs/tdeclaredlocs.nim @@ -0,0 +1,92 @@ +discard """ + action: reject + matrix: "--declaredLocs --hints:off" + nimoutFull: true + nimout: ''' +tdeclaredlocs.nim(92, 3) Error: type mismatch: got +but expected one of: +proc fn(a: Bam) [proc declared in tdeclaredlocs.nim(86, 6)] + first type mismatch at position: 1 + required type for a: Bam [object declared in tdeclaredlocs.nim(78, 3)] + but expression 'a' is of type: seq[MyInt2{char}] [char declared in tdeclaredlocs.nim(73, 3)] +proc fn(a: Goo[MyInt2]) [proc declared in tdeclaredlocs.nim(89, 6)] + first type mismatch at position: 1 + required type for a: Goo[MyInt2{char}] [object declared in tdeclaredlocs.nim(79, 3)] + but expression 'a' is of type: seq[MyInt2{char}] [char declared in tdeclaredlocs.nim(73, 3)] +proc fn(a: Goo[cint]) [proc declared in tdeclaredlocs.nim(88, 6)] + first type mismatch at position: 1 + required type for a: Goo[cint{int32}] [object declared in tdeclaredlocs.nim(79, 3)] + but expression 'a' is of type: seq[MyInt2{char}] [char declared in tdeclaredlocs.nim(73, 3)] +proc fn(a: array[3, Bar]) [proc declared in tdeclaredlocs.nim(82, 6)] + first type mismatch at position: 1 + required type for a: array[0..2, Bar] [object declared in tdeclaredlocs.nim(74, 3)] + but expression 'a' is of type: seq[MyInt2{char}] [char declared in tdeclaredlocs.nim(73, 3)] +proc fn(a: seq[Bar]) [proc declared in tdeclaredlocs.nim(81, 6)] + first type mismatch at position: 1 + required type for a: seq[Bar] [object declared in tdeclaredlocs.nim(74, 3)] + but expression 'a' is of type: seq[MyInt2{char}] [char declared in tdeclaredlocs.nim(73, 3)] +proc fn(a: seq[MyInt1]) [proc declared in tdeclaredlocs.nim(80, 6)] + first type mismatch at position: 1 + required type for a: seq[MyInt1{int}] [int declared in tdeclaredlocs.nim(72, 3)] + but expression 'a' is of type: seq[MyInt2{char}] [char declared in tdeclaredlocs.nim(73, 3)] +proc fn(a: set[Baz]) [proc declared in tdeclaredlocs.nim(84, 6)] + first type mismatch at position: 1 + required type for a: set[Baz{enum}] [enum declared in tdeclaredlocs.nim(75, 3)] + but expression 'a' is of type: seq[MyInt2{char}] [char declared in tdeclaredlocs.nim(73, 3)] +proc fn(a: set[MyInt2]) [proc declared in tdeclaredlocs.nim(83, 6)] + first type mismatch at position: 1 + required type for a: set[MyInt2{char}] [char declared in tdeclaredlocs.nim(73, 3)] + but expression 'a' is of type: seq[MyInt2{char}] [char declared in tdeclaredlocs.nim(73, 3)] +proc fn(a: var SetBaz) [proc declared in tdeclaredlocs.nim(85, 6)] + first type mismatch at position: 1 + required type for a: var SetBaz [enum declared in tdeclaredlocs.nim(75, 3)] + but expression 'a' is of type: seq[MyInt2{char}] [char declared in tdeclaredlocs.nim(73, 3)] +proc fn(a: var ref ptr Bam) [proc declared in tdeclaredlocs.nim(87, 6)] + first type mismatch at position: 1 + required type for a: var ref ptr Bam [object declared in tdeclaredlocs.nim(78, 3)] + but expression 'a' is of type: seq[MyInt2{char}] [char declared in tdeclaredlocs.nim(73, 3)] + +expression: fn(a) +''' +""" + +#[ +see also: tests/errmsgs/tsigmatch.nim +]# + + + + + + + + + + + + + + +# line 70 +type + MyInt1 = int + MyInt2 = char + Bar = object + Baz = enum k0, k1 + Baz2 = Baz + SetBaz = set[Baz2] + Bam = ref object + Goo[T] = object +proc fn(a: seq[MyInt1]) = discard +proc fn(a: seq[Bar]) = discard +proc fn(a: array[3, Bar]) = discard +proc fn(a: set[MyInt2]) = discard +proc fn(a: set[Baz]) = discard +proc fn(a: var SetBaz) = discard +proc fn(a: Bam) = discard +proc fn(a: var ref ptr Bam) = discard +proc fn(a: Goo[cint]) = discard +proc fn(a: Goo[MyInt2]) = discard + +var a: seq[MyInt2] +fn(a) diff --git a/tests/errmsgs/tproc_mismatch.nim b/tests/errmsgs/tproc_mismatch.nim index 400c3d441..4ddc7635e 100644 --- a/tests/errmsgs/tproc_mismatch.nim +++ b/tests/errmsgs/tproc_mismatch.nim @@ -6,7 +6,7 @@ discard """ tproc_mismatch.nim(35, 52) Error: type mismatch: got but expected 'proc (a: int, c: float){.closure, noSideEffect.}' Calling convention mismatch: got '{.cdecl.}', but expected '{.closure.}'. tproc_mismatch.nim(39, 6) Error: type mismatch: got -but expected one of: +but expected one of: proc bar(a: proc ()) first type mismatch at position: 1 required type for a: proc (){.closure.} diff --git a/tests/errmsgs/tsigmatch.nim b/tests/errmsgs/tsigmatch.nim index 4f40be2d4..023b7d518 100644 --- a/tests/errmsgs/tsigmatch.nim +++ b/tests/errmsgs/tsigmatch.nim @@ -89,9 +89,9 @@ expression: fun1(default(Mystring), "asdf") - - - +#[ +see also: tests/errmsgs/tdeclaredlocs.nim +]# From 0d74f60742ecff2ac96d97e9ad421aa1b2e8fc85 Mon Sep 17 00:00:00 2001 From: Miran Date: Thu, 8 Jul 2021 16:09:56 +0200 Subject: [PATCH 1023/2472] Revert "Make 'echo' raise IOErrors when appropriate (#16367)" (#18460) This reverts commit 23d23ecb081be6702d74024be8f96d92d9f88a59. --- changelog.md | 3 --- lib/system/io.nim | 24 ++++++++---------------- tests/exception/t16366.nim | 11 ----------- 3 files changed, 8 insertions(+), 30 deletions(-) delete mode 100644 tests/exception/t16366.nim diff --git a/changelog.md b/changelog.md index f4c31fbc9..849640e06 100644 --- a/changelog.md +++ b/changelog.md @@ -24,9 +24,6 @@ - Type mismatch errors now show more context, use `-d:nimLegacyTypeMismatch` for previous behavior. -- `echo` and `debugEcho` will now raise `IOError` if writing to stdout fails. Previous behavior - silently ignored errors. See #16366. Use `-d:nimLegacyEchoNoRaise` for previous behavior. - - `math.round` now is rounded "away from zero" in JS backend which is consistent with other backends. See #9125. Use `-d:nimLegacyJsRound` for previous behavior. diff --git a/lib/system/io.nim b/lib/system/io.nim index 62a9d1e18..591ea3e92 100644 --- a/lib/system/io.nim +++ b/lib/system/io.nim @@ -227,9 +227,6 @@ when defined(windows): # But we cannot call printf directly as the string might contain \0. # So we have to loop over all the sections separated by potential \0s. var i = c_fprintf(f, "%s", s) - if i < 0: - if doRaise: raiseEIO("cannot write string to file") - return while i < s.len: if s[i] == '\0': let w = c_fputc('\0', f) @@ -787,13 +784,6 @@ when declared(stdout): not defined(nintendoswitch) and not defined(freertos) and hostOS != "any" - const echoDoRaise = not defined(nimLegacyEchoNoRaise) and not defined(guiapp) # see PR #16366 - - template checkErrMaybe(succeeded: bool): untyped = - if not succeeded: - when echoDoRaise: - checkErr(stdout) - proc echoBinSafe(args: openArray[string]) {.compilerproc.} = when defined(androidNDK): var s = "" @@ -806,18 +796,20 @@ when declared(stdout): proc flockfile(f: File) {.importc, nodecl.} proc funlockfile(f: File) {.importc, nodecl.} flockfile(stdout) - defer: funlockfile(stdout) when defined(windows) and compileOption("threads"): acquireSys echoLock - defer: releaseSys echoLock for s in args: when defined(windows): - writeWindows(stdout, s, doRaise = echoDoRaise) + writeWindows(stdout, s) else: - checkErrMaybe(c_fwrite(s.cstring, cast[csize_t](s.len), 1, stdout) == s.len) + discard c_fwrite(s.cstring, cast[csize_t](s.len), 1, stdout) const linefeed = "\n" - checkErrMaybe(c_fwrite(linefeed.cstring, linefeed.len, 1, stdout) == linefeed.len) - checkErrMaybe(c_fflush(stdout) == 0) + discard c_fwrite(linefeed.cstring, linefeed.len, 1, stdout) + discard c_fflush(stdout) + when stdOutLock: + funlockfile(stdout) + when defined(windows) and compileOption("threads"): + releaseSys echoLock when defined(windows) and not defined(nimscript) and not defined(js): diff --git a/tests/exception/t16366.nim b/tests/exception/t16366.nim deleted file mode 100644 index 4bcad79ed..000000000 --- a/tests/exception/t16366.nim +++ /dev/null @@ -1,11 +0,0 @@ -discard """ - action: run - exitcode: 0 - targets: "c cpp" - disabled: openbsd -""" - -echo "foo1" -close stdout -doAssertRaises(IOError): - echo "foo" From 0e44d137f97a8206af1f2747391615e2f237f000 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Fri, 9 Jul 2021 03:24:57 +0300 Subject: [PATCH 1024/2472] rm redundant blank lines before literal blocks (#18465) --- lib/packages/docutils/rst.nim | 2 +- lib/packages/docutils/rstgen.nim | 2 +- nimdoc/rst2html/expected/rst_examples.html | 15 +++++---------- tests/stdlib/trst.nim | 16 ++++++++++++++++ 4 files changed, 23 insertions(+), 12 deletions(-) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 419cf50e7..9abdf3fc3 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -1762,7 +1762,7 @@ proc parseLiteralBlock(p: var RstParser): PRstNode = var n = newLeaf("") if currentTok(p).kind == tkIndent: var indent = currentTok(p).ival - inc p.idx + while currentTok(p).kind == tkIndent: inc p.idx # skip blank lines while true: case currentTok(p).kind of tkEof: diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index 829177263..b9c8f688b 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -1258,7 +1258,7 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = doAssert false, "renderRstToOut" of rnLiteralBlock: renderAux(d, n, "$1\n", - "\n\n\\begin{rstpre}\n$2\n$1\n\\end{rstpre}\n\n", result) + "\n\n$2\\begin{rstpre}\n$1\n\\end{rstpre}\n\n", result) of rnQuotedLiteralBlock: doAssert false, "renderRstToOut" of rnLineBlock: diff --git a/nimdoc/rst2html/expected/rst_examples.html b/nimdoc/rst2html/expected/rst_examples.html index 68b796be7..1df91ff1f 100644 --- a/nimdoc/rst2html/expected/rst_examples.html +++ b/nimdoc/rst2html/expected/rst_examples.html @@ -133,8 +133,7 @@ window.addEventListener('DOMContentLoaded', main);

    Note: Assignments, moves, and destruction are specified in the destructors document.

    The language constructs are explained using an extended BNF, in which (a)* means 0 or more a's, a+ means 1 or more a's, and (a)? means an optional a. Parentheses may be used to group elements.

    & is the lookahead operator; &a means that an a is expected but not consumed. It will be consumed in the following rule.

    -

    Non-terminals start with a lowercase letter, abstract terminal symbols are in UPPERCASE. Verbatim terminal symbols (including keywords) are quoted with '. An example:

    -ifStmt = 'if' expr ':' stmts ('elif' expr ':' stmts)* ('else' stmts)?
    +

    Non-terminals start with a lowercase letter, abstract terminal symbols are in UPPERCASE. Verbatim terminal symbols (including keywords) are quoted with '. An example:

    ifStmt = 'if' expr ':' stmts ('elif' expr ':' stmts)* ('else' stmts)?

    In a typical Nim program, most of the code is compiled into the executable. However, some of the code may be executed at compile-time. This can include constant expressions, macro definitions, and Nim procedures used by macro definitions. Most of the Nim language is supported at compile-time, but there are some restrictions -- see Restrictions on Compile-Time Execution for details. We use the term runtime to cover both compile-time execution and code execution in the executable.

    var a: array[0..1, char]
@@ -146,8 +145,7 @@ ifStmt = 'if' expr ':' stmts ('elif' expr ':' stmts)* ('else' stmts)?

    Encoding

    All Nim source files are in the UTF-8 encoding (or its ASCII subset). Other encodings are not supported. Any of the standard platform line termination sequences can be used - the Unix form using ASCII LF (linefeed), the Windows form using the ASCII sequence CR LF (return followed by linefeed), or the old Macintosh form using the ASCII CR (return) character. All of these forms can be used equally, regardless of the platform.

    Indentation

    Nim's standard grammar describes an indentation sensitive language. This means that all the control structures are recognized by indentation. Indentation consists only of spaces; tabulators are not allowed.

    -

    With this notation we can now easily define the core of the grammar: A block of statements (simplified example):

    -ifStmt = 'if' expr ':' stmt
+
    With this notation we can now easily define the core of the grammar: A block of statements (simplified example):
    ifStmt = 'if' expr ':' stmt
          (IND{=} 'elif' expr ':' stmt)*
          (IND{=} 'else' ':' stmt)?
 
@@ -174,18 +172,15 @@ stmt = IND{>} stmt ^+ IND{=} DED  # list of statements
 \x HHcharacter with hex value HH; exactly two hex digits are allowed
 \u HHHHunicode codepoint with hex value HHHH; exactly four hex digits are allowed
 \u {H+}unicode codepoint; all hex digits enclosed in {} are used for the codepoint
-
    """"long string within quotes""""
    Produces:
    -"long string within quotes"
    
+
    """"long string within quotes""""
    Produces:
    "long string within quotes"
    
 
    
 
-
    Operators
    Nim allows user defined operators. An operator is any combination of the following characters:
    -=     +     -     *     /     <     >
+
    Operators
    Nim allows user defined operators. An operator is any combination of the following characters:
    =     +     -     *     /     <     >
 @     $     ~     &     %     |
 !     ?     ^     .     :     \
    
 
    
 
    (The grammar uses the terminal OPR to refer to operator symbols as defined here.)
    
-
    The following strings denote other tokens:
    -`   (    )     {    }     [    ]    ,  ;   [.    .]  {.   .}  (.  .)  [:
    
+
    The following strings denote other tokens:
    `   (    )     {    }     [    ]    ,  ;   [.    .]  {.   .}  (.  .)  [:
    
 
    
 
    Otherwise, precedence is determined by the first character.
    
 
diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim
index fb9552479..fe99d6cfa 100644
--- a/tests/stdlib/trst.nim
+++ b/tests/stdlib/trst.nim
@@ -196,6 +196,22 @@ suite "RST parsing":
               rnLeaf  'someBlockQuote'
         """)
 
+  test "no redundant blank lines in literal blocks":
+    check(dedent"""
+      Check::
+
+
+        code
+
+      """.toAst ==
+      dedent"""
+        rnInner
+          rnLeaf  'Check'
+          rnLeaf  ':'
+          rnLiteralBlock
+            rnLeaf  'code'
+      """)
+
 suite "RST indentation":
   test "nested bullet lists":
     let input = dedent """

From 86f5a56fcdbdbd4748c979e6f517d8dfc64857f7 Mon Sep 17 00:00:00 2001
From: Ivan Bobev 
Date: Fri, 9 Jul 2021 03:26:49 +0300
Subject: [PATCH 1025/2472] Fix a bug with starting of asynchronous processes
 (#18464)

The asynchronous process completion handler callback should be called
only once. This is achieved by passing `WT_EXECUTEONLYONCE` flag to the
`registerWaitForSingleObject` Windows API procedure.

Related to cheatfate/asynctools#35
---
 lib/pure/asyncdispatch.nim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim
index 17fd5f824..d98ca77df 100644
--- a/lib/pure/asyncdispatch.nim
+++ b/lib/pure/asyncdispatch.nim
@@ -1022,7 +1022,7 @@ when defined(windows) or defined(nimdoc):
       raiseOSError(osLastError())
 
     var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData)))
-    var flags = WT_EXECUTEINWAITTHREAD.DWORD
+    var flags = WT_EXECUTEINWAITTHREAD.DWORD or WT_EXECUTEONLYONCE.DWORD
 
     proc proccb(fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) =
       closeWaitable(hProcess)

From 6869157cd17229dbadb8b1b43c218d5fb28e8eb2 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Fri, 9 Jul 2021 13:44:38 +0800
Subject: [PATCH 1026/2472] [docs] improve std/encodings (#18458)

* [docs] improve std/encodings

* tiny

* shrink

* Apply suggestions from code review

Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com>

* Apply suggestions from code review

* Update lib/pure/encodings.nim

Co-authored-by: Varriount 

Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com>
Co-authored-by: Varriount 
---
 lib/pure/encodings.nim | 53 ++++++++++++++++++++++++++++++++----------
 1 file changed, 41 insertions(+), 12 deletions(-)

diff --git a/lib/pure/encodings.nim b/lib/pure/encodings.nim
index 89db0a278..dcdc474ae 100644
--- a/lib/pure/encodings.nim
+++ b/lib/pure/encodings.nim
@@ -7,15 +7,42 @@
 #    distribution, for details about the copyright.
 #
 
-## Converts between different character encodings. On UNIX, this uses
+## Routines for converting between different character encodings. On UNIX, this uses
 ## the `iconv`:idx: library, on Windows the Windows API.
+##
+## The following example shows how to change character encodings.
+runnableExamples:
+  let
+    orig = "öäüß"
+    # convert `orig` from "UTF-8" to "CP1252"
+    cp1252 = convert(orig, "CP1252", "UTF-8")
+    # convert `cp1252` from "CP1252" to "ibm850"
+    ibm850 = convert(cp1252, "ibm850", "CP1252")
+    current = getCurrentEncoding()
+  assert orig == "\195\182\195\164\195\188\195\159"
+  assert ibm850 == "\148\132\129\225"
+  assert convert(ibm850, current, "ibm850") == orig
+
+## The example below uses a reuseable `EncodingConverter` object which is
+## created by `open` with `destEncoding` and `srcEncoding` specified. You can use
+## `convert` on this object multiple times.
+runnableExamples:
+  var fromGB2312 = open("utf-8", "gb2312")
+  let first = "\203\173\197\194\163\191\210\187" &
+      "\203\242\209\204\211\234\200\206\198\189\201\250"
+  assert fromGB2312.convert(first) == "谁怕？一蓑烟雨任平生"
+
+  let second = "\211\208\176\215\205\183\200\231" &
+      "\208\194\163\172\199\227\184\199\200\231\185\202"
+  assert fromGB2312.convert(second) == "有白头如新，倾盖如故"
+
 
 import os
 
 when not defined(windows):
   type
     ConverterObj = object
-    EncodingConverter* = ptr ConverterObj ## can convert between two character sets
+    EncodingConverter* = ptr ConverterObj ## Can convert between two character sets.
 
 else:
   type
@@ -24,8 +51,8 @@ else:
       dest, src: CodePage
 
 type
-  EncodingError* = object of ValueError ## exception that is raised
-                                        ## for encoding errors
+  EncodingError* = object of ValueError ## Exception that is raised
+                                        ## for encoding errors.
 
 when defined(windows):
   import parseutils, strutils
@@ -298,7 +325,7 @@ else:
     importc: "iconv", importIconv.}
 
 proc getCurrentEncoding*(uiApp = false): string =
-  ## retrieves the current encoding. On Unix, always "UTF-8" is returned.
+  ## Retrieves the current encoding. On Unix, "UTF-8" is always returned.
   ## The `uiApp` parameter is Windows specific. If true, the UI's code-page
   ## is returned, if false, the Console's code-page is returned.
   when defined(windows):
@@ -307,7 +334,7 @@ proc getCurrentEncoding*(uiApp = false): string =
     result = "UTF-8"
 
 proc open*(destEncoding = "UTF-8", srcEncoding = "CP1252"): EncodingConverter =
-  ## opens a converter that can convert from `srcEncoding` to `destEncoding`.
+  ## Opens a converter that can convert from `srcEncoding` to `destEncoding`.
   ## Raises `IOError` if it cannot fulfill the request.
   when not defined(windows):
     result = iconvOpen(destEncoding, srcEncoding)
@@ -326,7 +353,7 @@ proc open*(destEncoding = "UTF-8", srcEncoding = "CP1252"): EncodingConverter =
         "cannot find encoding " & srcEncoding)
 
 proc close*(c: EncodingConverter) =
-  ## frees the resources the converter `c` holds.
+  ## Frees the resources the converter `c` holds.
   when not defined(windows):
     iconvClose(c)
 
@@ -421,12 +448,13 @@ when defined(windows):
            else: convertFromWideString(codePageTo, wideString)
 
   proc convert*(c: EncodingConverter, s: string): string =
-    ## converts `s` to `destEncoding` that was given to the converter `c`. It
-    ## assumed that `s` is in `srcEncoding`.
-    ## utf-16BE, utf-32 conversions not supported on windows
     result = convertWin(c.src, c.dest, s)
 else:
   proc convert*(c: EncodingConverter, s: string): string =
+    ## Converts `s` to `destEncoding` that was given to the converter `c`. It
+    ## assumes that `s` is in `srcEncoding`.
+    ##
+    ## .. warning:: UTF-16BE and UTF-32 conversions are not supported on Windows.
     result = newString(s.len)
     var inLen = csize_t len(s)
     var outLen = csize_t len(result)
@@ -467,10 +495,11 @@ else:
 
 proc convert*(s: string, destEncoding = "UTF-8",
                          srcEncoding = "CP1252"): string =
-  ## converts `s` to `destEncoding`. It assumed that `s` is in `srcEncoding`.
+  ## Converts `s` to `destEncoding`. It assumed that `s` is in `srcEncoding`.
   ## This opens a converter, uses it and closes it again and is thus more
   ## convenient but also likely less efficient than re-using a converter.
-  ## utf-16BE, utf-32 conversions not supported on windows
+  ##
+  ## .. warning:: UTF-16BE and UTF-32 conversions are not supported on Windows.
   var c = open(destEncoding, srcEncoding)
   try:
     result = convert(c, s)

From ae7e7756fea146126ffc5200b2e66bfe2dab4cd4 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Fri, 9 Jul 2021 02:41:28 -0700
Subject: [PATCH 1027/2472] runnableExamples now show originating location in
 stacktraces on failure (#18457)

* runnableExamples now show originating location in stacktraces on failure

* PRTEMP

* fix indentation inside multiline strings
---
 compiler/docgen.nim         | 25 +++++++++++++++++++------
 compiler/renderverbatim.nim | 11 ++++++-----
 2 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/compiler/docgen.nim b/compiler/docgen.nim
index e6bb7cef8..1ae00e22e 100644
--- a/compiler/docgen.nim
+++ b/compiler/docgen.nim
@@ -468,6 +468,10 @@ proc runAllExamples(d: PDoc) =
 
 proc quoted(a: string): string = result.addQuoted(a)
 
+proc toInstantiationInfo(conf: ConfigRef, info: TLineInfo): auto =
+  # xxx expose in compiler/lineinfos.nim
+  (conf.toMsgFilename(info), info.line.int, info.col.int + ColOffset)
+
 proc prepareExample(d: PDoc; n: PNode, topLevel: bool): tuple[rdoccmd: string, code: string] =
   ## returns `rdoccmd` and source code for this runnableExamples
   var rdoccmd = ""
@@ -481,6 +485,7 @@ proc prepareExample(d: PDoc; n: PNode, topLevel: bool): tuple[rdoccmd: string, c
   let useRenderModule = false
   let loc = d.conf.toFileLineCol(n.info)
   let code = extractRunnableExamplesSource(d.conf, n)
+  let codeIndent = extractRunnableExamplesSource(d.conf, n, indent = 2)
 
   if d.conf.errorCounter > 0:
     return (rdoccmd, code)
@@ -489,7 +494,7 @@ proc prepareExample(d: PDoc; n: PNode, topLevel: bool): tuple[rdoccmd: string, c
   let outputDir = d.exampleOutputDir
   createDir(outputDir)
   inc d.exampleCounter
-  let outp = outputDir / RelativeFile(extractFilename(d.filename.changeFileExt"" & ("_examples$1.nim" % $d.exampleCounter)))
+  let outp = outputDir / RelativeFile("$#_examples_$#.nim" % [d.filename.extractFilename.changeFileExt"", $d.exampleCounter])
 
   if useRenderModule:
     var docComment = newTree(nkCommentStmt)
@@ -507,13 +512,21 @@ proc prepareExample(d: PDoc; n: PNode, topLevel: bool): tuple[rdoccmd: string, c
     renderModule(runnableExamples, outp.string, conf = d.conf)
 
   else:
-    let code2 = """
+    var code2 = code
+    if code.len > 0 and "codeReordering" notin code:
+      # hacky but simplest solution, until we devise a way to make `{.line.}`
+      # work without introducing a scope
+      code2 = """
+{.line: $#.}:
+$#
+""" % [$toInstantiationInfo(d.conf, n.info), codeIndent]
+    code2 = """
 #[
-$1
+$#
 ]#
-import $2
-$3
-""" % [comment, d.filename.quoted, code]
+import $#
+$#
+""" % [comment, d.filename.quoted, code2]
     writeFile(outp.string, code2)
 
   if rdoccmd notin d.exampleGroups:
diff --git a/compiler/renderverbatim.nim b/compiler/renderverbatim.nim
index a20c8873d..fc74f2322 100644
--- a/compiler/renderverbatim.nim
+++ b/compiler/renderverbatim.nim
@@ -83,7 +83,7 @@ proc startOfLineInsideTriple(ldata: LineData, line: int): bool =
   if index >= ldata.lines.len: false
   else: ldata.lines[index]
 
-proc extractRunnableExamplesSource*(conf: ConfigRef; n: PNode): string =
+proc extractRunnableExamplesSource*(conf: ConfigRef; n: PNode, indent = 0): string =
   ## TLineInfo.offsetA,offsetB would be cleaner but it's only enabled for nimpretty,
   ## we'd need to check performance impact to enable it for nimdoc.
   var first = n.lastSon.info
@@ -102,7 +102,7 @@ proc extractRunnableExamplesSource*(conf: ConfigRef; n: PNode): string =
 
   let last = n.lastNodeRec.info
   var info = first
-  var indent = info.col
+  var indent2 = info.col
   let numLines = numLines(conf, info.fileIndex).uint16
   var lastNonemptyPos = 0
 
@@ -118,14 +118,15 @@ proc extractRunnableExamplesSource*(conf: ConfigRef; n: PNode): string =
     info.line = line
     let src = sourceLine(conf, info)
     let special = startOfLineInsideTriple(ldata, line.int)
-    if line > last.line and not special and not isInIndentationBlock(src, indent):
+    if line > last.line and not special and not isInIndentationBlock(src, indent2):
       break
     if line > first.line: result.add "\n"
     if special:
       result.add src
       lastNonemptyPos = result.len
-    elif src.len > indent:
-      result.add src[indent..^1]
+    elif src.len > indent2:
+      for i in 0..
Date: Fri, 9 Jul 2021 15:15:49 +0200
Subject: [PATCH 1028/2472] ORC: support for custom =trace procs (#18459)

* ORC: support custom =trace procs (WIP)
* Update tests/arc/tcustomtrace.nim

Co-authored-by: Clyybber 

* =trace is now documented and seems to work
* make test green

Co-authored-by: Clyybber 
---
 changelog.md                 |   2 +
 compiler/ast.nim             |   2 +-
 compiler/ccgexprs.nim        |   1 +
 compiler/condsyms.nim        |   1 +
 compiler/jsgen.nim           |   2 +-
 compiler/liftdestructors.nim |  16 ++-
 compiler/semmagic.nim        |   6 +
 compiler/semstmts.nim        |  13 +-
 compiler/vmgen.nim           |   2 +-
 doc/destructors.rst          |  37 ++++++
 doc/manual.rst               |   5 +-
 lib/system.nim               |   5 +
 tests/arc/tcustomtrace.nim   | 240 +++++++++++++++++++++++++++++++++++
 13 files changed, 321 insertions(+), 11 deletions(-)
 create mode 100644 tests/arc/tcustomtrace.nim

diff --git a/changelog.md b/changelog.md
index 849640e06..bc64e9579 100644
--- a/changelog.md
+++ b/changelog.md
@@ -388,6 +388,8 @@
 
 - `typeof(voidStmt)` now works and returns `void`.
 
+- The `gc:orc` algorithm was refined so that custom container types can participate in the
+  cycle collection process.
 
 
 ## Compiler changes
diff --git a/compiler/ast.nim b/compiler/ast.nim
index 26050426a..f96e464af 100644
--- a/compiler/ast.nim
+++ b/compiler/ast.nim
@@ -667,7 +667,7 @@ type
     mIsPartOf, mAstToStr, mParallel,
     mSwap, mIsNil, mArrToSeq,
     mNewString, mNewStringOfCap, mParseBiggestFloat,
-    mMove, mWasMoved, mDestroy,
+    mMove, mWasMoved, mDestroy, mTrace,
     mDefault, mUnown, mIsolate, mAccessEnv, mReset,
     mArray, mOpenArray, mRange, mSet, mSeq, mVarargs,
     mRef, mPtr, mVar, mDistinct, mVoid, mTuple,
diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim
index c07ddf1db..09326ceeb 100644
--- a/compiler/ccgexprs.nim
+++ b/compiler/ccgexprs.nim
@@ -2422,6 +2422,7 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
   of mDestroy: genDestroy(p, e)
   of mAccessEnv: unaryExpr(p, e, d, "$1.ClE_0")
   of mSlice: genSlice(p, e, d)
+  of mTrace: discard "no code to generate"
   else:
     when defined(debugMagics):
       echo p.prc.name.s, " ", p.prc.id, " ", p.prc.flags, " ", p.prc.ast[genericParamsPos].kind
diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim
index b3f8465f0..33cd1818b 100644
--- a/compiler/condsyms.nim
+++ b/compiler/condsyms.nim
@@ -136,3 +136,4 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimHasTypeofVoid")
   defineSymbol("nimHasDragonBox")
   defineSymbol("nimHasHintAll")
+  defineSymbol("nimHasTrace")
diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim
index a9339c7fd..82fba02a6 100644
--- a/compiler/jsgen.nim
+++ b/compiler/jsgen.nim
@@ -2079,7 +2079,7 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) =
       gen(p, n[1], x)
       useMagic(p, "nimCopy")
       r.res = "nimCopy(null, $1, $2)" % [x.rdLoc, genTypeInfo(p, n.typ)]
-  of mDestroy: discard "ignore calls to the default destructor"
+  of mDestroy, mTrace: discard "ignore calls to the default destructor"
   of mOrd: genOrd(p, n, r)
   of mLengthStr, mLengthSeq, mLengthOpenArray, mLengthArray:
     var x: TCompRes
diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim
index 1824557e6..b2b70edee 100644
--- a/compiler/liftdestructors.nim
+++ b/compiler/liftdestructors.nim
@@ -416,11 +416,23 @@ proc considerUserDefinedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
       body.add destructorCall(c, op, x)
       result = true
     #result = addDestructorCall(c, t, body, x)
-  of attachedAsgn, attachedSink, attachedTrace:
+  of attachedAsgn, attachedSink:
     var op = getAttachedOp(c.g, t, c.kind)
     result = considerAsgnOrSink(c, t, body, x, y, op)
     if op != nil:
       setAttachedOp(c.g, c.idgen.module, t, c.kind, op)
+  of attachedTrace:
+    var op = getAttachedOp(c.g, t, c.kind)
+    if op != nil and sfOverriden in op.flags:
+      if op.ast.isGenericRoutine:
+        # patch generic =trace:
+        op = instantiateGeneric(c, op, t, t.typeInst)
+        setAttachedOp(c.g, c.idgen.module, t, c.kind, op)
+
+    result = considerAsgnOrSink(c, t, body, x, y, op)
+    if op != nil:
+      setAttachedOp(c.g, c.idgen.module, t, c.kind, op)
+
   of attachedDeepCopy:
     let op = getAttachedOp(c.g, t, attachedDeepCopy)
     if op != nil:
@@ -1065,7 +1077,7 @@ proc createTypeBoundOps(g: ModuleGraph; c: PContext; orig: PType; info: TLineInf
   # 4. We have a custom destructor.
   # 5. We have a (custom) generic destructor.
 
-  # we do not generate '=trace' nor '=dispose' procs if we
+  # we do not generate '=trace' procs if we
   # have the cycle detection disabled, saves code size.
   let lastAttached = if g.config.selectedGC == gcOrc: attachedTrace
                      else: attachedSink
diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim
index d3f26e630..04f4c4729 100644
--- a/compiler/semmagic.nim
+++ b/compiler/semmagic.nim
@@ -551,6 +551,12 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode,
     let op = getAttachedOp(c.graph, t, attachedDestructor)
     if op != nil:
       result[0] = newSymNode(op)
+  of mTrace:
+    result = n
+    let t = n[1].typ.skipTypes(abstractVar)
+    let op = getAttachedOp(c.graph, t, attachedTrace)
+    if op != nil:
+      result[0] = newSymNode(op)
   of mUnown:
     result = semUnown(c, n)
   of mExists, mForall:
diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim
index c3402aee3..fd2f8a1d9 100644
--- a/compiler/semstmts.nim
+++ b/compiler/semstmts.nim
@@ -1649,6 +1649,8 @@ proc bindTypeHook(c: PContext; s: PSym; n: PNode; op: TTypeAttachedOp) =
   var noError = false
   let cond = if op == attachedDestructor:
                t.len == 2 and t[0] == nil and t[1].kind == tyVar
+             elif op == attachedTrace:
+               t.len == 3 and t[0] == nil and t[1].kind == tyVar and t[2].kind == tyPointer
              else:
                t.len >= 2 and t[0] == nil
 
@@ -1673,8 +1675,12 @@ proc bindTypeHook(c: PContext; s: PSym; n: PNode; op: TTypeAttachedOp) =
         localError(c.config, n.info, errGenerated,
           "type bound operation `" & s.name.s & "` can be defined only in the same module with its type (" & obj.typeToString() & ")")
   if not noError and sfSystemModule notin s.owner.flags:
-    localError(c.config, n.info, errGenerated,
-      "signature for '" & s.name.s & "' must be proc[T: object](x: var T)")
+    if op == attachedTrace:
+      localError(c.config, n.info, errGenerated,
+        "signature for '=trace' must be proc[T: object](x: var T; env: pointer)")
+    else:
+      localError(c.config, n.info, errGenerated,
+        "signature for '" & s.name.s & "' must be proc[T: object](x: var T)")
   incl(s.flags, sfUsed)
   incl(s.flags, sfOverriden)
 
@@ -1752,7 +1758,8 @@ proc semOverride(c: PContext, s: PSym, n: PNode) =
       localError(c.config, n.info, errGenerated,
                 "signature for '" & s.name.s & "' must be proc[T: object](x: var T; y: T)")
   of "=trace":
-    bindTypeHook(c, s, n, attachedTrace)
+    if s.magic != mTrace:
+      bindTypeHook(c, s, n, attachedTrace)
   else:
     if sfOverriden in s.flags:
       localError(c.config, n.info, errGenerated,
diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim
index 14f9e0b30..e5e4a854e 100644
--- a/compiler/vmgen.nim
+++ b/compiler/vmgen.nim
@@ -1362,7 +1362,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
     globalError(c.config, n.info, sizeOfLikeMsg("offsetof"))
   of mRunnableExamples:
     discard "just ignore any call to runnableExamples"
-  of mDestroy: discard "ignore calls to the default destructor"
+  of mDestroy, mTrace: discard "ignore calls to the default destructor"
   of mMove:
     let arg = n[1]
     let a = c.genx(arg)
diff --git a/doc/destructors.rst b/doc/destructors.rst
index c99606b95..a94ccb9e7 100644
--- a/doc/destructors.rst
+++ b/doc/destructors.rst
@@ -42,6 +42,12 @@ written as:
       for i in 0..`_.
diff --git a/lib/system.nim b/lib/system.nim
index 090ab309e..8aa9bd718 100644
--- a/lib/system.nim
+++ b/lib/system.nim
@@ -488,6 +488,11 @@ proc `=sink`*[T](x: var T; y: T) {.inline, magic: "Asgn".} =
   ## Generic `sink`:idx: implementation that can be overridden.
   shallowCopy(x, y)
 
+when defined(nimHasTrace):
+  proc `=trace`*[T](x: var T; env: pointer) {.inline, magic: "Trace".} =
+    ## Generic `trace`:idx: implementation that can be overridden.
+    discard
+
 type
   HSlice*[T, U] = object   ## "Heterogeneous" slice type.
     a*: T                  ## The lower bound (inclusive).
diff --git a/tests/arc/tcustomtrace.nim b/tests/arc/tcustomtrace.nim
new file mode 100644
index 000000000..3977194d9
--- /dev/null
+++ b/tests/arc/tcustomtrace.nim
@@ -0,0 +1,240 @@
+discard """
+  outputsub: '''1
+2
+3
+4
+5
+6
+89
+90
+90
+0 0 1
+0 1 2
+0 2 3
+1 0 4
+1 1 5
+1 2 6
+1 3 7
+after 6 6
+MEM 0'''
+joinable: false
+  cmd: "nim c --gc:orc -d:useMalloc $file"
+  valgrind: "true"
+"""
+
+import typetraits
+
+type
+  myseq*[T] = object
+    len, cap: int
+    data: ptr UncheckedArray[T]
+
+# XXX make code memory safe for overflows in '*'
+var
+  allocCount, deallocCount: int
+
+proc `=destroy`*[T](x: var myseq[T]) =
+  if x.data != nil:
+    when not supportsCopyMem(T):
+      for i in 0..= x.cap: resize(x)
+  result = addr(x.data[x.len])
+  inc x.len
+
+template add*[T](x: var myseq[T]; y: T) =
+  reserveSlot(x)[] = y
+
+proc shrink*[T](x: var myseq[T]; newLen: int) =
+  assert newLen <= x.len
+  assert newLen >= 0
+  when not supportsCopyMem(T):
+    for i in countdown(x.len - 1, newLen - 1):
+      `=destroy`(x.data[i])
+  x.len = newLen
+
+proc grow*[T](x: var myseq[T]; newLen: int; value: T) =
+  if newLen <= x.len: return
+  assert newLen >= 0
+  let oldCap = x.cap
+  if oldCap == 0: x.cap = newLen
+  else: x.cap = max(newLen, (oldCap * 3) shr 1)
+  if x.data == nil: inc allocCount
+  x.data = cast[type(x.data)](realloc0(x.data, oldCap * sizeof(T), x.cap * sizeof(T)))
+  for i in x.len..
Date: Sat, 10 Jul 2021 09:41:07 +0300
Subject: [PATCH 1029/2472] sync with the same template from locks module
 (#18414)

---
 lib/core/rlocks.nim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/core/rlocks.nim b/lib/core/rlocks.nim
index a1ee0a6ec..0444b9a83 100644
--- a/lib/core/rlocks.nim
+++ b/lib/core/rlocks.nim
@@ -48,7 +48,7 @@ proc release*(lock: var RLock) {.inline.} =
   ## Releases the given lock.
   releaseSys(lock)
 
-template withRLock*(lock: var RLock, code: untyped): untyped =
+template withRLock*(lock: RLock, code: untyped) =
   ## Acquires the given lock and then executes the code.
   acquire(lock)
   {.locks: [lock].}:

From 3645f5fc7ae885e22cb52b9f865edf062cfb8e59 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Sat, 10 Jul 2021 15:55:01 +0800
Subject: [PATCH 1030/2472] more important packages (#18472)

---
 testament/important_packages.nim | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/testament/important_packages.nim b/testament/important_packages.nim
index 3d3c340eb..afceffb2e 100644
--- a/testament/important_packages.nim
+++ b/testament/important_packages.nim
@@ -37,6 +37,7 @@ pkg "alea", allowFailure = true
 pkg "argparse"
 pkg "arraymancer", "nim c tests/tests_cpu.nim", allowFailure = true
 pkg "ast_pattern_matching", "nim c -r --oldgensym:on tests/test1.nim", allowFailure = true
+pkg "asyncthreadpool"
 pkg "awk"
 pkg "bigints", url = "https://github.com/Araq/nim-bigints"
 pkg "binaryheap", "nim c -r binaryheap.nim"
@@ -56,6 +57,7 @@ pkg "combparser", "nimble test --gc:orc"
 pkg "compactdict"
 pkg "comprehension", "nimble test", "https://github.com/alehander42/comprehension"
 pkg "criterion", allowFailure = true # pending https://github.com/disruptek/criterion/issues/3 (wrongly closed)
+pkg "datamancer"
 pkg "dashing", "nim c tests/functional.nim"
 pkg "delaunay"
 pkg "docopt"
@@ -81,6 +83,7 @@ pkg "jstin"
 pkg "karax", "nim c -r tests/tester.nim"
 pkg "kdtree", "nimble test -d:nimLegacyRandomInitRand", "https://github.com/jblindsay/kdtree"
 pkg "loopfusion"
+pkg "lockfreequeues"
 pkg "macroutils"
 pkg "manu"
 pkg "markdown"
@@ -89,6 +92,7 @@ pkg "msgpack4nim", "nim c -r tests/test_spec.nim"
 pkg "nake", "nim c nakefile.nim"
 pkg "neo", "nim c -d:blas=openblas tests/all.nim"
 pkg "nesm", "nimble tests", allowFailure = true # notice plural 'tests'
+pkg "netty"
 pkg "nico", allowFailure = true
 pkg "nicy", "nim c -r src/nicy.nim"
 pkg "nigui", "nim c -o:niguii -r src/nigui.nim"
@@ -134,10 +138,12 @@ pkg "rosencrantz", "nim c -o:rsncntz -r rosencrantz.nim"
 pkg "sdl1", "nim c -r src/sdl.nim"
 pkg "sdl2_nim", "nim c -r sdl2/sdl.nim"
 pkg "sigv4", "nim c --gc:arc -r sigv4.nim", "https://github.com/disruptek/sigv4"
+pkg "sim"
 pkg "snip", "nimble test", "https://github.com/genotrance/snip"
 pkg "stint", "nim r stint.nim"
 pkg "strslice"
 pkg "strunicode", "nim c -r src/strunicode.nim"
+pkg "supersnappy"
 pkg "synthesis"
 pkg "telebot", "nim c -o:tbot -r src/telebot.nim", allowFailure = true # pending https://github.com/ba0f3/telebot.nim/issues/59
 pkg "tempdir"

From 980a9ed5239ef03e73946ffcc84671a6eaa35616 Mon Sep 17 00:00:00 2001
From: sivchari 
Date: Sun, 11 Jul 2021 15:19:20 +0900
Subject: [PATCH 1031/2472] fix comment (#18473)

---
 lib/impure/re.nim | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/lib/impure/re.nim b/lib/impure/re.nim
index 504d8f22e..d6922f4f1 100644
--- a/lib/impure/re.nim
+++ b/lib/impure/re.nim
@@ -41,12 +41,12 @@ const
 
 type
   RegexFlag* = enum     ## options for regular expressions
-    reIgnoreCase = 0,    ## do caseless matching
-    reMultiLine = 1,     ## `^` and `$` match newlines within data
-    reDotAll = 2,        ## `.` matches anything including NL
-    reExtended = 3,      ## ignore whitespace and `#` comments
-    reStudy = 4          ## study the expression (may be omitted if the
-                         ## expression will be used only once)
+    reIgnoreCase = 0,   ## do caseless matching
+    reMultiLine = 1,    ## `^` and `$` match newlines within data
+    reDotAll = 2,       ## `.` matches anything including NL
+    reExtended = 3,     ## ignore whitespace and `#` comments
+    reStudy = 4         ## study the expression (may be omitted if the
+                        ## expression will be used only once)
 
   RegexDesc = object
     h: ptr Pcre

From 9ffc70851b586dbdc579d8c6af30a6a3604d7313 Mon Sep 17 00:00:00 2001
From: GordonBGood 
Date: Mon, 12 Jul 2021 16:55:40 +0700
Subject: [PATCH 1032/2472] thamming_orc test created/destroyed counts match
 (#18471)

The thamming_orc.nim code now counts all created objects being tested, not just the ones following the "first 20" test, and the position of the `destroyed += 1` counter has been adjusted so it counts all the calls that are as a result of `=trace` tracing and not just the original destruction calls.
---
 tests/arc/thamming_orc.nim | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/arc/thamming_orc.nim b/tests/arc/thamming_orc.nim
index 463f7117e..777efb38e 100644
--- a/tests/arc/thamming_orc.nim
+++ b/tests/arc/thamming_orc.nim
@@ -1,6 +1,6 @@
 discard """
   output: '''(allocCount: 1114, deallocCount: 1112)
-created 439 destroyed 402'''
+created 491 destroyed 491'''
   cmd: "nim c --gc:orc -d:nimAllocStats $file"
 """
 
@@ -94,8 +94,8 @@ type
 var destroyed = 0
 
 proc `=destroy`(ll: var LazyListObj) =
-  if ll.tlf == nil and ll.tl == nil: return
   destroyed += 1
+  if ll.tlf == nil and ll.tl == nil: return
 
   when defined(trace20):
     echo "destroying:  ", (destroyed, ll.hd[1].convertTriVal2BigInt)
@@ -140,7 +140,7 @@ proc main =
            "Algorithmic error finding first 20 Hamming numbers!!!"
 
   when not defined(trace20):
-    var lsth: TriVal; created = 0; destroyed = 0
+    var lsth: TriVal
     for h in hammings(200): lsth = h
     doAssert $lsth.convertTriVal2BigInt == "16200",
              "Algorithmic error finding 200th Hamming number!!!"
@@ -149,7 +149,7 @@ let mem = getOccupiedMem()
 main()
 GC_FullCollect()
 let mb = getOccupiedMem() - mem
-#doAssert mb == 0, "Found memory leak of " & $mb & " bytes!!!"
+doAssert mb == 0, "Found memory leak of " & $mb & " bytes!!!"
 
 echo getAllocStats()
 echo "created ", created, " destroyed ", destroyed

From 195300a938c7c8dd4ec7db3ca7cc76dddeab953e Mon Sep 17 00:00:00 2001
From: flywind 
Date: Mon, 12 Jul 2021 20:26:46 +0800
Subject: [PATCH 1033/2472] [source code filter]fix Nightlies bug (#18475)

* workaround Nightlies bug

* Update tools/niminst/makefile.nimf
---
 tools/niminst/makefile.nimf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/niminst/makefile.nimf b/tools/niminst/makefile.nimf
index 8c40cf50f..d14960ac5 100644
--- a/tools/niminst/makefile.nimf
+++ b/tools/niminst/makefile.nimf
@@ -25,7 +25,7 @@ endif
 
 ifeq ($(uos),linux)
   myos = linux
-  # add -lrt to avoid "undefined reference to `clock_gettime'" with glibc<2.17
+  ## add -lrt to avoid "undefined reference to `clock_gettime'" with glibc<2.17
   LDFLAGS += -ldl -lm -lrt
 endif
 ifeq ($(uos),dragonfly)

From 1b128ac04533421547298ecca0d9b8e7de477280 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Tue, 13 Jul 2021 06:18:07 +0800
Subject: [PATCH 1034/2472] close #17986  add testcase (#18477)

---
 tests/stdlib/tjsonutils.nim | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/tests/stdlib/tjsonutils.nim b/tests/stdlib/tjsonutils.nim
index 31ec4316f..b0c444cbf 100644
--- a/tests/stdlib/tjsonutils.nim
+++ b/tests/stdlib/tjsonutils.nim
@@ -311,6 +311,18 @@ template fn() =
       doAssert guide == AboutLifeUniverseAndEverythingElse(
         question: "6*9=?", answer: 42)
 
+      block refObject: #bug 17986
+        type A = ref object
+          case is_a: bool
+          of true:
+            a: int
+          else:
+            b: int
+
+        var a = A()
+        fromJson(a, """{"is_a": true, "a":1, "extra_key": 1}""".parse_json, Joptions(allowExtraKeys: true))
+        doAssert $a[] == "(is_a: true, a: 1)"
+
     block testAllowMissingKeys:
       var guide = AboutLifeUniverseAndEverythingElse(
         question: "6*9=?", answer: 54)

From 15fdcd0c4d4d5b44a08766120d88b085eb358ae1 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Tue, 13 Jul 2021 15:22:33 +0800
Subject: [PATCH 1035/2472] workaround #18481 (#18482)

---
 lib/pure/encodings.nim | 36 +++++++++++++++++++-----------------
 1 file changed, 19 insertions(+), 17 deletions(-)

diff --git a/lib/pure/encodings.nim b/lib/pure/encodings.nim
index dcdc474ae..36e891dd3 100644
--- a/lib/pure/encodings.nim
+++ b/lib/pure/encodings.nim
@@ -12,29 +12,31 @@
 ##
 ## The following example shows how to change character encodings.
 runnableExamples:
-  let
-    orig = "öäüß"
-    # convert `orig` from "UTF-8" to "CP1252"
-    cp1252 = convert(orig, "CP1252", "UTF-8")
-    # convert `cp1252` from "CP1252" to "ibm850"
-    ibm850 = convert(cp1252, "ibm850", "CP1252")
-    current = getCurrentEncoding()
-  assert orig == "\195\182\195\164\195\188\195\159"
-  assert ibm850 == "\148\132\129\225"
-  assert convert(ibm850, current, "ibm850") == orig
+  when defined(windows):
+    let
+      orig = "öäüß"
+      # convert `orig` from "UTF-8" to "CP1252"
+      cp1252 = convert(orig, "CP1252", "UTF-8")
+      # convert `cp1252` from "CP1252" to "ibm850"
+      ibm850 = convert(cp1252, "ibm850", "CP1252")
+      current = getCurrentEncoding()
+    assert orig == "\195\182\195\164\195\188\195\159"
+    assert ibm850 == "\148\132\129\225"
+    assert convert(ibm850, current, "ibm850") == orig
 
 ## The example below uses a reuseable `EncodingConverter` object which is
 ## created by `open` with `destEncoding` and `srcEncoding` specified. You can use
 ## `convert` on this object multiple times.
 runnableExamples:
-  var fromGB2312 = open("utf-8", "gb2312")
-  let first = "\203\173\197\194\163\191\210\187" &
-      "\203\242\209\204\211\234\200\206\198\189\201\250"
-  assert fromGB2312.convert(first) == "谁怕？一蓑烟雨任平生"
+  when defined(windows):
+    var fromGB2312 = open("utf-8", "gb2312")
+    let first = "\203\173\197\194\163\191\210\187" &
+        "\203\242\209\204\211\234\200\206\198\189\201\250"
+    assert fromGB2312.convert(first) == "谁怕？一蓑烟雨任平生"
 
-  let second = "\211\208\176\215\205\183\200\231" &
-      "\208\194\163\172\199\227\184\199\200\231\185\202"
-  assert fromGB2312.convert(second) == "有白头如新，倾盖如故"
+    let second = "\211\208\176\215\205\183\200\231" &
+        "\208\194\163\172\199\227\184\199\200\231\185\202"
+    assert fromGB2312.convert(second) == "有白头如新，倾盖如故"
 
 
 import os

From b3aca78e228feb93d5dcd6f00cdd81e4178c24cc Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Tue, 13 Jul 2021 12:44:29 +0200
Subject: [PATCH 1036/2472] closes #18433 (#18484)

* beneficial refactoring; use system.Endianness

* closes #18433
---
 compiler/options.nim   |  4 ++--
 compiler/platform.nim  |  6 ++----
 tests/arc/tfuncobj.nim | 38 ++++++++++++++++++++++++++++++++++++++
 3 files changed, 42 insertions(+), 6 deletions(-)
 create mode 100644 tests/arc/tfuncobj.nim

diff --git a/compiler/options.nim b/compiler/options.nim
index 135ecf1de..45ca14e76 100644
--- a/compiler/options.nim
+++ b/compiler/options.nim
@@ -592,8 +592,8 @@ proc isDefined*(conf: ConfigRef; symbol: string): bool =
       result = conf.target.targetOS == osNintendoSwitch
     of "freertos", "lwip":
       result = conf.target.targetOS == osFreeRTOS
-    of "littleendian": result = CPU[conf.target.targetCPU].endian == platform.littleEndian
-    of "bigendian": result = CPU[conf.target.targetCPU].endian == platform.bigEndian
+    of "littleendian": result = CPU[conf.target.targetCPU].endian == littleEndian
+    of "bigendian": result = CPU[conf.target.targetCPU].endian == bigEndian
     of "cpu8": result = CPU[conf.target.targetCPU].bit == 8
     of "cpu16": result = CPU[conf.target.targetCPU].bit == 16
     of "cpu32": result = CPU[conf.target.targetCPU].bit == 32
diff --git a/compiler/platform.nim b/compiler/platform.nim
index 413a91304..7bc77d809 100644
--- a/compiler/platform.nim
+++ b/compiler/platform.nim
@@ -196,13 +196,11 @@ type
     cpuSparc64, cpuMips64, cpuMips64el, cpuRiscV32, cpuRiscV64, cpuEsp, cpuWasm32
 
 type
-  TEndian* = enum
-    littleEndian, bigEndian
-  TInfoCPU* = tuple[name: string, intSize: int, endian: TEndian,
+  TInfoCPU* = tuple[name: string, intSize: int, endian: Endianness,
                     floatSize, bit: int]
 
 const
-  EndianToStr*: array[TEndian, string] = ["littleEndian", "bigEndian"]
+  EndianToStr*: array[Endianness, string] = ["littleEndian", "bigEndian"]
   CPU*: array[succ(low(TSystemCPU))..high(TSystemCPU), TInfoCPU] = [
     (name: "i386", intSize: 32, endian: littleEndian, floatSize: 64, bit: 32),
     (name: "m68k", intSize: 32, endian: bigEndian, floatSize: 64, bit: 32),
diff --git a/tests/arc/tfuncobj.nim b/tests/arc/tfuncobj.nim
new file mode 100644
index 000000000..50cd9425e
--- /dev/null
+++ b/tests/arc/tfuncobj.nim
@@ -0,0 +1,38 @@
+discard """
+  outputsub: '''f1
+f2
+f3'''
+  cmd: "nim c --gc:orc $file"
+  valgrind: true
+"""
+
+type
+  FuncObj = object
+    fn: proc (env: pointer) {.cdecl.}
+    env: pointer
+
+proc `=destroy`(x: var FuncObj) =
+  GC_unref(cast[RootRef](x.env))
+
+proc `=copy`(x: var FuncObj, y: FuncObj) {.error.}
+
+# bug #18433
+
+proc main =
+  var fs: seq[FuncObj]
+
+  proc wrap(p: proc()) =
+    proc closeOver() = p()
+    let env = rawEnv closeOver
+    GC_ref(cast[RootRef](env))
+    fs.add(FuncObj(fn: cast[proc(env: pointer){.cdecl.}](rawProc closeOver), env: env))
+
+  wrap(proc() {.closure.} = echo "f1")
+  wrap(proc() {.closure.} = echo "f2")
+  wrap(proc() {.closure.} = echo "f3")
+
+  for a in fs:
+    a.fn(a.env)
+
+main()
+

From 12da32a891b08f34acbb676001f0c211ecfe1af8 Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Tue, 13 Jul 2021 14:17:59 +0200
Subject: [PATCH 1037/2472] fixes #17893 (#18485)

* fixes #17893
---
 compiler/liftdestructors.nim    |  7 +---
 tests/arc/tshared_ptr_crash.nim | 67 +++++++++++++++++++++++++++++++++
 2 files changed, 68 insertions(+), 6 deletions(-)
 create mode 100644 tests/arc/tshared_ptr_crash.nim

diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim
index b2b70edee..cc7e36cdb 100644
--- a/compiler/liftdestructors.nim
+++ b/compiler/liftdestructors.nim
@@ -416,12 +416,7 @@ proc considerUserDefinedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
       body.add destructorCall(c, op, x)
       result = true
     #result = addDestructorCall(c, t, body, x)
-  of attachedAsgn, attachedSink:
-    var op = getAttachedOp(c.g, t, c.kind)
-    result = considerAsgnOrSink(c, t, body, x, y, op)
-    if op != nil:
-      setAttachedOp(c.g, c.idgen.module, t, c.kind, op)
-  of attachedTrace:
+  of attachedAsgn, attachedSink, attachedTrace:
     var op = getAttachedOp(c.g, t, c.kind)
     if op != nil and sfOverriden in op.flags:
       if op.ast.isGenericRoutine:
diff --git a/tests/arc/tshared_ptr_crash.nim b/tests/arc/tshared_ptr_crash.nim
new file mode 100644
index 000000000..1794834db
--- /dev/null
+++ b/tests/arc/tshared_ptr_crash.nim
@@ -0,0 +1,67 @@
+discard """
+  cmd: "nim c --threads:on --gc:arc $file"
+  action: compile
+"""
+
+# bug #17893
+
+type
+  SharedPtr*[T] = object
+    val: ptr tuple[value: T, atomicCounter: int]
+
+proc `=destroy`*[T](p: var SharedPtr[T]) =
+  mixin `=destroy`
+  if p.val != nil:
+    if atomicLoadN(addr p.val[].atomicCounter, AtomicConsume) == 0:
+      `=destroy`(p.val[])
+      deallocShared(p.val)
+    else:
+      discard atomicDec(p.val[].atomicCounter)
+
+proc `=copy`*[T](dest: var SharedPtr[T], src: SharedPtr[T]) =
+  if src.val != nil:
+    discard atomicInc(src.val[].atomicCounter)
+  if dest.val != nil:
+    `=destroy`(dest)
+  dest.val = src.val
+
+proc newSharedPtr*[T](val: sink T): SharedPtr[T] {.nodestroy.} =
+  result.val = cast[typeof(result.val)](allocShared(sizeof(result.val[])))
+  result.val.atomicCounter = 0
+  result.val.value = val
+
+proc isNil*[T](p: SharedPtr[T]): bool {.inline.} =
+  p.val == nil
+
+proc `[]`*[T](p: SharedPtr[T]): var T {.inline.} =
+  when compileOption("boundChecks"):
+    doAssert(p.val != nil, "deferencing nil shared pointer")
+  result = p.val.value
+
+type
+  Sender*[T] = object
+    queue: SharedPtr[seq[T]]
+
+proc newSender*[T](queue: sink SharedPtr[seq[T]]): Sender[T] =
+  result = Sender[T](queue: queue)
+
+proc send*[T](self: Sender[T]; t: sink T) =
+  self.queue[].add t
+
+proc newChannel*(): Sender[int] =
+  let queue = newSharedPtr(newSeq[int]())
+  result = newSender(queue)
+
+
+var
+  p: Thread[Sender[int]]
+
+proc threadFn(tx: Sender[int]) =
+  send tx, 0
+
+proc multiThreadedChannel =
+  let tx = newChannel()
+  createThread(p, threadFn, tx)
+  joinThread(p)
+
+multiThreadedChannel()

From a6ff6e7871d757821ba2b84cb8f175f48721bb9a Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Wed, 14 Jul 2021 10:52:09 -0700
Subject: [PATCH 1038/2472] fix https://github.com/dom96/choosenim/issues/256
 WSL CRLF (#18452)

* fix https://github.com/dom96/choosenim/issues/256 WSL CRLF

* fixup
---
 .gitattributes | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.gitattributes b/.gitattributes
index 674da7939..2c9ff103c 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -4,3 +4,6 @@
 # duplicated, which is easily identifiable and fixable.
 /changelog.md merge=union
 
+# bug https://github.com/dom96/choosenim/issues/256 for WSL CRLF
+*.sh text eol=lf
+/config/build_config.txt text eol=lf

From 8c6dd2b9a9157c0e0bd822d8a80d1014fc87ecd7 Mon Sep 17 00:00:00 2001
From: GordonBGood 
Date: Thu, 15 Jul 2021 19:35:53 +0700
Subject: [PATCH 1039/2472] Improve description of how =trace is used (#18491)

* [skip ci] thamming_orc test created/destroyed counts match

The thamming_orc.nim code now counts all created objects being tested, not just the ones following the "first 20" test, and the position of the `destroyed += 1` counter has been adjusted so it counts all the calls that are as a result of `=trace` tracing and not just the original destruction calls.

* Improve description of how `=trace` is used

The following nuances weren't previously fully explained:

1. That `=trace` is only used by `--gc:orc`.
2. That `=trace` is almost certainly used along with `=destroy` when manual resource allocation has been used, but it is only required if there is a possibility of cyclic references in the wrapped types (ie. generic types).
3. That, currently, a forward definition is required for the second of the pair to avoid an auto compiler generation conflict.

The pattern of the use of `=trace` has also been made more extensive, showing how both a custom `=destroy` and `=trace` are used for manual allocation of resources when there is any possibility of a cyclic reference in the resource-wrapped values.

* Update doc/destructors.rst

Co-authored-by: Andreas Rumpf 
---
 doc/destructors.rst | 27 ++++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/doc/destructors.rst b/doc/destructors.rst
index a94ccb9e7..a3fd7514e 100644
--- a/doc/destructors.rst
+++ b/doc/destructors.rst
@@ -222,16 +222,33 @@ The prototype of this hook for a type `T` needs to be:
 `env` is used by ORC to keep track of its internal state, it should be passed around
 to calls of the built-in `=trace` operation.
 
-The general pattern in `=trace` looks like:
+Usually there will only be a need for a custom `=trace` when a custom `=destroy` that deallocates manually allocated resources is also used, and then only when there is a chance of cyclic references from items within the manually allocated resources when it is desired that `--gc:orc` be able to break and collect these cyclic referenced resources.  Currently however, there is a mutual use problem in that whichever of `=destroy`/`=trace` is used first will automatically create a version of the other which will then conflict with the creation of the second of the pair.  The work around for this problem is to forward declare the second of the "hooks" to prevent the automatic creation.
+
+The general pattern in using `=destroy` with `=trace` looks like:
 
 .. code-block:: nim
 
-  proc `=trace`(dest: var T; env: pointer) =
-    for child in childrenThatCanContainPointers(dest):
-      `=trace`(child, env)
+  type
+    Test[T] = object
+      size: Natural
+      arr: ptr UncheckedArray[T] # raw pointer field
+
+  proc makeTest[T](size: Natural): Test[T] = # custom allocation...
+    Test[T](size: size, arr: cast[ptr UncheckedArray[T]](alloc0(sizeof(T) * size)))
 
 
-**Note**: The `=trace` hooks is currently more experimental and less refined
+  proc `=destroy`[T](dest: var Test[T]) =
+    if dest.arr != nil:
+      for i in 0 ..< dest.size: dest.arr[i].`=destroy`
+      dest.arr.dealloc
+
+  proc `=trace`[T](dest: var Test[T]; env: pointer) =
+    if dest.arr != nil: # trace the `T`'s which may be cyclic
+      for i in 0 ..< dest.size: dest.arr[i].`=trace` env
+
+  # following may be other custom "hooks" as required...
+
+**Note**: The `=trace` hooks (which are only used by `--gc:orc`) are currently more experimental and less refined
 than the other hooks.
 
 

From af3d2d8ad90c31161c34f0d4f82cd0ac4a9b3d00 Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Thu, 15 Jul 2021 17:58:47 +0200
Subject: [PATCH 1040/2472] added `nimAllocPagesViaMalloc` switch (#18490)

* added  switch

* alloc.nim needs page aligned memory blocks
---
 changelog.md           |  3 +++
 doc/nimc.rst           | 14 +++++++++++++-
 lib/system/alloc.nim   |  3 ++-
 lib/system/osalloc.nim | 31 +++++++++++++++++++++++++++----
 tests/arc/tarcmisc.nim |  2 +-
 5 files changed, 46 insertions(+), 7 deletions(-)

diff --git a/changelog.md b/changelog.md
index bc64e9579..d44f28b94 100644
--- a/changelog.md
+++ b/changelog.md
@@ -391,6 +391,9 @@
 - The `gc:orc` algorithm was refined so that custom container types can participate in the
   cycle collection process.
 
+- On embedded devices `malloc` can now be used instead of `mmap` via `-d:nimAllocPagesViaMalloc`.
+  This is only supported for `--gc:orc` or `--gc:arc`.
+
 
 ## Compiler changes
 
diff --git a/doc/nimc.rst b/doc/nimc.rst
index 2b9f33cb5..070956860 100644
--- a/doc/nimc.rst
+++ b/doc/nimc.rst
@@ -176,7 +176,7 @@ directories (in this order; later files overwrite previous settings):
    ``%APPDATA%/nim/nim.cfg`` (Windows).
    This file can be skipped with the `--skipUserCfg`:option: command line
    option.
-3) ``$parentDir/nim.cfg`` where ``$parentDir`` stands for any parent 
+3) ``$parentDir/nim.cfg`` where ``$parentDir`` stands for any parent
    directory of the project file's path.
    These files can be skipped with the `--skipParentCfg`:option:
    command-line option.
@@ -634,6 +634,18 @@ optimization in the compiler and linker.
 Check the `Cross-compilation`_ section for instructions on how to compile the
 program for your target.
 
+
+nimAllocPagesViaMalloc
+----------------------
+
+Nim's default allocator is based on TLSF, this algorithm was designed for embedded
+devices. This allocator gets blocks/pages of memory via a currently undocumented
+`osalloc` API which usually uses POSIX's `mmap` call. On many environments `mmap`
+is not available but C's `malloc` is. You can use the `nimAllocPagesViaMalloc`
+define to use `malloc` instead of `mmap`. `nimAllocPagesViaMalloc` is currently
+only supported with `--gc:arc` or `--gc:orc`. (Since version 1.6)
+
+
 Nim for realtime systems
 ========================
 
diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim
index c598b8250..479458950 100644
--- a/lib/system/alloc.nim
+++ b/lib/system/alloc.nim
@@ -749,8 +749,9 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer =
     inc(a.allocCounter)
   sysAssert(allocInv(a), "rawAlloc: begin")
   sysAssert(roundup(65, 8) == 72, "rawAlloc: roundup broken")
-  sysAssert(requestedSize >= sizeof(FreeCell), "rawAlloc: requested size too small")
   var size = roundup(requestedSize, MemAlign)
+  sysAssert(size >= sizeof(FreeCell), "rawAlloc: requested size too small")
+
   sysAssert(size >= requestedSize, "insufficient allocated size!")
   #c_fprintf(stdout, "alloc; size: %ld; %ld\n", requestedSize, size)
   if size <= SmallChunkSize-smallChunkOverhead():
diff --git a/lib/system/osalloc.nim b/lib/system/osalloc.nim
index 32d3b166d..2830adb48 100644
--- a/lib/system/osalloc.nim
+++ b/lib/system/osalloc.nim
@@ -28,7 +28,30 @@ const doNotUnmap = not (defined(amd64) or defined(i386)) or
                    defined(windows) or defined(nimAllocNoUnmap)
 
 
-when defined(emscripten) and not defined(StandaloneHeapSize):
+when defined(nimAllocPagesViaMalloc):
+  when not defined(gcArc) and not defined(gcOrc):
+    {.error: "-d:nimAllocPagesViaMalloc is only supported with --gc:arc or --gc:orc".}
+
+  proc osTryAllocPages(size: int): pointer {.inline.} =
+    let base = c_malloc(csize_t size + PageSize - 1 + sizeof(uint32))
+    if base == nil: raiseOutOfMem()
+    # memory layout: padding + offset (4 bytes) + user_data
+    # in order to deallocate: read offset at user_data - 4 bytes,
+    # then deallocate user_data - offset
+    let offset = PageSize - (cast[int](base) and (PageSize - 1))
+    cast[ptr uint32](base +! (offset - sizeof(uint32)))[] = uint32(offset)
+    result = base +! offset
+
+  proc osAllocPages(size: int): pointer {.inline.} =
+    result = osTryAllocPages(size)
+    if result == nil: raiseOutOfMem()
+
+  proc osDeallocPages(p: pointer, size: int) {.inline.} =
+    # read offset at p - 4 bytes, then deallocate (p - offset) pointer
+    let offset = cast[ptr uint32](p -! sizeof(uint32))[]
+    c_free(p -! offset)
+
+elif defined(emscripten) and not defined(StandaloneHeapSize):
   const
     PROT_READ  = 1             # page can be read
     PROT_WRITE = 2             # page can be written
@@ -197,11 +220,11 @@ elif defined(posix) and not defined(StandaloneHeapSize):
     PROT_WRITE = 2             # page can be written
 
   when defined(netbsd) or defined(openbsd):
-      # OpenBSD security for setjmp/longjmp coroutines
-      var MAP_STACK {.importc: "MAP_STACK", header: "".}: cint
+    # OpenBSD security for setjmp/longjmp coroutines
+    var MAP_STACK {.importc: "MAP_STACK", header: "".}: cint
   else:
     const MAP_STACK = 0             # avoid sideeffects
-    
+
   when defined(macosx) or defined(freebsd):
     const MAP_ANONYMOUS = 0x1000
     const MAP_PRIVATE = 0x02        # Changes are private
diff --git a/tests/arc/tarcmisc.nim b/tests/arc/tarcmisc.nim
index 5d5d8e914..7daea62c8 100644
--- a/tests/arc/tarcmisc.nim
+++ b/tests/arc/tarcmisc.nim
@@ -34,7 +34,7 @@ closed
 destroying variable: 20
 destroying variable: 10
 '''
-  cmd: "nim c --gc:arc --deepcopy:on $file"
+  cmd: "nim c --gc:arc --deepcopy:on -d:nimAllocPagesViaMalloc $file"
 """
 
 proc takeSink(x: sink string): bool = true

From a9701f6531288ab3ba2c2a7ff0124802ae78122c Mon Sep 17 00:00:00 2001
From: quantimnot <54247259+quantimnot@users.noreply.github.com>
Date: Thu, 15 Jul 2021 14:43:57 -0400
Subject: [PATCH 1041/2472] Extended side effect error messages (#18418)

* Extended side effect error messages

* Applied feedback:
- refactored `markSideEffect`
- refactored string interpolations
- single message
- skip diagnostics in `system.compiles` context

Other:
- started a test of diagnostic messages

[ci skip] Tests aren't updated yet because messaging isn't nailed down.

* - Added hints of where for side effect call locations.
- Tried to clarify the reasons.

* fix tests

* Applied PR review feedback:
  - moved collection of side effects from TSym to TContext
  - used pragma shorthand form `.sideEffect` and `.noSideEffect` in messages
  - added leading '>' to structured messages for readability
  - changed `sempass2.markSideEffect` to a proc
  - replaced `system.echo` in the test to make the test compatible with Windows

* Applied NEP1 formatting suggestion

Co-authored-by: quantimnot 
---
 compiler/semdata.nim                   |  1 +
 compiler/sempass2.nim                  | 74 ++++++++++++++++++++------
 tests/effects/tdiagnostic_messages.nim | 37 +++++++++++++
 3 files changed, 95 insertions(+), 17 deletions(-)
 create mode 100644 tests/effects/tdiagnostic_messages.nim

diff --git a/compiler/semdata.nim b/compiler/semdata.nim
index bd863505c..12bb22ff9 100644
--- a/compiler/semdata.nim
+++ b/compiler/semdata.nim
@@ -159,6 +159,7 @@ type
     exportIndirections*: HashSet[(int, int)] # (module.id, symbol.id)
     importModuleMap*: Table[int, int] # (module.id, module.id)
     lastTLineInfo*: TLineInfo
+    sideEffects*: Table[int, seq[(TLineInfo, PSym)]] # symbol.id index
 
 template config*(c: PContext): ConfigRef = c.graph.config
 
diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim
index e124baf26..1e1672cad 100644
--- a/compiler/sempass2.nim
+++ b/compiler/sempass2.nim
@@ -10,7 +10,7 @@
 import
   intsets, ast, astalgo, msgs, renderer, magicsys, types, idents, trees,
   wordrecg, strutils, options, guards, lineinfos, semfold, semdata,
-  modulegraphs, varpartitions, typeallowed, nilcheck, errorhandling
+  modulegraphs, varpartitions, typeallowed, nilcheck, errorhandling, tables
 
 when defined(useDfa):
   import dfa
@@ -223,13 +223,21 @@ proc markGcUnsafe(a: PEffects; reason: PNode) =
         a.owner.gcUnsafetyReason = newSym(skUnknown, a.owner.name, nextSymId a.c.idgen,
                                           a.owner, reason.info, {})
 
-when true:
-  template markSideEffect(a: PEffects; reason: typed) =
-    if not a.inEnforcedNoSideEffects: a.hasSideEffect = true
-else:
-  template markSideEffect(a: PEffects; reason: typed) =
-    if not a.inEnforcedNoSideEffects: a.hasSideEffect = true
-    markGcUnsafe(a, reason)
+proc markSideEffect(a: PEffects; reason: PNode | PSym; useLoc: TLineInfo) =
+  if not a.inEnforcedNoSideEffects:
+    a.hasSideEffect = true
+    if a.owner.kind in routineKinds:
+      var sym: PSym
+      when reason is PNode:
+        if reason.kind == nkSym:
+          sym = reason.sym
+        else:
+          let kind = if reason.kind == nkHiddenDeref: skParam else: skUnknown
+          sym = newSym(kind, a.owner.name, nextSymId a.c.idgen, a.owner, reason.info, {})
+      else:
+        sym = reason
+      a.c.sideEffects.mgetOrPut(a.owner.id, @[]).add (useLoc, sym)
+    when false: markGcUnsafe(a, reason)
 
 proc listGcUnsafety(s: PSym; onlyWarning: bool; cycleCheck: var IntSet; conf: ConfigRef) =
   let u = s.gcUnsafetyReason
@@ -259,6 +267,31 @@ proc listGcUnsafety(s: PSym; onlyWarning: bool; conf: ConfigRef) =
   var cycleCheck = initIntSet()
   listGcUnsafety(s, onlyWarning, cycleCheck, conf)
 
+proc listSideEffects(result: var string; s: PSym; cycleCheck: var IntSet;
+                     conf: ConfigRef; context: PContext; indentLevel: int) =
+  template addHint(msg; lineInfo; sym; level = indentLevel) =
+    result.addf("$# $# Hint: '$#' $#\n", repeat(">", level), conf $ lineInfo, sym, msg)
+  if context.sideEffects.hasKey(s.id):
+    for (useLineInfo, u) in context.sideEffects[s.id]:
+      if u != nil and not cycleCheck.containsOrIncl(u.id):
+        case u.kind
+        of skLet, skVar:
+          addHint("accesses global state '$#'" % u.name.s, useLineInfo, s.name.s)
+          addHint("accessed by '$#'" % s.name.s, u.info, u.name.s, indentLevel + 1)
+        of routineKinds:
+          addHint("calls `.sideEffect` '$#'" % u.name.s, useLineInfo, s.name.s)
+          addHint("called by '$#'" % s.name.s, u.info, u.name.s, indentLevel + 1)
+          listSideEffects(result, u, cycleCheck, conf, context, indentLevel + 2)
+        of skParam, skForVar:
+          addHint("calls routine via hidden pointer indirection", useLineInfo, s.name.s)
+        else:
+          addHint("calls routine via pointer indirection", useLineInfo, s.name.s)
+
+proc listSideEffects(result: var string; s: PSym; conf: ConfigRef; context: PContext) =
+  var cycleCheck = initIntSet()
+  result.addf("'$#' can have side effects\n", s.name.s)
+  listSideEffects(result, s, cycleCheck, conf, context, 1)
+
 proc useVarNoInitCheck(a: PEffects; n: PNode; s: PSym) =
   if {sfGlobal, sfThread} * s.flags != {} and s.kind in {skVar, skLet} and
       s.magic != mNimvm:
@@ -267,9 +300,7 @@ proc useVarNoInitCheck(a: PEffects; n: PNode; s: PSym) =
         (tfHasGCedMem in s.typ.flags or s.typ.isGCedMem):
       #if a.config.hasWarn(warnGcUnsafe): warnAboutGcUnsafe(n)
       markGcUnsafe(a, s)
-      markSideEffect(a, s)
-    else:
-      markSideEffect(a, s)
+    markSideEffect(a, s, n.info)
   if s.owner != a.owner and s.kind in {skVar, skLet, skForVar, skResult, skParam} and
      {sfGlobal, sfThread} * s.flags == {}:
     a.isInnerProc = true
@@ -502,7 +533,7 @@ proc propagateEffects(tracked: PEffects, n: PNode, s: PSym) =
     if tracked.config.hasWarn(warnGcUnsafe): warnAboutGcUnsafe(n, tracked.config)
     markGcUnsafe(tracked, s)
   if tfNoSideEffect notin s.typ.flags:
-    markSideEffect(tracked, s)
+    markSideEffect(tracked, s, n.info)
   mergeLockLevels(tracked, n, s.getLockLevel)
 
 proc procVarCheck(n: PNode; conf: ConfigRef) =
@@ -584,7 +615,7 @@ proc trackOperandForIndirectCall(tracked: PEffects, n: PNode, paramType: PType;
         if tracked.config.hasWarn(warnGcUnsafe): warnAboutGcUnsafe(n, tracked.config)
         markGcUnsafe(tracked, a)
       elif tfNoSideEffect notin op.flags and not isOwnedProcVar(a, tracked.owner):
-        markSideEffect(tracked, a)
+        markSideEffect(tracked, a, n.info)
     else:
       mergeRaises(tracked, effectList[exceptionEffects], n)
       mergeTags(tracked, effectList[tagEffects], n)
@@ -592,7 +623,7 @@ proc trackOperandForIndirectCall(tracked: PEffects, n: PNode, paramType: PType;
         if tracked.config.hasWarn(warnGcUnsafe): warnAboutGcUnsafe(n, tracked.config)
         markGcUnsafe(tracked, a)
       elif tfNoSideEffect notin op.flags:
-        markSideEffect(tracked, a)
+        markSideEffect(tracked, a, n.info)
   if paramType != nil and paramType.kind in {tyVar}:
     invalidateFacts(tracked.guards, n)
     if n.kind == nkSym and isLocalVar(tracked, n.sym):
@@ -749,7 +780,7 @@ proc trackCall(tracked: PEffects; n: PNode) =
     if tfNoSideEffect notin op.flags and not importedFromC(a):
       # and it's not a recursive call:
       if not (a.kind == nkSym and a.sym == tracked.owner):
-        markSideEffect(tracked, a)
+        markSideEffect(tracked, a, n.info)
 
   # p's effects are ours too:
   var a = n[0]
@@ -771,7 +802,7 @@ proc trackCall(tracked: PEffects; n: PNode) =
         if a.sym == tracked.owner: tracked.isRecursive = true
         # even for recursive calls we need to check the lock levels (!):
         mergeLockLevels(tracked, n, a.sym.getLockLevel)
-        if sfSideEffect in a.sym.flags: markSideEffect(tracked, a)
+        if sfSideEffect in a.sym.flags: markSideEffect(tracked, a, n.info)
       else:
         mergeLockLevels(tracked, n, op.lockLevel)
       var effectList = op.n[0]
@@ -1336,6 +1367,7 @@ proc trackProc*(c: PContext; s: PSym, body: PNode) =
     effects[ensuresEffects] = ensuresSpec
 
   var mutationInfo = MutationInfo()
+  var hasMutationSideEffect = false
   if {strictFuncs, views} * c.features != {}:
     var goals: set[Goal] = {}
     if strictFuncs in c.features: goals.incl constParameters
@@ -1343,6 +1375,7 @@ proc trackProc*(c: PContext; s: PSym, body: PNode) =
     var partitions = computeGraphPartitions(s, body, g, goals)
     if not t.hasSideEffect and t.hasDangerousAssign:
       t.hasSideEffect = varpartitions.hasSideEffect(partitions, mutationInfo)
+      hasMutationSideEffect = t.hasSideEffect
     if views in c.features:
       checkBorrowedLocations(partitions, body, g.config)
 
@@ -1357,7 +1390,14 @@ proc trackProc*(c: PContext; s: PSym, body: PNode) =
     when false:
       listGcUnsafety(s, onlyWarning=false, g.config)
     else:
-      localError(g.config, s.info, ("'$1' can have side effects" % s.name.s) & (g.config $ mutationInfo))
+      if hasMutationSideEffect:
+        localError(g.config, s.info, "'$1' can have side effects$2" % [s.name.s, g.config $ mutationInfo])
+      elif c.compilesContextId == 0: # don't render extended diagnostic messages in `system.compiles` context
+        var msg = ""
+        listSideEffects(msg, s, g.config, t.c)
+        message(g.config, s.info, errGenerated, msg)
+      else:
+        localError(g.config, s.info, "") # simple error for `system.compiles` context
   if not t.gcUnsafe:
     s.typ.flags.incl tfGcSafe
   if not t.hasSideEffect and sfSideEffect notin s.flags:
diff --git a/tests/effects/tdiagnostic_messages.nim b/tests/effects/tdiagnostic_messages.nim
new file mode 100644
index 000000000..2ce4895a3
--- /dev/null
+++ b/tests/effects/tdiagnostic_messages.nim
@@ -0,0 +1,37 @@
+discard """
+  nimoutFull: true
+  action: "reject"
+  cmd: "nim r --hint:Conf:off $file"
+  nimout: '''
+tdiagnostic_messages.nim(36, 6) Error: 'a' can have side effects
+> tdiagnostic_messages.nim(37, 30) Hint: 'a' calls `.sideEffect` 'callWithSideEffects'
+>> tdiagnostic_messages.nim(29, 6) Hint: 'callWithSideEffects' called by 'a'
+>>> tdiagnostic_messages.nim(31, 34) Hint: 'callWithSideEffects' calls `.sideEffect` 'indirectCallViaVarParam'
+>>>> tdiagnostic_messages.nim(25, 6) Hint: 'indirectCallViaVarParam' called by 'callWithSideEffects'
+>>>>> tdiagnostic_messages.nim(26, 7) Hint: 'indirectCallViaVarParam' calls routine via hidden pointer indirection
+>>> tdiagnostic_messages.nim(32, 33) Hint: 'callWithSideEffects' calls `.sideEffect` 'indirectCallViaPointer'
+>>>> tdiagnostic_messages.nim(27, 6) Hint: 'indirectCallViaPointer' called by 'callWithSideEffects'
+>>>>> tdiagnostic_messages.nim(28, 32) Hint: 'indirectCallViaPointer' calls routine via pointer indirection
+>>> tdiagnostic_messages.nim(33, 10) Hint: 'callWithSideEffects' calls `.sideEffect` 'myEcho'
+>>>> tdiagnostic_messages.nim(24, 6) Hint: 'myEcho' called by 'callWithSideEffects'
+>>> tdiagnostic_messages.nim(34, 3) Hint: 'callWithSideEffects' accesses global state 'globalVar'
+>>>> tdiagnostic_messages.nim(23, 5) Hint: 'globalVar' accessed by 'callWithSideEffects'
+
+'''
+"""
+
+var globalVar = 0
+proc myEcho(a: string) {.sideEffect.} = discard
+proc indirectCallViaVarParam(call: var proc(): int {.nimcall.}): int =
+  call()
+proc indirectCallViaPointer(call: pointer): int =
+  cast[ptr proc(): int](call)[]()
+proc callWithSideEffects(): int =
+  var p = proc (): int {.nimcall.} = 0
+  discard indirectCallViaVarParam(p)
+  discard indirectCallViaPointer(addr p)
+  myEcho ""
+  globalVar
+
+func a: int =
+  discard callWithSideEffects()

From 1d6863a7899fd87fd9eb017ae370ef37db18ad32 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Thu, 15 Jul 2021 11:54:07 -0700
Subject: [PATCH 1042/2472] deprecate `sequtils.delete` and add an overload
 with saner semantics consistent with `system.delete` (#18487)

* deprecate sequtils.delete and add an overload with saner semantics
* AssertionDefect => IndexDefect
* improve tsequtils
* add tests; use splice in js for optimization
---
 changelog.md                      |   3 +
 lib/pure/collections/sequtils.nim |  45 +++++++++-
 tests/stdlib/tsequtils.nim        | 144 ++++++++++++++++++++----------
 3 files changed, 144 insertions(+), 48 deletions(-)

diff --git a/changelog.md b/changelog.md
index d44f28b94..d07ddddb4 100644
--- a/changelog.md
+++ b/changelog.md
@@ -346,6 +346,9 @@
 
 - Added `dom.setInterval`, `dom.clearInterval` overloads.
 
+- Deprecated `sequtils.delete` and added an overload taking a `Slice` that raises a defect
+  if the slice is out of bounds.
+
 ## Language changes
 
 - `nimscript` now handles `except Exception as e`.
diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim
index 6facfcc32..5b41ea00d 100644
--- a/lib/pure/collections/sequtils.nim
+++ b/lib/pure/collections/sequtils.nim
@@ -509,12 +509,51 @@ proc keepIf*[T](s: var seq[T], pred: proc(x: T): bool {.closure.})
       inc(pos)
   setLen(s, pos)
 
-func delete*[T](s: var seq[T]; first, last: Natural) =
+func delete*[T](s: var seq[T]; slice: Slice[int]) =
+  ## Deletes the items `s[slice]`, raising `IndexDefect` if the slice contains
+  ## elements out of range.
+  ##
+  ## This operation moves all elements after `s[slice]` in linear time.
+  runnableExamples:
+    var a = @[10, 11, 12, 13, 14]
+    doAssertRaises(IndexDefect): a.delete(4..5)
+    assert a == @[10, 11, 12, 13, 14]
+    a.delete(4..4)
+    assert a == @[10, 11, 12, 13]
+    a.delete(1..2)
+    assert a == @[10, 13]
+    a.delete(1..<1) # empty slice
+    assert a == @[10, 13]
+  when compileOption("boundChecks"):
+    if not (slice.a < s.len and slice.a >= 0 and slice.b < s.len):
+      raise newException(IndexDefect, $(slice: slice, len: s.len))
+  if slice.b >= slice.a:
+    template defaultImpl =
+      var i = slice.a
+      var j = slice.b + 1
+      var newLen = s.len - j + i
+      while i < newLen:
+        when defined(gcDestructors):
+          s[i] = move(s[j])
+        else:
+          s[i].shallowCopy(s[j])
+        inc(i)
+        inc(j)
+      setLen(s, newLen)
+    when nimvm: defaultImpl()
+    else:
+      when defined(js):
+        let n = slice.b - slice.a + 1
+        let first = slice.a
+        {.emit: "`s`.splice(`first`, `n`);".}
+      else:
+        defaultImpl()
+
+func delete*[T](s: var seq[T]; first, last: Natural) {.deprecated: "use `delete(s, first..last)`".} =
   ## Deletes the items of a sequence `s` at positions `first..last`
   ## (including both ends of the range).
   ## This modifies `s` itself, it does not return a copy.
-  ##
-  runnableExamples:
+  runnableExamples("--warning:deprecated:off"):
     let outcome = @[1, 1, 1, 1, 1, 1, 1, 1]
     var dest = @[1, 1, 1, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1]
     dest.delete(3, 8)
diff --git a/tests/stdlib/tsequtils.nim b/tests/stdlib/tsequtils.nim
index 385e6e651..6c9c92873 100644
--- a/tests/stdlib/tsequtils.nim
+++ b/tests/stdlib/tsequtils.nim
@@ -1,3 +1,9 @@
+discard """
+  targets: "c js"
+"""
+
+# xxx move all tests under `main`
+
 import std/sequtils
 import strutils
 from algorithm import sorted
@@ -190,17 +196,6 @@ block: # keepIf test
   keepIf(floats, proc(x: float): bool = x > 10)
   doAssert floats == @[13.0, 12.5, 10.1]
 
-block: # delete tests
-  let outcome = @[1, 1, 1, 1, 1, 1, 1, 1]
-  var dest = @[1, 1, 1, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1]
-  dest.delete(3, 8)
-  doAssert outcome == dest, """\
-  Deleting range 3-9 from [1,1,1,2,2,2,2,2,2,1,1,1,1,1]
-  is [1,1,1,1,1,1,1,1]"""
-  var x = @[1, 2, 3]
-  x.delete(100, 100)
-  doAssert x == @[1, 2, 3]
-
 block: # insert tests
   var dest = @[1, 1, 1, 1, 1, 1, 1, 1]
   let
@@ -298,43 +293,45 @@ block: # toSeq test
     doAssert myIter.toSeq == @[1, 2]
     doAssert toSeq(myIter) == @[1, 2]
 
-  block:
-    iterator myIter(): int {.closure.} =
-      yield 1
-      yield 2
+  when not defined(js):
+    # pending #4695
+    block:
+        iterator myIter(): int {.closure.} =
+          yield 1
+          yield 2
 
-    doAssert myIter.toSeq == @[1, 2]
-    doAssert toSeq(myIter) == @[1, 2]
-
-  block:
-    proc myIter(): auto =
-      iterator ret(): int {.closure.} =
-        yield 1
-        yield 2
-      result = ret
-
-    doAssert myIter().toSeq == @[1, 2]
-    doAssert toSeq(myIter()) == @[1, 2]
-
-  block:
-    proc myIter(n: int): auto =
-      var counter = 0
-      iterator ret(): int {.closure.} =
-        while counter < n:
-          yield counter
-          counter.inc
-      result = ret
+        doAssert myIter.toSeq == @[1, 2]
+        doAssert toSeq(myIter) == @[1, 2]
 
     block:
-      let myIter3 = myIter(3)
-      doAssert myIter3.toSeq == @[0, 1, 2]
+      proc myIter(): auto =
+        iterator ret(): int {.closure.} =
+          yield 1
+          yield 2
+        result = ret
+
+      doAssert myIter().toSeq == @[1, 2]
+      doAssert toSeq(myIter()) == @[1, 2]
+
     block:
-      let myIter3 = myIter(3)
-      doAssert toSeq(myIter3) == @[0, 1, 2]
-    block:
-      # makes sure this does not hang forever
-      doAssert myIter(3).toSeq == @[0, 1, 2]
-      doAssert toSeq(myIter(3)) == @[0, 1, 2]
+      proc myIter(n: int): auto =
+        var counter = 0
+        iterator ret(): int {.closure.} =
+          while counter < n:
+            yield counter
+            counter.inc
+        result = ret
+
+      block:
+        let myIter3 = myIter(3)
+        doAssert myIter3.toSeq == @[0, 1, 2]
+      block:
+        let myIter3 = myIter(3)
+        doAssert toSeq(myIter3) == @[0, 1, 2]
+      block:
+        # makes sure this does not hang forever
+        doAssert myIter(3).toSeq == @[0, 1, 2]
+        doAssert toSeq(myIter(3)) == @[0, 1, 2]
 
 block:
   # tests https://github.com/nim-lang/Nim/issues/7187
@@ -452,4 +449,61 @@ block:
       for i in 0..
Date: Thu, 15 Jul 2021 23:13:01 +0300
Subject: [PATCH 1043/2472] Add changes required by Nimble lock file support
 (#12104)

Implemented support for Nimble local cache with package directories with
a checksum of the package at the end of their names. Now the compiler
supports package paths in the form:

 * /path_to_nimble_cache_dir/pkgs/package_name-1.2.3-
FEBADEAEA2345E777F0F6F8433F7F0A52EDD5D1B

 * /path_to_nimble_cache_dir/pkgs/package_name-#head-
042D4BE2B90ED0672E717D71850ABDB0A2D19CD2

 * /path_to_nimble_cache_dir/pkgs/package_name-#branch-name-
DBC1F902CB79946E990E38AF51F0BAD36ACFABD9

Related to nim-lang/nimble#127
---
 compiler/modulepaths.nim      |  2 +-
 compiler/nimblecmd.nim        | 78 +++++++++++++++++++-----------
 lib/std/sha1.nim              |  4 ++
 tests/compiler/tnimblecmd.nim | 91 +++++++++++++++++++++++++++--------
 tests/stdlib/tsha1.nim        | 10 ++++
 5 files changed, 135 insertions(+), 50 deletions(-)

diff --git a/compiler/modulepaths.nim b/compiler/modulepaths.nim
index 8511b1592..a16b669c4 100644
--- a/compiler/modulepaths.nim
+++ b/compiler/modulepaths.nim
@@ -21,7 +21,7 @@ when false:
     for k, p in os.walkDir(dir, relative=true):
       if k == pcDir and p.len > pkg.len+1 and
           p[pkg.len] == '-' and p.startsWith(pkg):
-        let (_, a) = getPathVersion(p)
+        let (_, a, _) = getPathVersionChecksum(p)
         if bestv.len == 0 or bestv < a:
           bestv = a
           best = dir / p
diff --git a/compiler/nimblecmd.nim b/compiler/nimblecmd.nim
index 28398289c..9cd2941ba 100644
--- a/compiler/nimblecmd.nim
+++ b/compiler/nimblecmd.nim
@@ -9,8 +9,8 @@
 
 ## Implements some helper procs for Nimble (Nim's package manager) support.
 
-import parseutils, strutils, strtabs, os, options, msgs, sequtils,
-  lineinfos, pathutils
+import parseutils, strutils, os, options, msgs, sequtils, lineinfos, pathutils,
+  std/sha1, tables
 
 proc addPath*(conf: ConfigRef; path: AbsoluteDir, info: TLineInfo) =
   if not conf.searchPaths.contains(path):
@@ -18,6 +18,7 @@ proc addPath*(conf: ConfigRef; path: AbsoluteDir, info: TLineInfo) =
 
 type
   Version* = distinct string
+  PackageInfo = Table[string, tuple[version, checksum: string]]
 
 proc `$`*(ver: Version): string {.borrow.}
 
@@ -62,43 +63,64 @@ proc `<`*(ver: Version, ver2: Version): bool =
     else:
       return false
 
-proc getPathVersion*(p: string): tuple[name, version: string] =
-  ## Splits path ``p`` in the format ``/home/user/.nimble/pkgs/package-0.1``
-  ## into ``(/home/user/.nimble/pkgs/package, 0.1)``
-  result.name = ""
-  result.version = ""
+proc getPathVersionChecksum*(p: string): tuple[name, version, checksum: string] =
+  ## Splits path ``p`` in the format
+  ## ``/home/user/.nimble/pkgs/package-0.1-febadeaea2345e777f0f6f8433f7f0a52edd5d1b`` into
+  ## ``("/home/user/.nimble/pkgs/package", "0.1", "febadeaea2345e777f0f6f8433f7f0a52edd5d1b")``
 
-  const specialSeparator = "-#"
-  let last = p.rfind(p.lastPathPart) # the index where the last path part begins
-  var sepIdx = p.find(specialSeparator, start = last)
-  if sepIdx == -1:
-    sepIdx = p.rfind('-', start = last)
+  const checksumSeparator = '-'
+  const versionSeparator = '-'
+  const specialVersionSepartator = "-#"
+  const separatorNotFound = -1
 
-  if sepIdx == -1:
-    result.name = p
-    return
+  var checksumSeparatorIndex = p.rfind(checksumSeparator)
+  if checksumSeparatorIndex != separatorNotFound:
+    result.checksum = p.substr(checksumSeparatorIndex + 1)
+    if not result.checksum.isValidSha1Hash():
+      result.checksum = ""
+      checksumSeparatorIndex = p.len()
+  else:
+    checksumSeparatorIndex = p.len()
 
-  for i in sepIdx..
Date: Fri, 16 Jul 2021 07:42:35 +0200
Subject: [PATCH 1044/2472] added Atlas helper tool (#18497)

* added Atlas helper tool
* further improvements
---
 koch.nim                       |   8 +-
 tools/atlas/atlas.md           |  77 +++++++
 tools/atlas/atlas.nim          | 390 +++++++++++++++++++++++++++++++++
 tools/atlas/osutils.nim        |  59 +++++
 tools/atlas/packagesjson.nim   | 117 ++++++++++
 tools/atlas/parse_requires.nim | 101 +++++++++
 6 files changed, 751 insertions(+), 1 deletion(-)
 create mode 100644 tools/atlas/atlas.md
 create mode 100644 tools/atlas/atlas.nim
 create mode 100644 tools/atlas/osutils.nim
 create mode 100644 tools/atlas/packagesjson.nim
 create mode 100644 tools/atlas/parse_requires.nim

diff --git a/koch.nim b/koch.nim
index 4ba0d0eb5..da7576e65 100644
--- a/koch.nim
+++ b/koch.nim
@@ -222,7 +222,13 @@ proc buildTools(args: string = "") =
   # `-d:nimDebugUtils` only makes sense when temporarily editing/debugging compiler
   # `-d:debug` should be changed to a flag that doesn't require re-compiling nim
   # `--opt:speed` is a sensible default even for a debug build, it doesn't affect nim stacktraces
-  nimCompileFold("Compile nim_dbg", "compiler/nim.nim", options = "--opt:speed --stacktrace -d:debug --stacktraceMsgs -d:nimCompilerStacktraceHints " & args, outputName = "nim_dbg")
+  nimCompileFold("Compile nim_dbg", "compiler/nim.nim", options =
+      "--opt:speed --stacktrace -d:debug --stacktraceMsgs -d:nimCompilerStacktraceHints " & args,
+      outputName = "nim_dbg")
+
+  nimCompileFold("Compile atlas", "tools/atlas/atlas.nim", options = "-d:release " & args,
+      outputName = "atlas")
+
 
 proc nsis(latest: bool; args: string) =
   bundleNimbleExe(latest, args)
diff --git a/tools/atlas/atlas.md b/tools/atlas/atlas.md
new file mode 100644
index 000000000..a36817dc5
--- /dev/null
+++ b/tools/atlas/atlas.md
@@ -0,0 +1,77 @@
+# Atlas Package Cloner
+
+Atlas is a simple package cloner tool that automates some of the
+workflows and needs for Nim's stdlib evolution.
+
+Atlas is compatible with Nimble in the sense that it supports the Nimble
+file format.
+
+
+## How it works
+
+Atlas uses git commits internally; version requirements are translated
+to git commits via `git show-ref --tags`.
+
+Atlas uses URLs internally; Nimble package names are translated to URLs
+via Nimble's  `packages.json` file.
+
+Atlas does not call the Nim compiler for a build, instead it creates/patches
+a `nim.cfg` file for the compiler. For example:
+
+```
+############# begin Atlas config section ##########
+--noNimblePath
+--path:"../nimx"
+--path:"../sdl2/src"
+--path:"../opengl/src"
+############# end Atlas config section   ##########
+```
+
+The version selection is deterministic, it picks up the *minimum* required
+version. Thanks to this design, lock files are not required.
+
+
+## Dependencies
+
+Dependencies are neither installed globally, nor locally into the current
+project. Instead a "workspace" is used. The workspace is the nearest parent
+directory of the current directory that does not contain a `.git` subdirectory.
+Dependencies are managed as **siblings**, not as children. Dependencies are
+kept as git repositories.
+
+Thanks to this setup, it's easy to develop multiple projects at the same time.
+
+A project plus its dependencies are stored in a workspace:
+
+  $workspace / main project
+  $workspace / dependency A
+  $workspace / dependency B
+
+
+No attempts are being made at keeping directory hygiene inside the
+workspace, you're supposed to create appropriate `$workspace` directories
+at your own leisure.
+
+
+## Commands
+
+Atlas supports the following commands:
+
+
+### Clone 
+
+Clones a URL and all of its dependencies (recursively) into the workspace.
+Creates or patches a `nim.cfg` file with the required `--path` entries.
+
+
+### Clone 
+
+The `` is translated into an URL via `packages.json` and
+then `clone ` is performed.
+
+
+### Search 
+
+Search the package index `packages.json` for a package that the given terms
+in its description (or name or list of tags).
+
diff --git a/tools/atlas/atlas.nim b/tools/atlas/atlas.nim
new file mode 100644
index 000000000..bc1cc6ae3
--- /dev/null
+++ b/tools/atlas/atlas.nim
@@ -0,0 +1,390 @@
+#
+#           Atlas Package Cloner
+#        (c) Copyright 2021 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Simple tool to automate frequent workflows: Can "clone"
+## a Nimble dependency and its dependencies recursively.
+
+import std/[parseopt, strutils, os, osproc, sequtils, unicode, tables, sets]
+import parse_requires, osutils, packagesjson
+
+const
+  Version = "0.1"
+  Usage = "atlas - Nim Package Manager Version " & Version & """
+
+  (c) 2021 Andreas Rumpf
+Usage:
+  atlas [options] [command] [arguments]
+Command:
+  clone url|pkgname     clone a package and all of its dependencies
+  search keyw keywB...  search for package that contains the given keywords
+
+Options:
+  --keepCommits         do not perform any `git checkouts`
+  --version             show the version
+  --help                show this help
+"""
+
+proc writeHelp() =
+  stdout.write(Usage)
+  stdout.flushFile()
+  quit(0)
+
+proc writeVersion() =
+  stdout.write(Version & "\n")
+  stdout.flushFile()
+  quit(0)
+
+type
+  PackageName = distinct string
+  DepRelation = enum
+    normal, strictlyLess, strictlyGreater
+
+  Dependency = object
+    name: PackageName
+    url, commit: string
+    rel: DepRelation # "requires x < 1.0" is silly, but Nimble allows it so we have too.
+  AtlasContext = object
+    projectDir, workspace: string
+    hasPackageList: bool
+    keepCommits: bool
+    p: Table[string, string] # name -> url mapping
+    processed: HashSet[string] # the key is (url / commit)
+    errors: int
+
+const
+  InvalidCommit = ""
+
+proc toDepRelation(s: string): DepRelation =
+  case s
+  of "<": strictlyLess
+  of ">": strictlyGreater
+  else: normal
+
+proc isCleanGit(dir: string): string =
+  result = ""
+  let (outp, status) = osproc.execCmdEx("git diff")
+  if outp.len != 0:
+    result = "'git diff' not empty"
+  elif status != 0:
+    result = "'git diff' returned non-zero"
+
+proc message(c: var AtlasContext; category: string; p: PackageName; args: varargs[string]) =
+  var msg = category & "(" & p.string & ")"
+  for a in args:
+    msg.add ' '
+    msg.add a
+  stdout.writeLine msg
+  inc c.errors
+
+proc warn(c: var AtlasContext; p: PackageName; args: varargs[string]) =
+  message(c, "[Warning] ", p, args)
+
+proc error(c: var AtlasContext; p: PackageName; args: varargs[string]) =
+  message(c, "[Error] ", p, args)
+
+proc sameVersionAs(tag, ver: string): bool =
+  const VersionChars = {'0'..'9', '.'}
+
+  proc safeCharAt(s: string; i: int): char {.inline.} =
+    if i >= 0 and i < s.len: s[i] else: '\0'
+
+  let idx = find(tag, ver)
+  if idx >= 0:
+    # we found the version as a substring inside the `tag`. But we
+    # need to watch out the the boundaries are not part of a
+    # larger/different version number:
+    result = safeCharAt(tag, idx-1) notin VersionChars and
+      safeCharAt(tag, idx+ver.len) notin VersionChars
+
+proc versionToCommit(d: Dependency): string =
+  let (outp, status) = osproc.execCmdEx("git show-ref --tags")
+  if status == 0:
+    var useNextOne = false
+    for line in splitLines(outp):
+      let commitsAndTags = strutils.splitWhitespace(line)
+      if commitsAndTags.len == 2:
+        case d.rel
+        of normal:
+          if commitsAndTags[1].sameVersionAs(d.commit):
+            return commitsAndTags[0]
+        of strictlyLess:
+          if d.commit == InvalidCommit or not commitsAndTags[1].sameVersionAs(d.commit):
+            return commitsAndTags[0]
+        of strictlyGreater:
+          if commitsAndTags[1].sameVersionAs(d.commit):
+            useNextOne = true
+          elif useNextOne:
+            return commitsAndTags[0]
+
+  return ""
+
+proc checkoutGitCommit(c: var AtlasContext; p: PackageName; commit: string) =
+  let (outp, status) = osproc.execCmdEx("git checkout " & quoteShell(commit))
+  if status != 0:
+    error(c, p, "could not checkout commit", commit)
+
+proc gitPull(c: var AtlasContext; p: PackageName) =
+  let (_, status) = osproc.execCmdEx("git pull")
+  if status != 0:
+    error(c, p, "could not 'git pull'")
+
+proc updatePackages(c: var AtlasContext) =
+  if dirExists(c.workspace / PackagesDir):
+    withDir(c.workspace / PackagesDir):
+      gitPull(c, PackageName PackagesDir)
+  else:
+    withDir c.workspace:
+      let err = cloneUrl("https://github.com/nim-lang/packages", PackagesDir, false)
+      if err != "":
+        error c, PackageName(PackagesDir), err
+
+proc fillPackageLookupTable(c: var AtlasContext) =
+  if not c.hasPackageList:
+    c.hasPackageList = true
+    updatePackages(c)
+    let plist = getPackages(c.workspace)
+    for entry in plist:
+      c.p[unicode.toLower entry.name] = entry.url
+
+proc toUrl(c: var AtlasContext; p: string): string =
+  if p.isUrl:
+    result = p
+  else:
+    fillPackageLookupTable(c)
+    result = c.p.getOrDefault(unicode.toLower p)
+  if result.len == 0:
+    inc c.errors
+
+proc toName(p: string): PackageName =
+  if p.isUrl:
+    result = PackageName splitFile(p).name
+  else:
+    result = PackageName p
+
+proc needsCommitLookup(commit: string): bool {.inline} =
+  '.' in commit or commit == InvalidCommit
+
+proc checkoutCommit(c: var AtlasContext; w: Dependency) =
+  let dir = c.workspace / w.name.string
+  withDir dir:
+    if w.commit.len == 0 or cmpIgnoreCase(w.commit, "#head") == 0:
+      gitPull(c, w.name)
+    else:
+      let err = isCleanGit(dir)
+      if err != "":
+        warn c, w.name, err
+      else:
+        let requiredCommit = if needsCommitLookup(w.commit): versionToCommit(w) else: w.commit
+        let (cc, status) = osproc.execCmdEx("git log -n 1 --format=%H")
+        let currentCommit = strutils.strip(cc)
+        if requiredCommit == "" or status != 0:
+          if requiredCommit == "" and w.commit == InvalidCommit:
+            warn c, w.name, "package has no tagged releases"
+          else:
+            warn c, w.name, "cannot find specified version/commit", w.commit
+        else:
+          if currentCommit != requiredCommit:
+            # checkout the later commit:
+            # git merge-base --is-ancestor  
+            let (mergeBase, status) = osproc.execCmdEx("git merge-base " &
+                currentCommit.quoteShell & " " & requiredCommit.quoteShell)
+            if status == 0 and (mergeBase == currentCommit or mergeBase == requiredCommit):
+              # conflict resolution: pick the later commit:
+              if mergeBase == currentCommit:
+                checkoutGitCommit(c, w.name, requiredCommit)
+            else:
+              checkoutGitCommit(c, w.name, requiredCommit)
+              when false:
+                warn c, w.name, "do not know which commit is more recent:",
+                  currentCommit, "(current) or", w.commit, " =", requiredCommit, "(required)"
+
+proc findNimbleFile(c: AtlasContext; dep: Dependency): string =
+  result = c.workspace / dep.name.string / (dep.name.string & ".nimble")
+  if not fileExists(result):
+    result = ""
+    for x in walkFiles(c.workspace / dep.name.string / "*.nimble"):
+      if result.len == 0:
+        result = x
+      else:
+        # ambiguous .nimble file
+        return ""
+
+proc addUniqueDep(c: var AtlasContext; work: var seq[Dependency];
+                  tokens: seq[string]) =
+  let oldErrors = c.errors
+  let url = toUrl(c, tokens[0])
+  if oldErrors != c.errors:
+    warn c, toName(tokens[0]), "cannot resolve package name"
+  elif not c.processed.containsOrIncl(url / tokens[2]):
+    work.add Dependency(name: toName(tokens[0]), url: url, commit: tokens[2],
+                        rel: toDepRelation(tokens[1]))
+
+proc collectNewDeps(c: var AtlasContext; work: var seq[Dependency];
+                    dep: Dependency; result: var seq[string];
+                    isMainProject: bool) =
+  let nimbleFile = findNimbleFile(c, dep)
+  if nimbleFile != "":
+    let nimbleInfo = extractRequiresInfo(nimbleFile)
+    for r in nimbleInfo.requires:
+      var tokens: seq[string] = @[]
+      for token in tokenizeRequires(r):
+        tokens.add token
+      if tokens.len == 1:
+        # nimx uses dependencies like 'requires "sdl2"'.
+        # Via this hack we map them to the first tagged release.
+        # (See the `isStrictlySmallerThan` logic.)
+        tokens.add "<"
+        tokens.add InvalidCommit
+      elif tokens.len == 2 and tokens[1].startsWith("#"):
+        # Dependencies can also look like 'requires "sdl2#head"
+        var commit = tokens[1]
+        tokens[1] = "=="
+        tokens.add commit
+
+      if tokens.len >= 3 and cmpIgnoreCase(tokens[0], "nim") != 0:
+        c.addUniqueDep work, tokens
+
+    result.add dep.name.string / nimbleInfo.srcDir
+  else:
+    result.add dep.name.string
+
+proc clone(c: var AtlasContext; start: string): seq[string] =
+  # non-recursive clone.
+  let oldErrors = c.errors
+  var work = @[Dependency(name: toName(start), url: toUrl(c, start), commit: "")]
+
+  if oldErrors != c.errors:
+    error c, toName(start), "cannot resolve package name"
+    return
+
+  c.projectDir = work[0].name.string
+  result = @[]
+  var i = 0
+  while i < work.len:
+    let w = work[i]
+    let oldErrors = c.errors
+    if not dirExists(c.workspace / w.name.string):
+      withDir c.workspace:
+        let err = cloneUrl(w.url, w.name.string, false)
+        if err != "":
+          error c, w.name, err
+    if oldErrors == c.errors:
+      if not c.keepCommits: checkoutCommit(c, w)
+      # even if the checkout fails, we can make use of the somewhat
+      # outdated .nimble file to clone more of the most likely still relevant
+      # dependencies:
+      collectNewDeps(c, work, w, result, i == 0)
+    inc i
+
+const
+  configPatternBegin = "############# begin Atlas config section ##########\n"
+  configPatternEnd =   "############# end Atlas config section   ##########\n"
+
+proc patchNimCfg(c: AtlasContext; deps: seq[string]) =
+  var paths = "--noNimblePath\n"
+  for d in deps:
+    paths.add "--path:\"../" & d.replace("\\", "/") & "\"\n"
+
+  let cfg = c.projectDir / "nim.cfg"
+  var cfgContent = configPatternBegin & paths & configPatternEnd
+  if not fileExists(cfg):
+    writeFile(cfg, cfgContent)
+  else:
+    let content = readFile(cfg)
+    let start = content.find(configPatternBegin)
+    if start >= 0:
+      cfgContent = content.substr(0, start-1) & cfgContent
+      let theEnd = content.find(configPatternEnd, start)
+      if theEnd >= 0:
+        cfgContent.add content.substr(theEnd+len(configPatternEnd))
+    else:
+      cfgContent = content & "\n" & cfgContent
+    if cfgContent != content:
+      # do not touch the file if nothing changed
+      # (preserves the file date information):
+      writeFile(cfg, cfgContent)
+
+proc error*(msg: string) =
+  when defined(debug):
+    writeStackTrace()
+  quit "[Error] " & msg
+
+proc main =
+  var action = ""
+  var args: seq[string] = @[]
+  template singleArg() =
+    if args.len != 1:
+      error action & " command takes a single package name"
+
+  template noArgs() =
+    if args.len != 0:
+      error action & " command takes no arguments"
+
+  var c = AtlasContext(
+    projectDir: getCurrentDir(),
+    workspace: getCurrentDir())
+
+  for kind, key, val in getopt():
+    case kind
+    of cmdArgument:
+      if action.len == 0:
+        action = key.normalize
+      else:
+        args.add key
+    of cmdLongOption, cmdShortOption:
+      case normalize(key)
+      of "help", "h": writeHelp()
+      of "version", "v": writeVersion()
+      of "keepcommits": c.keepCommits = true
+      else: writeHelp()
+    of cmdEnd: assert false, "cannot happen"
+
+  while c.workspace.len > 0 and dirExists(c.workspace / ".git"):
+    c.workspace = c.workspace.parentDir()
+
+  case action
+  of "":
+    error "No action."
+  of "clone":
+    singleArg()
+    let deps = clone(c, args[0])
+    patchNimCfg c, deps
+    if c.errors > 0:
+      error "There were problems."
+  of "refresh":
+    noArgs()
+    updatePackages(c)
+  of "search", "list":
+    updatePackages(c)
+    search getPackages(c.workspace), args
+  else:
+    error "Invalid action: " & action
+
+when isMainModule:
+  main()
+
+when false:
+  # some testing code for the `patchNimCfg` logic:
+  var c = AtlasContext(
+    projectDir: getCurrentDir(),
+    workspace: getCurrentDir().parentDir)
+
+  patchNimCfg(c, @[PackageName"abc", PackageName"xyz"])
+
+when false:
+  assert sameVersionAs("v0.2.0", "0.2.0")
+  assert sameVersionAs("v1", "1")
+
+  assert sameVersionAs("1.90", "1.90")
+
+  assert sameVersionAs("v1.2.3-zuzu", "1.2.3")
+  assert sameVersionAs("foo-1.2.3.4", "1.2.3.4")
+
+  assert not sameVersionAs("foo-1.2.3.4", "1.2.3")
+  assert not sameVersionAs("foo", "1.2.3")
+  assert not sameVersionAs("", "1.2.3")
diff --git a/tools/atlas/osutils.nim b/tools/atlas/osutils.nim
new file mode 100644
index 000000000..31392b113
--- /dev/null
+++ b/tools/atlas/osutils.nim
@@ -0,0 +1,59 @@
+## OS utilities like 'withDir'.
+## (c) 2021 Andreas Rumpf
+
+import os, strutils, osproc
+
+template withDir*(dir, body) =
+  let oldDir = getCurrentDir()
+  try:
+    setCurrentDir(dir)
+    body
+  finally:
+    setCurrentDir(oldDir)
+
+proc isUrl*(x: string): bool =
+  x.startsWith("git://") or x.startsWith("https://") or x.startsWith("http://")
+
+proc cloneUrl*(url, dest: string; cloneUsingHttps: bool): string =
+  ## Returns an error message on error or else "".
+  result = ""
+  var modUrl =
+    if url.startsWith("git://") and cloneUsingHttps:
+      "https://" & url[6 .. ^1]
+    else: url
+
+  # github + https + trailing url slash causes a
+  # checkout/ls-remote to fail with Repository not found
+  var isGithub = false
+  if modUrl.contains("github.com") and modUrl.endswith("/"):
+    modUrl = modUrl[0 .. ^2]
+    isGithub = true
+
+  let (_, exitCode) = execCmdEx("git ls-remote --quiet --tags " & modUrl)
+  var xcode = exitCode
+  if isGithub and exitCode != QuitSuccess:
+    # retry multiple times to avoid annoying github timeouts:
+    for i in 0..4:
+      os.sleep(4000)
+      xcode = execCmdEx("git ls-remote --quiet --tags " & modUrl)[1]
+      if xcode == QuitSuccess: break
+
+  if xcode == QuitSuccess:
+    # retry multiple times to avoid annoying github timeouts:
+    let cmd = "git clone " & modUrl & " " & dest
+    for i in 0..4:
+      if execShellCmd(cmd) == 0: return ""
+      os.sleep(4000)
+    result = "exernal program failed: " & cmd
+  elif not isGithub:
+    let (_, exitCode) = execCmdEx("hg identify " & modUrl)
+    if exitCode == QuitSuccess:
+      let cmd = "hg clone " & modUrl & " " & dest
+      for i in 0..4:
+        if execShellCmd(cmd) == 0: return ""
+        os.sleep(4000)
+      result = "exernal program failed: " & cmd
+    else:
+      result = "Unable to identify url: " & modUrl
+  else:
+    result = "Unable to identify url: " & modUrl
diff --git a/tools/atlas/packagesjson.nim b/tools/atlas/packagesjson.nim
new file mode 100644
index 000000000..e1e23ee29
--- /dev/null
+++ b/tools/atlas/packagesjson.nim
@@ -0,0 +1,117 @@
+
+import std / [json, os, sets, strutils]
+import osutils
+
+type
+  Package* = ref object
+    # Required fields in a package.
+    name*: string
+    url*: string # Download location.
+    license*: string
+    downloadMethod*: string
+    description*: string
+    tags*: seq[string] # \
+    # From here on, optional fields set to the empty string if not available.
+    version*: string
+    dvcsTag*: string
+    web*: string # Info url for humans.
+
+proc optionalField(obj: JsonNode, name: string, default = ""): string =
+  if hasKey(obj, name) and obj[name].kind == JString:
+    result = obj[name].str
+  else:
+    result = default
+
+proc requiredField(obj: JsonNode, name: string): string =
+  result = optionalField(obj, name, "")
+
+proc fromJson*(obj: JSonNode): Package =
+  result = Package()
+  result.name = obj.requiredField("name")
+  if result.name.len == 0: return nil
+  result.version = obj.optionalField("version")
+  result.url = obj.requiredField("url")
+  if result.url.len == 0: return nil
+  result.downloadMethod = obj.requiredField("method")
+  if result.downloadMethod.len == 0: return nil
+  result.dvcsTag = obj.optionalField("dvcs-tag")
+  result.license = obj.optionalField("license")
+  result.tags = @[]
+  for t in obj["tags"]:
+    result.tags.add(t.str)
+  result.description = obj.requiredField("description")
+  result.web = obj.optionalField("web")
+
+const PackagesDir* = "packages"
+
+proc getPackages*(workspaceDir: string): seq[Package] =
+  result = @[]
+  var uniqueNames = initHashSet[string]()
+  var jsonFiles = 0
+  for kind, path in walkDir(workspaceDir / PackagesDir):
+    if kind == pcFile and path.endsWith(".json"):
+      inc jsonFiles
+      let packages = json.parseFile(path)
+      for p in packages:
+        let pkg = p.fromJson()
+        if pkg != nil and not uniqueNames.containsOrIncl(pkg.name):
+          result.add(pkg)
+
+proc `$`*(pkg: Package): string =
+  result = pkg.name & ":\n"
+  result &= "  url:         " & pkg.url & " (" & pkg.downloadMethod & ")\n"
+  result &= "  tags:        " & pkg.tags.join(", ") & "\n"
+  result &= "  description: " & pkg.description & "\n"
+  result &= "  license:     " & pkg.license & "\n"
+  if pkg.web.len > 0:
+    result &= "  website:     " & pkg.web & "\n"
+
+proc search*(pkgList: seq[Package]; terms: seq[string]) =
+  var found = false
+  template onFound =
+    echo pkg
+    echo("")
+    found = true
+    break forPackage
+
+  for pkg in pkgList:
+    if terms.len > 0:
+      block forPackage:
+        for term in terms:
+          let word = term.toLower
+          # Search by name.
+          if word in pkg.name.toLower:
+            onFound()
+          # Search by tag.
+          for tag in pkg.tags:
+            if word in tag.toLower:
+              onFound()
+    else:
+      echo(pkg)
+      echo(" ")
+
+  if not found and terms.len > 0:
+    echo("No package found.")
+
+type PkgCandidates* = array[3, seq[Package]]
+
+proc determineCandidates*(pkgList: seq[Package];
+                         terms: seq[string]): PkgCandidates =
+  result[0] = @[]
+  result[1] = @[]
+  result[2] = @[]
+  for pkg in pkgList:
+    block termLoop:
+      for term in terms:
+        let word = term.toLower
+        if word == pkg.name.toLower:
+          result[0].add pkg
+          break termLoop
+        elif word in pkg.name.toLower:
+          result[1].add pkg
+          break termLoop
+        else:
+          for tag in pkg.tags:
+            if word in tag.toLower:
+              result[2].add pkg
+              break termLoop
diff --git a/tools/atlas/parse_requires.nim b/tools/atlas/parse_requires.nim
new file mode 100644
index 000000000..7e26a1656
--- /dev/null
+++ b/tools/atlas/parse_requires.nim
@@ -0,0 +1,101 @@
+## Utility API for Nim package managers.
+## (c) 2021 Andreas Rumpf
+
+import std / strutils
+import ".." / compiler / [ast, idents, msgs, syntaxes, options, pathutils]
+
+type
+  NimbleFileInfo* = object
+    requires*: seq[string]
+    srcDir*: string
+    tasks*: seq[(string, string)]
+
+proc extract(n: PNode; conf: ConfigRef; result: var NimbleFileInfo) =
+  case n.kind
+  of nkStmtList, nkStmtListExpr:
+    for child in n:
+      extract(child, conf, result)
+  of nkCallKinds:
+    if n[0].kind == nkIdent:
+      case n[0].ident.s
+      of "requires":
+        for i in 1.. 0: ch = ch.lastSon
+          if ch.kind in {nkStrLit..nkTripleStrLit}:
+            result.requires.add ch.strVal
+          else:
+            localError(conf, ch.info, "'requires' takes string literals")
+      of "task":
+        if n.len >= 3 and n[1].kind == nkIdent and n[2].kind in {nkStrLit..nkTripleStrLit}:
+          result.tasks.add((n[1].ident.s, n[2].strVal))
+      else: discard
+  of nkAsgn, nkFastAsgn:
+    if n[0].kind == nkIdent and cmpIgnoreCase(n[0].ident.s, "srcDir") == 0:
+      if n[1].kind in {nkStrLit..nkTripleStrLit}:
+        result.srcDir = n[1].strVal
+      else:
+        localError(conf, n[1].info, "assignments to 'srcDir' must be string literals")
+  else:
+    discard
+
+proc extractRequiresInfo*(nimbleFile: string): NimbleFileInfo =
+  ## Extract the `requires` information from a Nimble file. This does **not**
+  ## evaluate the Nimble file. Errors are produced on stderr/stdout and are
+  ## formatted as the Nim compiler does it. The parser uses the Nim compiler
+  ## as an API. The result can be empty, this is not an error, only parsing
+  ## errors are reported.
+  var conf = newConfigRef()
+  conf.foreignPackageNotes = {}
+  conf.notes = {}
+  conf.mainPackageNotes = {}
+
+  let fileIdx = fileInfoIdx(conf, AbsoluteFile nimbleFile)
+  var parser: Parser
+  if setupParser(parser, fileIdx, newIdentCache(), conf):
+    extract(parseAll(parser), conf, result)
+    closeParser(parser)
+
+const Operators* = {'<', '>', '=', '&', '@', '!', '^'}
+
+proc token(s: string; idx: int; lit: var string): int =
+  var i = idx
+  if i >= s.len: return i
+  while s[i] in Whitespace: inc(i)
+  case s[i]
+  of Letters, '#':
+    lit.add s[i]
+    inc i
+    while i < s.len and s[i] notin (Whitespace + {'@', '#'}):
+      lit.add s[i]
+      inc i
+  of '0'..'9':
+    while i < s.len and s[i] in {'0'..'9', '.'}:
+      lit.add s[i]
+      inc i
+  of '"':
+    inc i
+    while i < s.len and s[i] != '"':
+      lit.add s[i]
+      inc i
+    inc i
+  of Operators:
+    while i < s.len and s[i] in Operators:
+      lit.add s[i]
+      inc i
+  else:
+    lit.add s[i]
+    inc i
+  result = i
+
+iterator tokenizeRequires*(s: string): string =
+  var start = 0
+  var tok = ""
+  while start < s.len:
+    tok.setLen 0
+    start = token(s, start, tok)
+    yield tok
+
+when isMainModule:
+  for x in tokenizeRequires("jester@#head >= 1.5 & <= 1.8"):
+    echo x

From 96a7f9b31c84f14906b201d5e3b17c93d3878af6 Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Fri, 16 Jul 2021 15:50:02 +0200
Subject: [PATCH 1045/2472] deprecate cuchar, don't redefine it (#18505)

---
 changelog.md                       |  3 ++-
 lib/posix/posix_linux_amd64.nim    |  2 +-
 lib/posix/posix_nintendoswitch.nim | 12 ++++-----
 lib/pure/bitops.nim                | 42 +++++++++++++++---------------
 lib/pure/net.nim                   |  6 ++---
 lib/pure/terminal.nim              |  2 +-
 lib/std/sysrand.nim                |  2 +-
 lib/system.nim                     |  4 +--
 lib/windows/winlean.nim            |  4 +--
 lib/wrappers/openssl.nim           | 24 ++++++++---------
 tests/stdlib/topenssl.nim          |  8 +++---
 11 files changed, 55 insertions(+), 54 deletions(-)

diff --git a/changelog.md b/changelog.md
index d07ddddb4..568ff0bba 100644
--- a/changelog.md
+++ b/changelog.md
@@ -6,7 +6,8 @@
 
 - Deprecated `std/mersenne`
 
--  `cuchar` now aliases `uint8` instead of `char`
+- `cuchar` is now deprecated as it aliased `char` where arguably it should have aliased `uint8`.
+  Please use `char` or `uint8` instead.
 
 - `repr` now doesn't insert trailing newline; previous behavior was very inconsistent,
   see #16034. Use `-d:nimLegacyReprWithNewline` for previous behavior.
diff --git a/lib/posix/posix_linux_amd64.nim b/lib/posix/posix_linux_amd64.nim
index 7f6a589f0..5c0b4d6cf 100644
--- a/lib/posix/posix_linux_amd64.nim
+++ b/lib/posix/posix_linux_amd64.nim
@@ -47,7 +47,7 @@ type
     d_ino*: Ino
     d_off*: Off
     d_reclen*: cushort
-    d_type*: int8  # cuchar really!
+    d_type*: int8  # uint8 really!
     d_name*: array[256, cchar]
 
   Tflock* {.importc: "struct flock", final, pure,
diff --git a/lib/posix/posix_nintendoswitch.nim b/lib/posix/posix_nintendoswitch.nim
index 44e431437..4cef80bb7 100644
--- a/lib/posix/posix_nintendoswitch.nim
+++ b/lib/posix/posix_nintendoswitch.nim
@@ -33,7 +33,7 @@ type
   Dirent* {.importc: "struct dirent",
             header: "", final, pure.} = object ## dirent_t struct
     d_ino*: Ino
-    d_type*: int8  # cuchar really!
+    d_type*: int8  # uint8 really!
     d_name*: array[256, cchar]
 
   Tflock* {.importc: "struct flock", final, pure,
@@ -347,20 +347,20 @@ type
 
   SockAddr* {.importc: "struct sockaddr", header: "",
               pure, final.} = object ## struct sockaddr
-    sa_len: cuchar
+    sa_len: uint8
     sa_family*: TSa_Family         ## Address family.
     sa_data*: array[14, char] ## Socket address (variable-length data).
 
   Sockaddr_storage* {.importc: "struct sockaddr_storage",
                        header: "",
                        pure, final.} = object ## struct sockaddr_storage
-    ss_len: cuchar
+    ss_len: uint8
     ss_family*: TSa_Family ## Address family.
-    ss_padding1: array[64 - sizeof(cuchar) - sizeof(cshort), char]
+    ss_padding1: array[64 - sizeof(uint8) - sizeof(cshort), char]
     ss_align: clonglong
     ss_padding2: array[
-      128 - sizeof(cuchar) - sizeof(cshort) -
-      (64 - sizeof(cuchar) - sizeof(cshort)) - 64, char]
+      128 - sizeof(uint8) - sizeof(cshort) -
+      (64 - sizeof(uint8) - sizeof(cshort)) - 64, char]
 
   Tif_nameindex* {.importc: "struct if_nameindex", final,
                    pure, header: "".} = object ## struct if_nameindex
diff --git a/lib/pure/bitops.nim b/lib/pure/bitops.nim
index b5b08985e..6d3865e5d 100644
--- a/lib/pure/bitops.nim
+++ b/lib/pure/bitops.nim
@@ -450,15 +450,15 @@ when useGCC_builtins:
 
 elif useVCC_builtins:
   # Search the mask data from most significant bit (MSB) to least significant bit (LSB) for a set bit (1).
-  func bitScanReverse(index: ptr culong, mask: culong): cuchar {.
+  func bitScanReverse(index: ptr culong, mask: culong): uint8 {.
       importc: "_BitScanReverse", header: "".}
-  func bitScanReverse64(index: ptr culong, mask: uint64): cuchar {.
+  func bitScanReverse64(index: ptr culong, mask: uint64): uint8 {.
       importc: "_BitScanReverse64", header: "".}
 
   # Search the mask data from least significant bit (LSB) to the most significant bit (MSB) for a set bit (1).
-  func bitScanForward(index: ptr culong, mask: culong): cuchar {.
+  func bitScanForward(index: ptr culong, mask: culong): uint8 {.
       importc: "_BitScanForward", header: "".}
-  func bitScanForward64(index: ptr culong, mask: uint64): cuchar {.
+  func bitScanForward64(index: ptr culong, mask: uint64): uint8 {.
       importc: "_BitScanForward64", header: "".}
 
   template vcc_scan_impl(fnc: untyped; v: untyped): int =
@@ -468,15 +468,15 @@ elif useVCC_builtins:
 
 elif useICC_builtins:
   # Returns the number of trailing 0-bits in x, starting at the least significant bit position. If x is 0, the result is undefined.
-  func bitScanForward(p: ptr uint32, b: uint32): cuchar {.
+  func bitScanForward(p: ptr uint32, b: uint32): uint8 {.
       importc: "_BitScanForward", header: "".}
-  func bitScanForward64(p: ptr uint32, b: uint64): cuchar {.
+  func bitScanForward64(p: ptr uint32, b: uint64): uint8 {.
       importc: "_BitScanForward64", header: "".}
 
   # Returns the number of leading 0-bits in x, starting at the most significant bit position. If x is 0, the result is undefined.
-  func bitScanReverse(p: ptr uint32, b: uint32): cuchar {.
+  func bitScanReverse(p: ptr uint32, b: uint32): uint8 {.
       importc: "_BitScanReverse", header: "".}
-  func bitScanReverse64(p: ptr uint32, b: uint64): cuchar {.
+  func bitScanReverse64(p: ptr uint32, b: uint64): uint8 {.
       importc: "_BitScanReverse64", header: "".}
 
   template icc_scan_impl(fnc: untyped; v: untyped): int =
@@ -664,7 +664,7 @@ when useBuiltinsRotate:
   when defined(gcc):
     # GCC was tested until version 4.8.1 and intrinsics were present. Not tested
     # in previous versions.
-    func builtin_rotl8(value: cuchar, shift: cint): cuchar
+    func builtin_rotl8(value: uint8, shift: cint): uint8
                       {.importc: "__rolb", header: "".}
     func builtin_rotl16(value: cushort, shift: cint): cushort
                        {.importc: "__rolw", header: "".}
@@ -674,7 +674,7 @@ when useBuiltinsRotate:
       func builtin_rotl64(value: culonglong, shift: cint): culonglong
                          {.importc: "__rolq", header: "".}
 
-    func builtin_rotr8(value: cuchar, shift: cint): cuchar
+    func builtin_rotr8(value: uint8, shift: cint): uint8
                       {.importc: "__rorb", header: "".}
     func builtin_rotr16(value: cushort, shift: cint): cushort
                        {.importc: "__rorw", header: "".}
@@ -690,7 +690,7 @@ when useBuiltinsRotate:
     # https://releases.llvm.org/8.0.0/tools/clang/docs/ReleaseNotes.html#non-comprehensive-list-of-changes-in-this-release
     # https://releases.llvm.org/8.0.0/tools/clang/docs/LanguageExtensions.html#builtin-rotateleft
     # source for correct declarations: https://github.com/llvm/llvm-project/blob/main/clang/include/clang/Basic/Builtins.def
-    func builtin_rotl8(value: cuchar, shift: cuchar): cuchar
+    func builtin_rotl8(value: uint8, shift: uint8): uint8
                       {.importc: "__builtin_rotateleft8", nodecl.}
     func builtin_rotl16(value: cushort, shift: cushort): cushort
                        {.importc: "__builtin_rotateleft16", nodecl.}
@@ -700,7 +700,7 @@ when useBuiltinsRotate:
       func builtin_rotl64(value: culonglong, shift: culonglong): culonglong
                          {.importc: "__builtin_rotateleft64", nodecl.}
 
-    func builtin_rotr8(value: cuchar, shift: cuchar): cuchar
+    func builtin_rotr8(value: uint8, shift: uint8): uint8
                       {.importc: "__builtin_rotateright8", nodecl.}
     func builtin_rotr16(value: cushort, shift: cushort): cushort
                        {.importc: "__builtin_rotateright16", nodecl.}
@@ -716,9 +716,9 @@ when useBuiltinsRotate:
     # https://docs.microsoft.com/en-us/cpp/intrinsics/rotl8-rotl16?view=msvc-160
     # https://docs.microsoft.com/en-us/cpp/intrinsics/rotr8-rotr16?view=msvc-160
     # https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/rotl-rotl64-rotr-rotr64?view=msvc-160
-    func builtin_rotl8(value: cuchar, shift: cuchar): cuchar
+    func builtin_rotl8(value: uint8, shift: uint8): uint8
                       {.importc: "_rotl8", header: "".}
-    func builtin_rotl16(value: cushort, shift: cuchar): cushort
+    func builtin_rotl16(value: cushort, shift: uint8): cushort
                        {.importc: "_rotl16", header: "".}
     func builtin_rotl32(value: cuint, shift: cint): cuint
                        {.importc: "_rotl", header: "".}
@@ -726,9 +726,9 @@ when useBuiltinsRotate:
       func builtin_rotl64(value: culonglong, shift: cint): culonglong
                          {.importc: "_rotl64", header: "".}
 
-    func builtin_rotr8(value: cuchar, shift: cuchar): cuchar
+    func builtin_rotr8(value: uint8, shift: uint8): uint8
                       {.importc: "_rotr8", header: "".}
-    func builtin_rotr16(value: cushort, shift: cuchar): cushort
+    func builtin_rotr16(value: cushort, shift: uint8): cushort
                        {.importc: "_rotr16", header: "".}
     func builtin_rotr32(value: cuint, shift: cint): cuint
                        {.importc: "_rotr", header: "".}
@@ -738,7 +738,7 @@ when useBuiltinsRotate:
   elif defined(icl):
     # Tested on Intel(R) C++ Intel(R) 64 Compiler Classic Version 2021.1.2 Build
     # 20201208_000000 x64 and x86. Not tested in previous versions.
-    func builtin_rotl8(value: cuchar, shift: cint): cuchar
+    func builtin_rotl8(value: uint8, shift: cint): uint8
                       {.importc: "__rolb", header: "".}
     func builtin_rotl16(value: cushort, shift: cint): cushort
                        {.importc: "__rolw", header: "".}
@@ -748,7 +748,7 @@ when useBuiltinsRotate:
       func builtin_rotl64(value: culonglong, shift: cint): culonglong
                          {.importc: "__rolq", header: "".}
 
-    func builtin_rotr8(value: cuchar, shift: cint): cuchar
+    func builtin_rotr8(value: uint8, shift: cint): uint8
                       {.importc: "__rorb", header: "".}
     func builtin_rotr16(value: cushort, shift: cint): cushort
                        {.importc: "__rorw", header: "".}
@@ -777,7 +777,7 @@ func shiftTypeTo(size: static int, shift: int): auto {.inline.} =
   when (defined(vcc) and (size in [4, 8])) or defined(gcc) or defined(icl):
     cint(shift)
   elif (defined(vcc) and (size in [1, 2])) or (defined(clang) and size == 1):
-    cuchar(shift)
+    uint8(shift)
   elif defined(clang):
     when size == 2:
       cushort(shift)
@@ -802,7 +802,7 @@ func rotateLeftBits*[T: SomeUnsignedInt](value: T, shift: range[0..(sizeof(T) *
     when useBuiltinsRotate:
       const size = sizeof(T)
       when size == 1:
-        builtin_rotl8(value.cuchar, shiftTypeTo(size, shift)).T
+        builtin_rotl8(value.uint8, shiftTypeTo(size, shift)).T
       elif size == 2:
         builtin_rotl16(value.cushort, shiftTypeTo(size, shift)).T
       elif size == 4:
@@ -830,7 +830,7 @@ func rotateRightBits*[T: SomeUnsignedInt](value: T, shift: range[0..(sizeof(T) *
     when useBuiltinsRotate:
       const size = sizeof(T)
       when size == 1:
-        builtin_rotr8(value.cuchar, shiftTypeTo(size, shift)).T
+        builtin_rotr8(value.uint8, shiftTypeTo(size, shift)).T
       elif size == 2:
         builtin_rotr16(value.cushort, shiftTypeTo(size, shift)).T
       elif size == 4:
diff --git a/lib/pure/net.nim b/lib/pure/net.nim
index 9c3827751..934dd034e 100644
--- a/lib/pure/net.nim
+++ b/lib/pure/net.nim
@@ -503,7 +503,7 @@ when defineSsl:
     ## Retrieve the ssl pointer of `socket`.
     ## Useful for interfacing with `openssl`.
     self.sslHandle
-    
+
   proc raiseSSLError*(s = "") =
     ## Raises a new SSL error.
     if s != "":
@@ -688,7 +688,7 @@ when defineSsl:
     return ctx.getExtraInternal().clientGetPskFunc
 
   proc pskClientCallback(ssl: SslPtr; hint: cstring; identity: cstring;
-      max_identity_len: cuint; psk: ptr cuchar;
+      max_identity_len: cuint; psk: ptr uint8;
       max_psk_len: cuint): cuint {.cdecl.} =
     let ctx = SslContext(context: ssl.SSL_get_SSL_CTX)
     let hintString = if hint == nil: "" else: $hint
@@ -714,7 +714,7 @@ when defineSsl:
   proc serverGetPskFunc*(ctx: SslContext): SslServerGetPskFunc =
     return ctx.getExtraInternal().serverGetPskFunc
 
-  proc pskServerCallback(ssl: SslCtx; identity: cstring; psk: ptr cuchar;
+  proc pskServerCallback(ssl: SslCtx; identity: cstring; psk: ptr uint8;
       max_psk_len: cint): cuint {.cdecl.} =
     let ctx = SslContext(context: ssl.SSL_get_SSL_CTX)
     let pskString = (ctx.serverGetPskFunc)($identity)
diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim
index 283652bb3..ce5660557 100644
--- a/lib/pure/terminal.nim
+++ b/lib/pure/terminal.nim
@@ -761,7 +761,7 @@ template styledWriteLine*(f: File, args: varargs[untyped]) =
   runnableExamples:
     proc error(msg: string) =
       styledWriteLine(stderr, fgRed, "Error: ", resetStyle, msg)
-  
+
   styledWrite(f, args)
   write(f, "\n")
 
diff --git a/lib/std/sysrand.nim b/lib/std/sysrand.nim
index 4c51577ac..75983e3c6 100644
--- a/lib/std/sysrand.nim
+++ b/lib/std/sysrand.nim
@@ -132,7 +132,7 @@ elif defined(windows):
   type
     PVOID = pointer
     BCRYPT_ALG_HANDLE = PVOID
-    PUCHAR = ptr cuchar
+    PUCHAR = ptr uint8
     NTSTATUS = clong
     ULONG = culong
 
diff --git a/lib/system.nim b/lib/system.nim
index 8aa9bd718..57d620e57 100644
--- a/lib/system.nim
+++ b/lib/system.nim
@@ -1427,8 +1427,8 @@ type # these work for most platforms:
     ## This is the same as the type `long double` in *C*.
     ## This C type is not supported by Nim's code generator.
 
-  cuchar* {.importc: "unsigned char", nodecl.} = uint8
-    ## This is the same as the type `unsigned char` in *C*.
+  cuchar* {.importc: "unsigned char", nodecl, deprecated: "use `char` or `uint8` instead".} = char
+    ## Deprecated: Use `uint8` instead.
   cushort* {.importc: "unsigned short", nodecl.} = uint16
     ## This is the same as the type `unsigned short` in *C*.
   cuint* {.importc: "unsigned int", nodecl.} = uint32
diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim
index a71dd97d8..d901af244 100644
--- a/lib/windows/winlean.nim
+++ b/lib/windows/winlean.nim
@@ -42,7 +42,7 @@ type
   PULONG_PTR* = ptr uint
   HDC* = Handle
   HGLRC* = Handle
-  BYTE* = cuchar
+  BYTE* = uint8
 
   SECURITY_ATTRIBUTES* {.final, pure.} = object
     nLength*: int32
@@ -717,7 +717,7 @@ const
   FILE_WRITE_DATA* = 0x00000002 # file & pipe
 
 # Error Constants
-const 
+const
   ERROR_FILE_NOT_FOUND* = 2 ## https://docs.microsoft.com/en-us/windows/win32/debug/system-error-codes--0-499-
   ERROR_PATH_NOT_FOUND* = 3
   ERROR_ACCESS_DENIED* = 5
diff --git a/lib/wrappers/openssl.nim b/lib/wrappers/openssl.nim
index 1a8893957..c756fa938 100644
--- a/lib/wrappers/openssl.nim
+++ b/lib/wrappers/openssl.nim
@@ -520,16 +520,16 @@ proc OPENSSL_sk_num*(stack: PSTACK): int {.cdecl, dynlib: DLLSSLName, importc.}
 proc OPENSSL_sk_value*(stack: PSTACK, index: int): pointer {.cdecl,
     dynlib: DLLSSLName, importc.}
 
-proc d2i_X509*(px: ptr PX509, i: ptr ptr cuchar, len: cint): PX509 {.cdecl,
+proc d2i_X509*(px: ptr PX509, i: ptr ptr uint8, len: cint): PX509 {.cdecl,
     dynlib: DLLUtilName, importc.}
 
-proc i2d_X509*(cert: PX509; o: ptr ptr cuchar): cint {.cdecl,
+proc i2d_X509*(cert: PX509; o: ptr ptr uint8): cint {.cdecl,
     dynlib: DLLUtilName, importc.}
 
 proc d2i_X509*(b: string): PX509 =
   ## decode DER/BER bytestring into X.509 certificate struct
   var bb = b.cstring
-  let i = cast[ptr ptr cuchar](addr bb)
+  let i = cast[ptr ptr uint8](addr bb)
   let ret = d2i_X509(addr result, i, b.len.cint)
   if ret.isNil:
     raise newException(Exception, "X.509 certificate decoding failed")
@@ -539,7 +539,7 @@ proc i2d_X509*(cert: PX509): string =
   let encoded_length = i2d_X509(cert, nil)
   result = newString(encoded_length)
   var q = result.cstring
-  let o = cast[ptr ptr cuchar](addr q)
+  let o = cast[ptr ptr uint8](addr q)
   let length = i2d_X509(cert, o)
   if length.int <= 0:
     raise newException(Exception, "X.509 certificate encoding failed")
@@ -599,11 +599,11 @@ proc SSL_CTX_set_tlsext_servername_arg*(ctx: SslCtx, arg: pointer): int =
 
 type
   PskClientCallback* = proc (ssl: SslPtr;
-    hint: cstring; identity: cstring; max_identity_len: cuint; psk: ptr cuchar;
+    hint: cstring; identity: cstring; max_identity_len: cuint; psk: ptr uint8;
     max_psk_len: cuint): cuint {.cdecl.}
 
   PskServerCallback* = proc (ssl: SslPtr;
-    identity: cstring; psk: ptr cuchar; max_psk_len: cint): cuint {.cdecl.}
+    identity: cstring; psk: ptr uint8; max_psk_len: cint): cuint {.cdecl.}
 
 proc SSL_CTX_set_psk_client_callback*(ctx: SslCtx; callback: PskClientCallback) {.cdecl, dynlib: DLLSSLName, importc.}
   ## Set callback called when OpenSSL needs PSK (for client).
@@ -672,13 +672,13 @@ proc PEM_read_bio_RSAPublicKey*(bp: BIO, x: ptr PRSA, cb: pem_password_cb, u: po
     dynlib: DLLUtilName, importc.}
 proc PEM_read_bio_RSAPrivateKey*(bp: BIO, x: ptr PRSA, cb: pem_password_cb, u: pointer): PRSA {.cdecl,
     dynlib: DLLUtilName, importc.}
-proc RSA_private_encrypt*(flen: cint, fr: ptr cuchar, to: ptr cuchar, rsa: PRSA, padding: PaddingType): cint {.cdecl,
+proc RSA_private_encrypt*(flen: cint, fr: ptr uint8, to: ptr uint8, rsa: PRSA, padding: PaddingType): cint {.cdecl,
     dynlib: DLLUtilName, importc.}
-proc RSA_public_encrypt*(flen: cint, fr: ptr cuchar, to: ptr cuchar, rsa: PRSA, padding: PaddingType): cint {.cdecl,
+proc RSA_public_encrypt*(flen: cint, fr: ptr uint8, to: ptr uint8, rsa: PRSA, padding: PaddingType): cint {.cdecl,
     dynlib: DLLUtilName, importc.}
-proc RSA_private_decrypt*(flen: cint, fr: ptr cuchar, to: ptr cuchar, rsa: PRSA, padding: PaddingType): cint {.cdecl,
+proc RSA_private_decrypt*(flen: cint, fr: ptr uint8, to: ptr uint8, rsa: PRSA, padding: PaddingType): cint {.cdecl,
     dynlib: DLLUtilName, importc.}
-proc RSA_public_decrypt*(flen: cint, fr: ptr cuchar, to: ptr cuchar, rsa: PRSA, padding: PaddingType): cint {.cdecl,
+proc RSA_public_decrypt*(flen: cint, fr: ptr uint8, to: ptr uint8, rsa: PRSA, padding: PaddingType): cint {.cdecl,
     dynlib: DLLUtilName, importc.}
 proc RSA_free*(rsa: PRSA) {.cdecl, dynlib: DLLUtilName, importc.}
 proc RSA_size*(rsa: PRSA): cint {.cdecl, dynlib: DLLUtilName, importc.}
@@ -744,8 +744,8 @@ type
 proc md5_Init*(c: var MD5_CTX): cint{.importc: "MD5_Init".}
 proc md5_Update*(c: var MD5_CTX; data: pointer; len: csize_t): cint{.importc: "MD5_Update".}
 proc md5_Final*(md: cstring; c: var MD5_CTX): cint{.importc: "MD5_Final".}
-proc md5*(d: ptr cuchar; n: csize_t; md: ptr cuchar): ptr cuchar{.importc: "MD5".}
-proc md5_Transform*(c: var MD5_CTX; b: ptr cuchar){.importc: "MD5_Transform".}
+proc md5*(d: ptr uint8; n: csize_t; md: ptr uint8): ptr uint8{.importc: "MD5".}
+proc md5_Transform*(c: var MD5_CTX; b: ptr uint8){.importc: "MD5_Transform".}
 {.pop.}
 
 from strutils import toHex, toLowerAscii
diff --git a/tests/stdlib/topenssl.nim b/tests/stdlib/topenssl.nim
index 75e1ba868..4c38b4216 100644
--- a/tests/stdlib/topenssl.nim
+++ b/tests/stdlib/topenssl.nim
@@ -12,8 +12,8 @@ proc rsaPublicEncrypt(fr: string): string =
   doAssert rsa != nil
   doAssert BIO_free(bio) >= 0
   result = newString(RSA_size(rsa))
-  let frdata = cast[ptr cuchar](fr.cstring)
-  var todata = cast[ptr cuchar](result.cstring)
+  let frdata = cast[ptr uint8](fr.cstring)
+  var todata = cast[ptr uint8](result.cstring)
   doAssert RSA_public_encrypt(fr.len.cint, frdata, todata, rsa, RSA_PKCS1_PADDING) != -1
   RSA_free(rsa)
 
@@ -26,8 +26,8 @@ proc rasPrivateDecrypt(fr: string): string =
   doAssert BIO_free(bio) >= 0
   let rsaLen = RSA_size(rsa)
   result = newString(rsaLen)
-  let frdata = cast[ptr cuchar](fr.cstring)
-  var todata = cast[ptr cuchar](result.cstring)
+  let frdata = cast[ptr uint8](fr.cstring)
+  var todata = cast[ptr uint8](result.cstring)
   let lenOrig = RSA_private_decrypt(rsaLen, frdata, todata, rsa, RSA_PKCS1_PADDING)
   doAssert lenOrig >= 0 and lenOrig < result.len
   doAssert result[lenOrig] == '\0'

From 25efb5386293540b0542833625d3fb6e22f3cfbc Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Fri, 16 Jul 2021 18:29:49 +0200
Subject: [PATCH 1046/2472] make dragonbox opt-in via -d:nimFpRoundtrips
 (#18504)

* make dragonbox opt-in via -d:nimFpRoundtrips

* make tests green again

* make tests green again
---
 changelog.md                    | 12 ++++++------
 lib/system/formatfloat.nim      |  2 +-
 lib/system/strmantle.nim        |  2 +-
 tests/errmsgs/treportunused.nim | 14 ++++++++------
 tests/float/nim.cfg             |  1 +
 tests/float/tfloats.nim         |  4 ++++
 tests/stdlib/tjson.nim          |  6 +++---
 tests/stdlib/tjsonutils.nim     | 21 +++++++++++----------
 8 files changed, 35 insertions(+), 27 deletions(-)
 create mode 100644 tests/float/nim.cfg

diff --git a/changelog.md b/changelog.md
index 568ff0bba..289202587 100644
--- a/changelog.md
+++ b/changelog.md
@@ -76,9 +76,6 @@
 - `json` and `jsonutils` now serialize NaN, Inf, -Inf as strings, so that
   `%[NaN, -Inf]` is the string `["nan","-inf"]` instead of `[nan,-inf]` which was invalid json.
 
-- `system.addFloat` now uses the "Dragonbox" algorithm, which ensures correct roundtrips of floating point
-  numbers, that the minimum length representation of a floating point number is used and correct rounding.
-  Use `-d:nimLegacyAddFloat` for a transition period.
 
 - `strformat` is now part of `include std/prelude`.
 
@@ -102,6 +99,12 @@
   added support for parenthesized expressions.
   added support for const string's instead of just string literals
 
+
+- `system.addFloat` and `system.$` now can produce string representations of floating point numbers
+  that are minimal in size and that "roundtrip" (via the "Dragonbox" algorithm). This currently has
+  to be enabled via `-d:nimFpRoundtrips`. It is expected that this behavior becomes the new default
+  in upcoming versions.
+
 - Fixed buffer overflow bugs in `net`
 
 - Exported `sslHandle` from `net` and `asyncnet`.
@@ -116,8 +119,6 @@
   the OpenSSL DLLs (e.g. libssl-1_1-x64.dll, libcrypto-1_1-x64.dll) you
   now also need to ship `cacert.pem` with your `.exe` file.
 
-- Make `{.requiresInit.}` pragma to work for `distinct` types.
-
 - `typetraits`:
   `distinctBase` now is identity instead of error for non distinct types.
   Added `enumLen` to return the number of elements in an enum.
@@ -468,7 +469,6 @@
   with `--styleCheck:error` or `--styleCheck:hint`.
 
 
-
 ## Tool changes
 
 - Latex doc generation is revised: output `.tex` files should be compiled
diff --git a/lib/system/formatfloat.nim b/lib/system/formatfloat.nim
index 41bc56adc..772f0a848 100644
--- a/lib/system/formatfloat.nim
+++ b/lib/system/formatfloat.nim
@@ -7,7 +7,7 @@
 #    distribution, for details about the copyright.
 #
 
-when not defined(nimLegacyAddFloat) and not defined(nimscript) and
+when defined(nimFpRoundtrips) and not defined(nimscript) and
     not defined(js) and defined(nimHasDragonBox):
   import dragonbox
 
diff --git a/lib/system/strmantle.nim b/lib/system/strmantle.nim
index 63e341763..041272175 100644
--- a/lib/system/strmantle.nim
+++ b/lib/system/strmantle.nim
@@ -97,7 +97,7 @@ proc nimFloatToStr(f: float): string {.compilerproc.} =
   result = newStringOfCap(8)
   result.addFloat f
 
-when not defined(nimLegacyAddFloat) and not defined(nimscript) and
+when defined(nimFpRoundtrips) and not defined(nimscript) and
     not defined(js) and defined(nimHasDragonBox):
   import schubfach
 
diff --git a/tests/errmsgs/treportunused.nim b/tests/errmsgs/treportunused.nim
index 65e00163b..f9b7c3d11 100644
--- a/tests/errmsgs/treportunused.nim
+++ b/tests/errmsgs/treportunused.nim
@@ -13,12 +13,12 @@ treportunused.nim(30, 5) Hint: 's8' is declared but not used [XDeclaredButNotUse
 treportunused.nim(31, 5) Hint: 's9' is declared but not used [XDeclaredButNotUsed]
 treportunused.nim(32, 6) Hint: 's10' is declared but not used [XDeclaredButNotUsed]
 treportunused.nim(33, 6) Hint: 's11' is declared but not used [XDeclaredButNotUsed]
-treportunused.nim(37, 3) Hint: 'v0.99' is declared but not used [XDeclaredButNotUsed]
-treportunused.nim(38, 3) Hint: 'v0.99.99' is declared but not used [XDeclaredButNotUsed]
 '''
 action: compile
 """
 
+#treportunused.nim(37, 3) Hint: 'v0.99' is declared but not used [XDeclaredButNotUsed]
+#treportunused.nim(38, 3) Hint: 'v0.99.99' is declared but not used [XDeclaredButNotUsed]
 # bug #9764
 iterator s1(a:string): int = discard
 iterator s2(): int = discard
@@ -32,7 +32,9 @@ var s9: int
 type s10 = object
 type s11 = type(1.2)
 
-# https://github.com/nim-lang/Nim/issues/14407
-let
-  `v0.99` = "0.99"
-  `v0.99.99` = "0.99.99"
+when false:
+  # enabled again when Nim bootstraps with -d:nimFpRoundtrips
+  # https://github.com/nim-lang/Nim/issues/14407
+  let
+    `v0.99` = "0.99"
+    `v0.99.99` = "0.99.99"
diff --git a/tests/float/nim.cfg b/tests/float/nim.cfg
new file mode 100644
index 000000000..d27bbf43b
--- /dev/null
+++ b/tests/float/nim.cfg
@@ -0,0 +1 @@
+-d:nimFpRoundtrips
diff --git a/tests/float/tfloats.nim b/tests/float/tfloats.nim
index 406ddb6d9..30d9c50d6 100644
--- a/tests/float/tfloats.nim
+++ b/tests/float/tfloats.nim
@@ -66,6 +66,8 @@ template main =
     var a = 1.1'f32
     doAssert $a == "1.1", $a # was failing
 
+proc runtimeOnlyTests =
+  # enable for 'static' once -d:nimFpRoundtrips became the default
   block: # bug #7717
     proc test(f: float) =
       let f2 = $f
@@ -82,3 +84,5 @@ template main =
 
 static: main()
 main()
+
+runtimeOnlyTests()
diff --git a/tests/stdlib/tjson.nim b/tests/stdlib/tjson.nim
index 289ef9d05..000f72038 100644
--- a/tests/stdlib/tjson.nim
+++ b/tests/stdlib/tjson.nim
@@ -303,7 +303,7 @@ let jsonNode = %*mynode
 doAssert $jsonNode == """{"kind":"P","pChildren":[{"kind":"Text","textStr":"mychild"},{"kind":"Br"}]}"""
 doAssert $jsonNode.to(ContentNode) == """(kind: P, pChildren: @[(kind: Text, textStr: "mychild"), (kind: Br)])"""
 
-block: # bug #17383
+when defined(nimFpRoundtrips): # bug #17383
   testRoundtrip(int32.high): "2147483647"
   testRoundtrip(uint32.high): "4294967295"
   when int.sizeof == 4:
@@ -316,7 +316,7 @@ block: # bug #17383
     testRoundtrip(int64.high): "9223372036854775807"
     testRoundtrip(uint64.high): "18446744073709551615"
 
-block: # bug #18007
+when defined(nimFpRoundtrips): # bug #18007
   testRoundtrip([NaN, Inf, -Inf, 0.0, -0.0, 1.0]): """["nan","inf","-inf",0.0,-0.0,1.0]"""
   # pending https://github.com/nim-lang/Nim/issues/18025 use:
   # testRoundtrip([float32(NaN), Inf, -Inf, 0.0, -0.0, 1.0])
@@ -332,7 +332,7 @@ block: # bug #18007
     testRoundtripVal(0.0): "0.0"
     testRoundtripVal(-0.0): "-0.0"
 
-block: # bug #15397, bug #13196
+when defined(nimFpRoundtrips): # bug #15397, bug #13196
   testRoundtripVal(1.0 + epsilon(float64)): "1.0000000000000002"
   testRoundtripVal(0.12345678901234567890123456789): "0.12345678901234568"
 
diff --git a/tests/stdlib/tjsonutils.nim b/tests/stdlib/tjsonutils.nim
index b0c444cbf..3e9c422e0 100644
--- a/tests/stdlib/tjsonutils.nim
+++ b/tests/stdlib/tjsonutils.nim
@@ -44,7 +44,7 @@ proc `$`(a: MyEnum): string =
   if a == me2: "me2Modif"
   else: system.`$`(a)
 
-template fn() = 
+template fn() =
   block: # toJson, jsonTo
     type Foo = distinct float
     testRoundtrip('x', """120""")
@@ -161,15 +161,16 @@ template fn() =
     doAssert b[2].signbit
     doAssert not b[3].signbit
 
-  block: # bug #15397, bug #13196
-    let a = 0.1
-    let x = 0.12345678901234567890123456789
-    let b = (a + 0.2, 0.3, x)
-    testRoundtripVal(b): "[0.30000000000000004,0.3,0.12345678901234568]"
+  when defined(nimFpRoundtrips):
+    block: # bug #15397, bug #13196
+      let a = 0.1
+      let x = 0.12345678901234567890123456789
+      let b = (a + 0.2, 0.3, x)
+      testRoundtripVal(b): "[0.30000000000000004,0.3,0.12345678901234568]"
 
-    testRoundtripVal(0.12345678901234567890123456789): "0.12345678901234568"
-    testRoundtripVal(epsilon(float64)): "2.220446049250313e-16"
-    testRoundtripVal(1.0 + epsilon(float64)): "1.0000000000000002"
+      testRoundtripVal(0.12345678901234567890123456789): "0.12345678901234568"
+      testRoundtripVal(epsilon(float64)): "2.220446049250313e-16"
+      testRoundtripVal(1.0 + epsilon(float64)): "1.0000000000000002"
 
   block: # case object
     type Foo = object
@@ -433,7 +434,7 @@ template fn() =
           """{"b": true, "bt": false, "btf": "test"}"""
         testRoundtrip(Variant(b: true, bt: true, btt: 'c')):
           """{"b": true, "bt": true, "btt": "c"}"""
-        
+
         # TODO: Add additional tests with missing and extra JSON keys, both when
         # allowed and forbidden analogous to the tests for the not nested
         # variant objects.

From 923a1c6ea7d9f45b6389680717692690218228fb Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Fri, 16 Jul 2021 22:54:47 -0700
Subject: [PATCH 1047/2472] fix nativeToUnixPath (#18501)

---
 lib/std/private/globs.nim | 11 ++++++++---
 tests/stdlib/tglobs.nim   | 20 ++++++++++++++++++++
 2 files changed, 28 insertions(+), 3 deletions(-)
 create mode 100644 tests/stdlib/tglobs.nim

diff --git a/lib/std/private/globs.nim b/lib/std/private/globs.nim
index 55f6a40bd..b3726c9c3 100644
--- a/lib/std/private/globs.nim
+++ b/lib/std/private/globs.nim
@@ -45,10 +45,15 @@ iterator walkDirRecFilter*(dir: string, follow: proc(entry: PathEntry): bool = n
 
 proc nativeToUnixPath*(path: string): string =
   # pending https://github.com/nim-lang/Nim/pull/13265
-  doAssert not path.isAbsolute # not implemented here; absolute files need more care for the drive
+  result = path
+  when defined(windows):
+    if path.len >= 2 and path[0] in {'a'..'z', 'A'..'Z'} and path[1] == ':':
+      result[0] = '/'
+      result[1] = path[0]
+      if path.len > 2 and path[2] != '\\':
+        doAssert false, "paths like `C:foo` are currently unsupported, path: " & path
   when DirSep == '\\':
-    result = replace(path, '\\', '/')
-  else: result = path
+    result = replace(result, '\\', '/')
 
 when isMainModule:
   import sugar
diff --git a/tests/stdlib/tglobs.nim b/tests/stdlib/tglobs.nim
new file mode 100644
index 000000000..739a127f8
--- /dev/null
+++ b/tests/stdlib/tglobs.nim
@@ -0,0 +1,20 @@
+import std/private/globs
+
+template main =
+  when defined(windows):
+    doAssert nativeToUnixPath("C:") == "/C"
+    doAssert nativeToUnixPath(r"D:\") == "/D/"
+    doAssert nativeToUnixPath(r"E:\a") == "/E/a"
+    doAssert nativeToUnixPath(r"E:\a1\") == "/E/a1/"
+    doAssert nativeToUnixPath(r"E:\a1\bc") == "/E/a1/bc"
+    doAssert nativeToUnixPath(r"\a1\bc") == "/a1/bc"
+    doAssert nativeToUnixPath(r"a1\bc") == "a1/bc"
+    doAssert nativeToUnixPath("a1") == "a1"
+    doAssert nativeToUnixPath("") == ""
+    doAssert nativeToUnixPath(".") == "."
+    doAssert nativeToUnixPath("..") == ".."
+    doAssert nativeToUnixPath(r"..\") == "../"
+    doAssert nativeToUnixPath(r"..\..\.\") == "../.././"
+
+static: main()
+main()

From ac5435ecd0a08d1a0bdd9d6d6cd89577634ecb0f Mon Sep 17 00:00:00 2001
From: konsumlamm <44230978+konsumlamm@users.noreply.github.com>
Date: Sun, 18 Jul 2021 10:49:03 +0200
Subject: [PATCH 1048/2472] Make error message for empty new-styled concept
 more descriptive (#18506)

* Allow empty new-styled concept

Slightly improve error messages

* Make empty new-styled concepts an error
---
 compiler/concepts.nim    | 15 +++++++--------
 compiler/parser.nim      |  2 ++
 tests/concepts/tspec.nim |  5 ++---
 3 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/compiler/concepts.nim b/compiler/concepts.nim
index de994f1b8..885b69c60 100644
--- a/compiler/concepts.nim
+++ b/compiler/concepts.nim
@@ -11,8 +11,7 @@
 ## for details. Note this is a first implementation and only the "Concept matching"
 ## section has been implemented.
 
-import ast, astalgo, semdata, lookups, lineinfos, idents, msgs, renderer,
-  types, intsets
+import ast, astalgo, semdata, lookups, lineinfos, idents, msgs, renderer, types, intsets
 
 from magicsys import addSonSkipIntLit
 
@@ -23,7 +22,7 @@ const
 ## --------------------------------------
 
 proc declareSelf(c: PContext; info: TLineInfo) =
-  ## adds the magical 'Self' symbols to the current scope.
+  ## Adds the magical 'Self' symbols to the current scope.
   let ow = getCurrOwner(c)
   let s = newSym(skType, getIdent(c.cache, "Self"), nextSymId(c.idgen), ow, info)
   s.typ = newType(tyTypeDesc, nextTypeId(c.idgen), ow)
@@ -32,7 +31,7 @@ proc declareSelf(c: PContext; info: TLineInfo) =
   addDecl(c, s, info)
 
 proc isSelf*(t: PType): bool {.inline.} =
-  ## is this the magical 'Self' type?
+  ## Is this the magical 'Self' type?
   t.kind == tyTypeDesc and tfPacked in t.flags
 
 proc makeTypeDesc*(c: PContext, typ: PType): PType =
@@ -45,8 +44,8 @@ proc makeTypeDesc*(c: PContext, typ: PType): PType =
 
 proc semConceptDecl(c: PContext; n: PNode): PNode =
   ## Recursive helper for semantic checking for the concept declaration.
-  ## Currently we only support lists of statements containing 'proc'
-  ## declarations and the like.
+  ## Currently we only support (possibly empty) lists of statements
+  ## containing 'proc' declarations and the like.
   case n.kind
   of nkStmtList, nkStmtListExpr:
     result = shallowCopy(n)
@@ -60,7 +59,7 @@ proc semConceptDecl(c: PContext; n: PNode): PNode =
       result[i] = n[i]
     result[^1] = semConceptDecl(c, n[^1])
   else:
-    localError(c.config, n.info, "unexpected construct in the new-styled concept " & renderTree(n))
+    localError(c.config, n.info, "unexpected construct in the new-styled concept: " & renderTree(n))
     result = n
 
 proc semConceptDeclaration*(c: PContext; n: PNode): PNode =
@@ -97,7 +96,7 @@ proc existingBinding(m: MatchCon; key: PType): PType =
 proc conceptMatchNode(c: PContext; n: PNode; m: var MatchCon): bool
 
 proc matchType(c: PContext; f, a: PType; m: var MatchCon): bool =
-  ## the heart of the concept matching process. 'f' is the formal parameter of some
+  ## The heart of the concept matching process. 'f' is the formal parameter of some
   ## routine inside the concept that we're looking for. 'a' is the formal parameter
   ## of a routine that might match.
   const
diff --git a/compiler/parser.nim b/compiler/parser.nim
index b6c01e9c6..d00a17e9f 100644
--- a/compiler/parser.nim
+++ b/compiler/parser.nim
@@ -2058,6 +2058,8 @@ proc parseTypeClass(p: var Parser): PNode =
     skipComment(p, result)
   # an initial IND{>} HAS to follow:
   if not realInd(p):
+    if result.isNewStyleConcept:
+      parMessage(p, "routine expected, but found '$1' (empty new-styled concepts are not allowed)", p.tok)
     result.add(p.emptyNode)
   else:
     result.add(parseStmt(p))
diff --git a/tests/concepts/tspec.nim b/tests/concepts/tspec.nim
index 1bca3c11b..52f13a40a 100644
--- a/tests/concepts/tspec.nim
+++ b/tests/concepts/tspec.nim
@@ -7,8 +7,7 @@ discard """
 2
 3
 yes int
-string int
-true'''
+string int'''
   joinable: false
 """
 
@@ -102,5 +101,5 @@ type Monoid = concept
 
 proc z(x: typedesc[int]): int = 0
 
-echo(int is Monoid)
+doAssert int is Monoid
 

From 8091545f95143d520c64b14f88bb5ce2818c893c Mon Sep 17 00:00:00 2001
From: Miran 
Date: Sun, 18 Jul 2021 11:04:19 +0200
Subject: [PATCH 1049/2472] Revert #17398 and #17402 (#18480)

* Revert "followup #17398: `getTempDir`, `getConfigDir` now do not have trailing DirSep (#17402)"

This reverts commit 2356d0603f70cad90f76fa57999054bf5c0a9157.

* Revert "fix #17393 getHomeDir and expandTilde should not include trailing `/` (#17398)"

This reverts commit bebf2ce24a43bef4cde5c90c4010631a1e4a7927.

* fix test
---
 changelog.md                |  3 ---
 lib/pure/os.nim             | 40 ++++++++++++++-----------------------
 tests/stdlib/tos.nim        |  6 +++---
 tests/stdlib/ttempfiles.nim |  2 +-
 4 files changed, 19 insertions(+), 32 deletions(-)

diff --git a/changelog.md b/changelog.md
index 289202587..16f2192dc 100644
--- a/changelog.md
+++ b/changelog.md
@@ -39,9 +39,6 @@
   underlying code is also updated the same way.
 - Custom pragma values have now an API for use in macros.
 
-- In `std/os`, `getHomeDir`, `expandTilde`, `getTempDir`, `getConfigDir` now do not include trailing `DirSep`,
-  unless `-d:nimLegacyHomeDir` is specified (for a transition period).
-
 - On POSIX systems, the default signal handlers used for Nim programs (it's
   used for printing the stacktrace on fatal signals) will now re-raise the
   signal for the OS default handlers to handle.
diff --git a/lib/pure/os.nim b/lib/pure/os.nim
index 213da2aed..dadc380e6 100644
--- a/lib/pure/os.nim
+++ b/lib/pure/os.nim
@@ -901,14 +901,9 @@ proc getHomeDir*(): string {.rtl, extern: "nos$1",
   ## * `setCurrentDir proc <#setCurrentDir,string>`_
   runnableExamples:
     assert getHomeDir() == expandTilde("~")
-    # `getHomeDir()` doesn't end in `DirSep` even if `$HOME` (on posix) or
-    # `$USERPROFILE` (on windows) does, unless `-d:nimLegacyHomeDir` is specified.
-    from std/strutils import endsWith
-    assert not getHomeDir().endsWith DirSep
 
-  when defined(windows): result = getEnv("USERPROFILE")
-  else: result = getEnv("HOME")
-  result.normalizePathEnd(trailingSep = defined(nimLegacyHomeDir))
+  when defined(windows): return getEnv("USERPROFILE") & "\\"
+  else: return getEnv("HOME") & "/"
 
 proc getConfigDir*(): string {.rtl, extern: "nos$1",
   tags: [ReadEnvEffect, ReadIOEffect].} =
@@ -917,21 +912,23 @@ proc getConfigDir*(): string {.rtl, extern: "nos$1",
   ## On non-Windows OSs, this proc conforms to the XDG Base Directory
   ## spec. Thus, this proc returns the value of the `XDG_CONFIG_HOME` environment
   ## variable if it is set, otherwise it returns the default configuration
-  ## directory ("~/.config").
+  ## directory ("~/.config/").
+  ##
+  ## An OS-dependent trailing slash is always present at the end of the
+  ## returned string: `\\` on Windows and `/` on all other OSs.
   ##
   ## See also:
   ## * `getHomeDir proc <#getHomeDir>`_
   ## * `getTempDir proc <#getTempDir>`_
-  ## * `getCacheDir proc <#getCacheDir>`_
-  runnableExamples:
-    from std/strutils import endsWith
-    # See `getHomeDir` for behavior regarding trailing DirSep.
-    assert not getConfigDir().endsWith DirSep
+  ## * `expandTilde proc <#expandTilde,string>`_
+  ## * `getCurrentDir proc <#getCurrentDir>`_
+  ## * `setCurrentDir proc <#setCurrentDir,string>`_
   when defined(windows):
     result = getEnv("APPDATA")
   else:
     result = getEnv("XDG_CONFIG_HOME", getEnv("HOME") / ".config")
-  result.normalizePathEnd(trailingSep = defined(nimLegacyHomeDir))
+  result.normalizePathEnd(trailingSep = true)
+
 
 proc getCacheDir*(): string =
   ## Returns the cache directory of the current user for applications.
@@ -1006,11 +1003,9 @@ proc getTempDir*(): string {.rtl, extern: "nos$1",
   ## See also:
   ## * `getHomeDir proc <#getHomeDir>`_
   ## * `getConfigDir proc <#getConfigDir>`_
-  ## * `getCacheDir proc <#getCacheDir>`_
-  runnableExamples:
-    from std/strutils import endsWith
-    # See `getHomeDir` for behavior regarding trailing DirSep.
-    assert not getTempDir().endsWith(DirSep)
+  ## * `expandTilde proc <#expandTilde,string>`_
+  ## * `getCurrentDir proc <#getCurrentDir>`_
+  ## * `setCurrentDir proc <#setCurrentDir,string>`_
   const tempDirDefault = "/tmp"
   when defined(tempDir):
     const tempDir {.strdefine.}: string = tempDirDefault
@@ -1031,7 +1026,7 @@ proc getTempDir*(): string {.rtl, extern: "nos$1",
         getTempDirImpl(result)
     if result.len == 0:
       result = tempDirDefault
-  result.normalizePathEnd(trailingSep = defined(nimLegacyHomeDir))
+  normalizePathEnd(result, trailingSep=true)
 
 proc expandTilde*(path: string): string {.
   tags: [ReadEnvEffect, ReadIOEffect].} =
@@ -1041,8 +1036,6 @@ proc expandTilde*(path: string): string {.
   ## Windows: this is still supported despite the Windows platform not having this
   ## convention; also, both ``~/`` and ``~\`` are handled.
   ##
-  ## .. warning:: `~bob` and `~bob/` are not yet handled correctly.
-  ##
   ## See also:
   ## * `getHomeDir proc <#getHomeDir>`_
   ## * `getConfigDir proc <#getConfigDir>`_
@@ -1053,9 +1046,6 @@ proc expandTilde*(path: string): string {.
     assert expandTilde("~" / "appname.cfg") == getHomeDir() / "appname.cfg"
     assert expandTilde("~/foo/bar") == getHomeDir() / "foo/bar"
     assert expandTilde("/foo/bar") == "/foo/bar"
-    assert expandTilde("~") == getHomeDir()
-    from std/strutils import endsWith
-    assert not expandTilde("~").endsWith(DirSep)
 
   if len(path) == 0 or path[0] != '~':
     result = path
diff --git a/tests/stdlib/tos.nim b/tests/stdlib/tos.nim
index a4f030dde..912558461 100644
--- a/tests/stdlib/tos.nim
+++ b/tests/stdlib/tos.nim
@@ -594,12 +594,12 @@ block getTempDir:
       if existsEnv("TMPDIR"):
         let origTmpDir = getEnv("TMPDIR")
         putEnv("TMPDIR", "/mytmp")
-        doAssert getTempDir() == "/mytmp"
+        doAssert getTempDir() == "/mytmp/"
         delEnv("TMPDIR")
-        doAssert getTempDir() == "/tmp"
+        doAssert getTempDir() == "/tmp/"
         putEnv("TMPDIR", origTmpDir)
       else:
-        doAssert getTempDir() == "/tmp"
+        doAssert getTempDir() == "/tmp/"
 
 block: # getCacheDir
   doAssert getCacheDir().dirExists
diff --git a/tests/stdlib/ttempfiles.nim b/tests/stdlib/ttempfiles.nim
index d740e1850..297410686 100644
--- a/tests/stdlib/ttempfiles.nim
+++ b/tests/stdlib/ttempfiles.nim
@@ -40,7 +40,7 @@ block: # createTempDir
 
     doAssert dirExists(dir1)
     doAssert dir1.lastPathPart.contains(re"^D20210502T100442(\w+).tmp$")
-    doAssert dir1.parentDir == getTempDir()
+    doAssert dir1.parentDir == getTempDir().normalizePathEnd()
 
   block:
     let dir3 = createTempDir(prefix, "_mytmp", ".")

From 3723140044b99fde3f47df4f2e6a3ed4abdaba89 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Sun, 18 Jul 2021 02:05:35 -0700
Subject: [PATCH 1050/2472] add NIM_STATIC_ASSERT(CHAR_BIT == 8, "") to
 fail-fast where assumption is violated (#18512)

---
 lib/nimbase.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/nimbase.h b/lib/nimbase.h
index cbd35605b..e480c55fd 100644
--- a/lib/nimbase.h
+++ b/lib/nimbase.h
@@ -325,6 +325,9 @@ typedef unsigned char NIM_BOOL; // best effort
 #endif
 
 NIM_STATIC_ASSERT(sizeof(NIM_BOOL) == 1, ""); // check whether really needed
+NIM_STATIC_ASSERT(CHAR_BIT == 8, "");
+  // fail fast for (rare) environments where this doesn't hold, as some implicit
+  // assumptions would need revisiting (e.g. `uint8` or https://github.com/nim-lang/Nim/pull/18505)
 
 #define NIM_TRUE true
 #define NIM_FALSE false

From adba5eb45e0ae0d370aea4d653a4f00a4c075695 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Sun, 18 Jul 2021 02:10:08 -0700
Subject: [PATCH 1051/2472] deprecate strutils.delete and add an overload with
 saner semantics consistent with sequtils.delete; follows #18487 (#18510)

---
 changelog.md               |  2 +-
 lib/pure/strutils.nim      | 36 +++++++++++++++++++++++++++++++-----
 tests/stdlib/tstrutils.nim | 26 +++++++++++++++++++++++++-
 3 files changed, 57 insertions(+), 7 deletions(-)

diff --git a/changelog.md b/changelog.md
index 16f2192dc..6dddd8a01 100644
--- a/changelog.md
+++ b/changelog.md
@@ -346,7 +346,7 @@
 - Added `dom.setInterval`, `dom.clearInterval` overloads.
 
 - Deprecated `sequtils.delete` and added an overload taking a `Slice` that raises a defect
-  if the slice is out of bounds.
+  if the slice is out of bounds, likewise with `strutils.delete`.
 
 ## Language changes
 
diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim
index 9e0837942..ac078df4e 100644
--- a/lib/pure/strutils.nim
+++ b/lib/pure/strutils.nim
@@ -1476,12 +1476,39 @@ func dedent*(s: string, count: Natural = indentation(s)): string {.rtl,
     doAssert x == "Hello\n  There\n"
   unindent(s, count, " ")
 
-func delete*(s: var string, first, last: int) {.rtl, extern: "nsuDelete".} =
-  ## Deletes in `s` (must be declared as `var`) the characters at positions
-  ## `first .. last` (both ends included).
+func delete*(s: var string, slice: Slice[int]) =
+  ## Deletes the items `s[slice]`, raising `IndexDefect` if the slice contains
+  ## elements out of range.
   ##
-  ## This modifies `s` itself, it does not return a copy.
+  ## This operation moves all elements after `s[slice]` in linear time, and
+  ## is the string analog to `sequtils.delete`.
   runnableExamples:
+    var a = "abcde"
+    doAssertRaises(IndexDefect): a.delete(4..5)
+    assert a == "abcde"
+    a.delete(4..4)
+    assert a == "abcd"
+    a.delete(1..2)
+    assert a == "ad"
+    a.delete(1..<1) # empty slice
+    assert a == "ad"
+  when compileOption("boundChecks"):
+    if not (slice.a < s.len and slice.a >= 0 and slice.b < s.len):
+      raise newException(IndexDefect, $(slice: slice, len: s.len))
+  if slice.b >= slice.a:
+    var i = slice.a
+    var j = slice.b + 1
+    var newLen = s.len - j + i
+    # if j < s.len: moveMem(addr s[i], addr s[j], s.len - j) # pending benchmark
+    while i < newLen:
+      s[i] = s[j]
+      inc(i)
+      inc(j)
+    setLen(s, newLen)
+
+func delete*(s: var string, first, last: int) {.rtl, extern: "nsuDelete", deprecated: "use `delete(s, first..last)`".} =
+  ## Deletes in `s` the characters at positions `first .. last` (both ends included).
+  runnableExamples("--warning:deprecated:off"):
     var a = "abracadabra"
 
     a.delete(4, 5)
@@ -1502,7 +1529,6 @@ func delete*(s: var string, first, last: int) {.rtl, extern: "nsuDelete".} =
     inc(j)
   setLen(s, newLen)
 
-
 func startsWith*(s: string, prefix: char): bool {.inline.} =
   ## Returns true if `s` starts with character `prefix`.
   ##
diff --git a/tests/stdlib/tstrutils.nim b/tests/stdlib/tstrutils.nim
index 68ee5812b..234991bdb 100644
--- a/tests/stdlib/tstrutils.nim
+++ b/tests/stdlib/tstrutils.nim
@@ -198,7 +198,30 @@ template main() =
     s.removePrefix("")
     doAssert s == "\r\n\r\nhello"
 
-  block: # delete
+  block: # delete(slice)
+    var s = "0123456789ABCDEFGH"
+    delete(s, 4 .. 5)
+    doAssert s == "01236789ABCDEFGH"
+    delete(s, s.len-1 .. s.len-1)
+    doAssert s == "01236789ABCDEFG"
+    delete(s, 0..0)
+    doAssert s == "1236789ABCDEFG"
+    s = ""
+    doAssertRaises(IndexDefect): delete(s, 0..0)
+    doAssert s == ""
+    s = "abc"
+    doAssertRaises(IndexDefect): delete(s, -1 .. -2)
+    doAssertRaises(IndexDefect): delete(s, 2..3)
+    doAssertRaises(IndexDefect): delete(s, 3..2)
+    delete(s, 2..2)
+    doAssert s == "ab"
+    delete(s, 1..0)
+    doAssert s == "ab"
+    delete(s, 0..0)
+    doAssert s == "b"
+
+  block: # delete(first, last)
+    {.push warning[deprecated]:off.}
     var s = "0123456789ABCDEFGH"
     delete(s, 4, 5)
     doAssert s == "01236789ABCDEFGH"
@@ -206,6 +229,7 @@ template main() =
     doAssert s == "01236789ABCDEFG"
     delete(s, 0, 0)
     doAssert s == "1236789ABCDEFG"
+    {.pop.}
 
   block: # find
     doAssert "0123456789ABCDEFGH".find('A') == 10

From 99c4b69097638aca8377ea3746e52cc19c24ced8 Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Sun, 18 Jul 2021 15:16:26 +0200
Subject: [PATCH 1052/2472] fixed system.delete (#18507)

---
 changelog.md             | 13 ++++++++++++-
 lib/system.nim           | 17 ++++++++++-------
 tests/stdlib/tsystem.nim |  8 ++++----
 3 files changed, 26 insertions(+), 12 deletions(-)

diff --git a/changelog.md b/changelog.md
index 6dddd8a01..097beac2c 100644
--- a/changelog.md
+++ b/changelog.md
@@ -4,7 +4,18 @@
 
 ## Changes affecting backward compatibility
 
-- Deprecated `std/mersenne`
+- Deprecated `std/mersenne`.
+
+- `system.delete` had a most surprising behavior when the index passed to it was out of
+  bounds (it would delete the last entry then). Compile with `-d:nimStrictDelete` so
+  that an index error is produced instead. But be aware that your code might depend on
+  this quirky behavior so a review process is required on your part before you can
+  use `-d:nimStrictDelete`. To make this review easier, use the `-d:nimAuditDelete`
+  switch, it pretends that `system.delete` is deprecated so that it is easier to see
+  where it was used in your code.
+
+  `-d:nimStrictDelete` will become the default in upcoming versions.
+
 
 - `cuchar` is now deprecated as it aliased `char` where arguably it should have aliased `uint8`.
   Please use `char` or `uint8` instead.
diff --git a/lib/system.nim b/lib/system.nim
index 57d620e57..2790187f4 100644
--- a/lib/system.nim
+++ b/lib/system.nim
@@ -2134,7 +2134,12 @@ const
 import system/dollars
 export dollars
 
-proc delete*[T](x: var seq[T], i: Natural) {.noSideEffect.} =
+when defined(nimAuditDelete):
+  {.pragma: auditDelete, deprecated: "review this call for out of bounds behavior".}
+else:
+  {.pragma: auditDelete.}
+
+proc delete*[T](x: var seq[T], i: Natural) {.noSideEffect, auditDelete.} =
   ## Deletes the item at index `i` by moving all `x[i+1..^1]` items by one position.
   ##
   ## This is an `O(n)` operation.
@@ -2147,12 +2152,10 @@ proc delete*[T](x: var seq[T], i: Natural) {.noSideEffect.} =
     s.delete(2)
     doAssert s == @[1, 2, 4, 5]
 
-    doAssertRaises(IndexDefect):
-      s.delete(4)
-
-  if i > high(x):
-    # xxx this should call `raiseIndexError2(i, high(x))` after some refactoring
-    raise (ref IndexDefect)(msg: "index out of bounds: '" & $i & "' < '" & $x.len & "' failed")
+  when defined(nimStrictDelete):
+    if i > high(x):
+      # xxx this should call `raiseIndexError2(i, high(x))` after some refactoring
+      raise (ref IndexDefect)(msg: "index out of bounds: '" & $i & "' < '" & $x.len & "' failed")
 
   template defaultImpl =
     let xl = x.len
diff --git a/tests/stdlib/tsystem.nim b/tests/stdlib/tsystem.nim
index b48034043..00be16275 100644
--- a/tests/stdlib/tsystem.nim
+++ b/tests/stdlib/tsystem.nim
@@ -53,8 +53,8 @@ template main =
       var s = @["foo", "bar"]
       s.delete(1)
       doAssert s == @["foo"]
-  
-    block:
+
+    when false:
       var s: seq[string]
       doAssertRaises(IndexDefect):
         s.delete(0)
@@ -66,8 +66,8 @@ template main =
       var s = @["foo"]
       s.delete(0)
       doAssert s == @[]
-  
-    block: # bug #16544: deleting out of bounds index should raise
+
+    when false: # bug #16544: deleting out of bounds index should raise
       var s = @["foo"]
       doAssertRaises(IndexDefect):
         s.delete(1)

From 220b55c5d7c40aa93df7879ebc6c9f71147c3eda Mon Sep 17 00:00:00 2001
From: Antonis Geralis <43617260+planetis-m@users.noreply.github.com>
Date: Sun, 18 Jul 2021 16:29:49 +0300
Subject: [PATCH 1053/2472] attempt to support short commit hashes (#18514)

* attempt to support short commit hashes

Not sure if that's the correct way and what happens when the short hash matches "head"

* need to remove #

* Output needs to be stripped of newlines
---
 tools/atlas/atlas.nim | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/tools/atlas/atlas.nim b/tools/atlas/atlas.nim
index bc1cc6ae3..371a1c13f 100644
--- a/tools/atlas/atlas.nim
+++ b/tools/atlas/atlas.nim
@@ -123,8 +123,12 @@ proc versionToCommit(d: Dependency): string =
 
   return ""
 
+proc shortToCommit(short: string): string =
+  let (cc, status) = osproc.execCmdEx("git rev-parse " & quoteShell(short))
+  result = if status == 0: strutils.strip(cc) else: ""
+
 proc checkoutGitCommit(c: var AtlasContext; p: PackageName; commit: string) =
-  let (outp, status) = osproc.execCmdEx("git checkout " & quoteShell(commit))
+  let (_, status) = osproc.execCmdEx("git checkout " & quoteShell(commit))
   if status != 0:
     error(c, p, "could not checkout commit", commit)
 
@@ -169,17 +173,23 @@ proc toName(p: string): PackageName =
 proc needsCommitLookup(commit: string): bool {.inline} =
   '.' in commit or commit == InvalidCommit
 
+proc isShortCommitHash(commit: string): bool {.inline.} =
+  commit.len >= 4 and commit.len < 40
+
 proc checkoutCommit(c: var AtlasContext; w: Dependency) =
   let dir = c.workspace / w.name.string
   withDir dir:
-    if w.commit.len == 0 or cmpIgnoreCase(w.commit, "#head") == 0:
+    if w.commit.len == 0 or cmpIgnoreCase(w.commit, "head") == 0:
       gitPull(c, w.name)
     else:
       let err = isCleanGit(dir)
       if err != "":
         warn c, w.name, err
       else:
-        let requiredCommit = if needsCommitLookup(w.commit): versionToCommit(w) else: w.commit
+        let requiredCommit =
+          if needsCommitLookup(w.commit): versionToCommit(w)
+          elif isShortCommitHash(w.commit): shortToCommit(w.commit)
+          else: w.commit
         let (cc, status) = osproc.execCmdEx("git log -n 1 --format=%H")
         let currentCommit = strutils.strip(cc)
         if requiredCommit == "" or status != 0:
@@ -191,8 +201,9 @@ proc checkoutCommit(c: var AtlasContext; w: Dependency) =
           if currentCommit != requiredCommit:
             # checkout the later commit:
             # git merge-base --is-ancestor  
-            let (mergeBase, status) = osproc.execCmdEx("git merge-base " &
+            let (cc, status) = osproc.execCmdEx("git merge-base " &
                 currentCommit.quoteShell & " " & requiredCommit.quoteShell)
+            let mergeBase = strutils.strip(cc)
             if status == 0 and (mergeBase == currentCommit or mergeBase == requiredCommit):
               # conflict resolution: pick the later commit:
               if mergeBase == currentCommit:
@@ -242,7 +253,7 @@ proc collectNewDeps(c: var AtlasContext; work: var seq[Dependency];
         tokens.add InvalidCommit
       elif tokens.len == 2 and tokens[1].startsWith("#"):
         # Dependencies can also look like 'requires "sdl2#head"
-        var commit = tokens[1]
+        var commit = tokens[1][1 .. ^1]
         tokens[1] = "=="
         tokens.add commit
 

From 488e9c2991cd6f4ae946dd719271ea284a051891 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Mon, 19 Jul 2021 03:42:45 -0700
Subject: [PATCH 1054/2472] nim doc now correctly renders deprecated pragmas
 for routines and types (#18515)

---
 compiler/ast.nim                             | 31 ++++++++++++++++++++
 compiler/docgen.nim                          | 15 ++--------
 compiler/renderer.nim                        |  4 +--
 nimdoc/testproject/expected/testproject.html | 18 ++++++++++--
 nimdoc/testproject/expected/testproject.idx  |  1 +
 nimdoc/testproject/expected/theindex.html    |  4 +++
 nimdoc/testproject/testproject.nim           |  3 ++
 7 files changed, 59 insertions(+), 17 deletions(-)

diff --git a/compiler/ast.nim b/compiler/ast.nim
index f96e464af..f1f73dcee 100644
--- a/compiler/ast.nim
+++ b/compiler/ast.nim
@@ -1168,6 +1168,37 @@ template `[]=`*(n: Indexable, i: int; x: Indexable) = n.sons[i] = x
 template `[]`*(n: Indexable, i: BackwardsIndex): Indexable = n[n.len - i.int]
 template `[]=`*(n: Indexable, i: BackwardsIndex; x: Indexable) = n[n.len - i.int] = x
 
+proc getDeclPragma*(n: PNode): PNode =
+  ## return the `nkPragma` node for declaration `n`, or `nil` if no pragma was found.
+  ## Currently only supports routineDefs + {nkTypeDef}.
+  case n.kind
+  of routineDefs:
+    if n[pragmasPos].kind != nkEmpty: result = n[pragmasPos]
+  of nkTypeDef:
+    #[
+    type F3*{.deprecated: "x3".} = int
+
+    TypeSection
+      TypeDef
+        PragmaExpr
+          Postfix
+            Ident "*"
+            Ident "F3"
+          Pragma
+            ExprColonExpr
+              Ident "deprecated"
+              StrLit "x3"
+        Empty
+        Ident "int"
+    ]#
+    if n[0].kind == nkPragmaExpr:
+      result = n[0][1]
+  else:
+    # support as needed for `nkIdentDefs` etc.
+    result = nil
+  if result != nil:
+    assert result.kind == nkPragma, $(result.kind, n.kind)
+
 when defined(useNodeIds):
   const nodeIdToDebug* = -1 # 2322968
   var gNodeId: int
diff --git a/compiler/docgen.nim b/compiler/docgen.nim
index 1ae00e22e..9c75c0a6f 100644
--- a/compiler/docgen.nim
+++ b/compiler/docgen.nim
@@ -764,14 +764,6 @@ proc complexName(k: TSymKind, n: PNode, baseName: string): string =
       result.add(defaultParamSeparator)
       result.add(params)
 
-proc isCallable(n: PNode): bool =
-  ## Returns true if `n` contains a callable node.
-  case n.kind
-  of nkProcDef, nkMethodDef, nkIteratorDef, nkMacroDef, nkTemplateDef,
-    nkConverterDef, nkFuncDef: result = true
-  else:
-    result = false
-
 proc docstringSummary(rstText: string): string =
   ## Returns just the first line or a brief chunk of text from a rst string.
   ##
@@ -862,9 +854,8 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
       break
     plainName.add(literal)
 
-  var pragmaNode: PNode = nil
-  if n.isCallable and n[pragmasPos].kind != nkEmpty:
-    pragmaNode = findPragma(n[pragmasPos], wDeprecated)
+  var pragmaNode = getDeclPragma(n)
+  if pragmaNode != nil: pragmaNode = findPragma(pragmaNode, wDeprecated)
 
   inc(d.id)
   let
@@ -920,7 +911,7 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
   # because it doesn't include object fields or documentation comments. So we
   # use the plain one for callable elements, and the complex for the rest.
   var linkTitle = changeFileExt(extractFilename(d.filename), "") & ": "
-  if n.isCallable: linkTitle.add(xmltree.escape(plainName.strip))
+  if n.kind in routineDefs: linkTitle.add(xmltree.escape(plainName.strip))
   else: linkTitle.add(xmltree.escape(complexSymbol.strip))
 
   setIndexTerm(d[], external, symbolOrId, name, linkTitle,
diff --git a/compiler/renderer.nim b/compiler/renderer.nim
index ac3f47948..3346e629c 100644
--- a/compiler/renderer.nim
+++ b/compiler/renderer.nim
@@ -593,8 +593,8 @@ proc isHideable(config: ConfigRef, n: PNode): bool =
   # xxx compare `ident` directly with `getIdent(cache, wRaises)`, but
   # this requires a `cache`.
   case n.kind
-  of nkExprColonExpr: result = n[0].kind == nkIdent and n[0].ident.s in ["raises", "tags", "extern"]
-  of nkIdent: result = n.ident.s in ["gcsafe"]
+  of nkExprColonExpr: result = n[0].kind == nkIdent and n[0].ident.s in ["raises", "tags", "extern", "deprecated"]
+  of nkIdent: result = n.ident.s in ["gcsafe", "deprecated"]
   else: result = false
 
 proc gcommaAux(g: var TSrcGen, n: PNode, ind: int, start: int = 0,
diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html
index 65d9f83f5..7e5a685d2 100644
--- a/nimdoc/testproject/expected/testproject.html
+++ b/nimdoc/testproject/expected/testproject.html
@@ -107,7 +107,9 @@ window.addEventListener('DOMContentLoaded', main);
 
  • 
   Types
   
      
-      
    • FooBuzz
      • 
+  
    • A
      • 
   
    • 
 
      Types
      
 
      
+
+
      FooBuzz {....deprecated: "FooBuzz msg".} = int
      
+
      
+  
      
+    Deprecated: FooBuzz msg
+  
      
+
+
+
+
      
 
 
      A {.inject.} = enum
   aA
      
@@ -544,7 +556,7 @@ This should be visible.
 
 
 
-
      proc baz[T](a, b: T): T {.deprecated.}
      
+
      proc baz[T](a, b: T): T {....deprecated.}
      
 
      
   
      
     Deprecated 
@@ -554,7 +566,7 @@ This is deprecated without message.
 
 
      
 
-
      proc buzz[T](a, b: T): T {.deprecated: "since v0.20".}
      
+
      proc buzz[T](a, b: T): T {....deprecated: "since v0.20".}
      
 
      
   
      
     Deprecated: since v0.20
diff --git a/nimdoc/testproject/expected/testproject.idx b/nimdoc/testproject/expected/testproject.idx
index a8cdc3be2..4d0e63c0c 100644
--- a/nimdoc/testproject/expected/testproject.idx
+++ b/nimdoc/testproject/expected/testproject.idx
@@ -6,6 +6,7 @@ C_D	testproject.html#C_D	testproject: C_D
 bar	testproject.html#bar,T,T	testproject: bar[T](a, b: T): T	
 baz	testproject.html#baz,T,T	testproject: baz[T](a, b: T): T	
 buzz	testproject.html#buzz,T,T	testproject: buzz[T](a, b: T): T	
+FooBuzz	testproject.html#FooBuzz	testproject: FooBuzz	
 aVariable	testproject.html#aVariable	testproject: aVariable	
 A	testproject.html#A	testproject: A	
 B	testproject.html#B	testproject: B	
diff --git a/nimdoc/testproject/expected/theindex.html b/nimdoc/testproject/expected/theindex.html
index 55f7f33c9..cfaf6856d 100644
--- a/nimdoc/testproject/expected/theindex.html
+++ b/nimdoc/testproject/expected/theindex.html
@@ -179,6 +179,10 @@ window.addEventListener('DOMContentLoaded', main);
 
    • testproject: foo(a, b: SomeType)
      • 
           
    
+
    FooBuzz:
    
 
    fromUtils1:
      
 
    • testproject: fromUtils1(): int
      • 
diff --git a/nimdoc/testproject/testproject.nim b/nimdoc/testproject/testproject.nim
index b88b41098..57960c69f 100644
--- a/nimdoc/testproject/testproject.nim
+++ b/nimdoc/testproject/testproject.nim
@@ -39,6 +39,9 @@ proc buzz*[T](a, b: T): T {.deprecated: "since v0.20".} =
   ## This is deprecated with a message.
   result = a + b
 
+type
+  FooBuzz* {.deprecated: "FooBuzz msg".} = int
+
 import std/macros
 
 var aVariable*: array[1, int]

From bdfee32dc8e81dd6c727d711a7b1c1fad1728383 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Mon, 19 Jul 2021 18:43:31 +0800
Subject: [PATCH 1055/2472] close #18489 (#18524)

---
 tests/stdlib/tstrtabs2.nim | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 tests/stdlib/tstrtabs2.nim

diff --git a/tests/stdlib/tstrtabs2.nim b/tests/stdlib/tstrtabs2.nim
new file mode 100644
index 000000000..cb534f198
--- /dev/null
+++ b/tests/stdlib/tstrtabs2.nim
@@ -0,0 +1,15 @@
+discard """
+  targets: "c cpp js"
+"""
+
+import std/strtabs
+
+macro m =
+  var t = {"name": "John"}.newStringTable
+  doAssert t["name"] == "John"
+
+block:
+  var t = {"name": "John"}.newStringTable
+  doAssert t["name"] == "John"
+
+m()

From 1e7a10001a8f823dddabf10b59e0ef9893716787 Mon Sep 17 00:00:00 2001
From: ynfle <23086821+ynfle@users.noreply.github.com>
Date: Mon, 19 Jul 2021 13:44:06 +0300
Subject: [PATCH 1056/2472] Docs(manual): add clarification for default params
 (#18522)

* Docs(manual): add clarification for default params

* Docs(manual): fix wording to refer to call time

* Docs(manual): Clarify default parameter wording

Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com>

Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com>
---
 doc/manual.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/manual.rst b/doc/manual.rst
index 34781141e..f0bd85638 100644
--- a/doc/manual.rst
+++ b/doc/manual.rst
@@ -3543,7 +3543,8 @@ separation of types and subsequent identifiers more distinct.
   proc foo(a; b: int; c, d: bool): int
 
 A parameter may be declared with a default value which is used if the caller
-does not provide a value for the argument.
+does not provide a value for the argument. The value will be reevaluated
+every time the function is called.
 
 .. code-block:: nim
   # b is optional with 47 as its default value

From 73ce40aaf71196faa08072216bd826a94b5d40b1 Mon Sep 17 00:00:00 2001
From: konsumlamm <44230978+konsumlamm@users.noreply.github.com>
Date: Mon, 19 Jul 2021 12:45:13 +0200
Subject: [PATCH 1057/2472] Revert #17321 (#18521)

Use better names in example
---
 lib/pure/collections/lists.nim | 270 ++++++++++++++++-----------------
 1 file changed, 135 insertions(+), 135 deletions(-)

diff --git a/lib/pure/collections/lists.nim b/lib/pure/collections/lists.nim
index c83be7aa3..af53bb68b 100644
--- a/lib/pure/collections/lists.nim
+++ b/lib/pure/collections/lists.nim
@@ -19,15 +19,15 @@
 ##
 ## ## Lists
 runnableExamples:
-  var l = initDoublyLinkedList[int]()
+  var list = initDoublyLinkedList[int]()
   let
     a = newDoublyLinkedNode[int](3)
     b = newDoublyLinkedNode[int](7)
     c = newDoublyLinkedNode[int](9)
 
-  l.add(a)
-  l.add(b)
-  l.prepend(c)
+  list.add(a)
+  list.add(b)
+  list.prepend(c)
 
   assert a.next == b
   assert a.prev == c
@@ -38,15 +38,15 @@ runnableExamples:
 
 ## ## Rings
 runnableExamples:
-  var l = initSinglyLinkedRing[int]()
+  var ring = initSinglyLinkedRing[int]()
   let
     a = newSinglyLinkedNode[int](3)
     b = newSinglyLinkedNode[int](7)
     c = newSinglyLinkedNode[int](9)
 
-  l.add(a)
-  l.add(b)
-  l.prepend(c)
+  ring.add(a)
+  ring.add(b)
+  ring.prepend(c)
 
   assert c.next == a
   assert a.next == b
@@ -189,21 +189,21 @@ func toDoublyLinkedList*[T](elems: openArray[T]): DoublyLinkedList[T] {.since: (
     result.add(elem)
 
 template itemsListImpl() {.dirty.} =
-  var it = list.head
+  var it = L.head
   while it != nil:
     yield it.value
     it = it.next
 
 template itemsRingImpl() {.dirty.} =
-  var it = ring.head
+  var it = L.head
   if it != nil:
     while true:
       yield it.value
       it = it.next
-      if it == ring.head: break
+      if it == L.head: break
 
-iterator items*[T](list: SomeLinkedList[T]): T =
-  ## Yields every value of `list`.
+iterator items*[T](L: SomeLinkedList[T]): T =
+  ## Yields every value of `L`.
   ##
   ## **See also:**
   ## * `mitems iterator <#mitems.i,SomeLinkedList[T]>`_
@@ -218,8 +218,8 @@ iterator items*[T](list: SomeLinkedList[T]): T =
 
   itemsListImpl()
 
-iterator items*[T](ring: SomeLinkedRing[T]): T =
-  ## Yields every value of `ring`.
+iterator items*[T](L: SomeLinkedRing[T]): T =
+  ## Yields every value of `L`.
   ##
   ## **See also:**
   ## * `mitems iterator <#mitems.i,SomeLinkedRing[T]>`_
@@ -234,8 +234,8 @@ iterator items*[T](ring: SomeLinkedRing[T]): T =
 
   itemsRingImpl()
 
-iterator mitems*[T](list: var SomeLinkedList[T]): var T =
-  ## Yields every value of `list` so that you can modify it.
+iterator mitems*[T](L: var SomeLinkedList[T]): var T =
+  ## Yields every value of `L` so that you can modify it.
   ##
   ## **See also:**
   ## * `items iterator <#items.i,SomeLinkedList[T]>`_
@@ -251,8 +251,8 @@ iterator mitems*[T](list: var SomeLinkedList[T]): var T =
 
   itemsListImpl()
 
-iterator mitems*[T](ring: var SomeLinkedRing[T]): var T =
-  ## Yields every value of `ring` so that you can modify it.
+iterator mitems*[T](L: var SomeLinkedRing[T]): var T =
+  ## Yields every value of `L` so that you can modify it.
   ##
   ## **See also:**
   ## * `items iterator <#items.i,SomeLinkedRing[T]>`_
@@ -268,7 +268,7 @@ iterator mitems*[T](ring: var SomeLinkedRing[T]): var T =
 
   itemsRingImpl()
 
-iterator nodes*[T](list: SomeLinkedList[T]): SomeLinkedNode[T] =
+iterator nodes*[T](L: SomeLinkedList[T]): SomeLinkedNode[T] =
   ## Iterates over every node of `x`. Removing the current node from the
   ## list during traversal is supported.
   ##
@@ -287,13 +287,13 @@ iterator nodes*[T](list: SomeLinkedList[T]): SomeLinkedNode[T] =
         x.value = 5 * x.value - 1
     assert $a == "[49, 99, 199, 249]"
 
-  var it = list.head
+  var it = L.head
   while it != nil:
     let nxt = it.next
     yield it
     it = nxt
 
-iterator nodes*[T](ring: SomeLinkedRing[T]): SomeLinkedNode[T] =
+iterator nodes*[T](L: SomeLinkedRing[T]): SomeLinkedNode[T] =
   ## Iterates over every node of `x`. Removing the current node from the
   ## list during traversal is supported.
   ##
@@ -312,27 +312,27 @@ iterator nodes*[T](ring: SomeLinkedRing[T]): SomeLinkedNode[T] =
         x.value = 5 * x.value - 1
     assert $a == "[49, 99, 199, 249]"
 
-  var it = ring.head
+  var it = L.head
   if it != nil:
     while true:
       let nxt = it.next
       yield it
       it = nxt
-      if it == ring.head: break
+      if it == L.head: break
 
-proc `$`*[T](l: SomeLinkedCollection[T]): string =
+proc `$`*[T](L: SomeLinkedCollection[T]): string =
   ## Turns a list into its string representation for logging and printing.
   runnableExamples:
     let a = [1, 2, 3, 4].toSinglyLinkedList
     assert $a == "[1, 2, 3, 4]"
 
   result = "["
-  for x in nodes(l):
+  for x in nodes(L):
     if result.len > 1: result.add(", ")
     result.addQuoted(x.value)
   result.add("]")
 
-proc find*[T](l: SomeLinkedCollection[T], value: T): SomeLinkedNode[T] =
+proc find*[T](L: SomeLinkedCollection[T], value: T): SomeLinkedNode[T] =
   ## Searches in the list for a value. Returns `nil` if the value does not
   ## exist.
   ##
@@ -343,10 +343,10 @@ proc find*[T](l: SomeLinkedCollection[T], value: T): SomeLinkedNode[T] =
     assert a.find(9).value == 9
     assert a.find(1) == nil
 
-  for x in nodes(l):
+  for x in nodes(L):
     if x.value == value: return x
 
-proc contains*[T](l: SomeLinkedCollection[T], value: T): bool {.inline.} =
+proc contains*[T](L: SomeLinkedCollection[T], value: T): bool {.inline.} =
   ## Searches in the list for a value. Returns `false` if the value does not
   ## exist, `true` otherwise. This allows the usage of the `in` and `notin`
   ## operators.
@@ -360,7 +360,7 @@ proc contains*[T](l: SomeLinkedCollection[T], value: T): bool {.inline.} =
     assert(not a.contains(1))
     assert 2 notin a
 
-  result = find(l, value) != nil
+  result = find(L, value) != nil
 
 proc prepend*[T: SomeLinkedList](a: var T, b: T) {.since: (1, 5, 1).} =
   ## Prepends a shallow copy of `b` to the beginning of `a`.
@@ -411,8 +411,8 @@ proc prependMoved*[T: SomeLinkedList](a, b: var T) {.since: (1, 5, 1).} =
     (b, a) = (a, b)
   else: swap a, b
 
-proc add*[T](list: var SinglyLinkedList[T], n: SinglyLinkedNode[T]) {.inline.} =
-  ## Appends (adds to the end) a node `n` to `list`. Efficiency: O(1).
+proc add*[T](L: var SinglyLinkedList[T], n: SinglyLinkedNode[T]) {.inline.} =
+  ## Appends (adds to the end) a node `n` to `L`. Efficiency: O(1).
   ##
   ## **See also:**
   ## * `add proc <#add,SinglyLinkedList[T],T>`_ for appending a value
@@ -426,14 +426,14 @@ proc add*[T](list: var SinglyLinkedList[T], n: SinglyLinkedNode[T]) {.inline.} =
     assert a.contains(9)
 
   n.next = nil
-  if list.tail != nil:
-    assert(list.tail.next == nil)
-    list.tail.next = n
-  list.tail = n
-  if list.head == nil: list.head = n
+  if L.tail != nil:
+    assert(L.tail.next == nil)
+    L.tail.next = n
+  L.tail = n
+  if L.head == nil: L.head = n
 
-proc add*[T](list: var SinglyLinkedList[T], value: T) {.inline.} =
-  ## Appends (adds to the end) a value to `list`. Efficiency: O(1).
+proc add*[T](L: var SinglyLinkedList[T], value: T) {.inline.} =
+  ## Appends (adds to the end) a value to `L`. Efficiency: O(1).
   ##
   ## **See also:**
   ## * `add proc <#add,SinglyLinkedList[T],T>`_ for appending a value
@@ -446,11 +446,11 @@ proc add*[T](list: var SinglyLinkedList[T], value: T) {.inline.} =
     a.add(8)
     assert a.contains(9)
 
-  add(list, newSinglyLinkedNode(value))
+  add(L, newSinglyLinkedNode(value))
 
-proc prepend*[T](list: var SinglyLinkedList[T],
+proc prepend*[T](L: var SinglyLinkedList[T],
                  n: SinglyLinkedNode[T]) {.inline.} =
-  ## Prepends (adds to the beginning) a node to `list`. Efficiency: O(1).
+  ## Prepends (adds to the beginning) a node to `L`. Efficiency: O(1).
   ##
   ## **See also:**
   ## * `add proc <#add,SinglyLinkedList[T],SinglyLinkedNode[T]>`_
@@ -463,12 +463,12 @@ proc prepend*[T](list: var SinglyLinkedList[T],
     a.prepend(n)
     assert a.contains(9)
 
-  n.next = list.head
-  list.head = n
-  if list.tail == nil: list.tail = n
+  n.next = L.head
+  L.head = n
+  if L.tail == nil: L.tail = n
 
-proc prepend*[T](list: var SinglyLinkedList[T], value: T) {.inline.} =
-  ## Prepends (adds to the beginning) a node to `list`. Efficiency: O(1).
+proc prepend*[T](L: var SinglyLinkedList[T], value: T) {.inline.} =
+  ## Prepends (adds to the beginning) a node to `L`. Efficiency: O(1).
   ##
   ## **See also:**
   ## * `add proc <#add,SinglyLinkedList[T],SinglyLinkedNode[T]>`_
@@ -482,7 +482,7 @@ proc prepend*[T](list: var SinglyLinkedList[T], value: T) {.inline.} =
     a.prepend(8)
     assert a.contains(9)
 
-  prepend(list, newSinglyLinkedNode(value))
+  prepend(L, newSinglyLinkedNode(value))
 
 func copy*[T](a: SinglyLinkedList[T]): SinglyLinkedList[T] {.since: (1, 5, 1).} =
   ## Creates a shallow copy of `a`.
@@ -540,8 +540,8 @@ proc addMoved*[T](a, b: var SinglyLinkedList[T]) {.since: (1, 5, 1).} =
     b.head = nil
     b.tail = nil
 
-proc add*[T](list: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) =
-  ## Appends (adds to the end) a node `n` to `list`. Efficiency: O(1).
+proc add*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) =
+  ## Appends (adds to the end) a node `n` to `L`. Efficiency: O(1).
   ##
   ## **See also:**
   ## * `add proc <#add,DoublyLinkedList[T],T>`_ for appending a value
@@ -557,15 +557,15 @@ proc add*[T](list: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) =
     assert a.contains(9)
 
   n.next = nil
-  n.prev = list.tail
-  if list.tail != nil:
-    assert(list.tail.next == nil)
-    list.tail.next = n
-  list.tail = n
-  if list.head == nil: list.head = n
+  n.prev = L.tail
+  if L.tail != nil:
+    assert(L.tail.next == nil)
+    L.tail.next = n
+  L.tail = n
+  if L.head == nil: L.head = n
 
-proc add*[T](list: var DoublyLinkedList[T], value: T) =
-  ## Appends (adds to the end) a value to `list`. Efficiency: O(1).
+proc add*[T](L: var DoublyLinkedList[T], value: T) =
+  ## Appends (adds to the end) a value to `L`. Efficiency: O(1).
   ##
   ## **See also:**
   ## * `add proc <#add,DoublyLinkedList[T],DoublyLinkedNode[T]>`_
@@ -581,10 +581,10 @@ proc add*[T](list: var DoublyLinkedList[T], value: T) =
     a.add(8)
     assert a.contains(9)
 
-  add(list, newDoublyLinkedNode(value))
+  add(L, newDoublyLinkedNode(value))
 
-proc prepend*[T](list: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) =
-  ## Prepends (adds to the beginning) a node `n` to `list`. Efficiency: O(1).
+proc prepend*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) =
+  ## Prepends (adds to the beginning) a node `n` to `L`. Efficiency: O(1).
   ##
   ## **See also:**
   ## * `add proc <#add,DoublyLinkedList[T],DoublyLinkedNode[T]>`_
@@ -600,15 +600,15 @@ proc prepend*[T](list: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) =
     assert a.contains(9)
 
   n.prev = nil
-  n.next = list.head
-  if list.head != nil:
-    assert(list.head.prev == nil)
-    list.head.prev = n
-  list.head = n
-  if list.tail == nil: list.tail = n
+  n.next = L.head
+  if L.head != nil:
+    assert(L.head.prev == nil)
+    L.head.prev = n
+  L.head = n
+  if L.tail == nil: L.tail = n
 
-proc prepend*[T](list: var DoublyLinkedList[T], value: T) =
-  ## Prepends (adds to the beginning) a value to `list`. Efficiency: O(1).
+proc prepend*[T](L: var DoublyLinkedList[T], value: T) =
+  ## Prepends (adds to the beginning) a value to `L`. Efficiency: O(1).
   ##
   ## **See also:**
   ## * `add proc <#add,DoublyLinkedList[T],DoublyLinkedNode[T]>`_
@@ -624,7 +624,7 @@ proc prepend*[T](list: var DoublyLinkedList[T], value: T) =
     a.prepend(8)
     assert a.contains(9)
 
-  prepend(list, newDoublyLinkedNode(value))
+  prepend(L, newDoublyLinkedNode(value))
 
 func copy*[T](a: DoublyLinkedList[T]): DoublyLinkedList[T] {.since: (1, 5, 1).} =
   ## Creates a shallow copy of `a`.
@@ -705,9 +705,9 @@ proc add*[T: SomeLinkedList](a: var T, b: T) {.since: (1, 5, 1).} =
   var tmp = b.copy
   a.addMoved(tmp)
 
-proc remove*[T](list: var SinglyLinkedList[T], n: SinglyLinkedNode[T]): bool {.discardable.} =
-  ## Removes a node `n` from `list`.
-  ## Returns `true` if `n` was found in `list`.
+proc remove*[T](L: var SinglyLinkedList[T], n: SinglyLinkedNode[T]): bool {.discardable.} =
+  ## Removes a node `n` from `L`.
+  ## Returns `true` if `n` was found in `L`.
   ## Efficiency: O(n); the list is traversed until `n` is found.
   ## Attempting to remove an element not contained in the list is a no-op.
   ## When the list is cyclic, the cycle is preserved after removal.
@@ -728,12 +728,12 @@ proc remove*[T](list: var SinglyLinkedList[T], n: SinglyLinkedNode[T]): bool {.d
         ai
     assert s == [2, 2, 2, 2]
 
-  if n == list.head:
-    list.head = n.next
-    if list.tail.next == n:
-      list.tail.next = list.head # restore cycle
+  if n == L.head:
+    L.head = n.next
+    if L.tail.next == n:
+      L.tail.next = L.head # restore cycle
   else:
-    var prev = list.head
+    var prev = L.head
     while prev.next != n and prev.next != nil:
       prev = prev.next
     if prev.next == nil:
@@ -741,9 +741,9 @@ proc remove*[T](list: var SinglyLinkedList[T], n: SinglyLinkedNode[T]): bool {.d
     prev.next = n.next
   true
 
-proc remove*[T](list: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) =
-  ## Removes a node `n` from `list`. Efficiency: O(1).
-  ## This function assumes, for the sake of efficiency, that `n` is contained in `list`,
+proc remove*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) =
+  ## Removes a node `n` from `L`. Efficiency: O(1).
+  ## This function assumes, for the sake of efficiency, that `n` is contained in `L`,
   ## otherwise the effects are undefined.
   ## When the list is cyclic, the cycle is preserved after removal.
   runnableExamples:
@@ -763,15 +763,15 @@ proc remove*[T](list: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) =
         ai
     assert s == [2, 2, 2, 2]
 
-  if n == list.tail: list.tail = n.prev
-  if n == list.head: list.head = n.next
+  if n == L.tail: L.tail = n.prev
+  if n == L.head: L.head = n.next
   if n.next != nil: n.next.prev = n.prev
   if n.prev != nil: n.prev.next = n.next
 
 
 
-proc add*[T](ring: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) =
-  ## Appends (adds to the end) a node `n` to `ring`. Efficiency: O(1).
+proc add*[T](L: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) =
+  ## Appends (adds to the end) a node `n` to `L`. Efficiency: O(1).
   ##
   ## **See also:**
   ## * `add proc <#add,SinglyLinkedRing[T],T>`_ for appending a value
@@ -784,17 +784,17 @@ proc add*[T](ring: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) =
     a.add(n)
     assert a.contains(9)
 
-  if ring.head != nil:
-    n.next = ring.head
-    assert(ring.tail != nil)
-    ring.tail.next = n
+  if L.head != nil:
+    n.next = L.head
+    assert(L.tail != nil)
+    L.tail.next = n
   else:
     n.next = n
-    ring.head = n
-  ring.tail = n
+    L.head = n
+  L.tail = n
 
-proc add*[T](ring: var SinglyLinkedRing[T], value: T) =
-  ## Appends (adds to the end) a value to `ring`. Efficiency: O(1).
+proc add*[T](L: var SinglyLinkedRing[T], value: T) =
+  ## Appends (adds to the end) a value to `L`. Efficiency: O(1).
   ##
   ## **See also:**
   ## * `add proc <#add,SinglyLinkedRing[T],SinglyLinkedNode[T]>`_
@@ -808,10 +808,10 @@ proc add*[T](ring: var SinglyLinkedRing[T], value: T) =
     a.add(8)
     assert a.contains(9)
 
-  add(ring, newSinglyLinkedNode(value))
+  add(L, newSinglyLinkedNode(value))
 
-proc prepend*[T](ring: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) =
-  ## Prepends (adds to the beginning) a node `n` to `ring`. Efficiency: O(1).
+proc prepend*[T](L: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) =
+  ## Prepends (adds to the beginning) a node `n` to `L`. Efficiency: O(1).
   ##
   ## **See also:**
   ## * `add proc <#add,SinglyLinkedRing[T],SinglyLinkedNode[T]>`_
@@ -824,17 +824,17 @@ proc prepend*[T](ring: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) =
     a.prepend(n)
     assert a.contains(9)
 
-  if ring.head != nil:
-    n.next = ring.head
-    assert(ring.tail != nil)
-    ring.tail.next = n
+  if L.head != nil:
+    n.next = L.head
+    assert(L.tail != nil)
+    L.tail.next = n
   else:
     n.next = n
-    ring.tail = n
-  ring.head = n
+    L.tail = n
+  L.head = n
 
-proc prepend*[T](ring: var SinglyLinkedRing[T], value: T) =
-  ## Prepends (adds to the beginning) a value to `ring`. Efficiency: O(1).
+proc prepend*[T](L: var SinglyLinkedRing[T], value: T) =
+  ## Prepends (adds to the beginning) a value to `L`. Efficiency: O(1).
   ##
   ## **See also:**
   ## * `add proc <#add,SinglyLinkedRing[T],SinglyLinkedNode[T]>`_
@@ -848,12 +848,12 @@ proc prepend*[T](ring: var SinglyLinkedRing[T], value: T) =
     a.prepend(8)
     assert a.contains(9)
 
-  prepend(ring, newSinglyLinkedNode(value))
+  prepend(L, newSinglyLinkedNode(value))
 
 
 
-proc add*[T](ring: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) =
-  ## Appends (adds to the end) a node `n` to `ring`. Efficiency: O(1).
+proc add*[T](L: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) =
+  ## Appends (adds to the end) a node `n` to `L`. Efficiency: O(1).
   ##
   ## **See also:**
   ## * `add proc <#add,DoublyLinkedRing[T],T>`_ for appending a value
@@ -868,18 +868,18 @@ proc add*[T](ring: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) =
     a.add(n)
     assert a.contains(9)
 
-  if ring.head != nil:
-    n.next = ring.head
-    n.prev = ring.head.prev
-    ring.head.prev.next = n
-    ring.head.prev = n
+  if L.head != nil:
+    n.next = L.head
+    n.prev = L.head.prev
+    L.head.prev.next = n
+    L.head.prev = n
   else:
     n.prev = n
     n.next = n
-    ring.head = n
+    L.head = n
 
-proc add*[T](ring: var DoublyLinkedRing[T], value: T) =
-  ## Appends (adds to the end) a value to `ring`. Efficiency: O(1).
+proc add*[T](L: var DoublyLinkedRing[T], value: T) =
+  ## Appends (adds to the end) a value to `L`. Efficiency: O(1).
   ##
   ## **See also:**
   ## * `add proc <#add,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_
@@ -895,10 +895,10 @@ proc add*[T](ring: var DoublyLinkedRing[T], value: T) =
     a.add(8)
     assert a.contains(9)
 
-  add(ring, newDoublyLinkedNode(value))
+  add(L, newDoublyLinkedNode(value))
 
-proc prepend*[T](ring: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) =
-  ## Prepends (adds to the beginning) a node `n` to `ring`. Efficiency: O(1).
+proc prepend*[T](L: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) =
+  ## Prepends (adds to the beginning) a node `n` to `L`. Efficiency: O(1).
   ##
   ## **See also:**
   ## * `add proc <#add,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_
@@ -913,18 +913,18 @@ proc prepend*[T](ring: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) =
     a.prepend(n)
     assert a.contains(9)
 
-  if ring.head != nil:
-    n.next = ring.head
-    n.prev = ring.head.prev
-    ring.head.prev.next = n
-    ring.head.prev = n
+  if L.head != nil:
+    n.next = L.head
+    n.prev = L.head.prev
+    L.head.prev.next = n
+    L.head.prev = n
   else:
     n.prev = n
     n.next = n
-  ring.head = n
+  L.head = n
 
-proc prepend*[T](ring: var DoublyLinkedRing[T], value: T) =
-  ## Prepends (adds to the beginning) a value to `ring`. Efficiency: O(1).
+proc prepend*[T](L: var DoublyLinkedRing[T], value: T) =
+  ## Prepends (adds to the beginning) a value to `L`. Efficiency: O(1).
   ##
   ## **See also:**
   ## * `add proc <#add,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_
@@ -940,11 +940,11 @@ proc prepend*[T](ring: var DoublyLinkedRing[T], value: T) =
     a.prepend(8)
     assert a.contains(9)
 
-  prepend(ring, newDoublyLinkedNode(value))
+  prepend(L, newDoublyLinkedNode(value))
 
-proc remove*[T](ring: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) =
-  ## Removes `n` from `ring`. Efficiency: O(1).
-  ## This function assumes, for the sake of efficiency, that `n` is contained in `ring`,
+proc remove*[T](L: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) =
+  ## Removes `n` from `L`. Efficiency: O(1).
+  ## This function assumes, for the sake of efficiency, that `n` is contained in `L`,
   ## otherwise the effects are undefined.
   runnableExamples:
     var a = initDoublyLinkedRing[int]()
@@ -956,13 +956,13 @@ proc remove*[T](ring: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) =
 
   n.next.prev = n.prev
   n.prev.next = n.next
-  if n == ring.head:
-    let p = ring.head.prev
-    if p == ring.head:
+  if n == L.head:
+    let p = L.head.prev
+    if p == L.head:
       # only one element left:
-      ring.head = nil
+      L.head = nil
     else:
-      ring.head = p
+      L.head = p
 
 proc append*[T](a: var (SinglyLinkedList[T] | SinglyLinkedRing[T]),
                 b: SinglyLinkedList[T] | SinglyLinkedNode[T] | T) =

From db6e7fffba021c223fc67e1c0853e5839cd256d2 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Mon, 19 Jul 2021 20:07:47 -0700
Subject: [PATCH 1058/2472] minor clenup in vmops (#18529)

---
 compiler/vmops.nim | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/compiler/vmops.nim b/compiler/vmops.nim
index 67a196b8f..283306ecc 100644
--- a/compiler/vmops.nim
+++ b/compiler/vmops.nim
@@ -150,6 +150,9 @@ when defined(nimHasInvariant):
     of cincludes: copySeq(conf.cIncludes)
     of clibs: copySeq(conf.cLibs)
 
+proc stackTrace2(c: PCtx, msg: string, n: PNode) =
+  stackTrace(c, PStackFrame(prc: c.prc.sym, comesFrom: 0, next: nil), c.exceptionInstr, msg, n.info)
+
 proc registerAdditionalOps*(c: PCtx) =
   proc gorgeExWrapper(a: VmArgs) =
     let ret = opGorge(getString(a, 0), getString(a, 1), getString(a, 2),
@@ -238,15 +241,13 @@ proc registerAdditionalOps*(c: PCtx) =
   registerCallback c, "stdlib.macros.symBodyHash", proc (a: VmArgs) =
     let n = getNode(a, 0)
     if n.kind != nkSym:
-      stackTrace(c, PStackFrame(prc: c.prc.sym, comesFrom: 0, next: nil), c.exceptionInstr,
-                  "symBodyHash() requires a symbol. '" & $n & "' is of kind '" & $n.kind & "'", n.info)
+      stackTrace2(c, "symBodyHash() requires a symbol. '$#' is of kind '$#'" % [$n, $n.kind], n)
     setResult(a, $symBodyDigest(c.graph, n.sym))
 
   registerCallback c, "stdlib.macros.isExported", proc(a: VmArgs) =
     let n = getNode(a, 0)
     if n.kind != nkSym:
-      stackTrace(c, PStackFrame(prc: c.prc.sym, comesFrom: 0, next: nil), c.exceptionInstr,
-                  "isExported() requires a symbol. '" & $n & "' is of kind '" & $n.kind & "'", n.info)
+      stackTrace2(c, "isExported() requires a symbol. '$#' is of kind '$#'" % [$n, $n.kind], n)
     setResult(a, sfExported in n.sym.flags)
 
   registerCallback c, "stdlib.vmutils.vmTrace", proc (a: VmArgs) =

From 44c5afe448ecd9a7009982c16ec85fc8cabe3124 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Tue, 20 Jul 2021 11:14:04 +0800
Subject: [PATCH 1059/2472] [std/os]document `checkDir` and use
 `runnableExamples` (#18517)

Co-authored-by: Timothee Cour 
---
 lib/pure/os.nim | 36 +++++++++++++++---------------------
 1 file changed, 15 insertions(+), 21 deletions(-)

diff --git a/lib/pure/os.nim b/lib/pure/os.nim
index dadc380e6..6289c64f0 100644
--- a/lib/pure/os.nim
+++ b/lib/pure/os.nim
@@ -10,27 +10,14 @@
 ## This module contains basic operating system facilities like
 ## retrieving environment variables, reading command line arguments,
 ## working with directories, running shell commands, etc.
-##
-## .. code-block::
-##   import std/os
-##
-##   let myFile = "/path/to/my/file.nim"
-##
-##   let pathSplit = splitPath(myFile)
-##   assert pathSplit.head == "/path/to/my"
-##   assert pathSplit.tail == "file.nim"
-##
-##   assert parentDir(myFile) == "/path/to/my"
-##
-##   let fileSplit = splitFile(myFile)
-##   assert fileSplit.dir == "/path/to/my"
-##   assert fileSplit.name == "file"
-##   assert fileSplit.ext == ".nim"
-##
-##   assert myFile.changeFileExt("c") == "/path/to/my/file.c"
 
-##
-##
+runnableExamples("-r:off"):
+  let myFile = "/path/to/my/file.nim"
+  assert splitPath(myFile) == (head: "/path/to/my", tail: "file.nim")
+  assert parentDir(myFile) == "/path/to/my"
+  assert splitFile(myFile) == (dir: "/path/to/my", name: "file", ext: ".nim")
+  assert myFile.changeFileExt("c") == "/path/to/my/file.c"
+
 ## **See also:**
 ## * `osproc module `_ for process communication beyond
 ##   `execShellCmd proc <#execShellCmd,string>`_
@@ -2286,6 +2273,10 @@ iterator walkDir*(dir: string; relative = false, checkDir = false):
   ##
   ## Walking is not recursive. If ``relative`` is true (default: false)
   ## the resulting path is shortened to be relative to ``dir``.
+  ##
+  ## If `checkDir` is true, `OSError` is raised when `dir`
+  ## doesn't exist.
+  ##
   ## Example: This directory structure::
   ##   dirA / dirB / fileB1.txt
   ##        / dirC
@@ -2385,6 +2376,9 @@ iterator walkDirRec*(dir: string,
   ## If ``relative`` is true (default: false) the resulting path is
   ## shortened to be relative to ``dir``, otherwise the full path is returned.
   ##
+  ## If `checkDir` is true, `OSError` is raised when `dir`
+  ## doesn't exist.
+  ##
   ## .. warning:: Modifying the directory structure while the iterator
   ##   is traversing may result in undefined behavior!
   ##
@@ -2448,7 +2442,7 @@ proc removeDir*(dir: string, checkDir = false) {.rtl, extern: "nos$1", tags: [
   ## in `dir` (recursively).
   ##
   ## If this fails, `OSError` is raised. This does not fail if the directory never
-  ## existed in the first place, unless `checkDir` = true
+  ## existed in the first place, unless `checkDir` = true.
   ##
   ## See also:
   ## * `tryRemoveFile proc <#tryRemoveFile,string>`_

From 8c7ee96457769a8c8aa1d4bab306b53acf306c8e Mon Sep 17 00:00:00 2001
From: Andrey Makarov 
Date: Tue, 20 Jul 2021 09:32:22 +0300
Subject: [PATCH 1060/2472] rst: add missing line/column info for some warnings
 (#18383)

* rst: add missing line/column info for some warnings

* add workaround

* use TLineInfo/FileIndex for storing file names

* fix blank lines in include file (rm harmful strip)

* don't use ref TLineInfo

* return `hasToc` as output parameter for uniformity

* Update compiler/docgen.nim

Co-authored-by: Timothee Cour 

* Update compiler/docgen.nim

Co-authored-by: Timothee Cour 

* Update lib/packages/docutils/rst.nim

Co-authored-by: Timothee Cour 

* address review - stylistic things

* Update compiler/docgen.nim

Co-authored-by: Timothee Cour 

* unify RST warnings/errors names

* doAssert + minor name change

* fix a bug caught by doAssert

* apply strbasics.strip to final HTML/Latex

* rm redundant filename

* fix test after rebase

* delete `order` from rnFootnoteRef,

also display errors/warnings properly when footnote references are from
different files

* Update compiler/lineinfos.nim

Co-authored-by: Timothee Cour 

* Update lib/packages/docutils/rstast.nim

Co-authored-by: Timothee Cour 

* Update lib/packages/docutils/rstast.nim

Co-authored-by: Timothee Cour 

* Update lib/packages/docutils/rstast.nim

Co-authored-by: Timothee Cour 

* revert because of error:

Error: cannot prove that it's safe to initialize 'info' with the runtime value for the discriminator 'kind'

* Update lib/packages/docutils/rstgen.nim

Co-authored-by: Timothee Cour 

* apply suggestion

* Update lib/packages/docutils/rst.nim

Co-authored-by: Timothee Cour 

* add Table for string->file name mapping

* do not import compiler/lineinfos

* fix ambiguous calls

Co-authored-by: Timothee Cour 
Co-authored-by: narimiran 
---
 compiler/docgen.nim              |  54 ++++-----
 compiler/lineinfos.nim           |  49 +++++----
 compiler/main.nim                |  11 +-
 lib/packages/docutils/rst.nim    | 181 ++++++++++++++++++++-----------
 lib/packages/docutils/rstast.nim |  20 +++-
 lib/packages/docutils/rstgen.nim |  40 +++----
 tests/stdlib/trst.nim            |  58 +++++++++-
 tests/stdlib/trstgen.nim         | 122 ++++++++++++---------
 8 files changed, 342 insertions(+), 193 deletions(-)

diff --git a/compiler/docgen.nim b/compiler/docgen.nim
index 9c75c0a6f..f9e72850a 100644
--- a/compiler/docgen.nim
+++ b/compiler/docgen.nim
@@ -12,11 +12,12 @@
 
 import
   ast, strutils, strtabs, options, msgs, os, idents,
-  wordrecg, syntaxes, renderer, lexer, packages/docutils/rstast,
+  wordrecg, syntaxes, renderer, lexer,
   packages/docutils/rst, packages/docutils/rstgen,
   json, xmltree, trees, types,
   typesrenderer, astalgo, lineinfos, intsets,
   pathutils, tables, nimpaths, renderverbatim, osproc
+import packages/docutils/rstast except FileIndex, TLineInfo
 
 from uri import encodeUrl
 from std/private/globs import nativeToUnixPath
@@ -159,17 +160,18 @@ template declareClosures =
     case msgKind
     of meCannotOpenFile: k = errCannotOpenFile
     of meExpected: k = errXExpected
-    of meGridTableNotImplemented: k = errGridTableNotImplemented
-    of meMarkdownIllformedTable: k = errMarkdownIllformedTable
-    of meNewSectionExpected: k = errNewSectionExpected
-    of meGeneralParseError: k = errGeneralParseError
-    of meInvalidDirective: k = errInvalidDirectiveX
-    of meInvalidRstField: k = errInvalidRstField
-    of meFootnoteMismatch: k = errFootnoteMismatch
-    of mwRedefinitionOfLabel: k = warnRedefinitionOfLabel
-    of mwUnknownSubstitution: k = warnUnknownSubstitutionX
-    of mwUnsupportedLanguage: k = warnLanguageXNotSupported
-    of mwUnsupportedField: k = warnFieldXNotSupported
+    of meGridTableNotImplemented: k = errRstGridTableNotImplemented
+    of meMarkdownIllformedTable: k = errRstMarkdownIllformedTable
+    of meNewSectionExpected: k = errRstNewSectionExpected
+    of meGeneralParseError: k = errRstGeneralParseError
+    of meInvalidDirective: k = errRstInvalidDirectiveX
+    of meInvalidField: k = errRstInvalidField
+    of meFootnoteMismatch: k = errRstFootnoteMismatch
+    of mwRedefinitionOfLabel: k = warnRstRedefinitionOfLabel
+    of mwUnknownSubstitution: k = warnRstUnknownSubstitutionX
+    of mwBrokenLink: k = warnRstBrokenLink
+    of mwUnsupportedLanguage: k = warnRstLanguageXNotSupported
+    of mwUnsupportedField: k = warnRstFieldXNotSupported
     of mwRstStyle: k = warnRstStyle
     {.gcsafe.}:
       globalError(conf, newLineInfo(conf, AbsoluteFile filename, line, col), k, arg)
@@ -182,11 +184,9 @@ template declareClosures =
 
 proc parseRst(text, filename: string,
               line, column: int,
-              rstOptions: RstParseOptions;
               conf: ConfigRef, sharedState: PRstSharedState): PRstNode =
   declareClosures()
-  result = rstParsePass1(text, filename, line, column, rstOptions,
-                         sharedState)
+  result = rstParsePass1(text, line, column, sharedState)
 
 proc getOutFile2(conf: ConfigRef; filename: RelativeFile,
                  ext: string, guessTarget: bool): AbsoluteFile =
@@ -202,20 +202,22 @@ proc getOutFile2(conf: ConfigRef; filename: RelativeFile,
 proc isLatexCmd(conf: ConfigRef): bool = conf.cmd in {cmdRst2tex, cmdDoc2tex}
 
 proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
-                    outExt: string = HtmlExt, module: PSym = nil): PDoc =
+                    outExt: string = HtmlExt, module: PSym = nil,
+                    isPureRst = false): PDoc =
   declareClosures()
   new(result)
   result.module = module
   result.conf = conf
   result.cache = cache
   result.outDir = conf.outDir.string
-  const options = {roSupportRawDirective, roSupportMarkdown,
-                   roPreferMarkdown, roNimFile}
+  result.isPureRst = isPureRst
+  var options= {roSupportRawDirective, roSupportMarkdown, roPreferMarkdown}
+  if not isPureRst: options.incl roNimFile
   result.sharedState = newRstSharedState(
       options, filename.string,
       docgenFindFile, compilerMsgHandler)
   initRstGenerator(result[], (if conf.isLatexCmd: outLatex else: outHtml),
-                   conf.configVars, filename.string, options,
+                   conf.configVars, filename.string,
                    docgenFindFile, compilerMsgHandler)
 
   if conf.configVars.hasKey("doc.googleAnalytics"):
@@ -299,8 +301,7 @@ proc genComment(d: PDoc, n: PNode): PRstNode =
     result = parseRst(n.comment, toFullPath(d.conf, n.info),
                       toLinenumber(n.info),
                       toColumn(n.info) + DocColOffset,
-                      d.options, d.conf,
-                      d.sharedState)
+                      d.conf, d.sharedState)
 
 proc genRecCommentAux(d: PDoc, n: PNode): PRstNode =
   if n == nil: return nil
@@ -1123,6 +1124,9 @@ proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) =
 
 proc finishGenerateDoc*(d: var PDoc) =
   ## Perform 2nd RST pass for resolution of links/footnotes/headings...
+  # copy file map `filenames` to ``rstgen.nim`` for its warnings
+  d.filenames = d.sharedState.filenames
+
   # Main title/subtitle are allowed only in the first RST fragment of document
   var firstRst = PRstNode(nil)
   for fragment in d.modDescPre:
@@ -1417,14 +1421,10 @@ proc commandDoc*(cache: IdentCache, conf: ConfigRef) =
 proc commandRstAux(cache: IdentCache, conf: ConfigRef;
                    filename: AbsoluteFile, outExt: string) =
   var filen = addFileExt(filename, "txt")
-  var d = newDocumentor(filen, cache, conf, outExt)
-
-  d.isPureRst = true
+  var d = newDocumentor(filen, cache, conf, outExt, isPureRst = true)
   let rst = parseRst(readFile(filen.string), filen.string,
                      line=LineRstInit, column=ColRstInit,
-                     {roSupportRawDirective, roSupportMarkdown,
-                      roPreferMarkdown}, conf,
-                     d.sharedState)
+                     conf, d.sharedState)
   d.modDescPre = @[ItemFragment(isRst: true, rst: rst)]
   finishGenerateDoc(d)
   writeOutput(d)
diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim
index 801c20d25..8ab52a452 100644
--- a/compiler/lineinfos.nim
+++ b/compiler/lineinfos.nim
@@ -32,13 +32,13 @@ type
     # non-fatal errors
     errIllFormedAstX, errCannotOpenFile,
     errXExpected,
-    errGridTableNotImplemented,
-    errMarkdownIllformedTable,
-    errGeneralParseError,
-    errNewSectionExpected,
-    errInvalidDirectiveX,
-    errInvalidRstField,
-    errFootnoteMismatch,
+    errRstGridTableNotImplemented,
+    errRstMarkdownIllformedTable,
+    errRstNewSectionExpected,
+    errRstGeneralParseError,
+    errRstInvalidDirectiveX,
+    errRstInvalidField,
+    errRstFootnoteMismatch,
     errProveInit, # deadcode
     errGenerated,
     errUser,
@@ -47,10 +47,13 @@ type
     warnXIsNeverRead = "XIsNeverRead", warnXmightNotBeenInit = "XmightNotBeenInit",
     warnDeprecated = "Deprecated", warnConfigDeprecated = "ConfigDeprecated",
     warnSmallLshouldNotBeUsed = "SmallLshouldNotBeUsed", warnUnknownMagic = "UnknownMagic",
-    warnRedefinitionOfLabel = "RedefinitionOfLabel", warnUnknownSubstitutionX = "UnknownSubstitutionX",
-    warnLanguageXNotSupported = "LanguageXNotSupported",
-    warnFieldXNotSupported = "FieldXNotSupported",
-    warnRstStyle = "warnRstStyle", warnCommentXIgnored = "CommentXIgnored",
+    warnRstRedefinitionOfLabel = "RedefinitionOfLabel",
+    warnRstUnknownSubstitutionX = "UnknownSubstitutionX",
+    warnRstBrokenLink = "BrokenLink",
+    warnRstLanguageXNotSupported = "LanguageXNotSupported",
+    warnRstFieldXNotSupported = "FieldXNotSupported",
+    warnRstStyle = "warnRstStyle",
+    warnCommentXIgnored = "CommentXIgnored",
     warnTypelessParam = "TypelessParam",
     warnUseBase = "UseBase", warnWriteToForeignHeap = "WriteToForeignHeap",
     warnUnsafeCode = "UnsafeCode", warnUnusedImportX = "UnusedImport",
@@ -93,13 +96,13 @@ const
     errIllFormedAstX: "illformed AST: $1",
     errCannotOpenFile: "cannot open '$1'",
     errXExpected: "'$1' expected",
-    errGridTableNotImplemented: "grid table is not implemented",
-    errMarkdownIllformedTable: "illformed delimiter row of a markdown table",
-    errGeneralParseError: "general parse error",
-    errNewSectionExpected: "new section expected $1",
-    errInvalidDirectiveX: "invalid directive: '$1'",
-    errInvalidRstField: "invalid field: $1",
-    errFootnoteMismatch: "number of footnotes and their references don't match: $1",
+    errRstGridTableNotImplemented: "grid table is not implemented",
+    errRstMarkdownIllformedTable: "illformed delimiter row of a markdown table",
+    errRstNewSectionExpected: "new section expected $1",
+    errRstGeneralParseError: "general parse error",
+    errRstInvalidDirectiveX: "invalid directive: '$1'",
+    errRstInvalidField: "invalid field: $1",
+    errRstFootnoteMismatch: "number of footnotes and their references don't match: $1",
     errProveInit: "Cannot prove that '$1' is initialized.",  # deadcode
     errGenerated: "$1",
     errUser: "$1",
@@ -111,10 +114,11 @@ const
     warnConfigDeprecated: "config file '$1' is deprecated",
     warnSmallLshouldNotBeUsed: "'l' should not be used as an identifier; may look like '1' (one)",
     warnUnknownMagic: "unknown magic '$1' might crash the compiler",
-    warnRedefinitionOfLabel: "redefinition of label '$1'",
-    warnUnknownSubstitutionX: "unknown substitution '$1'",
-    warnLanguageXNotSupported: "language '$1' not supported",
-    warnFieldXNotSupported: "field '$1' not supported",
+    warnRstRedefinitionOfLabel: "redefinition of label '$1'",
+    warnRstUnknownSubstitutionX: "unknown substitution '$1'",
+    warnRstBrokenLink: "broken link '$1'",
+    warnRstLanguageXNotSupported: "language '$1' not supported",
+    warnRstFieldXNotSupported: "field '$1' not supported",
     warnRstStyle: "RST style: $1",
     warnCommentXIgnored: "comment '$1' ignored",
     warnTypelessParam: "", # deadcode
@@ -196,6 +200,7 @@ const
   warnMax* = pred(hintSuccess)
   hintMin* = hintSuccess
   hintMax* = high(TMsgKind)
+  rstWarnings* = {warnRstRedefinitionOfLabel..warnRstStyle}
 
 type
   TNoteKind* = range[warnMin..hintMax] # "notes" are warnings or hints
diff --git a/compiler/main.nim b/compiler/main.nim
index 1c8034c6c..e4ec4d729 100644
--- a/compiler/main.nim
+++ b/compiler/main.nim
@@ -285,7 +285,7 @@ proc mainCommand*(graph: ModuleGraph) =
   of cmdDoc:
     docLikeCmd():
       conf.setNoteDefaults(warnLockLevel, false) # issue #13218
-      conf.setNoteDefaults(warnRedefinitionOfLabel, false) # issue #13218
+      conf.setNoteDefaults(warnRstRedefinitionOfLabel, false) # issue #13218
         # because currently generates lots of false positives due to conflation
         # of labels links in doc comments, e.g. for random.rand:
         #  ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer
@@ -295,19 +295,16 @@ proc mainCommand*(graph: ModuleGraph) =
         commandBuildIndex(conf, $conf.outDir)
   of cmdRst2html:
     # XXX: why are warnings disabled by default for rst2html and rst2tex?
-    for warn in [warnUnknownSubstitutionX, warnLanguageXNotSupported,
-                 warnFieldXNotSupported, warnRstStyle]:
+    for warn in rstWarnings:
       conf.setNoteDefaults(warn, true)
-    conf.setNoteDefaults(warnRedefinitionOfLabel, false) # similar to issue #13218
+    conf.setNoteDefaults(warnRstRedefinitionOfLabel, false) # similar to issue #13218
     when defined(leanCompiler):
       conf.quitOrRaise "compiler wasn't built with documentation generator"
     else:
       loadConfigs(DocConfig, cache, conf, graph.idgen)
       commandRst2Html(cache, conf)
   of cmdRst2tex, cmdDoc2tex:
-    for warn in [warnRedefinitionOfLabel, warnUnknownSubstitutionX,
-                 warnLanguageXNotSupported,
-                 warnFieldXNotSupported, warnRstStyle]:
+    for warn in rstWarnings:
       conf.setNoteDefaults(warn, true)
     when defined(leanCompiler):
       conf.quitOrRaise "compiler wasn't built with documentation generator"
diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim
index 9abdf3fc3..abea026dd 100644
--- a/lib/packages/docutils/rst.nim
+++ b/lib/packages/docutils/rst.nim
@@ -196,7 +196,7 @@
 
 import
   os, strutils, rstast, std/enumutils, algorithm, lists, sequtils,
-  std/private/miscdollars
+  std/private/miscdollars, tables
 from highlite import SourceLanguage, getSourceLanguage
 
 type
@@ -217,6 +217,7 @@ type
     mcWarning = "Warning",
     mcError = "Error"
 
+  # keep the order in sync with compiler/docgen.nim and compiler/lineinfos.nim:
   MsgKind* = enum          ## the possible messages
     meCannotOpenFile = "cannot open '$1'",
     meExpected = "'$1' expected",
@@ -225,10 +226,11 @@ type
     meNewSectionExpected = "new section expected $1",
     meGeneralParseError = "general parse error",
     meInvalidDirective = "invalid directive: '$1'",
-    meInvalidRstField = "invalid field: $1",
+    meInvalidField = "invalid field: $1",
     meFootnoteMismatch = "mismatch in number of footnotes and their refs: $1",
     mwRedefinitionOfLabel = "redefinition of label '$1'",
     mwUnknownSubstitution = "unknown substitution '$1'",
+    mwBrokenLink = "broken link '$1'",
     mwUnsupportedLanguage = "language '$1' not supported",
     mwUnsupportedField = "field '$1' not supported",
     mwRstStyle = "RST style: $1"
@@ -489,7 +491,9 @@ type
     autoNumIdx: int     # order of occurence: fnAutoNumber, fnAutoNumberLabel
     autoSymIdx: int     # order of occurence: fnAutoSymbol
     label: string       # valid for fnAutoNumberLabel
-
+  RstFileTable* = object
+    filenameToIdx*: Table[string, FileIndex]
+    idxToFilename*: seq[string]
   RstSharedState = object
     options: RstParseOptions    # parsing options
     hLevels: LevelMap           # hierarchy of heading styles
@@ -501,15 +505,19 @@ type
     subs: seq[Substitution]     # substitutions
     refs*: seq[Substitution]    # references
     anchors*: seq[AnchorSubst]  # internal target substitutions
-    lineFootnoteNum: seq[int]     # footnote line, auto numbers .. [#]
-    lineFootnoteNumRef: seq[int]  # footnote line, their reference [#]_
-    lineFootnoteSym: seq[int]     # footnote line, auto symbols .. [*]
-    lineFootnoteSymRef: seq[int]  # footnote line, their reference [*]_
+    lineFootnoteNum: seq[TLineInfo]     # footnote line, auto numbers .. [#]
+    lineFootnoteNumRef: seq[TLineInfo]  # footnote line, their reference [#]_
+    currFootnoteNumRef: int             # ... their counter for `resolveSubs`
+    lineFootnoteSym: seq[TLineInfo]     # footnote line, auto symbols .. [*]
+    lineFootnoteSymRef: seq[TLineInfo]  # footnote line, their reference [*]_
+    currFootnoteSymRef: int             # ... their counter for `resolveSubs`
     footnotes: seq[FootnoteSubst] # correspondence b/w footnote label,
                                   # number, order of occurrence
     msgHandler: MsgHandler      # How to handle errors.
     findFile: FindFileHandler   # How to find files.
-    filename: string
+    filenames*: RstFileTable    # map file name <-> FileIndex (for storing
+                                # file names for warnings after 1st stage)
+    currFileIdx: FileIndex      # current index in `filesnames`
     hasToc*: bool
 
   PRstSharedState* = ref RstSharedState
@@ -579,6 +587,25 @@ proc whichRoleAux(sym: string): RstNodeKind =
   else:  # unknown role
     result = rnUnknownRole
 
+proc len(filenames: RstFileTable): int = filenames.idxToFilename.len
+
+proc setCurrFilename(s: PRstSharedState, file1: string) =
+  let nextIdx = s.filenames.len.FileIndex
+  let v = getOrDefault(s.filenames.filenameToIdx, file1, default = nextIdx)
+  if v == nextIdx:
+    s.filenames.filenameToIdx[file1] = v
+    s.filenames.idxToFilename.add file1
+  s.currFileIdx = v
+
+proc getFilename(filenames: RstFileTable, fid: FileIndex): string =
+  doAssert(0 <= fid.int and fid.int < filenames.len,
+      "incorrect FileIndex $1 (range 0..$2)" % [
+        $fid.int, $(filenames.len - 1)])
+  result = filenames.idxToFilename[fid.int]
+
+proc currFilename(s: PRstSharedState): string =
+  getFilename(s.filenames, s.currFileIdx)
+
 proc newRstSharedState*(options: RstParseOptions,
                         filename: string,
                         findFile: FindFileHandler,
@@ -589,32 +616,37 @@ proc newRstSharedState*(options: RstParseOptions,
       currRoleKind: whichRoleAux(r),
       options: options,
       msgHandler: if not isNil(msgHandler): msgHandler else: defaultMsgHandler,
-      filename: filename,
       findFile: if not isNil(findFile): findFile else: defaultFindFile
   )
+  setCurrFilename(result, filename)
 
 proc curLine(p: RstParser): int = p.line + currentTok(p).line
 
 proc findRelativeFile(p: RstParser; filename: string): string =
-  result = p.s.filename.splitFile.dir / filename
+  result = p.s.currFilename.splitFile.dir / filename
   if not fileExists(result):
     result = p.s.findFile(filename)
 
 proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string) =
-  p.s.msgHandler(p.s.filename, curLine(p),
+  p.s.msgHandler(p.s.currFilename, curLine(p),
                              p.col + currentTok(p).col, msgKind, arg)
 
 proc rstMessage(s: PRstSharedState, msgKind: MsgKind, arg: string) =
-  ## Print warnings for footnotes/substitutions.
-  ## TODO: their line/column info is not known, to fix it.
-  s.msgHandler(s.filename, LineRstInit, ColRstInit, msgKind, arg)
+  s.msgHandler(s.currFilename, LineRstInit, ColRstInit, msgKind, arg)
+
+proc rstMessage*(filenames: RstFileTable, f: MsgHandler,
+                 info: TLineInfo, msgKind: MsgKind, arg: string) =
+  ## Print warnings using `info`, i.e. in 2nd-pass warnings for
+  ## footnotes/substitutions/references or from ``rstgen.nim``.
+  let file = getFilename(filenames, info.fileIndex)
+  f(file, info.line.int, info.col.int, msgKind, arg)
 
 proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string, line, col: int) =
-  p.s.msgHandler(p.s.filename, p.line + line,
+  p.s.msgHandler(p.s.currFilename, p.line + line,
                              p.col + col, msgKind, arg)
 
 proc rstMessage(p: RstParser, msgKind: MsgKind) =
-  p.s.msgHandler(p.s.filename, curLine(p),
+  p.s.msgHandler(p.s.currFilename, curLine(p),
                              p.col + currentTok(p).col, msgKind,
                              currentTok(p).symbol)
 
@@ -827,11 +859,18 @@ proc addFootnoteNumManual(p: var RstParser, num: int) =
       return
   p.s.footnotes.add((fnManualNumber, num, -1, -1, $num))
 
+proc lineInfo(p: RstParser, iTok: int): TLineInfo =
+  result.col = int16(p.col + p.tok[iTok].col)
+  result.line = uint16(p.line + p.tok[iTok].line)
+  result.fileIndex = p.s.currFileIdx
+
+proc lineInfo(p: RstParser): TLineInfo = lineInfo(p, p.idx)
+
 proc addFootnoteNumAuto(p: var RstParser, label: string) =
   ## add auto-numbered footnote.
   ## Empty label [#] means it'll be resolved by the occurrence.
   if label == "":  # simple auto-numbered [#]
-    p.s.lineFootnoteNum.add curLine(p)
+    p.s.lineFootnoteNum.add lineInfo(p)
     p.s.footnotes.add((fnAutoNumber, -1, p.s.lineFootnoteNum.len, -1, label))
   else:           # auto-numbered with label [#label]
     for fnote in p.s.footnotes:
@@ -841,7 +880,7 @@ proc addFootnoteNumAuto(p: var RstParser, label: string) =
     p.s.footnotes.add((fnAutoNumberLabel, -1, -1, -1, label))
 
 proc addFootnoteSymAuto(p: var RstParser) =
-  p.s.lineFootnoteSym.add curLine(p)
+  p.s.lineFootnoteSym.add lineInfo(p)
   p.s.footnotes.add((fnAutoSymbol, -1, -1, p.s.lineFootnoteSym.len, ""))
 
 proc orderFootnotes(s: PRstSharedState) =
@@ -850,7 +889,15 @@ proc orderFootnotes(s: PRstSharedState) =
   ## Save the result back to `s.footnotes`.
 
   # Report an error if found any mismatch in number of automatic footnotes
-  proc listFootnotes(lines: seq[int]): string =
+  proc listFootnotes(locations: seq[TLineInfo]): string =
+    var lines: seq[string]
+    for info in locations:
+      if s.filenames.len > 1:
+        let file = getFilename(s.filenames, info.fileIndex)
+        lines.add file & ":"
+      else:  # no need to add file name here if there is only 1
+        lines.add ""
+      lines[^1].add $info.line
     result.add $lines.len & " (lines " & join(lines, ", ") & ")"
   if s.lineFootnoteNum.len != s.lineFootnoteNumRef.len:
     rstMessage(s, meFootnoteMismatch,
@@ -1157,7 +1204,7 @@ proc whichRole(p: RstParser, sym: string): RstNodeKind =
 
 proc toInlineCode(n: PRstNode, language: string): PRstNode =
   ## Creates rnInlineCode and attaches `n` contents as code (in 3rd son).
-  result = newRstNode(rnInlineCode)
+  result = newRstNode(rnInlineCode, info=n.info)
   let args = newRstNode(rnDirArg)
   var lang = language
   if language == "cpp": lang = "c++"
@@ -1179,6 +1226,7 @@ proc toOtherRole(n: PRstNode, kind: RstNodeKind, roleName: string): PRstNode =
   result = newRstNode(kind, newSons)
 
 proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode =
+  ## Finalizes node `n` that was tentatively determined as interpreted text.
   var newKind = n.kind
   var newSons = n.sons
 
@@ -1207,12 +1255,10 @@ proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode =
         newKind = rnHyperlink
         newSons = @[a, b]
         setRef(p, rstnodeToRefname(a), b)
-    elif n.kind == rnInterpretedText:
+      result = newRstNode(newKind, newSons)
+    else:  # some link that will be resolved in `resolveSubs`
       newKind = rnRef
-    else:
-      newKind = rnRef
-      newSons = @[n]
-    result = newRstNode(newKind, newSons)
+      result = newRstNode(newKind, sons=newSons, info=n.info)
   elif match(p, p.idx, ":w:"):
     # a role:
     let (roleName, lastIdx) = getRefname(p, p.idx+1)
@@ -1300,20 +1346,19 @@ proc parseWordOrRef(p: var RstParser, father: PRstNode) =
   else:
     # check for reference (probably, long one like some.ref.with.dots_ )
     var saveIdx = p.idx
-    var isRef = false
+    var reference: PRstNode = nil
     inc p.idx
     while currentTok(p).kind in {tkWord, tkPunct}:
       if currentTok(p).kind == tkPunct:
         if isInlineMarkupEnd(p, "_", exact=true):
-          isRef = true
+          reference = newRstNode(rnRef, info=lineInfo(p, saveIdx))
           break
         if not validRefnamePunct(currentTok(p).symbol):
           break
       inc p.idx
-    if isRef:
-      let r = newRstNode(rnRef)
-      for i in saveIdx..p.idx-1: r.add newLeaf(p.tok[i].symbol)
-      father.add r
+    if reference != nil:
+      for i in saveIdx..p.idx-1: reference.add newLeaf(p.tok[i].symbol)
+      father.add reference
       inc p.idx  # skip final _
     else:  # 1 normal word
       father.add newLeaf(p.tok[saveIdx].symbol)
@@ -1387,6 +1432,8 @@ proc parseUntil(p: var RstParser, father: PRstNode, postfix: string,
     else: rstMessage(p, meExpected, postfix, line, col)
 
 proc parseMarkdownCodeblock(p: var RstParser): PRstNode =
+  result = newRstNodeA(p, rnCodeBlock)
+  result.info = lineInfo(p)
   var args = newRstNode(rnDirArg)
   if currentTok(p).kind == tkWord:
     args.add(newLeaf(p))
@@ -1411,7 +1458,6 @@ proc parseMarkdownCodeblock(p: var RstParser): PRstNode =
       inc p.idx
   var lb = newRstNode(rnLiteralBlock)
   lb.add(n)
-  result = newRstNodeA(p, rnCodeBlock)
   result.add(args)
   result.add(PRstNode(nil))
   result.add(lb)
@@ -1495,6 +1541,7 @@ proc parseFootnoteName(p: var RstParser, reference: bool): PRstNode =
 
 proc parseInline(p: var RstParser, father: PRstNode) =
   var n: PRstNode  # to be used in `if` condition
+  let saveIdx = p.idx
   case currentTok(p).kind
   of tkPunct:
     if isInlineMarkupStart(p, "***"):
@@ -1537,12 +1584,12 @@ proc parseInline(p: var RstParser, father: PRstNode) =
         n = n.toOtherRole(k, roleName)
       father.add(n)
     elif isInlineMarkupStart(p, "`"):
-      var n = newRstNode(rnInterpretedText)
+      var n = newRstNode(rnInterpretedText, info=lineInfo(p, p.idx+1))
       parseUntil(p, n, "`", false) # bug #17260
       n = parsePostfix(p, n)
       father.add(n)
     elif isInlineMarkupStart(p, "|"):
-      var n = newRstNode(rnSubstitutionReferences)
+      var n = newRstNode(rnSubstitutionReferences, info=lineInfo(p, p.idx+1))
       parseUntil(p, n, "|", false)
       father.add(n)
     elif roSupportMarkdown in p.s.options and
@@ -1552,15 +1599,14 @@ proc parseInline(p: var RstParser, father: PRstNode) =
     elif isInlineMarkupStart(p, "[") and nextTok(p).symbol != "[" and
          (n = parseFootnoteName(p, reference=true); n != nil):
       var nn = newRstNode(rnFootnoteRef)
+      nn.info = lineInfo(p, saveIdx+1)
       nn.add n
       let (fnType, _) = getFootnoteType(n)
       case fnType
       of fnAutoSymbol:
-        p.s.lineFootnoteSymRef.add curLine(p)
-        nn.order = p.s.lineFootnoteSymRef.len
+        p.s.lineFootnoteSymRef.add lineInfo(p)
       of fnAutoNumber:
-        p.s.lineFootnoteNumRef.add curLine(p)
-        nn.order = p.s.lineFootnoteNumRef.len
+        p.s.lineFootnoteNumRef.add lineInfo(p)
       else: discard
       father.add(nn)
     else:
@@ -1693,7 +1739,7 @@ proc parseField(p: var RstParser): PRstNode =
   ## Returns a parsed rnField node.
   ##
   ## rnField nodes have two children nodes, a rnFieldName and a rnFieldBody.
-  result = newRstNode(rnField)
+  result = newRstNode(rnField, info=lineInfo(p))
   var col = currentTok(p).col
   var fieldname = newRstNode(rnFieldName)
   parseUntil(p, fieldname, ":", false)
@@ -2469,6 +2515,7 @@ proc parseDirective(p: var RstParser, k: RstNodeKind, flags: DirFlags): PRstNode
   ## Both rnDirArg and rnFieldList children nodes might be nil, so you need to
   ## check them before accessing.
   result = newRstNodeA(p, k)
+  if k == rnCodeBlock: result.info = lineInfo(p)
   var args: PRstNode = nil
   var options: PRstNode = nil
   if hasArg in flags:
@@ -2593,14 +2640,16 @@ proc dirInclude(p: var RstParser): PRstNode =
 
       var q: RstParser
       initParser(q, p.s)
-      q.s.filename = path
+      let saveFileIdx = p.s.currFileIdx
+      setCurrFilename(p.s, path)
       getTokens(
-        inputString[startPosition..endPosition].strip(),
+        inputString[startPosition..endPosition],
         q.tok)
       # workaround a GCC bug; more like the interior pointer bug?
       #if find(q.tok[high(q.tok)].symbol, "\0\x01\x02") > 0:
       #  InternalError("Too many binary zeros in include file")
       result = parseDoc(q)
+      p.s.currFileIdx = saveFileIdx
 
 proc dirCodeBlock(p: var RstParser, nimExtension = false): PRstNode =
   ## Parses a code block.
@@ -2634,7 +2683,7 @@ proc dirCodeBlock(p: var RstParser, nimExtension = false): PRstNode =
     if result.sons[1].isNil: result.sons[1] = newRstNode(rnFieldList)
     assert result.sons[1].kind == rnFieldList
     # Hook the extra field and specify the Nim language as value.
-    var extraNode = newRstNode(rnField)
+    var extraNode = newRstNode(rnField, info=lineInfo(p))
     extraNode.add(newRstNode(rnFieldName))
     extraNode.add(newRstNode(rnFieldBody))
     extraNode.sons[0].add newLeaf("default-language")
@@ -2837,9 +2886,8 @@ proc parseDotDot(p: var RstParser): PRstNode =
   else:
     result = parseComment(p, col)
 
-proc rstParsePass1*(fragment, filename: string,
+proc rstParsePass1*(fragment: string,
                     line, column: int,
-                    options: RstParseOptions,
                     sharedState: PRstSharedState): PRstNode =
   ## Parses an RST `fragment`.
   ## The result should be further processed by
@@ -2872,7 +2920,8 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode =
       var key = addNodes(n)
       var e = getEnv(key)
       if e != "": result = newLeaf(e)
-      else: rstMessage(s, mwUnknownSubstitution, key)
+      else: rstMessage(s.filenames, s.msgHandler, n.info,
+                       mwUnknownSubstitution, key)
   of rnHeadline, rnOverline:
     # fix up section levels depending on presence of a title and subtitle
     if s.hTitleCnt == 2:
@@ -2890,12 +2939,14 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode =
       let text = newRstNode(rnInner, n.sons)
       result.sons = @[text, y]
     else:
-      let s = findMainAnchor(s, refn)
-      if s != "":
+      let anchor = findMainAnchor(s, refn)
+      if anchor != "":
         result = newRstNode(rnInternalRef)
         let text = newRstNode(rnInner, n.sons)
-        result.sons = @[text,        # visible text of reference
-                        newLeaf(s)]  # link itself
+        result.sons = @[text,             # visible text of reference
+                        newLeaf(anchor)]  # link itself
+      else:
+        rstMessage(s.filenames, s.msgHandler, n.info, mwBrokenLink, refn)
   of rnFootnote:
     var (fnType, num) = getFootnoteType(n.sons[0])
     case fnType
@@ -2922,20 +2973,22 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode =
       result.add(nn)
     var refn = fnType.prefix
     # create new rnFootnoteRef, add final label, and finalize target refn:
-    result = newRstNode(rnFootnoteRef)
+    result = newRstNode(rnFootnoteRef, info = n.info)
     case fnType
     of fnManualNumber:
       addLabel num
       refn.add $num
     of fnAutoNumber:
-      addLabel getFootnoteNum(s, n.order)
-      refn.add $n.order
+      inc s.currFootnoteNumRef
+      addLabel getFootnoteNum(s, s.currFootnoteNumRef)
+      refn.add $s.currFootnoteNumRef
     of fnAutoNumberLabel:
       addLabel getFootnoteNum(s, rstnodeToRefname(n))
       refn.add rstnodeToRefname(n)
     of fnAutoSymbol:
-      addLabel getAutoSymbol(s, n.order)
-      refn.add $n.order
+      inc s.currFootnoteSymRef
+      addLabel getAutoSymbol(s, s.currFootnoteSymRef)
+      refn.add $s.currFootnoteSymRef
     of fnCitation:
       result.add n.sons[0]
       refn.add rstnodeToRefname(n)
@@ -2943,7 +2996,7 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode =
     if anch != "":
       result.add newLeaf(anch)     # add link
     else:
-      rstMessage(s, mwUnknownSubstitution, refn)
+      rstMessage(s.filenames, s.msgHandler, n.info, mwBrokenLink, refn)
       result.add newLeaf(refn)  # add link
   of rnLeaf:
     discard
@@ -2971,14 +3024,18 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode =
       result.sons = newSons
 
 proc rstParse*(text, filename: string,
-               line, column: int, hasToc: var bool,
+               line, column: int,
                options: RstParseOptions,
                findFile: FindFileHandler = nil,
-               msgHandler: MsgHandler = nil): PRstNode =
-  ## Parses the whole `text`. The result is ready for `rstgen.renderRstToOut`.
+               msgHandler: MsgHandler = nil):
+              tuple[node: PRstNode, filenames: RstFileTable, hasToc: bool] =
+  ## Parses the whole `text`. The result is ready for `rstgen.renderRstToOut`,
+  ## note that 2nd tuple element should be fed to `initRstGenerator`
+  ## argument `filenames` (it is being filled here at least with `filename`
+  ## and possibly with other files from RST ``.. include::`` statement).
   var sharedState = newRstSharedState(options, filename, findFile, msgHandler)
-  let unresolved = rstParsePass1(text, filename, line, column,
-                                 options, sharedState)
+  let unresolved = rstParsePass1(text, line, column, sharedState)
   preparePass2(sharedState, unresolved)
-  result = resolveSubs(sharedState, unresolved)
-  hasToc = sharedState.hasToc
+  result.node = resolveSubs(sharedState, unresolved)
+  result.filenames = sharedState.filenames
+  result.hasToc = sharedState.hasToc
diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim
index 2489ce40c..fa0620f44 100644
--- a/lib/packages/docutils/rstast.nim
+++ b/lib/packages/docutils/rstast.nim
@@ -74,6 +74,11 @@ type
     rnLeaf                    # a leaf; the node's text field contains the
                               # leaf val
 
+  FileIndex* = distinct int32
+  TLineInfo* = object
+    line*: uint16
+    col*: int16
+    fileIndex*: FileIndex
 
   PRstNode* = ref RstNode    ## an RST node
   RstNodeSeq* = seq[PRstNode]
@@ -92,21 +97,32 @@ type
       level*: int             ## level of headings starting from 1 (main
                               ## chapter) to larger ones (minor sub-sections)
                               ## level=0 means it's document title or subtitle
-    of rnFootnote, rnCitation, rnFootnoteRef, rnOptionListItem:
+    of rnFootnote, rnCitation, rnOptionListItem:
       order*: int             ## footnote order (for auto-symbol footnotes and
                               ## auto-numbered ones without a label)
+    of rnRef, rnSubstitutionReferences,
+        rnInterpretedText, rnField, rnInlineCode, rnCodeBlock, rnFootnoteRef:
+      info*: TLineInfo        ## To have line/column info for warnings at
+                              ## nodes that are post-processed after parsing
     else:
       discard
     anchor*: string           ## anchor, internal link target
                               ## (aka HTML id tag, aka Latex label/hypertarget)
     sons*: RstNodeSeq        ## the node's sons
 
+proc `==`*(a, b: FileIndex): bool {.borrow.}
+
 proc len*(n: PRstNode): int =
   result = len(n.sons)
 
 proc newRstNode*(kind: RstNodeKind, sons: seq[PRstNode] = @[],
                  anchor = ""): PRstNode =
+  result = PRstNode(kind: kind, sons: sons, anchor: anchor)
+
+proc newRstNode*(kind: RstNodeKind, info: TLineInfo,
+                 sons: seq[PRstNode] = @[]): PRstNode =
   result = PRstNode(kind: kind, sons: sons)
+  result.info = info
 
 proc newRstNode*(kind: RstNodeKind, s: string): PRstNode {.deprecated.} =
   assert kind in {rnLeaf, rnSmiley}
@@ -388,7 +404,7 @@ proc renderRstToStr*(node: PRstNode, indent=0): string =
     result.add "  adType=" & node.adType
   of rnHeadline, rnOverline, rnMarkdownHeadline:
     result.add "  level=" & $node.level
-  of rnFootnote, rnCitation, rnFootnoteRef, rnOptionListItem:
+  of rnFootnote, rnCitation, rnOptionListItem:
     result.add (if node.order == 0:   "" else: "  order=" & $node.order)
   else:
     discard
diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim
index b9c8f688b..10a229479 100644
--- a/lib/packages/docutils/rstgen.nim
+++ b/lib/packages/docutils/rstgen.nim
@@ -40,7 +40,7 @@
 ##   can be done by simply searching for [footnoteName].
 
 import strutils, os, hashes, strtabs, rstast, rst, highlite, tables, sequtils,
-  algorithm, parseutils
+  algorithm, parseutils, std/strbasics
 
 import ../../std/private/since
 
@@ -72,11 +72,11 @@ type
     tocPart*: seq[TocEntry]
     hasToc*: bool
     theIndex: string # Contents of the index file to be dumped at the end.
-    options*: RstParseOptions
     findFile*: FindFileHandler
     msgHandler*: MsgHandler
     outDir*: string      ## output directory, initialized by docgen.nim
     destFile*: string    ## output (HTML) file, initialized by docgen.nim
+    filenames*: RstFileTable
     filename*: string         ## source Nim or Rst file
     meta*: array[MetaEnum, string]
     currentSection: string ## \
@@ -112,9 +112,9 @@ proc init(p: var CodeBlockParams) =
 
 proc initRstGenerator*(g: var RstGenerator, target: OutputTarget,
                        config: StringTableRef, filename: string,
-                       options: RstParseOptions,
                        findFile: FindFileHandler = nil,
-                       msgHandler: MsgHandler = nil) =
+                       msgHandler: MsgHandler = nil,
+                       filenames = default(RstFileTable)) =
   ## Initializes a ``RstGenerator``.
   ##
   ## You need to call this before using a ``RstGenerator`` with any other
@@ -160,9 +160,9 @@ proc initRstGenerator*(g: var RstGenerator, target: OutputTarget,
   g.target = target
   g.tocPart = @[]
   g.filename = filename
+  g.filenames = filenames
   g.splitAfter = 20
   g.theIndex = ""
-  g.options = options
   g.findFile = findFile
   g.currentSection = ""
   g.id = 0
@@ -908,9 +908,8 @@ proc renderSmiley(d: PDoc, n: PRstNode, result: var string) =
     [d.config.getOrDefault"doc.smiley_format" % n.text])
 
 proc getField1Int(d: PDoc, n: PRstNode, fieldName: string): int =
-  # TODO: proper column/line info
   template err(msg: string) =
-    d.msgHandler(d.filename, 1, 0, meInvalidRstField, msg)
+    rstMessage(d.filenames, d.msgHandler, n.info, meInvalidField, msg)
   let value = n.getFieldValue
   var number: int
   let nChars = parseInt(value, number)
@@ -958,7 +957,8 @@ proc parseCodeBlockField(d: PDoc, n: PRstNode, params: var CodeBlockParams) =
     params.langStr = n.getFieldValue.strip
     params.lang = params.langStr.getSourceLanguage
   else:
-    d.msgHandler(d.filename, 1, 0, mwUnsupportedField, n.getArgument)
+    rstMessage(d.filenames, d.msgHandler, n.info, mwUnsupportedField,
+               n.getArgument)
 
 proc parseCodeBlockParams(d: PDoc, n: PRstNode): CodeBlockParams =
   ## Iterates over all code block fields and returns processed params.
@@ -1069,7 +1069,8 @@ proc renderCode(d: PDoc, n: PRstNode, result: var string) =
   dispA(d.target, result, blockStart, blockStart, [])
   if params.lang == langNone:
     if len(params.langStr) > 0:
-      d.msgHandler(d.filename, 1, 0, mwUnsupportedLanguage, params.langStr)
+      rstMessage(d.filenames, d.msgHandler, n.info, mwUnsupportedLanguage,
+                 params.langStr)
     for letter in m.text: escChar(d.target, result, letter, emText)
   else:
     renderCodeLang(result, params.lang, m.text, d.target)
@@ -1564,23 +1565,24 @@ proc rstToHtml*(s: string, options: RstParseOptions,
     result = ""
 
   const filen = "input"
+  let (rst, filenames, _) = rstParse(s, filen,
+                                     line=LineRstInit, column=ColRstInit,
+                                     options, myFindFile, msgHandler)
   var d: RstGenerator
-  initRstGenerator(d, outHtml, config, filen, options, myFindFile, msgHandler)
-  var dummyHasToc = false
-  var rst = rstParse(s, filen, line=LineRstInit, column=ColRstInit,
-                     dummyHasToc, options, myFindFile, msgHandler)
+  initRstGenerator(d, outHtml, config, filen, myFindFile, msgHandler, filenames)
   result = ""
   renderRstToOut(d, rst, result)
+  strbasics.strip(result)
 
 
 proc rstToLatex*(rstSource: string; options: RstParseOptions): string {.inline, since: (1, 3).} =
   ## Convenience proc for `renderRstToOut` and `initRstGenerator`.
   runnableExamples: doAssert rstToLatex("*Hello* **world**", {}) == """\emph{Hello} \textbf{world}"""
   if rstSource.len == 0: return
-  var option: bool
+  let (rst, filenames, _) = rstParse(rstSource, "",
+                                     line=LineRstInit, column=ColRstInit,
+                                     options)
   var rstGenera: RstGenerator
-  rstGenera.initRstGenerator(outLatex, defaultConfig(), "input", options)
-  rstGenera.renderRstToOut(
-      rstParse(rstSource, "", line=LineRstInit, column=ColRstInit,
-               option, options),
-      result)
+  rstGenera.initRstGenerator(outLatex, defaultConfig(), "input", filenames=filenames)
+  rstGenera.renderRstToOut(rst, result)
+  strbasics.strip(result)
diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim
index fe99d6cfa..fc3ccbf4e 100644
--- a/tests/stdlib/trst.nim
+++ b/tests/stdlib/trst.nim
@@ -5,6 +5,8 @@ discard """
 
 [Suite] RST indentation
 
+[Suite] Warnings
+
 [Suite] RST include directive
 
 [Suite] RST escaping
@@ -51,9 +53,8 @@ proc toAst(input: string,
       # we don't find any files in online mode:
       result = ""
 
-    var dummyHasToc = false
-    var rst = rstParse(input, filen, line=LineRstInit, column=ColRstInit,
-                       dummyHasToc, rstOptions, myFindFile, testMsgHandler)
+    var (rst, _, _) = rstParse(input, filen, line=LineRstInit, column=ColRstInit,
+                               rstOptions, myFindFile, testMsgHandler)
     result = renderRstToStr(rst)
   except EParseError as e:
     if e.msg != "":
@@ -356,6 +357,53 @@ suite "RST indentation":
     # "template..." should be parsed as a definition list attached to ":test:":
     check inputWrong.toAst != ast
 
+suite "Warnings":
+  test "warnings for broken footnotes/links/substitutions":
+    let input = dedent"""
+      firstParagraph
+
+      footnoteRef [som]_
+
+      link `a broken Link`_
+
+      substitution |undefined subst|
+
+      link short.link_
+
+      lastParagraph
+      """
+    var warnings = new seq[string]
+    let output = input.toAst(warnings=warnings)
+    check(warnings[] == @[
+        "input(3, 14) Warning: broken link 'citation-som'",
+        "input(5, 7) Warning: broken link 'a-broken-link'",
+        "input(7, 15) Warning: unknown substitution 'undefined subst'",
+        "input(9, 6) Warning: broken link 'shortdotlink'"
+        ])
+
+  test "With include directive and blank lines at the beginning":
+    "other.rst".writeFile(dedent"""
+
+
+        firstParagraph
+
+        here brokenLink_""")
+    let input = ".. include:: other.rst"
+    var warnings = new seq[string]
+    let output = input.toAst(warnings=warnings)
+    check warnings[] == @["other.rst(5, 6) Warning: broken link 'brokenlink'"]
+    check(output == dedent"""
+      rnInner
+        rnParagraph
+          rnLeaf  'firstParagraph'
+        rnParagraph
+          rnLeaf  'here'
+          rnLeaf  ' '
+          rnRef
+            rnLeaf  'brokenLink'
+      """)
+    removeFile("other.rst")
+
 suite "RST include directive":
   test "Include whole":
     "other.rst".writeFile("**test1**")
@@ -374,7 +422,7 @@ OtherStart
 .. include:: other.rst
              :start-after: OtherStart
 """
-    doAssert "Visible" == rstTohtml(input, {}, defaultConfig())
+    check "Visible" == rstTohtml(input, {}, defaultConfig())
     removeFile("other.rst")
 
   test "Include everything before":
@@ -406,7 +454,7 @@ And this should **NOT** be visible in `docs.html`
              :start-after: OtherStart
              :end-before: OtherEnd
 """
-    doAssert "Visible" == rstTohtml(input, {}, defaultConfig())
+    check "Visible" == rstTohtml(input, {}, defaultConfig())
     removeFile("other.rst")
 
 
diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim
index d5d902e1d..867677404 100644
--- a/tests/stdlib/trstgen.nim
+++ b/tests/stdlib/trstgen.nim
@@ -237,8 +237,7 @@ not in table"""
 
    • 
-
    Precedence levelOperatorsFirst characterTerminal symbol
    D1 """ & id"code" & " " & op"\|" & """D2
    
 
    E1 | text
    
 
    F2 without pipe
    not in table
    
-""")
+
    not in table
    """)
     let input2 = """
 | A1 header | A2 |
 | --- | --- |"""
@@ -420,9 +419,10 @@ Some chapter
             "the following intermediate section level(s) are missing on " &
             "lines 12..15: underline -----)")
 
+  test "RST sections overline":
     # the same as input9good but with overline headings
     # first overline heading has a special meaning: document title
-    let input10 = dedent """
+    let input = dedent """
       ======
       Title0
       ======
@@ -454,22 +454,23 @@ Some chapter
       ~~~~~
 
       """
-    var option: bool
     var rstGenera: RstGenerator
-    var output10: string
-    rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", {})
-    rstGenera.renderRstToOut(rstParse(input10, "", 1, 1, option, {}), output10)
+    var output: string
+    let (rst, files, _) = rstParse(input, "", 1, 1, {})
+    rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", filenames = files)
+    rstGenera.renderRstToOut(rst, output)
     doAssert rstGenera.meta[metaTitle] == "Title0"
     doAssert rstGenera.meta[metaSubTitle] == "SubTitle0"
-    doAssert "
    Level1
    " in output10
-    doAssert "
    Level2
    " in output10
-    doAssert "
    Level3
    " in output10
-    doAssert "
    L1
    " in output10
-    doAssert "
    Another2
    " in output10
-    doAssert "
    More3
    " in output10
+    doAssert "
    Level1
    " in output
+    doAssert "
    Level2
    " in output
+    doAssert "
    Level3
    " in output
+    doAssert "
    L1
    " in output
+    doAssert "
    Another2
    " in output
+    doAssert "
    More3
    " in output
 
+  test "RST sections overline 2":
     # check that a paragraph prevents interpreting overlines as document titles
-    let input11 = dedent """
+    let input = dedent """
       Paragraph
 
       ======
@@ -480,18 +481,19 @@ Some chapter
       SubTitle0
       +++++++++
       """
-    var option11: bool
-    var rstGenera11: RstGenerator
-    var output11: string
-    rstGenera11.initRstGenerator(outHtml, defaultConfig(), "input", {})
-    rstGenera11.renderRstToOut(rstParse(input11, "", 1, 1, option11, {}), output11)
-    doAssert rstGenera11.meta[metaTitle] == ""
-    doAssert rstGenera11.meta[metaSubTitle] == ""
-    doAssert "
    Title0
    " in output11
-    doAssert "
    SubTitle0
    " in output11
+    var rstGenera: RstGenerator
+    var output: string
+    let (rst, files, _) = rstParse(input, "", 1, 1, {})
+    rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", filenames=files)
+    rstGenera.renderRstToOut(rst, output)
+    doAssert rstGenera.meta[metaTitle] == ""
+    doAssert rstGenera.meta[metaSubTitle] == ""
+    doAssert "
    Title0
    " in output
+    doAssert "
    SubTitle0
    " in output
 
+  test "RST+Markdown sections":
     # check that RST and Markdown headings don't interfere
-    let input12 = dedent """
+    let input = dedent """
       ======
       Title0
       ======
@@ -509,14 +511,14 @@ Some chapter
       MySection2a
       -----------
       """
-    var option12: bool
-    var rstGenera12: RstGenerator
-    var output12: string
-    rstGenera12.initRstGenerator(outHtml, defaultConfig(), "input", {})
-    rstGenera12.renderRstToOut(rstParse(input12, "", 1, 1, option12, {roSupportMarkdown}), output12)
-    doAssert rstGenera12.meta[metaTitle] == "Title0"
-    doAssert rstGenera12.meta[metaSubTitle] == ""
-    doAssert output12 ==
+    var rstGenera: RstGenerator
+    var output: string
+    let (rst, files, _) = rstParse(input, "", 1, 1, {roSupportMarkdown})
+    rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", filenames=files)
+    rstGenera.renderRstToOut(rst, output)
+    doAssert rstGenera.meta[metaTitle] == "Title0"
+    doAssert rstGenera.meta[metaSubTitle] == ""
+    doAssert output ==
              "\n
    MySection1a
    " & # RST
              "\n
    MySection1b
    " & # Markdown
              "\n
    MySection1c
    " & # RST
@@ -628,7 +630,7 @@ let x = 1
     let p2 = """
    Par2 value2.
    """
     let p3 = """
    Par3 """ & id"value3" & ".
    "
     let p4 = """
    Par4 value4.
    """
-    let expected = p1 & p2 & "\n" & p3 & "\n" & p4 & "\n"
+    let expected = p1 & p2 & "\n" & p3 & "\n" & p4
     check(input.toHtml == expected)
 
   test "role directive":
@@ -653,8 +655,8 @@ Check that comment disappears:
     let output1 = input1.toHtml
     doAssert output1 == "Check that comment disappears:"
 
-  test "RST line blocks":
-    let input1 = """
+  test "RST line blocks + headings":
+    let input = """
 =====
 Test1
 =====
@@ -665,19 +667,20 @@ Test1
 | other line
 
 """
-    var option: bool
     var rstGenera: RstGenerator
-    var output1: string
-    rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", {})
-    rstGenera.renderRstToOut(rstParse(input1, "", 1, 1, option, {}), output1)
+    var output: string
+    let (rst, files, _) = rstParse(input, "", 1, 1, {})
+    rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", filenames=files)
+    rstGenera.renderRstToOut(rst, output)
     doAssert rstGenera.meta[metaTitle] == "Test1"
       # check that title was not overwritten to '|'
-    doAssert output1 == "


    line block
    other line
    "
-    let output1l = rstToLatex(input1, {})
+    doAssert output == "


    line block
    other line
    "
+    let output1l = rstToLatex(input, {})
     doAssert "line block\n\n" in output1l
     doAssert "other line\n\n" in output1l
     doAssert output1l.count("\\vspace") == 2 + 2  # +2 surrounding paddings
 
+  test "RST line blocks":
     let input2 = dedent"""
       Paragraph1
       
@@ -686,7 +689,7 @@ Test1
       Paragraph2"""
 
     let output2 = input2.toHtml
-    doAssert "Paragraph1

     
    Paragraph2
    \n" == output2
+    doAssert "Paragraph1

     
    Paragraph2
    " == output2
 
     let input3 = dedent"""
       | xxx
@@ -853,7 +856,7 @@ Test1
     check("input(6, 1) Warning: RST style: \n" &
           "not enough indentation on line 6" in warnings8[0])
     doAssert output8 == "Paragraph.
      " &
-        "
    1. stringA
      2. \n
    2. stringB
      3. \n
    \n
    C. string1 string2 
    \n"
+        "
  • stringA
    • \n
  • stringB
    • \n\n
    C. string1 string2 
    "
 
   test "Markdown enumerated lists":
     let input1 = dedent """
@@ -926,7 +929,7 @@ Test1
       Not references[#note]_[1 #]_ [wrong citation]_ and [not&allowed]_.
       """
     let output2 = input2.toHtml
-    doAssert output2 == "Not references[#note]_[1 #]_ [wrong citation]_ and [not&allowed]_. "
+    doAssert output2 == "Not references[#note]_[1 #]_ [wrong citation]_ and [not&allowed]_."
 
     # check that auto-symbol footnotes work:
     let input3 = dedent """
@@ -1034,9 +1037,7 @@ Test1
       """
     var warnings8 = new seq[string]
     let output8 = input8.toHtml(warnings=warnings8)
-    # TODO: the line 1 is arbitrary because reference lines are not preserved
-    check(warnings8[] == @["input(1, 1) Warning: unknown substitution " &
-            "\'citation-som\'"])
+    check(warnings8[] == @["input(3, 7) Warning: broken link 'citation-som'"])
 
     # check that footnote group does not break parsing of other directives:
     let input9 = dedent """
@@ -1144,10 +1145,33 @@ Test1
       """
     var error = new string
     let output = input.toHtml(error=error)
-    check(error[] == "input(1, 1) Error: invalid field: " &
+    check(error[] == "input(2, 3) Error: invalid field: " &
                      "extra arguments were given to number-lines: ' let a = 1'")
     check "" == output
 
+  test "code-block warning":
+    let input = dedent """
+      .. code:: Nim
+         :unsupportedField: anything
+
+      .. code:: unsupportedLang
+
+         anything
+
+      ```anotherLang
+      someCode
+      ```
+      """
+    let warnings = new seq[string]
+    let output = input.toHtml(warnings=warnings)
+    check(warnings[] == @[
+        "input(2, 4) Warning: field 'unsupportedField' not supported",
+        "input(4, 11) Warning: language 'unsupportedLang' not supported",
+        "input(8, 4) Warning: language 'anotherLang' not supported"
+        ])
+    check(output == "
    anything
    " &
+                    "
    \nsomeCode\n
     
    ")
+
   test "RST admonitions":
     # check that all admonitions are implemented
     let input0 = dedent """
@@ -1477,7 +1501,7 @@ Test1
     check "(3, 15) Warning: " in warnings[1]
     check "language 'py:class' not supported" in warnings[1]
     check("""
    See function spam.
    """ & "\n" &
-          """
    See also egg. 
    """ & "\n" ==
+          """
    See also egg. 
    """ ==
           output)
 
   test "(not) Roles: check escaping 1":

From e38388eb4e92b0e7c7db204fe36d7969967cd609 Mon Sep 17 00:00:00 2001
From: Araq 
Date: Tue, 20 Jul 2021 08:35:53 +0200
Subject: [PATCH 1061/2472] use .} instead of } for code highlighters

---
 lib/system/threads.nim | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/system/threads.nim b/lib/system/threads.nim
index 7fbfe54ab..2b5eb07fd 100644
--- a/lib/system/threads.nim
+++ b/lib/system/threads.nim
@@ -41,7 +41,7 @@
 ##  for i in 0..high(thr):
 ##    createThread(thr[i], threadFunc, (i*10, i*10+5))
 ##  joinThreads(thr)
-## 
+##
 ##  deinitLock(L)
 
 when not declared(ThisIsSystem):
@@ -395,7 +395,7 @@ elif defined(netbsd):
 
 elif defined(freebsd):
   proc syscall(arg: cint, arg0: ptr cint): cint {.varargs, importc: "syscall", header: "".}
-  var SYS_thr_self {.importc:"SYS_thr_self", header:""}: cint
+  var SYS_thr_self {.importc:"SYS_thr_self", header:"".}: cint
 
   proc getThreadId*(): int =
     ## Gets the ID of the currently running thread.

From 5e011e8e42b2e4e322e9b40d4075da6d4a9103e5 Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Tue, 20 Jul 2021 10:37:36 +0200
Subject: [PATCH 1062/2472] updated the changelog (#18539)

---
 changelog.md | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/changelog.md b/changelog.md
index 097beac2c..a633c1fb3 100644
--- a/changelog.md
+++ b/changelog.md
@@ -39,9 +39,6 @@
 - `math.round` now is rounded "away from zero" in JS backend which is consistent
   with other backends. See #9125. Use `-d:nimLegacyJsRound` for previous behavior.
 
-- Instead of deleting the element at the last index,
-  `system.delete()` now raises `IndexDefect` when given index is out of bounds.
-
 - Changed the behavior of `uri.decodeQuery` when there are unencoded `=`
   characters in the decoded values. Prior versions would raise an error. This is
   no longer the case to comply with the HTML spec and other languages

From 6286b5599c7d9fc03f3a6809f5c17fbefc23427e Mon Sep 17 00:00:00 2001
From: Paul Roberts 
Date: Tue, 20 Jul 2021 11:50:43 +0100
Subject: [PATCH 1063/2472] termux ssl (#18520)

---
 lib/pure/net.nim       | 2 +-
 lib/pure/ssl_certs.nim | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/pure/net.nim b/lib/pure/net.nim
index 934dd034e..c2e4810de 100644
--- a/lib/pure/net.nim
+++ b/lib/pure/net.nim
@@ -647,7 +647,7 @@ when defineSsl:
       if verifyMode != CVerifyNone:
         # Use the caDir and caFile parameters if set
         if caDir != "" or caFile != "":
-          if newCTX.SSL_CTX_load_verify_locations(caFile, caDir) != VerifySuccess:
+          if newCTX.SSL_CTX_load_verify_locations(if caFile == "": nil else: caFile.cstring, if caDir == "": nil else: caDir.cstring) != VerifySuccess:
             raise newException(IOError, "Failed to load SSL/TLS CA certificate(s).")
 
         else:
diff --git a/lib/pure/ssl_certs.nim b/lib/pure/ssl_certs.nim
index 2d2644ebe..c7ce04ffa 100644
--- a/lib/pure/ssl_certs.nim
+++ b/lib/pure/ssl_certs.nim
@@ -34,6 +34,7 @@ elif defined(linux):
     # Fedora/RHEL
     "/etc/pki/tls/certs",
     # Android
+    "/data/data/com.termux/files/usr/etc/tls/cert.pem",
     "/system/etc/security/cacerts",
   ]
 elif defined(bsd):

From dbf8d0b8946419311ac710b1adc995ad206b52c1 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Tue, 20 Jul 2021 20:12:15 +0800
Subject: [PATCH 1064/2472] use more `.}` (#18542)

---
 changelogs/changelog_0_19_0.md  | 2 +-
 doc/manual_experimental.rst     | 4 ++--
 lib/posix/posix_linux_amd64.nim | 2 +-
 lib/system.nim                  | 8 ++++----
 lib/windows/winlean.nim         | 2 +-
 5 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/changelogs/changelog_0_19_0.md b/changelogs/changelog_0_19_0.md
index 18d3ca2b3..45cefe933 100644
--- a/changelogs/changelog_0_19_0.md
+++ b/changelogs/changelog_0_19_0.md
@@ -15,7 +15,7 @@
   at runtime. Use ``ref object`` with inheritance rather than ``object`` with
   inheritance to prevent this issue.
 - The ``not nil`` type annotation now has to be enabled explicitly
-  via ``{.experimental: "notnil"}`` as we are still not pleased with how this
+  via ``{.experimental: "notnil".}`` as we are still not pleased with how this
   feature works with Nim's containers.
 - The parser now warns about inconsistent spacing around binary operators as
   these can easily be confused with unary operators. This warning will likely
diff --git a/doc/manual_experimental.rst b/doc/manual_experimental.rst
index d10ef8d1b..22db184a8 100644
--- a/doc/manual_experimental.rst
+++ b/doc/manual_experimental.rst
@@ -474,13 +474,13 @@ Not nil annotation
 ==================
 
 **Note:** This is an experimental feature. It can be enabled with
-`{.experimental: "notnil"}`.
+`{.experimental: "notnil".}`.
 
 All types for which `nil` is a valid value can be annotated with the
 `not nil` annotation to exclude `nil` as a valid value:
 
 .. code-block:: nim
-  {.experimental: "notnil"}
+  {.experimental: "notnil".}
 
   type
     PObject = ref TObj not nil
diff --git a/lib/posix/posix_linux_amd64.nim b/lib/posix/posix_linux_amd64.nim
index 5c0b4d6cf..2309e19a9 100644
--- a/lib/posix/posix_linux_amd64.nim
+++ b/lib/posix/posix_linux_amd64.nim
@@ -334,7 +334,7 @@ type
     si_status*: cint   ## Exit value or signal.
     si_band*: int      ## Band event for SIGPOLL.
     si_value*: SigVal  ## Signal value.
-    pad {.importc: "_pad"}: array[128 - 56, uint8]
+    pad {.importc: "_pad".}: array[128 - 56, uint8]
 
   Nl_item* {.importc: "nl_item", header: "".} = cint
   Nl_catd* {.importc: "nl_catd", header: "".} = pointer
diff --git a/lib/system.nim b/lib/system.nim
index 2790187f4..e878f9f2f 100644
--- a/lib/system.nim
+++ b/lib/system.nim
@@ -536,7 +536,7 @@ include "system/arithmetics"
 include "system/comparisons"
 
 const
-  appType* {.magic: "AppType"}: string = ""
+  appType* {.magic: "AppType".}: string = ""
     ## A string that describes the application type. Possible values:
     ## `"console"`, `"gui"`, `"lib"`.
 
@@ -1101,15 +1101,15 @@ const
     ## True only when accessed in the main module. This works thanks to
     ## compiler magic. It is useful to embed testing code in a module.
 
-  CompileDate* {.magic: "CompileDate"}: string = "0000-00-00"
+  CompileDate* {.magic: "CompileDate".}: string = "0000-00-00"
     ## The date (in UTC) of compilation as a string of the form
     ## `YYYY-MM-DD`. This works thanks to compiler magic.
 
-  CompileTime* {.magic: "CompileTime"}: string = "00:00:00"
+  CompileTime* {.magic: "CompileTime".}: string = "00:00:00"
     ## The time (in UTC) of compilation as a string of the form
     ## `HH:MM:SS`. This works thanks to compiler magic.
 
-  cpuEndian* {.magic: "CpuEndian"}: Endianness = littleEndian
+  cpuEndian* {.magic: "CpuEndian".}: Endianness = littleEndian
     ## The endianness of the target CPU. This is a valuable piece of
     ## information for low-level code only. This works thanks to compiler
     ## magic.
diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim
index d901af244..b8f3886d7 100644
--- a/lib/windows/winlean.nim
+++ b/lib/windows/winlean.nim
@@ -1143,7 +1143,7 @@ proc setFileTime*(hFile: Handle, lpCreationTime: LPFILETIME,
 type
   # https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-sid_identifier_authority
   SID_IDENTIFIER_AUTHORITY* {.importc, header: "".} = object
-    value* {.importc: "Value"}: array[6, BYTE]
+    value* {.importc: "Value".}: array[6, BYTE]
   # https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-sid
   SID* {.importc, header: "".} = object
     Revision: BYTE

From a8b3e7c05919511db62f1aabd706c46316b4f7b6 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Tue, 20 Jul 2021 21:32:55 +0800
Subject: [PATCH 1065/2472] follow up #18517 (#18537)

* follow up #18517

* Update lib/pure/os.nim

Co-authored-by: Timothee Cour 

* Update lib/pure/os.nim

Co-authored-by: Timothee Cour 
---
 lib/pure/os.nim | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/pure/os.nim b/lib/pure/os.nim
index 6289c64f0..73e3fcb31 100644
--- a/lib/pure/os.nim
+++ b/lib/pure/os.nim
@@ -11,10 +11,11 @@
 ## retrieving environment variables, reading command line arguments,
 ## working with directories, running shell commands, etc.
 
-runnableExamples("-r:off"):
+runnableExamples:
   let myFile = "/path/to/my/file.nim"
   assert splitPath(myFile) == (head: "/path/to/my", tail: "file.nim")
-  assert parentDir(myFile) == "/path/to/my"
+  when defined(posix):
+    assert parentDir(myFile) == "/path/to/my"
   assert splitFile(myFile) == (dir: "/path/to/my", name: "file", ext: ".nim")
   assert myFile.changeFileExt("c") == "/path/to/my/file.c"
 

From cf0cf32d276002e850a87667fff62c4df12999d6 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Tue, 20 Jul 2021 13:13:52 -0700
Subject: [PATCH 1066/2472] make -d:nimFpRoundtrips work consistently in vm vs
 rt, fix #18400, etc (#18531)

* compiler/vmhooks: add getVar to allow vmops with var params
* addFloat vmops with var param
* cgen now renders float32 literals in c backend using roundtrip float to string
---
 compiler/ast.nim                |   4 +-
 compiler/ccgexprs.nim           |  12 +--
 compiler/jsgen.nim              |   7 +-
 compiler/nim.cfg                |   1 +
 compiler/rodutils.nim           |  10 +-
 compiler/semfold.nim            |   1 -
 compiler/vmgen.nim              |   3 +-
 compiler/vmhooks.nim            |  32 +++---
 compiler/vmops.nim              |  11 +++
 lib/system.nim                  |   3 -
 lib/system/dollars.nim          |  13 +--
 lib/system/formatfloat.nim      | 166 ++++++++++++++++++++++----------
 lib/system/jssys.nim            |  18 +---
 lib/system/repr_v2.nim          |   6 +-
 lib/system/strmantle.nim        |  41 --------
 tests/config.nims               |   2 +
 tests/errmsgs/treportunused.nim |  14 ++-
 tests/float/nim.cfg             |   1 -
 tests/float/tfloats.nim         | 117 ++++++++++++++++++----
 tests/stdlib/tjson.nim          |   6 +-
 tests/stdlib/tjsonutils.nim     |  17 ++--
 21 files changed, 286 insertions(+), 199 deletions(-)
 delete mode 100644 tests/float/nim.cfg

diff --git a/compiler/ast.nim b/compiler/ast.nim
index f1f73dcee..8b836e088 100644
--- a/compiler/ast.nim
+++ b/compiler/ast.nim
@@ -652,7 +652,7 @@ type
     mUnaryMinusI, mUnaryMinusI64, mAbsI, mNot,
     mUnaryPlusI, mBitnotI,
     mUnaryPlusF64, mUnaryMinusF64,
-    mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mFloatToStr, mCStrToStr,
+    mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mCStrToStr,
     mStrToStr, mEnumToStr,
     mAnd, mOr,
     mImplies, mIff, mExists, mForall, mOld,
@@ -720,7 +720,7 @@ const
     mEqRef, mEqProc, mLePtr, mLtPtr, mEqCString, mXor,
     mUnaryMinusI, mUnaryMinusI64, mAbsI, mNot, mUnaryPlusI, mBitnotI,
     mUnaryPlusF64, mUnaryMinusF64,
-    mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mFloatToStr, mCStrToStr,
+    mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mCStrToStr,
     mStrToStr, mEnumToStr,
     mAnd, mOr,
     mEqStr, mLeStr, mLtStr,
diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim
index 09326ceeb..79c73b602 100644
--- a/compiler/ccgexprs.nim
+++ b/compiler/ccgexprs.nim
@@ -94,9 +94,12 @@ proc genLiteral(p: BProc, n: PNode, ty: PType): Rope =
     else:
       result = makeCString(n.strVal)
   of nkFloatLit, nkFloat64Lit:
-    result = rope(n.floatVal.toStrMaxPrecision)
+    if ty.kind == tyFloat32:
+      result = rope(n.floatVal.float32.toStrMaxPrecision)
+    else:
+      result = rope(n.floatVal.toStrMaxPrecision)
   of nkFloat32Lit:
-    result = rope(n.floatVal.toStrMaxPrecision("f"))
+    result = rope(n.floatVal.float32.toStrMaxPrecision)
   else:
     internalError(p.config, n.info, "genLiteral(" & $n.kind & ')')
     result = nil
@@ -2300,11 +2303,6 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
   of mInt64ToStr: genDollar(p, e, d, "#nimInt64ToStr($1)")
   of mBoolToStr: genDollar(p, e, d, "#nimBoolToStr($1)")
   of mCharToStr: genDollar(p, e, d, "#nimCharToStr($1)")
-  of mFloatToStr:
-    if e[1].typ.skipTypes(abstractInst).kind == tyFloat32:
-      genDollar(p, e, d, "#nimFloat32ToStr($1)")
-    else:
-      genDollar(p, e, d, "#nimFloatToStr($1)")
   of mCStrToStr: genDollar(p, e, d, "#cstrToNimstr($1)")
   of mStrToStr, mUnown: expr(p, e[1], d)
   of mIsolate: genCall(p, e, d)
diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim
index 82fba02a6..4b89b0cde 100644
--- a/compiler/jsgen.nim
+++ b/compiler/jsgen.nim
@@ -434,7 +434,6 @@ const # magic checked op; magic unchecked op;
     mBoolToStr: ["nimBoolToStr", "nimBoolToStr"],
     mIntToStr: ["cstrToNimstr", "cstrToNimstr"],
     mInt64ToStr: ["cstrToNimstr", "cstrToNimstr"],
-    mFloatToStr: ["cstrToNimstr", "cstrToNimstr"],
     mCStrToStr: ["cstrToNimstr", "cstrToNimstr"],
     mStrToStr: ["", ""]]
 
@@ -656,9 +655,6 @@ proc arithAux(p: PProc, n: PNode, r: var TCompRes, op: TMagic) =
   of mBoolToStr: applyFormat("nimBoolToStr($1)", "nimBoolToStr($1)")
   of mIntToStr: applyFormat("cstrToNimstr(($1) + \"\")", "cstrToNimstr(($1) + \"\")")
   of mInt64ToStr: applyFormat("cstrToNimstr(($1) + \"\")", "cstrToNimstr(($1) + \"\")")
-  of mFloatToStr:
-    useMagic(p, "nimFloatToString")
-    applyFormat "cstrToNimstr(nimFloatToString($1))"
   of mCStrToStr: applyFormat("cstrToNimstr($1)", "cstrToNimstr($1)")
   of mStrToStr, mUnown, mIsolate: applyFormat("$1", "$1")
   else:
@@ -682,8 +678,7 @@ proc arith(p: PProc, n: PNode, r: var TCompRes, op: TMagic) =
     gen(p, n[1], x)
     gen(p, n[2], y)
     r.res = "($1 >>> $2)" % [x.rdLoc, y.rdLoc]
-  of mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mFloatToStr,
-      mCStrToStr, mStrToStr, mEnumToStr:
+  of mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mCStrToStr, mStrToStr, mEnumToStr:
     arithAux(p, n, r, op)
   of mEqRef:
     if mapType(n[1].typ) != etyBaseIndex:
diff --git a/compiler/nim.cfg b/compiler/nim.cfg
index 9ecd00b0b..f10d847ac 100644
--- a/compiler/nim.cfg
+++ b/compiler/nim.cfg
@@ -4,6 +4,7 @@ hint:XDeclaredButNotUsed:off
 
 define:booting
 define:nimcore
+define:nimFpRoundtrips
 
 #import:"$projectpath/testability"
 
diff --git a/compiler/rodutils.nim b/compiler/rodutils.nim
index e13b08e05..a4f7a5146 100644
--- a/compiler/rodutils.nim
+++ b/compiler/rodutils.nim
@@ -39,7 +39,10 @@ when not declared(signbit):
   proc signbit*(x: SomeFloat): bool {.inline.} =
     result = c_signbit(x) != 0
 
-proc toStrMaxPrecision*(f: BiggestFloat, literalPostfix = ""): string =
+import system/formatfloat
+
+proc toStrMaxPrecision*(f: BiggestFloat | float32): string =
+  const literalPostfix = when f is float32: "f" else: ""
   case classify(f)
   of fcNan:
     if signbit(f):
@@ -55,9 +58,8 @@ proc toStrMaxPrecision*(f: BiggestFloat, literalPostfix = ""): string =
   of fcNegInf:
     result = "-INF"
   else:
-    result = newString(81)
-    let n = c_snprintf(result.cstring, result.len.uint, "%#.16e%s", f, literalPostfix.cstring)
-    setLen(result, n)
+    result.addFloatRoundtrip(f)
+    result.add literalPostfix
 
 proc encodeStr*(s: string, result: var string) =
   for i in 0..", discardable.}
 
-  proc writeFloatToBuffer*(buf: var array[65, char]; value: BiggestFloat): int =
-    ## This is the implementation to format floats.
-    ##
-    ## returns the amount of bytes written to `buf` not counting the
-    ## terminating '\0' character.
-    result = toChars(buf, value, forceTrailingDotZero=true)
-    buf[result] = '\0'
+proc addCstringN(result: var string, buf: cstring; buflen: int) =
+  # no nimvm support needed, so it doesn't need to be fast here either
+  let oldLen = result.len
+  let newLen = oldLen + buflen
+  result.setLen newLen
+  c_memcpy(result[oldLen].addr, buf, buflen.csize_t)
 
-else:
-  proc c_sprintf(buf, frmt: cstring): cint {.header: "",
-                                      importc: "sprintf", varargs, noSideEffect.}
+import dragonbox, schubfach
 
-  proc writeToBuffer(buf: var array[65, char]; value: cstring) =
-    var i = 0
-    while value[i] != '\0':
-      buf[i] = value[i]
-      inc i
+proc writeFloatToBufferRoundtrip*(buf: var array[65, char]; value: BiggestFloat): int =
+  ## This is the implementation to format floats.
+  ##
+  ## returns the amount of bytes written to `buf` not counting the
+  ## terminating '\0' character.
+  result = toChars(buf, value, forceTrailingDotZero=true)
+  buf[result] = '\0'
 
-  proc writeFloatToBuffer*(buf: var array[65, char]; value: BiggestFloat): int =
-    ## This is the implementation to format floats.
-    ##
-    ## returns the amount of bytes written to `buf` not counting the
-    ## terminating '\0' character.
-    var n: int = c_sprintf(addr buf, "%.16g", value)
-    var hasDot = false
-    for i in 0..n-1:
-      if buf[i] == ',':
-        buf[i] = '.'
-        hasDot = true
-      elif buf[i] in {'a'..'z', 'A'..'Z', '.'}:
-        hasDot = true
-    if not hasDot:
-      buf[n] = '.'
-      buf[n+1] = '0'
-      buf[n+2] = '\0'
-      result = n + 2
+proc writeFloatToBufferRoundtrip*(buf: var array[65, char]; value: float32): int =
+  result = float32ToChars(buf, value, forceTrailingDotZero=true)
+  buf[result] = '\0'
+
+proc c_sprintf(buf, frmt: cstring): cint {.header: "",
+                                    importc: "sprintf", varargs, noSideEffect.}
+
+proc writeToBuffer(buf: var array[65, char]; value: cstring) =
+  var i = 0
+  while value[i] != '\0':
+    buf[i] = value[i]
+    inc i
+
+proc writeFloatToBufferSprintf*(buf: var array[65, char]; value: BiggestFloat): int =
+  ## This is the implementation to format floats.
+  ##
+  ## returns the amount of bytes written to `buf` not counting the
+  ## terminating '\0' character.
+  var n: int = c_sprintf(addr buf, "%.16g", value)
+  var hasDot = false
+  for i in 0..n-1:
+    if buf[i] == ',':
+      buf[i] = '.'
+      hasDot = true
+    elif buf[i] in {'a'..'z', 'A'..'Z', '.'}:
+      hasDot = true
+  if not hasDot:
+    buf[n] = '.'
+    buf[n+1] = '0'
+    buf[n+2] = '\0'
+    result = n + 2
+  else:
+    result = n
+  # On Windows nice numbers like '1.#INF', '-1.#INF' or '1.#NAN' or 'nan(ind)'
+  # of '-1.#IND' are produced.
+  # We want to get rid of these here:
+  if buf[n-1] in {'n', 'N', 'D', 'd', ')'}:
+    writeToBuffer(buf, "nan")
+    result = 3
+  elif buf[n-1] == 'F':
+    if buf[0] == '-':
+      writeToBuffer(buf, "-inf")
+      result = 4
     else:
-      result = n
-    # On Windows nice numbers like '1.#INF', '-1.#INF' or '1.#NAN' or 'nan(ind)'
-    # of '-1.#IND' are produced.
-    # We want to get rid of these here:
-    if buf[n-1] in {'n', 'N', 'D', 'd', ')'}:
-      writeToBuffer(buf, "nan")
+      writeToBuffer(buf, "inf")
       result = 3
-    elif buf[n-1] == 'F':
-      if buf[0] == '-':
-        writeToBuffer(buf, "-inf")
-        result = 4
-      else:
-        writeToBuffer(buf, "inf")
-        result = 3
+
+proc writeFloatToBuffer*(buf: var array[65, char]; value: BiggestFloat | float32): int {.inline.} =
+  when defined(nimFpRoundtrips):
+    writeFloatToBufferRoundtrip(buf, value)
+  else:
+    writeFloatToBufferSprintf(buf, value)
+
+proc addFloatRoundtrip*(result: var string; x: float | float32) =
+  when nimvm:
+    doAssert false
+  else:
+    var buffer {.noinit.}: array[65, char]
+    let n = writeFloatToBufferRoundtrip(buffer, x)
+    result.addCstringN(cstring(buffer[0].addr), n)
+
+proc addFloatSprintf*(result: var string; x: float) =
+  when nimvm:
+    doAssert false
+  else:
+    var buffer {.noinit.}: array[65, char]
+    let n = writeFloatToBufferSprintf(buffer, x)
+    result.addCstringN(cstring(buffer[0].addr), n)
+
+proc nimFloatToString(a: float): cstring =
+  ## ensures the result doesn't print like an integer, i.e. return 2.0, not 2
+  # print `-0.0` properly
+  asm """
+    function nimOnlyDigitsOrMinus(n) {
+      return n.toString().match(/^-?\d+$/);
+    }
+    if (Number.isSafeInteger(`a`))
+      `result` = `a` === 0 && 1 / `a` < 0 ? "-0.0" : `a`+".0"
+    else {
+      `result` = `a`+""
+      if(nimOnlyDigitsOrMinus(`result`)){
+        `result` = `a`+".0"
+      }
+    }
+  """
+
+proc addFloat*(result: var string; x: float | float32) {.inline.} =
+  ## Converts float to its string representation and appends it to `result`.
+  runnableExamples:
+    var
+      s = "foo:"
+      b = 45.67
+    s.addFloat(45.67)
+    assert s == "foo:45.67"
+  template impl =
+    when defined(nimFpRoundtrips):
+      addFloatRoundtrip(result, x)
+    else:
+      addFloatSprintf(result, x)
+  when defined(js):
+    when nimvm: impl()
+    else:
+      result.add nimFloatToString(x)
+  else: impl()
diff --git a/lib/system/jssys.nim b/lib/system/jssys.nim
index 7cb5652c5..b42dc3a20 100644
--- a/lib/system/jssys.nim
+++ b/lib/system/jssys.nim
@@ -496,23 +496,6 @@ proc negInt(a: int): int {.compilerproc.} =
 proc negInt64(a: int64): int64 {.compilerproc.} =
   result = a*(-1)
 
-proc nimFloatToString(a: float): cstring {.compilerproc.} =
-  ## ensures the result doesn't print like an integer, i.e. return 2.0, not 2
-  # print `-0.0` properly
-  asm """
-    function nimOnlyDigitsOrMinus(n) {
-      return n.toString().match(/^-?\d+$/);
-    }
-    if (Number.isSafeInteger(`a`))
-      `result` = `a` === 0 && 1 / `a` < 0 ? "-0.0" : `a`+".0"
-    else {
-      `result` = `a`+""
-      if(nimOnlyDigitsOrMinus(`result`)){
-        `result` = `a`+".0"
-      }
-    }
-  """
-
 proc absInt(a: int): int {.compilerproc.} =
   result = if a < 0: a*(-1) else: a
 
@@ -703,6 +686,7 @@ proc addChar(x: string, c: char) {.compilerproc, asmNoStackFrame.} =
 {.pop.}
 
 proc tenToThePowerOf(b: int): BiggestFloat =
+  # xxx deadcode
   var b = b
   var a = 10.0
   result = 1.0
diff --git a/lib/system/repr_v2.nim b/lib/system/repr_v2.nim
index 618cc2b40..8471ea148 100644
--- a/lib/system/repr_v2.nim
+++ b/lib/system/repr_v2.nim
@@ -19,9 +19,9 @@ proc repr*(x: uint64): string {.noSideEffect.} =
   ## converted to a decimal string.
   $x #Calls `$` from system/strmantle.nim
 
-proc repr*(x: float): string {.magic: "FloatToStr", noSideEffect.}
-  ## repr for a float argument. Returns `x`
-  ## converted to a decimal string.
+proc repr*(x: float): string =
+  ## Same as $x
+  $x
 
 proc repr*(x: bool): string {.magic: "BoolToStr", noSideEffect.}
   ## repr for a boolean argument. Returns `x`
diff --git a/lib/system/strmantle.nim b/lib/system/strmantle.nim
index 041272175..81ac5c751 100644
--- a/lib/system/strmantle.nim
+++ b/lib/system/strmantle.nim
@@ -69,47 +69,6 @@ proc nimIntToStr(x: int): string {.compilerRtl.} =
   result = newStringOfCap(sizeof(x)*4)
   result.addInt x
 
-proc addCstringN(result: var string, buf: cstring; buflen: int) =
-  # no nimvm support needed, so it doesn't need to be fast here either
-  let oldLen = result.len
-  let newLen = oldLen + buflen
-  result.setLen newLen
-  copyMem(result[oldLen].addr, buf, buflen)
-
-import formatfloat
-
-proc addFloat*(result: var string; x: float) =
-  ## Converts float to its string representation and appends it to `result`.
-  ##
-  ## .. code-block:: Nim
-  ##   var
-  ##     a = "123"
-  ##     b = 45.67
-  ##   a.addFloat(b) # a <- "12345.67"
-  when nimvm:
-    result.add $x
-  else:
-    var buffer {.noinit.}: array[65, char]
-    let n = writeFloatToBuffer(buffer, x)
-    result.addCstringN(cstring(buffer[0].addr), n)
-
-proc nimFloatToStr(f: float): string {.compilerproc.} =
-  result = newStringOfCap(8)
-  result.addFloat f
-
-when defined(nimFpRoundtrips) and not defined(nimscript) and
-    not defined(js) and defined(nimHasDragonBox):
-  import schubfach
-
-proc nimFloat32ToStr(f: float32): string {.compilerproc.} =
-  when declared(float32ToChars):
-    result = newString(65)
-    let L = float32ToChars(result, f, forceTrailingDotZero=true)
-    setLen(result, L)
-  else:
-    result = newStringOfCap(8)
-    result.addFloat f
-
 proc c_strtod(buf: cstring, endptr: ptr cstring): float64 {.
   importc: "strtod", header: "", noSideEffect.}
 
diff --git a/tests/config.nims b/tests/config.nims
index 539de5e8d..12b303318 100644
--- a/tests/config.nims
+++ b/tests/config.nims
@@ -34,3 +34,5 @@ hint("Processing", off)
 switch("define", "nimExperimentalAsyncjsThen")
 switch("define", "nimExperimentalJsfetch")
 switch("define", "nimExperimentalLinenoiseExtra")
+
+switch("define", "nimFpRoundtrips")
diff --git a/tests/errmsgs/treportunused.nim b/tests/errmsgs/treportunused.nim
index f9b7c3d11..3105b35ea 100644
--- a/tests/errmsgs/treportunused.nim
+++ b/tests/errmsgs/treportunused.nim
@@ -13,12 +13,12 @@ treportunused.nim(30, 5) Hint: 's8' is declared but not used [XDeclaredButNotUse
 treportunused.nim(31, 5) Hint: 's9' is declared but not used [XDeclaredButNotUsed]
 treportunused.nim(32, 6) Hint: 's10' is declared but not used [XDeclaredButNotUsed]
 treportunused.nim(33, 6) Hint: 's11' is declared but not used [XDeclaredButNotUsed]
+treportunused.nim(37, 3) Hint: 'v0.99' is declared but not used [XDeclaredButNotUsed]
+treportunused.nim(38, 3) Hint: 'v0.99.99' is declared but not used [XDeclaredButNotUsed]
 '''
 action: compile
 """
 
-#treportunused.nim(37, 3) Hint: 'v0.99' is declared but not used [XDeclaredButNotUsed]
-#treportunused.nim(38, 3) Hint: 'v0.99.99' is declared but not used [XDeclaredButNotUsed]
 # bug #9764
 iterator s1(a:string): int = discard
 iterator s2(): int = discard
@@ -32,9 +32,7 @@ var s9: int
 type s10 = object
 type s11 = type(1.2)
 
-when false:
-  # enabled again when Nim bootstraps with -d:nimFpRoundtrips
-  # https://github.com/nim-lang/Nim/issues/14407
-  let
-    `v0.99` = "0.99"
-    `v0.99.99` = "0.99.99"
+# bug #14407 (requires `compiler/nim.cfg` containing define:nimFpRoundtrips)
+let
+  `v0.99` = "0.99"
+  `v0.99.99` = "0.99.99"
diff --git a/tests/float/nim.cfg b/tests/float/nim.cfg
deleted file mode 100644
index d27bbf43b..000000000
--- a/tests/float/nim.cfg
+++ /dev/null
@@ -1 +0,0 @@
--d:nimFpRoundtrips
diff --git a/tests/float/tfloats.nim b/tests/float/tfloats.nim
index 30d9c50d6..63987bb8d 100644
--- a/tests/float/tfloats.nim
+++ b/tests/float/tfloats.nim
@@ -1,13 +1,14 @@
 discard """
+  matrix: "-d:nimFpRoundtrips; -u:nimFpRoundtrips"
   targets: "c cpp js"
 """
-# disabled: "windows"
 
 #[
 xxx merge all or most float tests into this file
 ]#
 
 import std/[fenv, math, strutils]
+import stdtest/testutils
 
 proc equalsOrNaNs(a, b: float): bool =
   if isNaN(a): isNaN(b)
@@ -62,27 +63,103 @@ template main =
     reject "1_.0"
     reject "1.0_"
 
-  block: # bug #18148
-    var a = 1.1'f32
-    doAssert $a == "1.1", $a # was failing
+  block: # bugs mentioned in https://github.com/nim-lang/Nim/pull/18504#issuecomment-881635317
+    block: # example 1
+      let a = 0.1+0.2
+      doAssert a != 0.3
+      when defined(nimFpRoundtrips):
+        doAssert $a == "0.30000000000000004"
+      else:
+        whenRuntimeJs: discard
+        do: doAssert $a == "0.3"
+    block: # example 2
+      const a = 0.1+0.2
+      when defined(nimFpRoundtrips):
+        doAssert $($a, a) == """("0.30000000000000004", 0.30000000000000004)"""
+      else:
+        whenRuntimeJs: discard
+        do: doAssert $($a, a) == """("0.3", 0.3)"""
+    block: # example 3
+      const a1 = 0.1+0.2
+      let a2 = a1
+      doAssert a1 != 0.3
+      when defined(nimFpRoundtrips):
+        doAssert $[$a1, $a2] == """["0.30000000000000004", "0.30000000000000004"]"""
+      else:
+        whenRuntimeJs: discard
+        do: doAssert $[$a1, $a2] == """["0.3", "0.3"]"""
 
-proc runtimeOnlyTests =
-  # enable for 'static' once -d:nimFpRoundtrips became the default
-  block: # bug #7717
-    proc test(f: float) =
-      let f2 = $f
-      let f3 = parseFloat(f2)
-      doAssert equalsOrNaNs(f, f3), $(f, f2, f3)
+  when defined(nimFpRoundtrips):
+    block: # bug #18148
+      var a = 1.1'f32
+      doAssert $a == "1.1", $a # was failing
 
-    test 1.0 + epsilon(float64)
-    test 1000000.0000000123
-    test log2(100000.0)
-    test maximumPositiveValue(float32)
-    test maximumPositiveValue(float64)
-    test minimumPositiveValue(float32)
-    test minimumPositiveValue(float64)
+    block: # bug #18400
+      block:
+        let a1 = 0.1'f32
+        let a2 = 0.2'f32
+        let a3 = a1 + a2
+        var s = ""
+        s.addFloat(a3)
+        whenVMorJs: discard # xxx refs #12884
+        do:
+          doAssert a3 == 0.3'f32
+          doAssert $a3 == "0.3"
+
+      block:
+        let a1 = 0.1
+        let a2 = 0.2
+        let a3 = a1 + a2
+        var s = ""
+        s.addFloat(a3)
+        doAssert a3 != 0.3
+        doAssert $a3 == "0.30000000000000004"
+
+      block:
+        var s = [-13.888888'f32]
+        whenRuntimeJs: discard
+        do:
+          doAssert $s == "[-13.888888]"
+          doAssert $s[0] == "-13.888888"
+
+    block: # bug #7717
+      proc test(f: float) =
+        let f2 = $f
+        let f3 = parseFloat(f2)
+        doAssert equalsOrNaNs(f, f3), $(f, f2, f3)
+      test 1.0 + epsilon(float64)
+      test 1000000.0000000123
+      test log2(100000.0)
+      test maximumPositiveValue(float32)
+      test maximumPositiveValue(float64)
+      test minimumPositiveValue(float32)
+      test minimumPositiveValue(float64)
+
+    block: # bug #12884
+      block: # example 1
+        const x0: float32 = 1.32
+        let x1 = 1.32
+        let x2 = 1.32'f32
+        var x3: float32 = 1.32
+        doAssert $(x0, x1, x2, x3) == "(1.32, 1.32, 1.32, 1.32)"
+      block: # example https://github.com/nim-lang/Nim/issues/12884#issuecomment-564967962
+        let x = float(1.32'f32)
+        when nimvm: discard # xxx prints 1.3
+        else:
+          when not defined(js):
+            doAssert $x == "1.3200000524520874"
+        doAssert $1.32 == "1.32"
+        doAssert $1.32'f32 == "1.32"
+        let x2 = 1.32'f32
+        doAssert $x2 == "1.32"
+      block:
+        var x = 1.23456789012345'f32
+        when nimvm:
+          discard # xxx, refs #12884
+        else:
+          when not defined(js):
+            doAssert x == 1.2345679'f32
+            doAssert $x == "1.2345679"
 
 static: main()
 main()
-
-runtimeOnlyTests()
diff --git a/tests/stdlib/tjson.nim b/tests/stdlib/tjson.nim
index 000f72038..289ef9d05 100644
--- a/tests/stdlib/tjson.nim
+++ b/tests/stdlib/tjson.nim
@@ -303,7 +303,7 @@ let jsonNode = %*mynode
 doAssert $jsonNode == """{"kind":"P","pChildren":[{"kind":"Text","textStr":"mychild"},{"kind":"Br"}]}"""
 doAssert $jsonNode.to(ContentNode) == """(kind: P, pChildren: @[(kind: Text, textStr: "mychild"), (kind: Br)])"""
 
-when defined(nimFpRoundtrips): # bug #17383
+block: # bug #17383
   testRoundtrip(int32.high): "2147483647"
   testRoundtrip(uint32.high): "4294967295"
   when int.sizeof == 4:
@@ -316,7 +316,7 @@ when defined(nimFpRoundtrips): # bug #17383
     testRoundtrip(int64.high): "9223372036854775807"
     testRoundtrip(uint64.high): "18446744073709551615"
 
-when defined(nimFpRoundtrips): # bug #18007
+block: # bug #18007
   testRoundtrip([NaN, Inf, -Inf, 0.0, -0.0, 1.0]): """["nan","inf","-inf",0.0,-0.0,1.0]"""
   # pending https://github.com/nim-lang/Nim/issues/18025 use:
   # testRoundtrip([float32(NaN), Inf, -Inf, 0.0, -0.0, 1.0])
@@ -332,7 +332,7 @@ when defined(nimFpRoundtrips): # bug #18007
     testRoundtripVal(0.0): "0.0"
     testRoundtripVal(-0.0): "-0.0"
 
-when defined(nimFpRoundtrips): # bug #15397, bug #13196
+block: # bug #15397, bug #13196
   testRoundtripVal(1.0 + epsilon(float64)): "1.0000000000000002"
   testRoundtripVal(0.12345678901234567890123456789): "0.12345678901234568"
 
diff --git a/tests/stdlib/tjsonutils.nim b/tests/stdlib/tjsonutils.nim
index 3e9c422e0..205160471 100644
--- a/tests/stdlib/tjsonutils.nim
+++ b/tests/stdlib/tjsonutils.nim
@@ -161,16 +161,15 @@ template fn() =
     doAssert b[2].signbit
     doAssert not b[3].signbit
 
-  when defined(nimFpRoundtrips):
-    block: # bug #15397, bug #13196
-      let a = 0.1
-      let x = 0.12345678901234567890123456789
-      let b = (a + 0.2, 0.3, x)
-      testRoundtripVal(b): "[0.30000000000000004,0.3,0.12345678901234568]"
+  block: # bug #15397, bug #13196
+    let a = 0.1
+    let x = 0.12345678901234567890123456789
+    let b = (a + 0.2, 0.3, x)
+    testRoundtripVal(b): "[0.30000000000000004,0.3,0.12345678901234568]"
 
-      testRoundtripVal(0.12345678901234567890123456789): "0.12345678901234568"
-      testRoundtripVal(epsilon(float64)): "2.220446049250313e-16"
-      testRoundtripVal(1.0 + epsilon(float64)): "1.0000000000000002"
+    testRoundtripVal(0.12345678901234567890123456789): "0.12345678901234568"
+    testRoundtripVal(epsilon(float64)): "2.220446049250313e-16"
+    testRoundtripVal(1.0 + epsilon(float64)): "1.0000000000000002"
 
   block: # case object
     type Foo = object

From f8519657c43f458db9c915cec62c59022041eb05 Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Tue, 20 Jul 2021 22:15:06 +0200
Subject: [PATCH 1067/2472] fixes #18469 (#18544)

* fixes #18469

* Update compiler/injectdestructors.nim
---
 compiler/injectdestructors.nim         | 25 ++++++++++++---
 compiler/jsgen.nim                     | 10 ------
 compiler/types.nim                     | 10 ++++++
 tests/arc/tcursor_field_obj_constr.nim | 44 ++++++++++++++++++++++++++
 4 files changed, 74 insertions(+), 15 deletions(-)
 create mode 100644 tests/arc/tcursor_field_obj_constr.nim

diff --git a/compiler/injectdestructors.nim b/compiler/injectdestructors.nim
index 6e4eaa817..cf09b3ff5 100644
--- a/compiler/injectdestructors.nim
+++ b/compiler/injectdestructors.nim
@@ -774,7 +774,7 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode): PNode =
       result = passCopyToSink(n, c, s)
   else:
     case n.kind
-    of nkBracket, nkObjConstr, nkTupleConstr, nkClosure, nkCurly:
+    of nkBracket, nkTupleConstr, nkClosure, nkCurly:
       # Let C(x) be the construction, 'x' the vector of arguments.
       # C(x) either owns 'x' or it doesn't.
       # If C(x) owns its data, we must consume C(x).
@@ -785,13 +785,11 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode): PNode =
       # don't destroy it"
       # but if C(x) is a ref it MUST own its data since we must destroy it
       # so then we have no choice but to use 'sinkArg'.
-      let isRefConstr = n.kind == nkObjConstr and n.typ.skipTypes(abstractInst).kind == tyRef
-      let m = if isRefConstr: sinkArg
-              elif mode == normal: normal
+      let m = if mode == normal: normal
               else: sinkArg
 
       result = copyTree(n)
-      for i in ord(n.kind in {nkObjConstr, nkClosure})..
Date: Wed, 21 Jul 2021 08:51:03 +0200
Subject: [PATCH 1068/2472] =?UTF-8?q?undo=20RFC=20#294,=20it=20breaks=20co?=
 =?UTF-8?q?de=20for=20no=20good=20reason,=20the=20compiler=20can=20wa?=
 =?UTF-8?q?=E2=80=A6=20(#18546)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* undo RFC #294, it breaks code for no good reason, the compiler can warn about the construct instead

* Update changelog.md

Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com>

* enable test case

Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com>
---
 changelog.md           | 9 ++++-----
 compiler/lineinfos.nim | 2 ++
 compiler/semexprs.nim  | 4 +---
 compiler/types.nim     | 2 --
 tests/misc/tconv.nim   | 8 +++-----
 5 files changed, 10 insertions(+), 15 deletions(-)

diff --git a/changelog.md b/changelog.md
index a633c1fb3..1a904d043 100644
--- a/changelog.md
+++ b/changelog.md
@@ -23,15 +23,14 @@
 - `repr` now doesn't insert trailing newline; previous behavior was very inconsistent,
   see #16034. Use `-d:nimLegacyReprWithNewline` for previous behavior.
 
-- An enum now can't be converted to another enum directly, you must use `ord` (or `cast`, but
-  compiler won't help if you misuse it).
+- A type conversion from one enum type to another now produces an `[EnumConv]` warning.
+  You should use `ord` (or `cast`, but the compiler won't help, if you misuse it) instead.
   ```
   type A = enum a1, a2
   type B = enum b1, b2
-  doAssert not compiles(a1.B)
-  doAssert compiles(a1.ord.B)
+  echo a1.B # produces a warning
+  echo a1.ord.B # produces no warning
   ```
-  for a transition period, use `-d:nimLegacyConvEnumEnum`.
 
 - Type mismatch errors now show more context, use `-d:nimLegacyTypeMismatch` for previous
   behavior.
diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim
index 8ab52a452..a3243d5fc 100644
--- a/compiler/lineinfos.nim
+++ b/compiler/lineinfos.nim
@@ -70,6 +70,7 @@ type
     warnResultUsed = "ResultUsed",
     warnCannotOpen = "CannotOpen",
     warnFileChanged = "FileChanged",
+    warnSuspiciousEnumConv = "EnumConv",
     warnUser = "User",
     # hints
     hintSuccess = "Success", hintSuccessX = "SuccessX",
@@ -153,6 +154,7 @@ const
     warnResultUsed: "used 'result' variable",
     warnCannotOpen: "cannot open: $1",
     warnFileChanged: "file changed: $1",
+    warnSuspiciousEnumConv: "$1",
     warnUser: "$1",
     hintSuccess: "operation successful: $#",
     # keep in sync with `testament.isSuccess`
diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim
index f46d44bf0..c25a4067f 100644
--- a/compiler/semexprs.nim
+++ b/compiler/semexprs.nim
@@ -168,9 +168,7 @@ proc checkConvertible(c: PContext, targetTyp: PType, src: PNode): TConvStatus =
   elif (targetBaseTyp.kind in IntegralTypes) and
       (srcBaseTyp.kind in IntegralTypes):
     if targetTyp.kind == tyEnum and srcBaseTyp.kind == tyEnum:
-      if c.config.isDefined("nimLegacyConvEnumEnum"):
-        message(c.config, src.info, warnUser, "enum to enum conversion is now deprecated")
-      else: result = convNotLegal
+      message(c.config, src.info, warnSuspiciousEnumConv, "suspicious code: enum to enum conversion")
     # `elif` would be incorrect here
     if targetTyp.kind == tyBool:
       discard "convOk"
diff --git a/compiler/types.nim b/compiler/types.nim
index 0a50af56e..798f872c2 100644
--- a/compiler/types.nim
+++ b/compiler/types.nim
@@ -1598,8 +1598,6 @@ proc typeMismatch*(conf: ConfigRef; info: TLineInfo, formal, actual: PType, n: P
       of efLockLevelsDiffer:
         msg.add "\nlock levels differ"
 
-    if formal.kind == tyEnum and actual.kind == tyEnum:
-      msg.add "\nmaybe use `-d:nimLegacyConvEnumEnum` for a transition period"
     localError(conf, info, msg)
 
 proc isTupleRecursive(t: PType, cycleDetector: var IntSet): bool =
diff --git a/tests/misc/tconv.nim b/tests/misc/tconv.nim
index c93fc57f8..94677b1bf 100644
--- a/tests/misc/tconv.nim
+++ b/tests/misc/tconv.nim
@@ -1,7 +1,5 @@
 discard """
-  nimout:'''
-tconv.nim(81, 15) Warning: enum to enum conversion is now deprecated [User]
-'''
+  matrix: "--warningAsError:EnumConv"
 """
 
 template reject(x) =
@@ -77,12 +75,12 @@ block: # https://github.com/nim-lang/RFCs/issues/294
   reject: k2.Goo
   reject: k2.string
 
-  {.define(nimLegacyConvEnumEnum).}
+  {.push warningAsError[EnumConv]:off.}
   discard Goo(k2)
   accept: Goo(k2)
   accept: k2.Goo
   reject: k2.string
-  {.undef(nimLegacyConvEnumEnum).}
+  {.pop.}
 
   reject: Goo(k2)
   reject: k2.Goo

From b6f9f7a33eb550a75350c2bce8423269734bb185 Mon Sep 17 00:00:00 2001
From: Araq 
Date: Wed, 21 Jul 2021 09:09:43 +0200
Subject: [PATCH 1069/2472] atlas: minor changes

---
 tools/atlas/atlas.nim | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/atlas/atlas.nim b/tools/atlas/atlas.nim
index 371a1c13f..b10ec0358 100644
--- a/tools/atlas/atlas.nim
+++ b/tools/atlas/atlas.nim
@@ -14,7 +14,7 @@ import parse_requires, osutils, packagesjson
 
 const
   Version = "0.1"
-  Usage = "atlas - Nim Package Manager Version " & Version & """
+  Usage = "atlas - Nim Package Cloner Version " & Version & """
 
   (c) 2021 Andreas Rumpf
 Usage:
@@ -170,7 +170,7 @@ proc toName(p: string): PackageName =
   else:
     result = PackageName p
 
-proc needsCommitLookup(commit: string): bool {.inline} =
+proc needsCommitLookup(commit: string): bool {.inline.} =
   '.' in commit or commit == InvalidCommit
 
 proc isShortCommitHash(commit: string): bool {.inline.} =

From 9c2442af94fc3db8486ee16e82ea27752e9ae4f5 Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Wed, 21 Jul 2021 09:46:18 +0200
Subject: [PATCH 1070/2472] unary slices get a deprecation period (#18549)

---
 changelog.md   |  3 +--
 lib/system.nim | 17 ++++++++---------
 2 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/changelog.md b/changelog.md
index 1a904d043..ece88f710 100644
--- a/changelog.md
+++ b/changelog.md
@@ -63,8 +63,7 @@
 - `hashes.hash(proc|ptr|ref|pointer)` now calls `hash(int)` and honors `-d:nimIntHash1`,
   `hashes.hash(closure)` has also been improved.
 
-- The unary slice `..b` was removed, use `0..b` instead or use `-d:nimLegacyUnarySlice`
-  for a deprecation period.
+- The unary slice `..b` was deprecated, use `0..b` instead.
 
 - Removed `.travis.yml`, `appveyor.yml.disabled`, `.github/workflows/ci.yml.disabled`.
 
diff --git a/lib/system.nim b/lib/system.nim
index 1da1444d2..e6a2439df 100644
--- a/lib/system.nim
+++ b/lib/system.nim
@@ -511,15 +511,14 @@ proc `..`*[T, U](a: sink T, b: sink U): HSlice[T, U] {.noSideEffect, inline, mag
   ##   echo a[2 .. 3] # @[30, 40]
   result = HSlice[T, U](a: a, b: b)
 
-when defined(nimLegacyUnarySlice):
-  proc `..`*[T](b: sink T): HSlice[int, T]
-    {.noSideEffect, inline, magic: "DotDot", deprecated: "replace `..b` with `0..b`".} =
-    ## Unary `slice`:idx: operator that constructs an interval `[default(int), b]`.
-    ##
-    ## .. code-block:: Nim
-    ##   let a = [10, 20, 30, 40, 50]
-    ##   echo a[.. 2] # @[10, 20, 30]
-    result = HSlice[int, T](a: 0, b: b)
+proc `..`*[T](b: sink T): HSlice[int, T]
+  {.noSideEffect, inline, magic: "DotDot", deprecated: "replace `..b` with `0..b`".} =
+  ## Unary `slice`:idx: operator that constructs an interval `[default(int), b]`.
+  ##
+  ## .. code-block:: Nim
+  ##   let a = [10, 20, 30, 40, 50]
+  ##   echo a[.. 2] # @[10, 20, 30]
+  result = HSlice[int, T](a: 0, b: b)
 
 when defined(hotCodeReloading):
   {.pragma: hcrInline, inline.}

From f048dad7c3af7d70fbb852a59cb5bdda332e96d7 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Wed, 21 Jul 2021 15:47:06 +0800
Subject: [PATCH 1071/2472] add testcase for #6499 #12229 #7172 (#18547)

---
 tests/magics/tmagics.nim |  8 ++++++++
 tests/threads/t7172.nim  | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+)
 create mode 100644 tests/threads/t7172.nim

diff --git a/tests/magics/tmagics.nim b/tests/magics/tmagics.nim
index fa138320c..70926a0d7 100644
--- a/tests/magics/tmagics.nim
+++ b/tests/magics/tmagics.nim
@@ -55,3 +55,11 @@ block t9442:
   GC_unref(v2)
   GC_ref(v3)
   GC_unref(v3)
+
+block: # bug #6499
+  let x = (chr, 0)
+  doAssert x[1] == 0
+
+block: # bug #12229
+  proc foo(T: typedesc) = discard
+  foo(ref)
diff --git a/tests/threads/t7172.nim b/tests/threads/t7172.nim
new file mode 100644
index 000000000..983765dba
--- /dev/null
+++ b/tests/threads/t7172.nim
@@ -0,0 +1,32 @@
+discard """
+  output: '''
+In doStuff()
+In initProcess()
+initProcess() done
+TEST
+Crashes before getting here!
+'''
+  joinable: false
+"""
+
+import std/os
+
+proc whatever() {.thread, nimcall.} =
+  echo("TEST")
+
+proc initProcess(): void =
+  echo("In initProcess()")
+  var thread: Thread[void]
+  createThread(thread, whatever)
+  echo("initProcess() done")
+  joinThread(thread)
+
+proc doStuff(): void =
+  echo("In doStuff()")
+  # ...
+  initProcess()
+  sleep(500)
+  # ...
+  echo("Crashes before getting here!")
+
+doStuff()

From f86a530214d56f39664d6aaca3ddddf3b2fdba0c Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Wed, 21 Jul 2021 10:22:30 +0200
Subject: [PATCH 1072/2472] breaking changes policies (#18541)

* document some std library evolution policies

In wanting to improve the standard library, it's helpful to have a set
of principles and guidelines to lean back on, that show how to introduce
such improvements while at the same time considering existing users of
the language.

A key idea behind documentation of this sort is that it should highlight
a path forwards in making changes rather than trying to prevent them,
and although the current snippet does include some language for what one
shouldn't do, it also shows a few options for what one can do.

This is a riff on https://github.com/nim-lang/Nim/pull/18468 mainly
based on what helps enable to the use of Nim productively in
environments and teams where the priority and focus is not always on the
tool (Nim in this case) but rather on the codebase itself, and its use
by end users.

We use similar guidance documentation to help coordinate the code of the
teams using Nim in https://status-im.github.io/nim-style-guide/ where it
is applied not as law, but rather as recommendations for the default
approach that should be considered first.

* document the new policies

* typo

* Update doc/contributing.rst

Co-authored-by: Timothee Cour 

* Update doc/contributing.rst

* Update doc/contributing.rst

Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com>

* Update doc/contributing.rst

Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com>

* Update doc/contributing.rst

* Update doc/contributing.rst

Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com>

* clarify some things

* Update doc/contributing.rst

Co-authored-by: Dominik Picheta 

Co-authored-by: Jacek Sieka 
Co-authored-by: Timothee Cour 
Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com>
Co-authored-by: Dominik Picheta 
---
 doc/contributing.rst | 114 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 114 insertions(+)

diff --git a/doc/contributing.rst b/doc/contributing.rst
index 5a31d85bd..07fa01714 100644
--- a/doc/contributing.rst
+++ b/doc/contributing.rst
@@ -671,3 +671,117 @@ Conventions
    Furthermore, module names should use `snake_case` and not use capital
    letters, which cause issues when going from an OS without case
    sensitivity to an OS with it.
+
+
+Breaking Changes
+================
+
+Introducing breaking changes, no matter how well intentioned,
+creates long-term problems for the community, in particular those looking to promote
+reusable Nim code in libraries: In the Nim distribution, critical security and bugfixes,
+language changes and community improvements are bundled in a single distribution - it is
+difficult to make partial upgrades with only benign changes. When one library depends on
+a legacy behavior, it can no longer be used together with another library that does not,
+breaking all downstream applications - the standard library is unique in that it sits at
+the root of **all** dependency trees.
+
+There is a big difference between compile-time breaking changes and run-time breaking
+changes.
+
+
+Run-time breaking changes
+-------------------------
+
+Run-time breaking changes are to be avoided at almost all costs: Nim is used for
+mission critical applications which depend on behaviours that
+are not covered by the test suite. As such, it's important that changes to the
+*stable* parts of the standard library are made avoiding changing the existing
+behaviors, even when the test suite continues to pass.
+
+Examples of run-time breaking changes:
+
+- Raising exceptions of a new type, compared to what's currently being raised.
+
+- Adding unconstrained or poorly constrained generic procs or macros
+  ("hash now works for all `ref T`"): This may cause code to behave differently
+  depending only on which modules are imported - common examples include `==` and `hash`.
+
+- Changing behavior of existing functions like:
+
+  * "Nim's path handling procs like `getXDir` now consistently lack the trailing slash"
+  * "Nim's strformat implementation is now more consistent with Python"
+
+Instead write new code that explicitly announces the feature you think we announced but
+didn't. For example, `strformat` does not say "it's compatible with Python", it
+says "inspired by Python's f-strings". This new code can be submitted to the stdlib
+and the old code can be deprecated or it can be published as a Nimble package.
+
+Sometimes, a run-time breaking change is most desirable: For example, a string
+representation of a floating point number that "roundtrips" is much better than
+a string represenation that doesn't. These run-time breaking changes must start in the
+state "opt-in" via a new `-d:nimPreviewX` or command line flag and then should become
+the new default later, in follow-up versions. This way users can track
+regressions more easily. ("git bisect" is not an acceptable alternative, that's for
+Nim compiler developers, not for Nim users.)
+
+Above all else, additive approaches that don't change existing behaviors
+should be preferred.
+
+
+Compile-time breaking changes
+-----------------------------
+
+Compile-time breaking changes are usually easier to handle, but for large code bases
+it can also be much work and it can hinder the adoption of a new Nim release.
+Additive approaches are to be preferred here as well.
+
+Examples of compile-time breaking changes include (but are not limited to):
+
+* Renaming functions and modules, or moving things. Instead of a direct rename,
+  deprecate the old name and introduce a new one.
+* Renaming the parameter names: Thanks to Nim's "named parameter" calling syntax
+  like `f(x = 0, y = 1)` this is a breaking change. Instead live with the existing
+  parameter names.
+* Adding an enum value to an existing enum. Nim's exhaustive case statements stop
+  compiling after such a change. Instead consider to introduce new `bool`
+  fields/parameters. This can be impractical though, so we use good judgement
+  and our list of "important packages" to see if it doesn't break too much code
+  out there in practice.
+* Adding a new proc to an existing stdlib module. However, for aesthetic reasons
+  this is often preferred over introducing a new module with just a single proc
+  inside. We use good judgement and our list of "important packages" to see if
+  it doesn't break too much code out there in practice. The new procs need to
+  be annotated with a `.since` annotation.
+
+
+Compiler/language spec bugfixes
+-------------------------------
+
+This can even be applied to compiler "bugfixes": If the compiler should have been
+"pickier" in its handling of `typedesc`, instead of "fixing typedesc handling bugs",
+consider the following solution:
+
+- Spec out how `typedesc` should really work and also spec out the cases where it
+  should not be allowed!
+- Deprecate `typedesc` and name the new metatype something new like `typeArg`.
+- Implement the spec.
+
+
+Non-breaking changes
+--------------------
+
+Examples of changes that are considered non-breaking (or acceptable breaking changes) include:
+
+* Creating a new module.
+* Adding an overload to an already overloaded proc.
+* Adding new default parameters to an existing proc. It is assumed that you do not
+  use Nim's stdlib procs's addresses (that you don't use them as first class entities).
+* Changing the calling convention from `nimcall` to `inline`
+  (but first RFC https://github.com/nim-lang/RFCs/issues/396 needs to be implemented).
+* Changing the behavior from "crashing" into some other, well documented result (including
+  raising a Defect, but not raising an exception that does not inherit from Defect).
+* Adding new fields to an existing object.
+
+Nim's introspection facilities imply that strictly speaking almost every addition can
+break somebody's code. It is impractical to care about these cases, a change that only
+affects introspection is not considered to be a breaking change.

From 6f34829ee5a060300859002fd41f0181c0adf754 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Wed, 21 Jul 2021 19:46:17 +0800
Subject: [PATCH 1073/2472] clean unused imports on runnableExamples (#18551)

---
 lib/std/monotimes.nim | 2 --
 1 file changed, 2 deletions(-)

diff --git a/lib/std/monotimes.nim b/lib/std/monotimes.nim
index 3bb4fa141..bd078d7c0 100644
--- a/lib/std/monotimes.nim
+++ b/lib/std/monotimes.nim
@@ -15,8 +15,6 @@ meaning that that the following is guaranteed to work:
 ]##
 
 runnableExamples:
-  import std/os
-
   let a = getMonoTime()
   let b = getMonoTime()
   assert a <= b

From 880353c0ad9300f2543c1b16548b4f2ab6784183 Mon Sep 17 00:00:00 2001
From: IterableTrucks <42210451+IterableTrucks@users.noreply.github.com>
Date: Wed, 21 Jul 2021 20:13:50 +0800
Subject: [PATCH 1074/2472] complement notes of two doc strings of tables
 module (#18527)

* complement notes of two doc strings warning the deprecated  procedure's side effect

* Update lib/pure/collections/tables.nim

Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com>

* Update lib/pure/collections/tables.nim

Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com>

Co-authored-by: BillyZee 
Co-authored-by: Andreas Rumpf 
Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com>
---
 lib/pure/collections/tables.nim | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim
index d617e06e4..b845f57cc 100644
--- a/lib/pure/collections/tables.nim
+++ b/lib/pure/collections/tables.nim
@@ -496,6 +496,8 @@ template tabCellHash(i)  = t.data[i].hcode
 proc del*[A, B](t: var Table[A, B], key: A) =
   ## Deletes `key` from hash table `t`. Does nothing if the key does not exist.
   ##
+  ## .. warning:: If duplicate keys were added, this may need to be called multiple times.
+  ##
   ## See also:
   ## * `pop proc<#pop,Table[A,B],A,B>`_
   ## * `clear proc<#clear,Table[A,B]>`_ to empty the whole table
@@ -514,6 +516,8 @@ proc pop*[A, B](t: var Table[A, B], key: A, val: var B): bool =
   ## mapping of the key. Otherwise, returns `false`, and the `val` is
   ## unchanged.
   ##
+  ## .. warning:: If duplicate keys were added, this may need to be called multiple times.
+  ##
   ## See also:
   ## * `del proc<#del,Table[A,B],A>`_
   ## * `clear proc<#clear,Table[A,B]>`_ to empty the whole table

From 58080525a1dcad23de6ad0cf9ca6988f103de350 Mon Sep 17 00:00:00 2001
From: itsumura-h <39766805+itsumura-h@users.noreply.github.com>
Date: Wed, 21 Jul 2021 22:03:48 +0900
Subject: [PATCH 1075/2472] add [1..2] for JArray (#18525)

* add [1..2] for JArray
* fix BackwardsIndex to int
* fix for BackwardsIndex
* fix for assert node kind check
* fix variable name
* Update lib/pure/json.nim
* fix for when x.a is BackwardsIndex

Co-authored-by: itsumura-h 
Co-authored-by: Dominik Picheta 
---
 lib/pure/json.nim | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/lib/pure/json.nim b/lib/pure/json.nim
index d5bbb86b8..85c3393b2 100644
--- a/lib/pure/json.nim
+++ b/lib/pure/json.nim
@@ -531,6 +531,24 @@ proc `[]`*(node: JsonNode, index: BackwardsIndex): JsonNode {.inline, since: (1,
 
   `[]`(node, node.len - int(index))
 
+proc `[]`*[U, V](a: JsonNode, x: HSlice[U, V]): JsonNode =
+  ## Slice operation for JArray.
+  ##
+  ## Returns the inclusive range `[a[x.a], a[x.b]]`:
+  runnableExamples:
+    import json
+    let arr = %[0,1,2,3,4,5]
+    doAssert arr[2..4] == %[2,3,4]
+    doAssert arr[2..^2] == %[2,3,4]
+    doAssert arr[^4..^2] == %[2,3,4]
+
+  assert(a.kind == JArray)
+  result = newJArray()
+  let xa = (when x.a is BackwardsIndex: a.len - int(x.a) else: int(x.a))
+  let L = (when x.b is BackwardsIndex: a.len - int(x.b) else: int(x.b)) - xa + 1
+  for i in 0..
Date: Wed, 21 Jul 2021 16:55:50 +0200
Subject: [PATCH 1076/2472] fixes #18550 (#18553)

* fixes #18550

* update the manual to reflect reality
---
 changelog.md            |  4 ++++
 compiler/lineinfos.nim  |  2 ++
 compiler/sempass2.nim   | 16 ++++++++++++++++
 compiler/sighashes.nim  |  4 +++-
 compiler/types.nim      | 15 +++++++++++++++
 doc/manual.rst          |  7 +++----
 lib/system/io.nim       |  8 ++++----
 lib/system/widestrs.nim |  2 +-
 tests/misc/tconv.nim    | 10 +++++++++-
 9 files changed, 57 insertions(+), 11 deletions(-)

diff --git a/changelog.md b/changelog.md
index ece88f710..f2a9ca914 100644
--- a/changelog.md
+++ b/changelog.md
@@ -32,6 +32,10 @@
   echo a1.ord.B # produces no warning
   ```
 
+- A dangerous implicit conversion to `cstring` now triggers a `[CStringConv]` warning.
+  This warning will become an error in future versions! Use an explicit conversion
+  like `cstring(x)` in order to silence the warning.
+
 - Type mismatch errors now show more context, use `-d:nimLegacyTypeMismatch` for previous
   behavior.
 
diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim
index a3243d5fc..3f31ceb97 100644
--- a/compiler/lineinfos.nim
+++ b/compiler/lineinfos.nim
@@ -71,6 +71,7 @@ type
     warnCannotOpen = "CannotOpen",
     warnFileChanged = "FileChanged",
     warnSuspiciousEnumConv = "EnumConv",
+    warnCstringConv = "CStringConv",
     warnUser = "User",
     # hints
     hintSuccess = "Success", hintSuccessX = "SuccessX",
@@ -155,6 +156,7 @@ const
     warnCannotOpen: "cannot open: $1",
     warnFileChanged: "file changed: $1",
     warnSuspiciousEnumConv: "$1",
+    warnCstringConv: "$1",
     warnUser: "$1",
     hintSuccess: "operation successful: $#",
     # keep in sync with `testament.isSuccess`
diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim
index 1e1672cad..ce21c3ed2 100644
--- a/compiler/sempass2.nim
+++ b/compiler/sempass2.nim
@@ -951,6 +951,15 @@ proc trackInnerProc(tracked: PEffects, n: PNode) =
   else:
     for ch in n: trackInnerProc(tracked, ch)
 
+proc allowCStringConv(n: PNode): bool =
+  case n.kind
+  of nkStrLit..nkTripleStrLit: result = true
+  of nkSym: result = n.sym.kind in {skConst, skParam}
+  of nkAddr: result = isCharArrayPtr(n.typ, true)
+  of nkCallKinds:
+    result = isCharArrayPtr(n.typ, n[0].kind == nkSym and n[0].sym.magic == mAddr)
+  else: result = isCharArrayPtr(n.typ, false)
+
 proc track(tracked: PEffects, n: PNode) =
   case n.kind
   of nkSym:
@@ -1157,6 +1166,13 @@ proc track(tracked: PEffects, n: PNode) =
       if tracked.owner.kind != skMacro:
         createTypeBoundOps(tracked, n.typ, n.info)
   of nkHiddenStdConv, nkHiddenSubConv, nkConv:
+    if n.kind in {nkHiddenStdConv, nkHiddenSubConv} and
+        n.typ.skipTypes(abstractInst).kind == tyCstring and
+        not allowCStringConv(n[1]):
+      message(tracked.config, n.info, warnCstringConv,
+        "implicit conversion to 'cstring' from a non-const location: $1; this will become a compile time error in the future" %
+          [$n[1]])
+
     if n.len == 2:
       track(tracked, n[1])
       if tracked.owner.kind != skMacro:
diff --git a/compiler/sighashes.nim b/compiler/sighashes.nim
index f6df422e6..0ff97017f 100644
--- a/compiler/sighashes.nim
+++ b/compiler/sighashes.nim
@@ -14,7 +14,9 @@ from hashes import Hash
 import types
 
 proc `&=`(c: var MD5Context, s: string) = md5Update(c, s, s.len)
-proc `&=`(c: var MD5Context, ch: char) = md5Update(c, unsafeAddr ch, 1)
+proc `&=`(c: var MD5Context, ch: char) =
+  # XXX suspicious code here; relies on ch being zero terminated?
+  md5Update(c, unsafeAddr ch, 1)
 proc `&=`(c: var MD5Context, r: Rope) =
   for l in leaves(r): md5Update(c, l, l.len)
 proc `&=`(c: var MD5Context, i: BiggestInt) =
diff --git a/compiler/types.nim b/compiler/types.nim
index 798f872c2..f0bd99a17 100644
--- a/compiler/types.nim
+++ b/compiler/types.nim
@@ -1664,3 +1664,18 @@ proc lookupFieldAgain*(ty: PType; field: PSym): PSym =
     if result != nil: break
     ty = ty[0]
   if result == nil: result = field
+
+proc isCharArrayPtr*(t: PType; allowPointerToChar: bool): bool =
+  let t = t.skipTypes(abstractInst)
+  if t.kind == tyPtr:
+    let pointsTo = t[0].skipTypes(abstractInst)
+    case pointsTo.kind
+    of tyUncheckedArray:
+      result = pointsTo[0].kind == tyChar
+    of tyArray:
+      result = pointsTo[1].kind == tyChar and firstOrd(nil, pointsTo[0]) == 0 and
+        skipTypes(pointsTo[0], {tyRange}).kind in {tyInt..tyInt64}
+    of tyChar:
+      result = allowPointerToChar
+    else:
+      discard
diff --git a/doc/manual.rst b/doc/manual.rst
index f0bd85638..9f614c2cd 100644
--- a/doc/manual.rst
+++ b/doc/manual.rst
@@ -1380,10 +1380,9 @@ variadic proc, it is implicitly converted to `cstring` too:
 
 Even though the conversion is implicit, it is not *safe*: The garbage collector
 does not consider a `cstring` to be a root and may collect the underlying
-memory. However, in practice, this almost never happens as the GC considers
-stack roots conservatively. One can use the builtin procs `GC_ref` and
-`GC_unref` to keep the string data alive for the rare cases where it does
-not work.
+memory. For this reason, the implicit conversion will be removed in future
+releases of the Nim compiler. Certain idioms like conversion of a `const` string
+to `cstring` are safe and will remain to be allowed.
 
 A `$` proc is defined for cstrings that returns a string. Thus to get a nim
 string from a cstring:
diff --git a/lib/system/io.nim b/lib/system/io.nim
index 591ea3e92..d8b2ac741 100644
--- a/lib/system/io.nim
+++ b/lib/system/io.nim
@@ -630,8 +630,8 @@ const
         ""
     else:
       ""
-  FormatOpen: array[FileMode, string] = [
-    "rb" & NoInheritFlag, "wb" & NoInheritFlag, "w+b" & NoInheritFlag,
+  FormatOpen: array[FileMode, cstring] = [
+    cstring("rb" & NoInheritFlag), "wb" & NoInheritFlag, "w+b" & NoInheritFlag,
     "r+b" & NoInheritFlag, "ab" & NoInheritFlag
   ]
     #"rt", "wt", "w+t", "r+t", "at"
@@ -678,7 +678,7 @@ proc open*(f: var File, filename: string,
   ## This throws no exception if the file could not be opened.
   ##
   ## The file handle associated with the resulting `File` is not inheritable.
-  var p = fopen(filename, FormatOpen[mode])
+  var p = fopen(filename.cstring, FormatOpen[mode])
   if p != nil:
     var f2 = cast[File](p)
     when defined(posix) and not defined(nimscript):
@@ -711,7 +711,7 @@ proc reopen*(f: File, filename: string, mode: FileMode = fmRead): bool {.
   ## Default mode is readonly. Returns true if the file could be reopened.
   ##
   ## The file handle associated with `f` won't be inheritable.
-  if freopen(filename, FormatOpen[mode], f) != nil:
+  if freopen(filename.cstring, FormatOpen[mode], f) != nil:
     when not defined(nimInheritHandles) and declared(setInheritable) and
          NoInheritFlag.len == 0:
       if not setInheritable(getOsFileHandle(f), false):
diff --git a/lib/system/widestrs.nim b/lib/system/widestrs.nim
index a999c9167..8b08959b5 100644
--- a/lib/system/widestrs.nim
+++ b/lib/system/widestrs.nim
@@ -171,7 +171,7 @@ proc newWideCString*(s: cstring): WideCStringObj =
   result = newWideCString(s, s.len)
 
 proc newWideCString*(s: string): WideCStringObj =
-  result = newWideCString(s, s.len)
+  result = newWideCString(cstring s, s.len)
 
 proc `$`*(w: WideCString, estimate: int, replacement: int = 0xFFFD): string =
   result = newStringOfCap(estimate + estimate shr 2)
diff --git a/tests/misc/tconv.nim b/tests/misc/tconv.nim
index 94677b1bf..1cb4b4fbf 100644
--- a/tests/misc/tconv.nim
+++ b/tests/misc/tconv.nim
@@ -1,5 +1,5 @@
 discard """
-  matrix: "--warningAsError:EnumConv"
+  matrix: "--warningAsError:EnumConv --warningAsError:CStringConv"
 """
 
 template reject(x) =
@@ -84,3 +84,11 @@ block: # https://github.com/nim-lang/RFCs/issues/294
 
   reject: Goo(k2)
   reject: k2.Goo
+
+reject:
+  # bug #18550
+  proc f(c: char): cstring =
+    var x = newString(109*1024*1024)
+    x[0] = c
+    x
+

From d5b5827bc2a3724290d3257b0f2fa70004c37c97 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Wed, 21 Jul 2021 23:00:29 -0700
Subject: [PATCH 1077/2472] fix setCommand so it behaves like regular nim
 invocation (#18555)

---
 compiler/cmdlinehelper.nim |  9 +--------
 compiler/options.nim       | 10 ++++++++++
 compiler/scriptconfig.nim  | 11 +----------
 3 files changed, 12 insertions(+), 18 deletions(-)

diff --git a/compiler/cmdlinehelper.nim b/compiler/cmdlinehelper.nim
index c6a0f200a..007dc2319 100644
--- a/compiler/cmdlinehelper.nim
+++ b/compiler/cmdlinehelper.nim
@@ -44,14 +44,7 @@ proc processCmdLineAndProjectPath*(self: NimProg, conf: ConfigRef) =
   elif self.supportsStdinFile and conf.projectName == "-":
     handleStdinInput(conf)
   elif conf.projectName != "":
-    try:
-      conf.projectFull = canonicalizePath(conf, AbsoluteFile conf.projectName)
-    except OSError:
-      conf.projectFull = AbsoluteFile conf.projectName
-    let p = splitFile(conf.projectFull)
-    let dir = if p.dir.isEmpty: AbsoluteDir getCurrentDir() else: p.dir
-    conf.projectPath = AbsoluteDir canonicalizePath(conf, AbsoluteFile dir)
-    conf.projectName = p.name
+    setFromProjectName(conf, conf.projectName)
   else:
     conf.projectPath = AbsoluteDir canonicalizePath(conf, AbsoluteFile getCurrentDir())
 
diff --git a/compiler/options.nim b/compiler/options.nim
index 45ca14e76..f02f02256 100644
--- a/compiler/options.nim
+++ b/compiler/options.nim
@@ -695,6 +695,16 @@ proc setDefaultLibpath*(conf: ConfigRef) =
 proc canonicalizePath*(conf: ConfigRef; path: AbsoluteFile): AbsoluteFile =
   result = AbsoluteFile path.string.expandFilename
 
+proc setFromProjectName*(conf: ConfigRef; projectName: string) =
+  try:
+    conf.projectFull = canonicalizePath(conf, AbsoluteFile projectName)
+  except OSError:
+    conf.projectFull = AbsoluteFile projectName
+  let p = splitFile(conf.projectFull)
+  let dir = if p.dir.isEmpty: AbsoluteDir getCurrentDir() else: p.dir
+  conf.projectPath = AbsoluteDir canonicalizePath(conf, AbsoluteFile dir)
+  conf.projectName = p.name
+
 proc removeTrailingDirSep*(path: string): string =
   if (path.len > 0) and (path[^1] == DirSep):
     result = substr(path, 0, path.len - 2)
diff --git a/compiler/scriptconfig.nim b/compiler/scriptconfig.nim
index ef1ceb12b..89510f6b2 100644
--- a/compiler/scriptconfig.nim
+++ b/compiler/scriptconfig.nim
@@ -151,18 +151,9 @@ proc setupVM*(module: PSym; cache: IdentCache; scriptName: string;
     setResult(a, strutils.cmpIgnoreCase(a.getString 0, a.getString 1))
   cbconf setCommand:
     conf.setCommandEarly(a.getString 0)
-    # xxx move remaining logic to commands.nim or other
     let arg = a.getString 1
     incl(conf.globalOptions, optWasNimscript)
-    if arg.len > 0:
-      conf.projectName = arg
-      let path =
-        if conf.projectName.isAbsolute: AbsoluteFile(conf.projectName)
-        else: conf.projectPath / RelativeFile(conf.projectName)
-      try:
-        conf.projectFull = canonicalizePath(conf, path)
-      except OSError:
-        conf.projectFull = path
+    if arg.len > 0: setFromProjectName(conf, arg)
   cbconf getCommand:
     setResult(a, conf.command)
   cbconf switch:

From 58e27ebd4a16ca9f2ba24826d7162384d5321101 Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Thu, 22 Jul 2021 09:37:41 +0200
Subject: [PATCH 1078/2472] fixes #12815 (#18554)

---
 changelog.md           |  3 +++
 compiler/ast.nim       |  2 +-
 compiler/lineinfos.nim |  4 +++-
 compiler/sempass2.nim  |  5 ++++-
 doc/nimc.rst           |  6 ++++++
 tests/misc/tconv.nim   | 13 +++++++++++++
 6 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/changelog.md b/changelog.md
index f2a9ca914..8e8b0a908 100644
--- a/changelog.md
+++ b/changelog.md
@@ -36,6 +36,9 @@
   This warning will become an error in future versions! Use an explicit conversion
   like `cstring(x)` in order to silence the warning.
 
+- There is a new warning for *any* type conversion to enum that can be enabled via
+  `.warning[AnyEnumConv]:on` or `--warning:AnyEnumConv:on`.
+
 - Type mismatch errors now show more context, use `-d:nimLegacyTypeMismatch` for previous
   behavior.
 
diff --git a/compiler/ast.nim b/compiler/ast.nim
index 8b836e088..0eeaa8133 100644
--- a/compiler/ast.nim
+++ b/compiler/ast.nim
@@ -1064,7 +1064,7 @@ proc getPIdent*(a: PNode): PIdent {.inline.} =
   # which may simplify code.
   case a.kind
   of nkSym: a.sym.name
-  of nkIdent:  a.ident
+  of nkIdent: a.ident
   else: nil
 
 proc getnimblePkg*(a: PSym): PSym =
diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim
index 3f31ceb97..8cb834156 100644
--- a/compiler/lineinfos.nim
+++ b/compiler/lineinfos.nim
@@ -71,6 +71,7 @@ type
     warnCannotOpen = "CannotOpen",
     warnFileChanged = "FileChanged",
     warnSuspiciousEnumConv = "EnumConv",
+    warnAnyEnumConv = "AnyEnumConv",
     warnCstringConv = "CStringConv",
     warnUser = "User",
     # hints
@@ -156,6 +157,7 @@ const
     warnCannotOpen: "cannot open: $1",
     warnFileChanged: "file changed: $1",
     warnSuspiciousEnumConv: "$1",
+    warnAnyEnumConv: "$1",
     warnCstringConv: "$1",
     warnUser: "$1",
     hintSuccess: "operation successful: $#",
@@ -211,7 +213,7 @@ type
   TNoteKinds* = set[TNoteKind]
 
 proc computeNotesVerbosity(): array[0..3, TNoteKinds] =
-  result[3] = {low(TNoteKind)..high(TNoteKind)} - {warnObservableStores, warnResultUsed}
+  result[3] = {low(TNoteKind)..high(TNoteKind)} - {warnObservableStores, warnResultUsed, warnAnyEnumConv}
   result[2] = result[3] - {hintStackTrace, warnUninit, hintExtendedContext, hintDeclaredLoc, hintProcessingStmt}
   result[1] = result[2] - {warnProveField, warnProveIndex,
     warnGcUnsafe, hintPath, hintDependency, hintCodeBegin, hintCodeEnd,
diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim
index ce21c3ed2..87ed81098 100644
--- a/compiler/sempass2.nim
+++ b/compiler/sempass2.nim
@@ -1171,7 +1171,10 @@ proc track(tracked: PEffects, n: PNode) =
         not allowCStringConv(n[1]):
       message(tracked.config, n.info, warnCstringConv,
         "implicit conversion to 'cstring' from a non-const location: $1; this will become a compile time error in the future" %
-          [$n[1]])
+          $n[1])
+
+    if n.typ.skipTypes(abstractInst).kind == tyEnum:
+      message(tracked.config, n.info, warnAnyEnumConv, "enum conversion: $1" % $n[1])
 
     if n.len == 2:
       track(tracked, n[1])
diff --git a/doc/nimc.rst b/doc/nimc.rst
index 070956860..3c451706b 100644
--- a/doc/nimc.rst
+++ b/doc/nimc.rst
@@ -70,6 +70,12 @@ SmallLshouldNotBeUsed            The letter 'l' should not be used as an
                                  identifier.
 EachIdentIsTuple                 The code contains a confusing `var`
                                  declaration.
+CStringConv                      Warn about dangerous implicit conversions
+                                 to `cstring`.
+EnumConv                         Warn about conversions from enum to enum.
+AnyEnumConv                      Warn about any conversions to an enum type.
+ResultUsed                       Warn about the usage of the
+                                 built-in `result` variable.
 User                             Some user-defined warning.
 ==========================       ============================================
 
diff --git a/tests/misc/tconv.nim b/tests/misc/tconv.nim
index 1cb4b4fbf..ff6875021 100644
--- a/tests/misc/tconv.nim
+++ b/tests/misc/tconv.nim
@@ -92,3 +92,16 @@ reject:
     x[0] = c
     x
 
+{.push warning[AnyEnumConv]:on, warningAsError[AnyEnumConv]:on.}
+
+reject:
+  # bug #12815
+  type
+    Foo = enum
+      one = 1
+      three = 3
+
+  var va = 2
+  var vb = va.Foo
+
+{.pop.}

From 0c4582c66524d58a3c72827d9546a5c5f1c40286 Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Thu, 22 Jul 2021 11:59:21 +0200
Subject: [PATCH 1079/2472] mitigates #12815 (#18557)

* mitigates #12815

* Update doc/nimc.rst

Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com>

Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com>
---
 compiler/lineinfos.nim |  2 ++
 compiler/sempass2.nim  |  8 ++++++--
 doc/nimc.rst           |  2 ++
 tests/misc/tconv.nim   | 19 ++++++++++++++++---
 4 files changed, 26 insertions(+), 5 deletions(-)

diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim
index 8cb834156..c64d7b9dd 100644
--- a/compiler/lineinfos.nim
+++ b/compiler/lineinfos.nim
@@ -72,6 +72,7 @@ type
     warnFileChanged = "FileChanged",
     warnSuspiciousEnumConv = "EnumConv",
     warnAnyEnumConv = "AnyEnumConv",
+    warnHoleEnumConv = "HoleEnumConv",
     warnCstringConv = "CStringConv",
     warnUser = "User",
     # hints
@@ -158,6 +159,7 @@ const
     warnFileChanged: "file changed: $1",
     warnSuspiciousEnumConv: "$1",
     warnAnyEnumConv: "$1",
+    warnHoleEnumConv: "$1",
     warnCstringConv: "$1",
     warnUser: "$1",
     hintSuccess: "operation successful: $#",
diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim
index 87ed81098..e2f85dd38 100644
--- a/compiler/sempass2.nim
+++ b/compiler/sempass2.nim
@@ -1173,8 +1173,12 @@ proc track(tracked: PEffects, n: PNode) =
         "implicit conversion to 'cstring' from a non-const location: $1; this will become a compile time error in the future" %
           $n[1])
 
-    if n.typ.skipTypes(abstractInst).kind == tyEnum:
-      message(tracked.config, n.info, warnAnyEnumConv, "enum conversion: $1" % $n[1])
+    let t = n.typ.skipTypes(abstractInst)
+    if t.kind == tyEnum:
+      if tfEnumHasHoles in t.flags:
+        message(tracked.config, n.info, warnHoleEnumConv, "conversion to enum with holes is unsafe: $1" % $n)
+      else:
+        message(tracked.config, n.info, warnAnyEnumConv, "enum conversion: $1" % $n)
 
     if n.len == 2:
       track(tracked, n[1])
diff --git a/doc/nimc.rst b/doc/nimc.rst
index 3c451706b..ed9077912 100644
--- a/doc/nimc.rst
+++ b/doc/nimc.rst
@@ -74,6 +74,8 @@ CStringConv                      Warn about dangerous implicit conversions
                                  to `cstring`.
 EnumConv                         Warn about conversions from enum to enum.
 AnyEnumConv                      Warn about any conversions to an enum type.
+HoleEnumConv                     Warn about conversion to an enum with
+                                 holes. These conversions are unsafe.
 ResultUsed                       Warn about the usage of the
                                  built-in `result` variable.
 User                             Some user-defined warning.
diff --git a/tests/misc/tconv.nim b/tests/misc/tconv.nim
index ff6875021..5e8eac729 100644
--- a/tests/misc/tconv.nim
+++ b/tests/misc/tconv.nim
@@ -95,13 +95,26 @@ reject:
 {.push warning[AnyEnumConv]:on, warningAsError[AnyEnumConv]:on.}
 
 reject:
-  # bug #12815
   type
     Foo = enum
-      one = 1
-      three = 3
+      one
+      three
 
   var va = 2
   var vb = va.Foo
 
 {.pop.}
+
+{.push warningAsError[HoleEnumConv]:on.}
+
+reject:
+  # bug #12815
+  type
+    Hole = enum
+      one = 1
+      three = 3
+
+  var va = 2
+  var vb = va.Hole
+
+{.pop.}

From 5386ae75ba91ee2fe280f1fe807676a7c9af4b13 Mon Sep 17 00:00:00 2001
From: Jason Beetham 
Date: Fri, 23 Jul 2021 00:46:13 -0600
Subject: [PATCH 1080/2472] Fixed template const tuple unpacking (#18562)

---
 compiler/semtempl.nim | 40 ++++++++++++++++++----------------------
 tests/vm/tconst.nim   | 10 ++++++++++
 2 files changed, 28 insertions(+), 22 deletions(-)

diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim
index 92cc0be8c..a2b0f99ba 100644
--- a/compiler/semtempl.nim
+++ b/compiler/semtempl.nim
@@ -209,7 +209,7 @@ proc addLocalDecl(c: var TemplCtx, n: var PNode, k: TSymKind) =
       let pragmaNode = n[1]
       for i in 0..
Date: Fri, 23 Jul 2021 08:04:29 +0000
Subject: [PATCH 1081/2472] Replace calls to `putenv` with `setenv` (#18530)

* Replace calls to C `putenv` with C `setenv` to remove possible memory leaks

* Add test of correct behaviour on invalid input

* Fix style in tests/stdlib/tos.nim

Co-authored-by: Timothee Cour 

* Update tests/stdlib/tos.nim

Co-authored-by: Timothee Cour 

* Update tests/stdlib/tos.nim

Co-authored-by: Timothee Cour 

* Add comment with bug number to tests/stdlib/tos.nim

Co-authored-by: Timothee Cour 

* Fix possible msvc arch issues

Co-authored-by: Timothee Cour 
---
 lib/pure/includes/osenv.nim | 11 ++++++++---
 tests/stdlib/tos.nim        |  3 +++
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/lib/pure/includes/osenv.nim b/lib/pure/includes/osenv.nim
index d0c92d566..db291c5fd 100644
--- a/lib/pure/includes/osenv.nim
+++ b/lib/pure/includes/osenv.nim
@@ -45,8 +45,10 @@ else:
 
   proc c_getenv(env: cstring): cstring {.
     importc: "getenv", header: "".}
-  proc c_putenv(env: cstring): cint {.
-    importc: "putenv", header: "".}
+  when defined(vcc):
+    proc c_putenv_s(envname: cstring, envval: cstring): cint {.importc: "_putenv_s", header: "".}
+  else:
+    proc c_setenv(envname: cstring, envval: cstring, overwrite: cint): cint {.importc: "setenv", header: "".}
   proc c_unsetenv(env: cstring): cint {.
     importc: "unsetenv", header: "".}
 
@@ -220,8 +222,11 @@ else:
           if setEnvironmentVariableW(k, v) == 0'i32: raiseOSError(osLastError())
         else:
           if setEnvironmentVariableA(key, val) == 0'i32: raiseOSError(osLastError())
+      elif defined(vcc):
+        if c_putenv_s(key, val) != 0'i32:
+          raiseOSError(osLastError())
       else:
-        if c_putenv(environment[indx]) != 0'i32:
+        if c_setenv(key, val, 1'i32) != 0'i32:
           raiseOSError(osLastError())
 
   proc delEnv*(key: string) {.tags: [WriteEnvEffect].} =
diff --git a/tests/stdlib/tos.nim b/tests/stdlib/tos.nim
index 912558461..8a12b9b90 100644
--- a/tests/stdlib/tos.nim
+++ b/tests/stdlib/tos.nim
@@ -614,6 +614,9 @@ block osenv:
     doAssert existsEnv(dummyEnvVar) == false
     delEnv(dummyEnvVar)         # deleting an already deleted env var
     doAssert existsEnv(dummyEnvVar) == false
+  block: # putEnv, bug #18502
+    doAssertRaises(OSError): putEnv("DUMMY_ENV_VAR_PUT=DUMMY_VALUE", "NEW_DUMMY_VALUE")
+    doAssertRaises(OSError): putEnv("", "NEW_DUMMY_VALUE")
   block:
     doAssert getEnv("DUMMY_ENV_VAR_NONEXISTENT", "") == ""
     doAssert getEnv("DUMMY_ENV_VAR_NONEXISTENT", " ") == " "

From 76f74fae88fba72f58e43ad5c8fd20a7c4d8b439 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Fri, 23 Jul 2021 04:41:16 -0700
Subject: [PATCH 1082/2472] std/random: fix overflow bugs; fixes #16360; fixes
 #16296; fixes #17670 (#18456)

---
 lib/pure/random.nim      | 32 ++++++++++++++------
 tests/stdlib/trandom.nim | 63 +++++++++++++++++++++++++++++++++++++++-
 2 files changed, 85 insertions(+), 10 deletions(-)

diff --git a/lib/pure/random.nim b/lib/pure/random.nim
index 07db0365c..a292386af 100644
--- a/lib/pure/random.nim
+++ b/lib/pure/random.nim
@@ -204,6 +204,19 @@ proc skipRandomNumbers*(s: var Rand) =
   s.a0 = s0
   s.a1 = s1
 
+proc rand[T: uint | uint64](r: var Rand; max: T): T =
+  # xxx export in future work
+  if max == 0: return
+  else:
+    let max = uint64(max)
+    when T.high.uint64 == uint64.high:
+      if max == uint64.high: return T(next(r))
+    while true:
+      let x = next(r)
+      # avoid `mod` bias
+      if x <= randMax - (randMax mod max):
+        return T(x mod (max + 1))
+
 proc rand*(r: var Rand; max: Natural): int {.benign.} =
   ## Returns a random integer in the range `0..max` using the given state.
   ##
@@ -213,15 +226,13 @@ proc rand*(r: var Rand; max: Natural): int {.benign.} =
   ## * `rand proc<#rand,Rand,HSlice[T: Ordinal or float or float32 or float64,T: Ordinal or float or float32 or float64]>`_
   ##   that accepts a slice
   ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer or range type
-  runnableExamples("-r:off"):
+  runnableExamples:
     var r = initRand(123)
-    assert r.rand(100) == 96 # implementation defined
-
-  if max == 0: return
-  while true:
-    let x = next(r)
-    if x <= randMax - (randMax mod Ui(max)):
-      return int(x mod (uint64(max) + 1u64))
+    if false:
+      assert r.rand(100) == 96 # implementation defined
+  # bootstrap: can't use `runnableExamples("-r:off")`
+  cast[int](rand(r, uint64(max)))
+    # xxx toUnsigned pending https://github.com/nim-lang/Nim/pull/18445
 
 proc rand*(max: int): int {.benign.} =
   ## Returns a random integer in the range `0..max`.
@@ -306,7 +317,10 @@ proc rand*[T: Ordinal or SomeFloat](r: var Rand; x: HSlice[T, T]): T =
   when T is SomeFloat:
     result = rand(r, x.b - x.a) + x.a
   else: # Integers and Enum types
-    result = T(rand(r, int(x.b) - int(x.a)) + int(x.a))
+    when defined(js):
+      result = cast[T](rand(r, cast[uint](x.b) - cast[uint](x.a)) + cast[uint](x.a))
+    else:
+      result = cast[T](rand(r, cast[uint64](x.b) - cast[uint64](x.a)) + cast[uint64](x.a))
 
 proc rand*[T: Ordinal or SomeFloat](x: HSlice[T, T]): T =
   ## For a slice `a..b`, returns a value in the range `a..b`.
diff --git a/tests/stdlib/trandom.nim b/tests/stdlib/trandom.nim
index e47ddad66..39ccca85b 100644
--- a/tests/stdlib/trandom.nim
+++ b/tests/stdlib/trandom.nim
@@ -3,7 +3,9 @@ discard """
   targets: "c js"
 """
 
-import std/[random, math, os, stats, sets, tables]
+import std/[random, math, stats, sets, tables]
+when not defined(js):
+  import std/os
 
 randomize(233)
 
@@ -187,3 +189,62 @@ block: # bug #17467
     doAssert x > 1e-4, $(x, i)
       # This used to fail for each i in 0..<26844, i.e. the 1st produced value
       # was predictable and < 1e-4, skewing distributions.
+
+const withUint = false # pending exporting `proc rand[T: uint | uint64](r: var Rand; max: T): T =`
+
+block: # bug #16360
+  var r = initRand()
+  template test(a) =
+    let a2 = a
+    block:
+      let a3 = r.rand(a2)
+      doAssert a3 <= a2
+      doAssert a3.type is a2.type
+    block:
+      let a3 = rand(a2)
+      doAssert a3 <= a2
+      doAssert a3.type is a2.type
+  when withUint:
+    test cast[uint](int.high)
+    test cast[uint](int.high) + 1
+    when not defined(js):
+      # pending bug #16411
+      test uint64.high
+      test uint64.high - 1
+    test uint.high - 2
+    test uint.high - 1
+    test uint.high
+  test int.high
+  test int.high - 1
+  test int.high - 2
+  test 0
+  when withUint:
+    test 0'u
+    test 0'u64
+
+block: # bug #16296
+  var r = initRand()
+  template test(x) =
+    let a2 = x
+    let a3 = r.rand(a2)
+    doAssert a3 <= a2.b
+    doAssert a3 >= a2.a
+    doAssert a3.type is a2.a.type
+  test(-2 .. int.high-1)
+  test(int.low .. int.high)
+  test(int.low+1 .. int.high)
+  test(int.low .. int.high-1)
+  test(int.low .. 0)
+  test(int.low .. -1)
+  test(int.low .. 1)
+  test(int64.low .. 1'i64)
+  when not defined(js):
+    # pending bug #16411
+    test(10'u64 .. uint64.high)
+
+block: # bug #17670
+  when not defined(js):
+    # pending bug #16411
+    type UInt48 = range[0'u64..2'u64^48-1]
+    let x = rand(UInt48)
+    doAssert x is UInt48

From faabcfa6432cd15ecfee44a061ebf7738b16ad6c Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Fri, 23 Jul 2021 16:15:40 +0200
Subject: [PATCH 1083/2472] fixes #18558 (#18563)

* fixes #18558

* better fix
---
 compiler/ccgcalls.nim | 44 +++++++++++++++++++++++++++----------------
 1 file changed, 28 insertions(+), 16 deletions(-)

diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim
index 616028ac6..510b8d9ec 100644
--- a/compiler/ccgcalls.nim
+++ b/compiler/ccgcalls.nim
@@ -317,36 +317,46 @@ proc skipTrivialIndirections(n: PNode): PNode =
   result = n
   while true:
     case result.kind
-    of {nkDerefExpr, nkHiddenDeref, nkAddr, nkHiddenAddr, nkObjDownConv, nkObjUpConv}:
+    of nkDerefExpr, nkHiddenDeref, nkAddr, nkHiddenAddr, nkObjDownConv, nkObjUpConv:
       result = result[0]
-    of {nkHiddenStdConv, nkHiddenSubConv}:
+    of nkHiddenStdConv, nkHiddenSubConv:
       result = result[1]
     else: break
 
-proc getPotentialWrites(n: PNode, mutate = false): seq[PNode] =
+proc getPotentialWrites(n: PNode; mutate: bool; result: var seq[PNode]) =
   case n.kind:
   of nkLiterals, nkIdent: discard
   of nkSym:
     if mutate: result.add n
   of nkAsgn, nkFastAsgn:
-    result.add getPotentialWrites(n[0], true)
-    result.add getPotentialWrites(n[1], mutate)
+    getPotentialWrites(n[0], true, result)
+    getPotentialWrites(n[1], mutate, result)
   of nkAddr, nkHiddenAddr:
-    result.add getPotentialWrites(n[0], true)
-  of nkCallKinds: #TODO: Find out why in f += 1, f is a nkSym and not a nkHiddenAddr
-    for s in n.sons:
-      result.add getPotentialWrites(s, true)
+    getPotentialWrites(n[0], true, result)
+  of nkCallKinds:
+    case n.getMagic:
+    of mIncl, mExcl, mInc, mDec, mAppendStrCh, mAppendStrStr, mAppendSeqElem,
+        mAddr, mNew, mNewFinalize, mWasMoved, mDestroy, mReset:
+      getPotentialWrites(n[1], true, result)
+      for i in 2..
Date: Fri, 23 Jul 2021 14:57:47 -0700
Subject: [PATCH 1084/2472] rename nimFpRoundtrips => nimPreviewFloatRoundtrip
 (#18566)

---
 changelog.md                    |  4 ++--
 compiler/nim.cfg                |  2 +-
 lib/system/formatfloat.nim      |  4 ++--
 tests/config.nims               |  2 +-
 tests/errmsgs/treportunused.nim |  2 +-
 tests/float/tfloats.nim         | 10 +++++-----
 6 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/changelog.md b/changelog.md
index 8e8b0a908..d33e24b65 100644
--- a/changelog.md
+++ b/changelog.md
@@ -112,8 +112,8 @@
 
 - `system.addFloat` and `system.$` now can produce string representations of floating point numbers
   that are minimal in size and that "roundtrip" (via the "Dragonbox" algorithm). This currently has
-  to be enabled via `-d:nimFpRoundtrips`. It is expected that this behavior becomes the new default
-  in upcoming versions.
+  to be enabled via `-d:nimPreviewFloatRoundtrip`. It is expected that this behavior becomes the new default
+  in upcoming versions, as with other `nimPreviewX` define flags.
 
 - Fixed buffer overflow bugs in `net`
 
diff --git a/compiler/nim.cfg b/compiler/nim.cfg
index f10d847ac..ec51bd463 100644
--- a/compiler/nim.cfg
+++ b/compiler/nim.cfg
@@ -4,7 +4,7 @@ hint:XDeclaredButNotUsed:off
 
 define:booting
 define:nimcore
-define:nimFpRoundtrips
+define:nimPreviewFloatRoundtrip
 
 #import:"$projectpath/testability"
 
diff --git a/lib/system/formatfloat.nim b/lib/system/formatfloat.nim
index cb46c6ea7..3bcd3257b 100644
--- a/lib/system/formatfloat.nim
+++ b/lib/system/formatfloat.nim
@@ -74,7 +74,7 @@ proc writeFloatToBufferSprintf*(buf: var array[65, char]; value: BiggestFloat):
       result = 3
 
 proc writeFloatToBuffer*(buf: var array[65, char]; value: BiggestFloat | float32): int {.inline.} =
-  when defined(nimFpRoundtrips):
+  when defined(nimPreviewFloatRoundtrip):
     writeFloatToBufferRoundtrip(buf, value)
   else:
     writeFloatToBufferSprintf(buf, value)
@@ -121,7 +121,7 @@ proc addFloat*(result: var string; x: float | float32) {.inline.} =
     s.addFloat(45.67)
     assert s == "foo:45.67"
   template impl =
-    when defined(nimFpRoundtrips):
+    when defined(nimPreviewFloatRoundtrip):
       addFloatRoundtrip(result, x)
     else:
       addFloatSprintf(result, x)
diff --git a/tests/config.nims b/tests/config.nims
index 12b303318..f876cb612 100644
--- a/tests/config.nims
+++ b/tests/config.nims
@@ -35,4 +35,4 @@ switch("define", "nimExperimentalAsyncjsThen")
 switch("define", "nimExperimentalJsfetch")
 switch("define", "nimExperimentalLinenoiseExtra")
 
-switch("define", "nimFpRoundtrips")
+switch("define", "nimPreviewFloatRoundtrip")
diff --git a/tests/errmsgs/treportunused.nim b/tests/errmsgs/treportunused.nim
index 3105b35ea..f5ee79afa 100644
--- a/tests/errmsgs/treportunused.nim
+++ b/tests/errmsgs/treportunused.nim
@@ -32,7 +32,7 @@ var s9: int
 type s10 = object
 type s11 = type(1.2)
 
-# bug #14407 (requires `compiler/nim.cfg` containing define:nimFpRoundtrips)
+# bug #14407 (requires `compiler/nim.cfg` containing define:nimPreviewFloatRoundtrip)
 let
   `v0.99` = "0.99"
   `v0.99.99` = "0.99.99"
diff --git a/tests/float/tfloats.nim b/tests/float/tfloats.nim
index 63987bb8d..480396e81 100644
--- a/tests/float/tfloats.nim
+++ b/tests/float/tfloats.nim
@@ -1,5 +1,5 @@
 discard """
-  matrix: "-d:nimFpRoundtrips; -u:nimFpRoundtrips"
+  matrix: "-d:nimPreviewFloatRoundtrip; -u:nimPreviewFloatRoundtrip"
   targets: "c cpp js"
 """
 
@@ -67,14 +67,14 @@ template main =
     block: # example 1
       let a = 0.1+0.2
       doAssert a != 0.3
-      when defined(nimFpRoundtrips):
+      when defined(nimPreviewFloatRoundtrip):
         doAssert $a == "0.30000000000000004"
       else:
         whenRuntimeJs: discard
         do: doAssert $a == "0.3"
     block: # example 2
       const a = 0.1+0.2
-      when defined(nimFpRoundtrips):
+      when defined(nimPreviewFloatRoundtrip):
         doAssert $($a, a) == """("0.30000000000000004", 0.30000000000000004)"""
       else:
         whenRuntimeJs: discard
@@ -83,13 +83,13 @@ template main =
       const a1 = 0.1+0.2
       let a2 = a1
       doAssert a1 != 0.3
-      when defined(nimFpRoundtrips):
+      when defined(nimPreviewFloatRoundtrip):
         doAssert $[$a1, $a2] == """["0.30000000000000004", "0.30000000000000004"]"""
       else:
         whenRuntimeJs: discard
         do: doAssert $[$a1, $a2] == """["0.3", "0.3"]"""
 
-  when defined(nimFpRoundtrips):
+  when defined(nimPreviewFloatRoundtrip):
     block: # bug #18148
       var a = 1.1'f32
       doAssert $a == "1.1", $a # was failing

From f4ff276a90091f2b88c191aac5a201dcb3ec9795 Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Sat, 24 Jul 2021 00:30:02 +0200
Subject: [PATCH 1085/2472] refactoring: removed dead code (#18567)

---
 compiler/semdata.nim      |  2 +-
 compiler/semexprs.nim     | 19 +++++++------------
 compiler/semobjconstr.nim |  5 ++---
 compiler/semstmts.nim     |  2 +-
 4 files changed, 11 insertions(+), 17 deletions(-)

diff --git a/compiler/semdata.nim b/compiler/semdata.nim
index 12bb22ff9..0a6d09a56 100644
--- a/compiler/semdata.nim
+++ b/compiler/semdata.nim
@@ -65,7 +65,7 @@ type
       # you may be in position to supply a better error message
       # to the user.
     efWantStmt, efAllowStmt, efDetermineType, efExplain,
-    efAllowDestructor, efWantValue, efOperand, efNoSemCheck,
+    efWantValue, efOperand, efNoSemCheck,
     efNoEvaluateGeneric, efInCall, efFromHlo, efNoSem2Check,
     efNoUndeclared
       # Use this if undeclared identifiers should not raise an error during
diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim
index c25a4067f..42c6b465e 100644
--- a/compiler/semexprs.nim
+++ b/compiler/semexprs.nim
@@ -623,10 +623,10 @@ proc semArrayConstr(c: PContext, n: PNode, flags: TExprFlags): PNode =
           localError(c.config, x.info, "invalid order in array constructor")
         x = x[1]
 
-      let xx = semExprWithType(c, x, flags*{efAllowDestructor})
+      let xx = semExprWithType(c, x, {})
       result.add xx
       typ = commonType(c, typ, xx.typ)
-      #n[i] = semExprWithType(c, x, flags*{efAllowDestructor})
+      #n[i] = semExprWithType(c, x, {})
       #result.add fitNode(c, typ, n[i])
       inc(lastIndex)
     addSonSkipIntLit(result.typ, typ, c.idgen)
@@ -1758,13 +1758,9 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode =
     localError(c.config, a.info, errXCannotBeAssignedTo %
                renderTree(a, {renderNoComments}))
   else:
-    let
-      lhs = n[0]
-      lhsIsResult = lhs.kind == nkSym and lhs.sym.kind == skResult
-    var
-      rhs = semExprWithType(c, n[1],
-        if lhsIsResult: {efAllowDestructor} else: {})
-    if lhsIsResult:
+    let lhs = n[0]
+    let rhs = semExprWithType(c, n[1], {})
+    if lhs.kind == nkSym and lhs.sym.kind == skResult:
       n.typ = c.enforceVoidContext
       if c.p.owner.kind != skMacro and resultTypeIsInferrable(lhs.sym.typ):
         var rhsTyp = rhs.typ
@@ -2512,8 +2508,7 @@ proc semTupleFieldsConstr(c: PContext, n: PNode, flags: TExprFlags): PNode =
     let id = considerQuotedIdent(c, n[i][0])
     if containsOrIncl(ids, id.id):
       localError(c.config, n[i].info, errFieldInitTwice % id.s)
-    n[i][1] = semExprWithType(c, n[i][1],
-                                        flags*{efAllowDestructor})
+    n[i][1] = semExprWithType(c, n[i][1], {})
 
     if n[i][1].typ.kind == tyTypeDesc:
       localError(c.config, n[i][1].info, "typedesc not allowed as tuple field.")
@@ -2533,7 +2528,7 @@ proc semTuplePositionsConstr(c: PContext, n: PNode, flags: TExprFlags): PNode =
   result.transitionSonsKind(nkTupleConstr)
   var typ = newTypeS(tyTuple, c)  # leave typ.n nil!
   for i in 0..
Date: Sun, 25 Jul 2021 09:06:04 -0700
Subject: [PATCH 1086/2472] fixes #18385 (#18571)

needs a more permanent fix by better incorporating IC, but fixes
nimsuggest highlighting for now.
---
 compiler/vmgen.nim | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim
index be86550ae..4957a3339 100644
--- a/compiler/vmgen.nim
+++ b/compiler/vmgen.nim
@@ -2254,8 +2254,16 @@ proc optimizeJumps(c: PCtx; start: int) =
     else: discard
 
 proc genProc(c: PCtx; s: PSym): int =
-  let pos = c.procToCodePos.getOrDefault(s.id)
-  if pos == 0:
+  let
+    pos = c.procToCodePos.getOrDefault(s.id)
+    wasNotGenProcBefore = pos == 0
+    noRegistersAllocated = s.offset == -1
+  if wasNotGenProcBefore or noRegistersAllocated:
+    # xxx: the noRegisterAllocated check is required in order to avoid issues
+    #      where nimsuggest can crash due as a macro with pos will be loaded
+    #      but it doesn't have offsets for register allocations see:
+    #      https://github.com/nim-lang/Nim/issues/18385
+    #      Improvements and further use of IC should remove the need for this.
     #if s.name.s == "outterMacro" or s.name.s == "innerProc":
     #  echo "GENERATING CODE FOR ", s.name.s
     let last = c.code.len-1

From e08ec0c6745e54de4e174970e513acef3b0d8f5e Mon Sep 17 00:00:00 2001
From: Juan Carlos 
Date: Sun, 25 Jul 2021 13:07:43 -0300
Subject: [PATCH 1087/2472] Documentation only, Sugar arrow and semicolons
 (#18574)

* Document that sugar arrow do not support semicolon as argument separator
---
 lib/pure/sugar.nim | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/pure/sugar.nim b/lib/pure/sugar.nim
index 7cfa932e6..f82e1706f 100644
--- a/lib/pure/sugar.nim
+++ b/lib/pure/sugar.nim
@@ -54,6 +54,8 @@ proc createProcType(p, b: NimNode): NimNode =
 
 macro `=>`*(p, b: untyped): untyped =
   ## Syntax sugar for anonymous procedures. It also supports pragmas.
+  ##
+  ## .. warning:: Semicolons can not be used to separate procedure arguments.
   runnableExamples:
     proc passTwoAndTwo(f: (int, int) -> int): int = f(2, 2)
 
@@ -133,6 +135,8 @@ macro `=>`*(p, b: untyped): untyped =
 
 macro `->`*(p, b: untyped): untyped =
   ## Syntax sugar for procedure types. It also supports pragmas.
+  ##
+  ## .. warning:: Semicolons can not be used to separate procedure arguments.
   runnableExamples:
     proc passTwoAndTwo(f: (int, int) -> int): int = f(2, 2)
     # is the same as:

From 5f7db652577523bdbb4e9b9157b4fb3bed25e01c Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Sun, 25 Jul 2021 12:55:33 -0700
Subject: [PATCH 1088/2472] followup #18453 (#18582)

---
 tests/stdlib/tos.nim |  4 ++--
 tests/vm/tvmmisc.nim | 30 +++++++++++++++---------------
 2 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/tests/stdlib/tos.nim b/tests/stdlib/tos.nim
index 8a12b9b90..d598a05b2 100644
--- a/tests/stdlib/tos.nim
+++ b/tests/stdlib/tos.nim
@@ -558,9 +558,9 @@ block ospaths:
   doAssert joinPath("foo","abc") == unixToNativePath"foo/abc"
   doAssert joinPath("","abc") == unixToNativePath"abc"
 
-  doAssert joinPath("gook/.","abc") == unixToNativePath"gook/abc"
+  doAssert joinPath("zook/.","abc") == unixToNativePath"zook/abc"
 
-  # controversial: inconsistent with `joinPath("gook/.","abc")`
+  # controversial: inconsistent with `joinPath("zook/.","abc")`
   # on linux, `./foo` and `foo` are treated a bit differently for executables
   # but not `./foo/bar` and `foo/bar`
   doAssert joinPath(".", "/lib") == unixToNativePath"./lib"
diff --git a/tests/vm/tvmmisc.nim b/tests/vm/tvmmisc.nim
index a4f6391cc..f6fbde0f9 100644
--- a/tests/vm/tvmmisc.nim
+++ b/tests/vm/tvmmisc.nim
@@ -376,7 +376,7 @@ block: # VM wrong register free causes errors in unrelated code
     in let prc = if not isClosure: bb.sym else: bb[0].sym
     ]#
     proc bar2(head: string): string = "asdf"
-    proc gook(u1: int) = discard
+    proc zook(u1: int) = discard
 
     type PathEntry = object
       kind: int
@@ -385,20 +385,20 @@ block: # VM wrong register free causes errors in unrelated code
     iterator globOpt(): int =
       var u1: int
 
-      gook(u1)
-      gook(u1)
-      gook(u1)
-      gook(u1)
-      gook(u1)
-      gook(u1)
-      gook(u1)
-      gook(u1)
-      gook(u1)
-      gook(u1)
-      gook(u1)
-      gook(u1)
-      gook(u1)
-      gook(u1)
+      zook(u1)
+      zook(u1)
+      zook(u1)
+      zook(u1)
+      zook(u1)
+      zook(u1)
+      zook(u1)
+      zook(u1)
+      zook(u1)
+      zook(u1)
+      zook(u1)
+      zook(u1)
+      zook(u1)
+      zook(u1)
 
       var entry = PathEntry()
       entry.path = bar2("")

From 10da888c075acb901b57ba51e4400df6cdf3efd6 Mon Sep 17 00:00:00 2001
From: Andrey Makarov 
Date: Mon, 26 Jul 2021 00:01:19 +0300
Subject: [PATCH 1089/2472] docgen: sort symbols (fix #17910) (#18560)

* docgen: sort symbols (fix #17910)

* add workaround + change naming

* switch to a dedicated sort comparator

* fix numbers with unequal string lengths

* dedicated `sortName` instead of `plainNameEsc`:

* more compact names for non-overloaded symbols
* more predictable Ascii sort (e.g. `<` instead of `<`)
---
 compiler/docgen.nim                           |  98 ++-
 .../expected/subdir/subdir_b/utils.html       |  10 +-
 nimdoc/testproject/expected/testproject.html  | 686 +++++++++---------
 3 files changed, 432 insertions(+), 362 deletions(-)

diff --git a/compiler/docgen.nim b/compiler/docgen.nim
index f9e72850a..492668543 100644
--- a/compiler/docgen.nim
+++ b/compiler/docgen.nim
@@ -11,7 +11,7 @@
 # by knowing how the anchors are going to be named.
 
 import
-  ast, strutils, strtabs, options, msgs, os, idents,
+  ast, strutils, strtabs, algorithm, sequtils, options, msgs, os, idents,
   wordrecg, syntaxes, renderer, lexer,
   packages/docutils/rst, packages/docutils/rstgen,
   json, xmltree, trees, types,
@@ -43,11 +43,15 @@ type
     descRst: ItemPre     ## Description of the item (may contain
                          ## runnableExamples).
     substitutions: seq[string]    ## Variable names in `doc.item`...
+    sortName: string    ## The string used for sorting in output
   ModSection = object  ## Section like Procs, Types, etc.
     secItems: seq[Item]  ## Pre-processed items.
     finalMarkup: string  ## The items, after RST pass 2 and rendering.
   ModSections = array[TSymKind, ModSection]
-  TSections = array[TSymKind, string]
+  TocItem = object  ## HTML TOC item
+    content: string
+    sortName: string
+  TocSectionsFinal = array[TSymKind, string]
   ExampleGroup = ref object
     ## a group of runnableExamples with same rdoccmd
     rdoccmd: string ## from 1st arg in `runnableExamples(rdoccmd): body`
@@ -64,8 +68,13 @@ type
     module: PSym
     modDeprecationMsg: string
     section: ModSections     # entries of ``.nim`` file (for `proc`s, etc)
-    toc, toc2: TSections     # toc2 - grouped TOC
-    tocTable: array[TSymKind, Table[string, string]]
+    tocSimple: array[TSymKind, seq[TocItem]]
+      # TOC entries for non-overloadable symbols (e.g. types, constants)...
+    tocTable:  array[TSymKind, Table[string, seq[TocItem]]]
+      # ...otherwise (e.g. procs)
+    toc2: TocSectionsFinal  # TOC `content`, which is probably wrapped
+                            # in `doc.section.toc2`
+    toc: TocSectionsFinal  # final TOC (wrapped in `doc.section.toc`)
     indexValFilename: string
     analytics: string  # Google Analytics javascript, "" if doesn't exist
     seenSymbols: StringTableRef # avoids duplicate symbol generation for HTML.
@@ -88,6 +97,51 @@ type
 proc add(dest: var ItemPre, rst: PRstNode) = dest.add ItemFragment(isRst: true, rst: rst)
 proc add(dest: var ItemPre, str: string) = dest.add ItemFragment(isRst: false, str: str)
 
+proc cmpDecimalsIgnoreCase(a, b: string): int =
+  ## For sorting with correct handling of cases like 'uint8' and 'uint16'.
+  ## Also handles leading zeros well (however note that leading zeros are
+  ## significant when lengths of numbers mismatch, e.g. 'bar08' > 'bar8' !).
+  runnableExamples:
+    doAssert cmpDecimalsIgnoreCase("uint8", "uint16") < 0
+    doAssert cmpDecimalsIgnoreCase("val00032", "val16suffix") > 0
+    doAssert cmpDecimalsIgnoreCase("val16suffix", "val16") > 0
+    doAssert cmpDecimalsIgnoreCase("val_08_32", "val_08_8") > 0
+    doAssert cmpDecimalsIgnoreCase("val_07_32", "val_08_8") < 0
+    doAssert cmpDecimalsIgnoreCase("ab8", "ab08") < 0
+    doAssert cmpDecimalsIgnoreCase("ab8de", "ab08c") < 0 # sanity check
+  let aLen = a.len
+  let bLen = b.len
+  var
+    iA = 0
+    iB = 0
+  while iA < aLen and iB < bLen:
+    if isDigit(a[iA]) and isDigit(b[iB]):
+      var
+        limitA = iA  # index after the last (least significant) digit
+        limitB = iB
+      while limitA < aLen and isDigit(a[limitA]): inc limitA
+      while limitB < bLen and isDigit(b[limitB]): inc limitB
+      var pos = max(limitA-iA, limitB-iA)
+      while pos > 0:
+        if limitA-pos < iA:  # digit in `a` is 0 effectively
+          result = ord('0') - ord(b[limitB-pos])
+        elif limitB-pos < iB:  # digit in `b` is 0 effectively
+          result = ord(a[limitA-pos]) - ord('0')
+        else:
+          result = ord(a[limitA-pos]) - ord(b[limitB-pos])
+        if result != 0: return
+        dec pos
+      result = (limitA - iA) - (limitB - iB)  # consider 'bar08' > 'bar8'
+      if result != 0: return
+      iA = limitA
+      iB = limitB
+    else:
+      result = ord(toLowerAscii(a[iA])) - ord(toLowerAscii(b[iB]))
+      if result != 0: return
+      inc iA
+      inc iB
+  result = (aLen - iA) - (bLen - iB)
+
 proc prettyString(a: object): string =
   # xxx pending std/prettyprint refs https://github.com/nim-lang/RFCs/issues/203#issuecomment-602534906
   for k, v in fieldPairs(a):
@@ -862,6 +916,7 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
   let
     plainNameEsc = esc(d.target, plainName.strip)
     uniqueName = if k in routineKinds: plainNameEsc else: name
+    sortName = if k in routineKinds: plainName.strip else: name
     cleanPlainSymbol = renderPlainSymbolName(nameNode)
     complexSymbol = complexName(k, n, cleanPlainSymbol)
     plainSymbolEnc = encodeUrl(cleanPlainSymbol)
@@ -874,7 +929,10 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
 
   let seeSrc = genSeeSrc(d, toFullPath(d.conf, n.info), n.info.line.int)
 
-  d.section[k].secItems.add Item(descRst: comm, substitutions: @[
+  d.section[k].secItems.add Item(
+    descRst: comm,
+    sortName: sortName,
+    substitutions: @[
      "name", name, "uniqueName", uniqueName,
      "header", result, "itemID", $d.id,
      "header_plain", plainNameEsc, "itemSym", cleanPlainSymbol,
@@ -898,12 +956,15 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
       setIndexTerm(d[], external, symbolOrId, plain, nameNode.sym.name.s & '.' & plain,
         xmltree.escape(getPlainDocstring(e).docstringSummary))
 
-  d.toc[k].add(getConfigVar(d.conf, "doc.item.toc") % [
+  d.tocSimple[k].add TocItem(
+    sortName: sortName,
+    content: getConfigVar(d.conf, "doc.item.toc") % [
       "name", name, "header_plain", plainNameEsc,
       "itemSymOrIDEnc", symbolOrIdEnc])
 
-  d.tocTable[k].mgetOrPut(cleanPlainSymbol, "").add(
-    getConfigVar(d.conf, "doc.item.tocTable") % [
+  d.tocTable[k].mgetOrPut(cleanPlainSymbol, newSeq[TocItem]()).add TocItem(
+    sortName: sortName,
+    content: getConfigVar(d.conf, "doc.item.tocTable") % [
       "name", name, "header_plain", plainNameEsc,
       "itemSymOrID", symbolOrId.replace(",", ","),
       "itemSymOrIDEnc", symbolOrIdEnc])
@@ -1143,8 +1204,9 @@ proc finishGenerateDoc*(d: var PDoc) =
         var resolved = resolveSubs(d.sharedState, f.rst)
         renderRstToOut(d[], resolved, result)
       of false: result &= f.str
+  proc cmp(x, y: Item): int = cmpDecimalsIgnoreCase(x.sortName, y.sortName)
   for k in TSymKind:
-    for item in d.section[k].secItems:
+    for item in d.section[k].secItems.sorted(cmp):
       var itemDesc: string
       renderItemPre(d, item.descRst, itemDesc)
       d.section[k].finalMarkup.add(
@@ -1262,18 +1324,26 @@ proc genSection(d: PDoc, kind: TSymKind, groupedToc = false) =
       "sectionid", $ord(kind), "sectionTitle", title,
       "sectionTitleID", $(ord(kind) + 50), "content", d.section[kind].finalMarkup]
 
-  var tocSource = d.toc
+  proc cmp(x, y: TocItem): int = cmpDecimalsIgnoreCase(x.sortName, y.sortName)
   if groupedToc:
-    for p in d.tocTable[kind].keys:
+    let overloadableNames = toSeq(keys(d.tocTable[kind]))
+    for plainName in overloadableNames.sorted(cmpDecimalsIgnoreCase):
+      var overloadChoices = d.tocTable[kind][plainName]
+      overloadChoices.sort(cmp)
+      var content: string
+      for item in overloadChoices:
+        content.add item.content
       d.toc2[kind].add getConfigVar(d.conf, "doc.section.toc2") % [
           "sectionid", $ord(kind), "sectionTitle", title,
           "sectionTitleID", $(ord(kind) + 50),
-          "content", d.tocTable[kind][p], "plainName", p]
-    tocSource = d.toc2
+          "content", content, "plainName", plainName]
+  else:
+    for item in d.tocSimple[kind].sorted(cmp):
+      d.toc2[kind].add item.content
 
   d.toc[kind] = getConfigVar(d.conf, "doc.section.toc") % [
       "sectionid", $ord(kind), "sectionTitle", title,
-      "sectionTitleID", $(ord(kind) + 50), "content", tocSource[kind]]
+      "sectionTitleID", $(ord(kind) + 50), "content", d.toc2[kind]]
 
 proc relLink(outDir: AbsoluteDir, destFile: AbsoluteFile, linkto: RelativeFile): string =
   $relativeTo(outDir / linkto, destFile.splitFile().dir, '/')
diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html
index 6aa71e8c2..939495ad1 100644
--- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html
+++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html
@@ -130,16 +130,16 @@ window.addEventListener('DOMContentLoaded', main);
       
  • aEnum(): untyped
    • 
 
-  
-  
   
+  
 
   
diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html
index 7e5a685d2..0f6289bbc 100644
--- a/nimdoc/testproject/expected/testproject.html
+++ b/nimdoc/testproject/expected/testproject.html
@@ -107,9 +107,7 @@ window.addEventListener('DOMContentLoaded', main);
 
  • 
   Types
   
      
-      
    • FooBuzz
      • 
-  
    • A
      • 
   
    • Foo
      • 
+  
    • FooBuzz
      • 
   
    • Vars
   
 
      • 
@@ -153,35 +153,57 @@ window.addEventListener('DOMContentLoaded', main);
 
    • 
   Procs
   
        
-      
          z10
-      
        • z10()
          • 
+      
-  
            tripleStrLitTest
-      
          • tripleStrLitTest()
            • 
+  
-  
              z17
-      
            • z17()
              • 
+  
+  
   
-  
   
+  
+  
+  
+  
   
-  
-  
   
-  
-  
-  
-  
   
-  
-  
-  
-  
-  
-  
-  
-  
-  
                z4
-      
              • z4()
                • 
+  
   
                  someFunc
@@ -285,29 +235,79 @@ window.addEventListener('DOMContentLoaded', main);
     title="someFunc()">someFunc()
 
   
                
-  
                  z8
-      
                • z8(): int
                  • 
+  
   
+  
+  
+  
   
-  
                    asyncFun2
-      
                  • asyncFun2(): owned(Future[void])
                    • 
+  
-  
                      asyncFun1
-      
                    • asyncFun1(): Future[int]
                      • 
+  
+  
+  
+  
+  
+  
+  
+  
 
@@ -316,7 +316,12 @@ window.addEventListener('DOMContentLoaded', main);
 
                    • 
   Methods
   
@@ -379,10 +379,25 @@ window.addEventListener('DOMContentLoaded', main);
 
                    • 
   Templates
   
                        
-      
@@ -447,16 +447,6 @@ window.addEventListener('DOMContentLoaded', main);
 
                        
 
                        Types
                        
 
                        
-
-
                        FooBuzz {....deprecated: "FooBuzz msg".} = int
                        
-
                        
-  
                        
-    Deprecated: FooBuzz msg
-  
                        
-
-
-
-
                        
 
 
                        A {.inject.} = enum
   aA
                        
@@ -480,6 +470,16 @@ The enum B.
 
 
 
+
+
+
                        FooBuzz {....deprecated: "FooBuzz msg".} = int
                        
+
                        
+  
                        
+    Deprecated: FooBuzz msg
+  
                        
+
+
+
 
                        
 
 
                        Shapes = enum
@@ -496,19 +496,19 @@ Some shapes.
 
                        
 
                        Vars
                        
 
                        
-
-
                        someVariable: bool
                        
-
                        
-
-This should be visible.
-
-
                        
 
 
                        aVariable: array[1, int]
                        
 
                        
 
 
 
+
                        
+
+
                        someVariable: bool
                        
+
                        
+
+This should be visible.
+
 
                        
 
 
                        
@@ -548,12 +548,69 @@ This should be visible.
 
                        
 
                        Procs
                        
 
                        
+
+
                        proc addfBug14485() {....raises: [], tags: [].}
                        
+
                        
+
+Some proc
+
                        Example:
                        
+
                        discard "foo() = " & $[1]
+#[
+0: let's also add some broken html to make sure this won't break in future
+1: </span>
+2: </span>
+3: </span
+4: </script>
+5: </script
+6: </script
+7: end of broken html
+]#
                        
+
+
                        
+
+
                        proc anything() {....raises: [], tags: [].}
                        
+
                        
+
+There is no block quote after blank lines at the beginning.
+
+
                        
+
+
                        proc asyncFun1(): Future[int] {....raises: [Exception, ValueError],
+                                tags: [RootEffect].}
                        
+
                        
+
+ok1
+
+
                        
+
+
                        proc asyncFun2(): owned(Future[void]) {....raises: [Exception], tags: [RootEffect].}
                        
+
                        
+
+
+
+
                        
+
+
                        proc asyncFun3(): owned(Future[void]) {....raises: [Exception], tags: [RootEffect].}
                        
+
                        
+
+
+
                        Example:
                        
+
                        discard
                        ok1
+
+
                        
 
 
                        proc bar[T](a, b: T): T
                        
 
                        
 
 
 
+
                        
+
+
                        proc baz() {....raises: [], tags: [].}
                        
+
                        
+
+
+
 
                        
 
 
                        proc baz[T](a, b: T): T {....deprecated.}
                        
@@ -575,11 +632,20 @@ This is deprecated without message.
 This is deprecated with a message.
 
 
-
-
                        func someFunc() {....raises: [], tags: [].}
                        
+
+
                        proc c_nonexistent(frmt: cstring): cint {.importc: "nonexistent",
+    header: "<stdio.h>", varargs, discardable.}
                        
 
                        
 
-My someFunc. Stuff in quotes here. Some link
+
+
+
                        
+
+
                        proc c_printf(frmt: cstring): cint {.importc: "printf", header: "<stdio.h>",
+                                     varargs, discardable.}
                        
+
                        
+
+the c printf. etc.
 
 
                        
 
@@ -598,6 +664,103 @@ came form utils but should be shown where 
+
+
                        proc low2[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect.}
                        
+
                        
+
+
                        Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.
                        
+
                        See also:
                        
+
+
                        low2(2) # => -9223372036854775808
                        
+
                        Example:
                        
+
                        discard "in low2"
                        
+
+
                        
+
+
                        proc low[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect.}
                        
+
                        
+
+
                        Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.
                        
+
                        See also:
                        
+
+
                        low(2) # => -9223372036854775808
                        
+
+
                        
+
+
                        proc p1() {....raises: [], tags: [].}
                        
+
                        
+
+cp1
+
                        Example:
                        
+
                        doAssert 1 == 1 # regular comments work here
                        c4
+
                        Example:
                        
+
                        # c5 regular comments before 1st token work
+# regular comment
+#[
+nested regular comment
+]#
+doAssert 2 == 2 # c8
+## this is a non-nested doc comment
+
+##[
+this is a nested doc comment
+]##
+discard "c9"
+# also work after
                        
+
+
                        
+
+
                        func someFunc() {....raises: [], tags: [].}
                        
+
                        
+
+My someFunc. Stuff in quotes here. Some link
+
+
                        
+
+
                        proc tripleStrLitTest() {....raises: [], tags: [].}
                        
+
                        
+
+
+
                        Example: cmd: --hint:XDeclaredButNotUsed:off
                        
+
                        ## mullitline string litterals are tricky as their indentation can span
+## below that of the runnableExamples
+let s1a = """
+should appear at indent 0
+  at indent 2
+at indent 0
+"""
+# make sure this works too
+let s1b = """start at same line
+  at indent 2
+at indent 0
+""" # comment after
+let s2 = """sandwich """
+let s3 = """"""
+when false:
+  let s5 = """
+        in s5 """
+
+let s3b = ["""
+%!? #[...] # inside a multiline ...
+""", "foo"]
+
+## make sure handles trailing spaces
+let s4 = """ 
+"""
+
+let s5 = """ x
+"""
+let s6 = """ ""
+"""
+let s7 = """"""""""
+let s8 = ["""""""""", """
+  """ ]
+discard
+# should be in
                        
+
 
                        
 
 
                        proc z1(): Foo {....raises: [], tags: [].}
                        
@@ -701,13 +864,6 @@ cz13
 
                        Example:
                        
 
                        discard
                        
 
-
-
-
                        proc baz() {....raises: [], tags: [].}
                        
-
                        
-
-
-
 
                        
 
 
                        proc z17() {....raises: [], tags: [].}
                        
@@ -717,162 +873,6 @@ cz17 rest
 
                        Example:
                        
 
                        discard 1
                        rest
 
-
-
-
                        proc p1() {....raises: [], tags: [].}
                        
-
                        
-
-cp1
-
                        Example:
                        
-
                        doAssert 1 == 1 # regular comments work here
                        c4
-
                        Example:
                        
-
                        # c5 regular comments before 1st token work
-# regular comment
-#[
-nested regular comment
-]#
-doAssert 2 == 2 # c8
-## this is a non-nested doc comment
-
-##[
-this is a nested doc comment
-]##
-discard "c9"
-# also work after
                        
-
-
                        
-
-
                        proc addfBug14485() {....raises: [], tags: [].}
                        
-
                        
-
-Some proc
-
                        Example:
                        
-
                        discard "foo() = " & $[1]
-#[
-0: let's also add some broken html to make sure this won't break in future
-1: </span>
-2: </span>
-3: </span
-4: </script>
-5: </script
-6: </script
-7: end of broken html
-]#
                        
-
-
                        
-
-
                        proc c_printf(frmt: cstring): cint {.importc: "printf", header: "<stdio.h>",
-                                     varargs, discardable.}
                        
-
                        
-
-the c printf. etc.
-
-
                        
-
-
                        proc c_nonexistent(frmt: cstring): cint {.importc: "nonexistent",
-    header: "<stdio.h>", varargs, discardable.}
                        
-
                        
-
-
-
-
                        
-
-
                        proc low[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect.}
                        
-
                        
-
-
                        Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.
                        
-
                        See also:
                        
-
-
                        low(2) # => -9223372036854775808
                        
-
-
                        
-
-
                        proc low2[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect.}
                        
-
                        
-
-
                        Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.
                        
-
                        See also:
                        
-
-
                        low2(2) # => -9223372036854775808
                        
-
                        Example:
                        
-
                        discard "in low2"
                        
-
-
                        
-
-
                        proc tripleStrLitTest() {....raises: [], tags: [].}
                        
-
                        
-
-
-
                        Example: cmd: --hint:XDeclaredButNotUsed:off
                        
-
                        ## mullitline string litterals are tricky as their indentation can span
-## below that of the runnableExamples
-let s1a = """
-should appear at indent 0
-  at indent 2
-at indent 0
-"""
-# make sure this works too
-let s1b = """start at same line
-  at indent 2
-at indent 0
-""" # comment after
-let s2 = """sandwich """
-let s3 = """"""
-when false:
-  let s5 = """
-        in s5 """
-
-let s3b = ["""
-%!? #[...] # inside a multiline ...
-""", "foo"]
-
-## make sure handles trailing spaces
-let s4 = """ 
-"""
-
-let s5 = """ x
-"""
-let s6 = """ ""
-"""
-let s7 = """"""""""
-let s8 = ["""""""""", """
-  """ ]
-discard
-# should be in
                        
-
-
                        
-
-
                        proc asyncFun1(): Future[int] {....raises: [Exception, ValueError],
-                                tags: [RootEffect].}
                        
-
                        
-
-ok1
-
-
                        
-
-
                        proc asyncFun2(): owned(Future[void]) {....raises: [Exception], tags: [RootEffect].}
                        
-
                        
-
-
-
-
                        
-
-
                        proc asyncFun3(): owned(Future[void]) {....raises: [Exception], tags: [RootEffect].}
                        
-
                        
-
-
-
                        Example:
                        
-
                        discard
                        ok1
-
-
                        
-
-
                        proc anything() {....raises: [], tags: [].}
                        
-
                        
-
-There is no block quote after blank lines at the beginning.
-
 
                        
 
 
                        
@@ -967,6 +967,13 @@ cz18
 
                        
 
                        Templates
                        
 
                        
+
+
                        template foo(a, b: SomeType)
                        
+
                        
+
+This does nothing 
+
+
                        
 
 
                        template fromUtils2()
                        
 
                        
@@ -976,20 +983,6 @@ ok3
 
                        discard """should be shown as examples for fromUtils2
        in module calling fromUtilsGen"""
                        
 
-
                        
-
-
                        template z6t(): int
                        
-
                        
-
-cz6t
-
-
                        
-
-
                        template foo(a, b: SomeType)
                        
-
                        
-
-This does nothing 
-
 
                        
 
 
                        template myfn()
                        
@@ -1012,6 +1005,22 @@ bar
   discard 0xff # elu par cette crapule
 # should be in
                        should be still in
 
+
+
+
                        template testNimDocTrailingExample()
                        
+
                        
+
+
+
                        Example:
                        
+
                        discard 2
                        
+
+
                        
+
+
                        template z6t(): int
                        
+
                        
+
+cz6t
+
 
                        
 
 
                        template z14()
                        
@@ -1038,15 +1047,6 @@ cz15
 
                        Example:
                        
 
                        discard 1
                        in or out?
 
-
-
-
                        template testNimDocTrailingExample()
                        
-
                        
-
-
-
                        Example:
                        
-
                        discard 2
                        
-
 
                        
 
 
                        

From ff280c012731b1978005a743eec96cd885331475 Mon Sep 17 00:00:00 2001
From: Araq 
Date: Mon, 26 Jul 2021 09:57:19 +0200
Subject: [PATCH 1090/2472] added missing .inline for toCChar

---
 compiler/msgs.nim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compiler/msgs.nim b/compiler/msgs.nim
index 242b06587..20a55daee 100644
--- a/compiler/msgs.nim
+++ b/compiler/msgs.nim
@@ -27,7 +27,7 @@ proc flushDot*(conf: ConfigRef) =
     conf.lastMsgWasDot.excl stdOrrKind
     write(stdOrr, "\n")
 
-proc toCChar*(c: char; result: var string) =
+proc toCChar*(c: char; result: var string) {.inline.} =
   case c
   of '\0'..'\x1F', '\x7F'..'\xFF':
     result.add '\\'

From 158d7c7a7a8977f5937311b98bc381399f31db8d Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Mon, 26 Jul 2021 14:15:55 +0200
Subject: [PATCH 1091/2472] fixes #18558 again (#18586)

---
 compiler/ccgcalls.nim | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim
index 510b8d9ec..749a7a3b6 100644
--- a/compiler/ccgcalls.nim
+++ b/compiler/ccgcalls.nim
@@ -333,6 +333,8 @@ proc getPotentialWrites(n: PNode; mutate: bool; result: var seq[PNode]) =
     getPotentialWrites(n[1], mutate, result)
   of nkAddr, nkHiddenAddr:
     getPotentialWrites(n[0], true, result)
+  of nkBracketExpr, nkDotExpr, nkCheckedFieldExpr:
+    getPotentialWrites(n[0], mutate, result)
   of nkCallKinds:
     case n.getMagic:
     of mIncl, mExcl, mInc, mDec, mAppendStrCh, mAppendStrStr, mAppendSeqElem,

From d4c3a09286d1cbb534ed5dbd21d99a18670b3427 Mon Sep 17 00:00:00 2001
From: Antonis Geralis <43617260+planetis-m@users.noreply.github.com>
Date: Mon, 26 Jul 2021 16:22:42 +0300
Subject: [PATCH 1092/2472] optimize for the non-throwing case (#18587)

---
 lib/pure/collections/tables.nim | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim
index b845f57cc..403345755 100644
--- a/lib/pure/collections/tables.nim
+++ b/lib/pure/collections/tables.nim
@@ -227,6 +227,12 @@ template dataLen(t): untyped = len(t.data)
 
 include tableimpl
 
+proc raiseKeyError[T](key: T) {.noinline, noreturn.} =
+  when compiles($key):
+    raise newException(KeyError, "key not found: " & $key)
+  else:
+    raise newException(KeyError, "key not found")
+
 template get(t, key): untyped =
   ## retrieves the value at `t[key]`. The value can be modified.
   ## If `key` is not in `t`, the `KeyError` exception is raised.
@@ -235,10 +241,7 @@ template get(t, key): untyped =
   var index = rawGet(t, key, hc)
   if index >= 0: result = t.data[index].val
   else:
-    when compiles($key):
-      raise newException(KeyError, "key not found: " & $key)
-    else:
-      raise newException(KeyError, "key not found")
+    raiseKeyError(key)
 
 proc enlarge[A, B](t: var Table[A, B]) =
   var n: KeyValuePairSeq[A, B]

From 664dbb6bfbe06ff0ddc4339c0fe80c49d0838ca4 Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Mon, 26 Jul 2021 18:00:03 +0200
Subject: [PATCH 1093/2472] atlas tool: testing via mocking (#18588)

* atlas: refactoring
* Atlas: do some basic testing via mocking
---
 koch.nim                                 |   2 +
 tools/atlas/atlas.nim                    | 187 +++++++++++++++++------
 tools/atlas/osutils.nim                  |   8 -
 tools/atlas/testdata.nim                 |  63 ++++++++
 tools/atlas/tests/balls.nimble           |  32 ++++
 tools/atlas/tests/grok.nimble            |   5 +
 tools/atlas/tests/nim-bytes2human.nimble |   7 +
 tools/atlas/tests/nim.cfg                |  11 ++
 tools/atlas/tests/npeg.nimble            |  48 ++++++
 tools/atlas/tests/packages/packages.json |  36 +++++
 tools/atlas/tests/sync.nimble            |  10 ++
 tools/atlas/tests/testes.nimble          |  23 +++
 tools/atlas/tests/ups.nimble             |  13 ++
 13 files changed, 387 insertions(+), 58 deletions(-)
 create mode 100644 tools/atlas/testdata.nim
 create mode 100644 tools/atlas/tests/balls.nimble
 create mode 100644 tools/atlas/tests/grok.nimble
 create mode 100644 tools/atlas/tests/nim-bytes2human.nimble
 create mode 100644 tools/atlas/tests/nim.cfg
 create mode 100644 tools/atlas/tests/npeg.nimble
 create mode 100644 tools/atlas/tests/packages/packages.json
 create mode 100644 tools/atlas/tests/sync.nimble
 create mode 100644 tools/atlas/tests/testes.nimble
 create mode 100644 tools/atlas/tests/ups.nimble

diff --git a/koch.nim b/koch.nim
index da7576e65..4296e4529 100644
--- a/koch.nim
+++ b/koch.nim
@@ -594,6 +594,8 @@ proc runCI(cmd: string) =
     when defined(posix):
       execFold("Run nimsuggest tests", "nim r nimsuggest/tester")
 
+    execFold("Run atlas tests", "nim c -r -d:atlasTests tools/atlas/atlas.nim clone https://github.com/disruptek/balls")
+
   when not defined(bsd):
     if not doUseCpp:
       # the BSDs are overwhelmed already, so only run this test on the other machines:
diff --git a/tools/atlas/atlas.nim b/tools/atlas/atlas.nim
index b10ec0358..d0990c1f7 100644
--- a/tools/atlas/atlas.nim
+++ b/tools/atlas/atlas.nim
@@ -39,6 +39,10 @@ proc writeVersion() =
   stdout.flushFile()
   quit(0)
 
+const
+  MockupRun = defined(atlasTests)
+  TestsDir = "tools/atlas/tests"
+
 type
   PackageName = distinct string
   DepRelation = enum
@@ -55,9 +59,79 @@ type
     p: Table[string, string] # name -> url mapping
     processed: HashSet[string] # the key is (url / commit)
     errors: int
+    when MockupRun:
+      currentDir: string
+      step: int
+      mockupSuccess: bool
 
 const
   InvalidCommit = ""
+  ProduceTest = false
+
+type
+  Command = enum
+    GitDiff = "git diff",
+    GitTags = "git show-ref --tags",
+    GitRevParse = "git rev-parse",
+    GitCheckout = "git checkout",
+    GitPull = "git pull",
+    GitCurrentCommit = "git log -n 1 --format=%H"
+    GitMergeBase = "git merge-base"
+
+include testdata
+
+proc exec(c: var AtlasContext; cmd: Command; args: openArray[string]): (string, int) =
+  when MockupRun:
+    assert TestLog[c.step].cmd == cmd
+    case cmd
+    of GitDiff, GitTags, GitRevParse, GitPull, GitCurrentCommit:
+      result = (TestLog[c.step].output, TestLog[c.step].exitCode)
+    of GitCheckout:
+      assert args[0] == TestLog[c.step].output
+    of GitMergeBase:
+      let tmp = TestLog[c.step].output.splitLines()
+      assert tmp.len == 4, $tmp.len
+      assert tmp[0] == args[0]
+      assert tmp[1] == args[1]
+      assert tmp[3] == ""
+      result[0] = tmp[2]
+      result[1] = TestLog[c.step].exitCode
+    inc c.step
+  else:
+    var cmdLine = $cmd
+    for i in 0.. ", result
+
+proc cloneUrl(c: var AtlasContext; url, dest: string; cloneUsingHttps: bool): string =
+  when MockupRun:
+    result = ""
+  else:
+    result = osutils.cloneUrl(url, dest, cloneUsingHttps)
+    when ProduceTest:
+      echo "cloned ", url, " into ", dest
+
+template withDir*(c: var AtlasContext; dir: string; body: untyped) =
+  when MockupRun:
+    c.currentDir = dir
+    body
+  else:
+    let oldDir = getCurrentDir()
+    try:
+      when ProduceTest:
+        echo "Current directory is now ", dir
+      setCurrentDir(dir)
+      body
+    finally:
+      setCurrentDir(oldDir)
+
+proc extractRequiresInfo(c: var AtlasContext; nimbleFile: string): NimbleFileInfo =
+  result = extractRequiresInfo(nimbleFile)
+  when ProduceTest:
+    echo "nimble ", nimbleFile, " info ", result
 
 proc toDepRelation(s: string): DepRelation =
   case s
@@ -65,9 +139,9 @@ proc toDepRelation(s: string): DepRelation =
   of ">": strictlyGreater
   else: normal
 
-proc isCleanGit(dir: string): string =
+proc isCleanGit(c: var AtlasContext; dir: string): string =
   result = ""
-  let (outp, status) = osproc.execCmdEx("git diff")
+  let (outp, status) = exec(c, GitDiff, [])
   if outp.len != 0:
     result = "'git diff' not empty"
   elif status != 0:
@@ -101,8 +175,8 @@ proc sameVersionAs(tag, ver: string): bool =
     result = safeCharAt(tag, idx-1) notin VersionChars and
       safeCharAt(tag, idx+ver.len) notin VersionChars
 
-proc versionToCommit(d: Dependency): string =
-  let (outp, status) = osproc.execCmdEx("git show-ref --tags")
+proc versionToCommit(c: var AtlasContext; d: Dependency): string =
+  let (outp, status) = exec(c, GitTags, [])
   if status == 0:
     var useNextOne = false
     for line in splitLines(outp):
@@ -123,35 +197,36 @@ proc versionToCommit(d: Dependency): string =
 
   return ""
 
-proc shortToCommit(short: string): string =
-  let (cc, status) = osproc.execCmdEx("git rev-parse " & quoteShell(short))
+proc shortToCommit(c: var AtlasContext; short: string): string =
+  let (cc, status) = exec(c, GitRevParse, [short])
   result = if status == 0: strutils.strip(cc) else: ""
 
 proc checkoutGitCommit(c: var AtlasContext; p: PackageName; commit: string) =
-  let (_, status) = osproc.execCmdEx("git checkout " & quoteShell(commit))
+  let (_, status) = exec(c, GitCheckout, [commit])
   if status != 0:
     error(c, p, "could not checkout commit", commit)
 
 proc gitPull(c: var AtlasContext; p: PackageName) =
-  let (_, status) = osproc.execCmdEx("git pull")
+  let (_, status) = exec(c, GitPull, [])
   if status != 0:
     error(c, p, "could not 'git pull'")
 
 proc updatePackages(c: var AtlasContext) =
   if dirExists(c.workspace / PackagesDir):
-    withDir(c.workspace / PackagesDir):
+    withDir(c, c.workspace / PackagesDir):
       gitPull(c, PackageName PackagesDir)
   else:
-    withDir c.workspace:
-      let err = cloneUrl("https://github.com/nim-lang/packages", PackagesDir, false)
+    withDir c, c.workspace:
+      let err = cloneUrl(c, "https://github.com/nim-lang/packages", PackagesDir, false)
       if err != "":
         error c, PackageName(PackagesDir), err
 
 proc fillPackageLookupTable(c: var AtlasContext) =
   if not c.hasPackageList:
     c.hasPackageList = true
-    updatePackages(c)
-    let plist = getPackages(c.workspace)
+    when not MockupRun:
+      updatePackages(c)
+    let plist = getPackages(when MockupRun: TestsDir else: c.workspace)
     for entry in plist:
       c.p[unicode.toLower entry.name] = entry.url
 
@@ -178,19 +253,19 @@ proc isShortCommitHash(commit: string): bool {.inline.} =
 
 proc checkoutCommit(c: var AtlasContext; w: Dependency) =
   let dir = c.workspace / w.name.string
-  withDir dir:
+  withDir c, dir:
     if w.commit.len == 0 or cmpIgnoreCase(w.commit, "head") == 0:
       gitPull(c, w.name)
     else:
-      let err = isCleanGit(dir)
+      let err = isCleanGit(c, dir)
       if err != "":
         warn c, w.name, err
       else:
         let requiredCommit =
-          if needsCommitLookup(w.commit): versionToCommit(w)
-          elif isShortCommitHash(w.commit): shortToCommit(w.commit)
+          if needsCommitLookup(w.commit): versionToCommit(c, w)
+          elif isShortCommitHash(w.commit): shortToCommit(c, w.commit)
           else: w.commit
-        let (cc, status) = osproc.execCmdEx("git log -n 1 --format=%H")
+        let (cc, status) = exec(c, GitCurrentCommit, [])
         let currentCommit = strutils.strip(cc)
         if requiredCommit == "" or status != 0:
           if requiredCommit == "" and w.commit == InvalidCommit:
@@ -201,8 +276,7 @@ proc checkoutCommit(c: var AtlasContext; w: Dependency) =
           if currentCommit != requiredCommit:
             # checkout the later commit:
             # git merge-base --is-ancestor  
-            let (cc, status) = osproc.execCmdEx("git merge-base " &
-                currentCommit.quoteShell & " " & requiredCommit.quoteShell)
+            let (cc, status) = exec(c, GitMergeBase, [currentCommit, requiredCommit])
             let mergeBase = strutils.strip(cc)
             if status == 0 and (mergeBase == currentCommit or mergeBase == requiredCommit):
               # conflict resolution: pick the later commit:
@@ -215,15 +289,19 @@ proc checkoutCommit(c: var AtlasContext; w: Dependency) =
                   currentCommit, "(current) or", w.commit, " =", requiredCommit, "(required)"
 
 proc findNimbleFile(c: AtlasContext; dep: Dependency): string =
-  result = c.workspace / dep.name.string / (dep.name.string & ".nimble")
-  if not fileExists(result):
-    result = ""
-    for x in walkFiles(c.workspace / dep.name.string / "*.nimble"):
-      if result.len == 0:
-        result = x
-      else:
-        # ambiguous .nimble file
-        return ""
+  when MockupRun:
+    result = TestsDir / dep.name.string & ".nimble"
+    doAssert fileExists(result), "file does not exist " & result
+  else:
+    result = c.workspace / dep.name.string / (dep.name.string & ".nimble")
+    if not fileExists(result):
+      result = ""
+      for x in walkFiles(c.workspace / dep.name.string / "*.nimble"):
+        if result.len == 0:
+          result = x
+        else:
+          # ambiguous .nimble file
+          return ""
 
 proc addUniqueDep(c: var AtlasContext; work: var seq[Dependency];
                   tokens: seq[string]) =
@@ -240,7 +318,7 @@ proc collectNewDeps(c: var AtlasContext; work: var seq[Dependency];
                     isMainProject: bool) =
   let nimbleFile = findNimbleFile(c, dep)
   if nimbleFile != "":
-    let nimbleInfo = extractRequiresInfo(nimbleFile)
+    let nimbleInfo = extractRequiresInfo(c, nimbleFile)
     for r in nimbleInfo.requires:
       var tokens: seq[string] = @[]
       for token in tokenizeRequires(r):
@@ -280,8 +358,8 @@ proc clone(c: var AtlasContext; start: string): seq[string] =
     let w = work[i]
     let oldErrors = c.errors
     if not dirExists(c.workspace / w.name.string):
-      withDir c.workspace:
-        let err = cloneUrl(w.url, w.name.string, false)
+      withDir c, c.workspace:
+        let err = cloneUrl(c, w.url, w.name.string, false)
         if err != "":
           error c, w.name, err
     if oldErrors == c.errors:
@@ -296,29 +374,34 @@ const
   configPatternBegin = "############# begin Atlas config section ##########\n"
   configPatternEnd =   "############# end Atlas config section   ##########\n"
 
-proc patchNimCfg(c: AtlasContext; deps: seq[string]) =
+proc patchNimCfg(c: var AtlasContext; deps: seq[string]) =
   var paths = "--noNimblePath\n"
   for d in deps:
     paths.add "--path:\"../" & d.replace("\\", "/") & "\"\n"
 
-  let cfg = c.projectDir / "nim.cfg"
   var cfgContent = configPatternBegin & paths & configPatternEnd
-  if not fileExists(cfg):
-    writeFile(cfg, cfgContent)
+
+  when MockupRun:
+    assert readFile(TestsDir / "nim.cfg") == cfgContent
+    c.mockupSuccess = true
   else:
-    let content = readFile(cfg)
-    let start = content.find(configPatternBegin)
-    if start >= 0:
-      cfgContent = content.substr(0, start-1) & cfgContent
-      let theEnd = content.find(configPatternEnd, start)
-      if theEnd >= 0:
-        cfgContent.add content.substr(theEnd+len(configPatternEnd))
-    else:
-      cfgContent = content & "\n" & cfgContent
-    if cfgContent != content:
-      # do not touch the file if nothing changed
-      # (preserves the file date information):
+    let cfg = c.projectDir / "nim.cfg"
+    if not fileExists(cfg):
       writeFile(cfg, cfgContent)
+    else:
+      let content = readFile(cfg)
+      let start = content.find(configPatternBegin)
+      if start >= 0:
+        cfgContent = content.substr(0, start-1) & cfgContent
+        let theEnd = content.find(configPatternEnd, start)
+        if theEnd >= 0:
+          cfgContent.add content.substr(theEnd+len(configPatternEnd))
+      else:
+        cfgContent = content & "\n" & cfgContent
+      if cfgContent != content:
+        # do not touch the file if nothing changed
+        # (preserves the file date information):
+        writeFile(cfg, cfgContent)
 
 proc error*(msg: string) =
   when defined(debug):
@@ -365,8 +448,12 @@ proc main =
     singleArg()
     let deps = clone(c, args[0])
     patchNimCfg c, deps
-    if c.errors > 0:
-      error "There were problems."
+    when MockupRun:
+      if not c.mockupSuccess:
+        error "There were problems."
+    else:
+      if c.errors > 0:
+        error "There were problems."
   of "refresh":
     noArgs()
     updatePackages(c)
diff --git a/tools/atlas/osutils.nim b/tools/atlas/osutils.nim
index 31392b113..1bb414c03 100644
--- a/tools/atlas/osutils.nim
+++ b/tools/atlas/osutils.nim
@@ -3,14 +3,6 @@
 
 import os, strutils, osproc
 
-template withDir*(dir, body) =
-  let oldDir = getCurrentDir()
-  try:
-    setCurrentDir(dir)
-    body
-  finally:
-    setCurrentDir(oldDir)
-
 proc isUrl*(x: string): bool =
   x.startsWith("git://") or x.startsWith("https://") or x.startsWith("http://")
 
diff --git a/tools/atlas/testdata.nim b/tools/atlas/testdata.nim
new file mode 100644
index 000000000..f960e3d08
--- /dev/null
+++ b/tools/atlas/testdata.nim
@@ -0,0 +1,63 @@
+
+type
+  PerDirData = object
+    dirname: string
+    cmd: Command
+    exitCode: int
+    output: string
+
+template toData(a, b, c, d): untyped =
+  PerDirData(dirname: a, cmd: b, exitCode: c, output: d)
+
+const
+  TestLog = [
+    toData("balls", GitPull, 0, "Already up to date.\n"),
+    toData("grok", GitDiff, 0, ""),
+    toData("grok", GitTags, 0, "2ca193c31fa2377c1e991a080d60ca3215ff6cf0 refs/tags/0.0.1\n48007554b21ba2f65c726ae2fdda88d621865b4a refs/tags/0.0.2\n7092a0286421c7818cd335cca9ebc72d03d866c2 refs/tags/0.0.3\n62707b8ac684efac35d301dbde57dc750880268e refs/tags/0.0.4\n876f2504e0c2f785ffd2cf65a78e2aea474fa8aa refs/tags/0.0.5\nb7eb1f2501aa2382cb3a38353664a13af62a9888 refs/tags/0.0.6\nf5d818bfd6038884b3d8b531c58484ded20a58a4 refs/tags/0.1.0\n961eaddea49c3144d130d105195583d3f11fb6c6 refs/tags/0.2.0\n15ab8ed8d4f896232a976a9008548bd53af72a66 refs/tags/0.2.1\n426a7d7d4603f77ced658e73ad7f3f582413f6cd refs/tags/0.3.0\n83cf7a39b2fe897786fb0fe01a7a5933c3add286 refs/tags/0.3.1\n8d2e3c900edbc95fa0c036fd76f8e4f814aef2c1 refs/tags/0.3.2\n48b43372f49a3bb4dc0969d82a0fca183fb94662 refs/tags/0.3.3\n9ca947a3009ea6ba17814b20eb953272064eb2e6 refs/tags/0.4.0\n1b5643d04fba6d996a16d1ffc13d034a40003f8f refs/tags/0.5.0\n486b0eb580b1c465453d264ac758cc490c19c33e refs/tags/0.5.1\naedb0d9497390e20b9d2541cef2bb05a5cda7a71 refs/tags/0.5.2\n"),
+    toData("grok", GitCurrentCommit, 0, "349c15fd1e03f1fcdd81a1edefba3fa6116ab911\n"),
+    toData("grok", GitMergeBase, 0, "349c15fd1e03f1fcdd81a1edefba3fa6116ab911\n1b5643d04fba6d996a16d1ffc13d034a40003f8f\n349c15fd1e03f1fcdd81a1edefba3fa6116ab911\n"),
+    toData("grok", GitCheckout, 0, "1b5643d04fba6d996a16d1ffc13d034a40003f8f"), # watch out!
+
+    toData("ups", GitDiff, 0, ""),
+    toData("ups", GitTags, 0, "4008f9339cd22b30e180bc87a6cca7270fd28ac1 refs/tags/0.0.2\n19bc490c22b4f5b0628c31cdedead1375b279356 refs/tags/0.0.3\nff34602aaea824cb46d6588cd5fe1178132e9702 refs/tags/0.0.4\n09de599138f20b745133b6e4fe563e204415a7e8 refs/tags/0.0.5\n85fee3b74798311108a105635df31f892150f5d0 refs/tags/0.0.6\nfd303913b22b121dc42f332109e9c44950b9acd4 refs/tags/0.0.7\n"),
+    toData("ups", GitCurrentCommit, 0, "74c31af8030112dac758440aa51ef175992f71f3\n"),
+    toData("ups", GitMergeBase, 0, "74c31af8030112dac758440aa51ef175992f71f3\n4008f9339cd22b30e180bc87a6cca7270fd28ac1\n74c31af8030112dac758440aa51ef175992f71f3\n"),
+    toData("ups", GitCheckout, 0, "4008f9339cd22b30e180bc87a6cca7270fd28ac1"),
+
+    toData("sync", GitDiff, 0, ""),
+    toData("sync", GitTags, 0, "c9811af2d03dc78a920d33592d0a3daae3d9e68f refs/tags/v0.1.0\nc99d3a73344c4c02335190b489e7ea8aaa4e9584 refs/tags/v0.2.0\n6186a38c97c0f26e1a0c8016979227d26c41717b refs/tags/v0.3.0\n5506e4e09b849bd59f930fcccee3096abb089b1d refs/tags/v0.4.0\n1e4ef046ed0447aec7ca4ab3b2e2a5347ac41448 refs/tags/v0.5.0\na95f7b0b22c77b98ed63c1c4f435d550de8dbb05 refs/tags/v0.6.0\n214b868ace4ebc9140ee56734147fbd16b238c71 refs/tags/v0.7.0\n786698bf9f597834f36cd530d32faee31545a1d7 refs/tags/v0.9.0\n810bd2d75e9f6e182534ae2488670b51a9f13fc3 refs/tags/v1.0.0\na133d37991e3d4e9553c5261eabcb07d471968f7 refs/tags/v1.1.0\n0872a7cd70ea43a26deb94b8ab12cc5b89a1c212 refs/tags/v1.2.0\n19ac734438b6c04fe89711728a97fcde591324eb refs/tags/v1.3.0\nde5c7337ebc22422190e8aeca37d05651735f440 refs/tags/v1.4.0\n"),
+    toData("sync", GitCurrentCommit, 0, "de5c7337ebc22422190e8aeca37d05651735f440\n"),
+    toData("sync", GitMergeBase, 0, "de5c7337ebc22422190e8aeca37d05651735f440\n810bd2d75e9f6e182534ae2488670b51a9f13fc3\n810bd2d75e9f6e182534ae2488670b51a9f13fc3\n"),
+
+    toData("npeg", GitDiff, 0, ""),
+    toData("npeg", GitTags, 0, "8df2f0c9391995fd086b8aab00e8ab7aded1e8f0 refs/tags/0.1.0\n4c959a72db5283b55eeef491076eefb5e02316f1 refs/tags/0.10.0\n802f47c0f7f4318a4f0858ba5a6a6ed2333bde71 refs/tags/0.11.0\n82c8d92837108dce225358ace2c416bf9a3f30ce refs/tags/0.12.0\n87d2f2c4f6ef7da350d45beb5a336611bde7f518 refs/tags/0.13.0\n39964f0d220bfaade47a568bf03c1cf28aa2bc37 refs/tags/0.14.0\nbe9f03f92304cbeab70572944a8563db9b23b2fb refs/tags/0.14.1\na933fb9832566fc95273e417597bfb4faf564ca6 refs/tags/0.15.0\n6aad2e438c52ff0636c7bfb64338e444ac3e83ba refs/tags/0.16.0\nf4ddffb5848c42c6151743dd9c7eddcaaabc56cc refs/tags/0.17.0\n30b446b39442cdbc53a97018ab8a54149aa7c3b7 refs/tags/0.17.1\n1a9d36aa3b34a6169d4530463f1c17a3fe1e075e refs/tags/0.18.0\ndd34f903a9a63b876cb2db19b7a4ce0bcc252134 refs/tags/0.19.0\nd93d49c81fc8722d7929ac463b435c0f2e10c53b refs/tags/0.2.0\neeae7746c9b1118bcf27744ab2aee26969051256 refs/tags/0.20.0\n8c3471a548129f3bf62df15cd0fd8cca1787d852 refs/tags/0.21.0\nc0e873a17bc713c80e74fec3c30cb62dcd5d194a refs/tags/0.21.1\nbae84c47a1bb259b209b6f6be1582327b784539d refs/tags/0.21.2\nbfcb4bcae76a917c3c88736ca773e4cb67dbb2d8 refs/tags/0.21.3\n0eabb7c462d30932049f0b7e6a030c1562cf9fee refs/tags/0.22.0\n2e75367095f54d4351005078bad98041a55b14c1 refs/tags/0.22.1\n814ea235dd398108d7b18f966694c3d951575701 refs/tags/0.22.2\na812064587d983c129737f8500bf74990e6b8dab refs/tags/0.23.0\nbd969ad3745db0d66022564cac76cf9424651104 refs/tags/0.23.1\na037c646a47623b92718efadc2bb74d03664b360 refs/tags/0.23.2\n078475ccceeaca0fac947492acdd24514da8d863 refs/tags/0.24.0\ne7bd87dc992512fd5825a557a56907647e03c979 refs/tags/0.24.1\n45ea601e1c7f64fb857bc99df984b86673621d2c refs/tags/0.3.0\n1ea9868a3fee3aa487ab7ec9129208a4dd483d0d refs/tags/0.4.0\n39afdb5733d3245386d29d08c5ff61c89268f499 refs/tags/0.5.0\n458c7b5910fcb157af3fc51bc3b3e663fdb3ed4a refs/tags/0.6.0\n06c38bd8563d822455bc237c2a98c153d938ed1b refs/tags/0.7.0\nf446b6056eef6d8dc9d8b47a79aca93d17dc8230 refs/tags/0.8.0\nbb25a195133f9f7af06386d0809793923cc5e8ab refs/tags/0.9.0\n"),
+    toData("npeg", GitCurrentCommit, 0, "5d80f93aa720898936668b3bc47d0fff101ec414\n"),
+    toData("npeg", GitMergeBase, 0, "5d80f93aa720898936668b3bc47d0fff101ec414\na037c646a47623b92718efadc2bb74d03664b360\na037c646a47623b92718efadc2bb74d03664b360\n"),
+
+    toData("testes", GitDiff, 0, ""),
+    toData("testes", GitTags, 0, "3ce9b2968b5f644755a0ced1baa3eece88c2f12e refs/tags/0.1.0\nf73af8318b54737678fab8b54bdcd8a451015e0d refs/tags/0.1.1\nd21d84d37b161a123a43318bae353108755916de refs/tags/0.1.2\n5c36b6095353ed03b08ac939d00aff2d73f79a35 refs/tags/0.1.3\na1220d11237ee8f135f772ff9731c11b2d91ba31 refs/tags/0.1.4\n574f741b90d04a7ce8c9b990e6077708d7ad076e refs/tags/0.1.5\nced0a9e58234b680def6931578e09165a32e6291 refs/tags/0.1.6\nbb248952e8742a6011eb1a45a9d2059aeb0341d7 refs/tags/0.1.7\nabb7d7c552da0a8e0ddc586c15ccf7e74b0d068b refs/tags/0.10.0\n6e42a768a90d6442196b344bcdcb6f834b76e7b7 refs/tags/0.2.0\n9d136c3a0851ca2c021f5fb4f7b63f0a0ef77232 refs/tags/0.2.1\ndcb282b2da863fd2939e1969cec7a99788feb456 refs/tags/0.2.2\nf708a632afaa40a322a1a61c1c13722edac8e8c5 refs/tags/0.3.0\n3213f59e3f9ba052452c59f01d1418360d856af6 refs/tags/0.3.1\nf7bb1743dffd327958dfcebae4cfb6f61cc1cb8c refs/tags/0.3.2\n6b64569ebecad6bc60cc8697713701e7659204f4 refs/tags/0.3.3\nb51c25a4367bd17f419f78cb5a27f319e9d820f5 refs/tags/0.3.4\nb265612710cbd5ddb1b173c94ece8ec5c7ceccac refs/tags/0.3.5\ne404bcfe42e92d7509717a2dfa115cacb4964c5d refs/tags/0.3.6\n5e4d0d5b7e7f314dde701c546c4365c59782d3dc refs/tags/0.3.7\ne13f91c9c913d2b81c59adeaad687efa2b35293a refs/tags/0.3.8\n17599625f09af0ae4b525e63ab726a3002540702 refs/tags/0.3.9\n13e907f70571dd146d8dc29ddec4599b40ba4e85 refs/tags/0.4.0\n155a74cf676495df1e0674dd07b5e4a0291a9a4a refs/tags/0.4.1\nf37abccdc148cb02ca637a6f0bc8821491cce358 refs/tags/0.4.2\n0250d29ebdd02f28f9020445adb5a4e51fd1902c refs/tags/0.5.0\n2fb87db6d9f34109a70205876030c53f815739b7 refs/tags/0.5.1\n629d17ba8d6a1a4eca8145eb089ed5bca4473dfc refs/tags/0.6.0\ne926130f5f1b7903f68be49cc1563225bd9d948d refs/tags/0.7.0\n7365303897e6185796c274425c079916047e3f14 refs/tags/0.7.1\na735c4adabeba637409f41c4325dd8fc5fb91e2d refs/tags/0.7.10\nfe023fd27404889c5122f902456cbba14b767405 refs/tags/0.7.11\n4430e72972c77a5e9c1555d59bba11d840682691 refs/tags/0.7.12\nf0e53eb490a9558c7f594d2e095b70665e36ca88 refs/tags/0.7.13\nf6520e25e7c329c2957cda447f149fc6a930db0d refs/tags/0.7.2\nd509762f7191757c240d3c79c9ecda53f8c0cfe3 refs/tags/0.7.3\nc02e7a783d1c42fd1f91bca7142f7c3733950c05 refs/tags/0.7.4\n8c8a9e496e9b86ba7602709438980ca31e6989d9 refs/tags/0.7.5\n29839c18b4ac83c0111a178322b57ebb8a8d402c refs/tags/0.7.6\n3b62973cf74fafd8ea906644d89ac34d29a8a6cf refs/tags/0.7.7\ne67ff99dc43c391e89a37f97a9d298c3428bbde2 refs/tags/0.7.8\n4b72ecda0d40ed8e5ab8ad4095a0691d30ec6cd0 refs/tags/0.7.9\n2512b8cc3d7f001d277e89978da2049a5feee5c4 refs/tags/0.8.0\n86c47029690bd2731d204245f3f54462227bba0d refs/tags/0.9.0\n9a7f94f78588e9b5ba7ca077e1f7eae0607c6cf6 refs/tags/0.9.1\n08c915dc016d16c1dfa9a77d0b045ec29c9f2074 refs/tags/0.9.2\n3fb658b1ce1e1efa37d6f9f14322bdac8def02a5 refs/tags/0.9.3\n738fda0add962379ffe6aa6ca5f01a6943a98a2e refs/tags/0.9.4\n48d821add361f7ad768ecb35a0b19c38f90c919e refs/tags/0.9.5\nff9ae890f597dac301b2ac6e6805eb9ac5afd49a refs/tags/0.9.6\n483c78f06e60b0ec5e79fc3476df075ee7286890 refs/tags/0.9.7\n416eec87a5ae39a1a6035552e9e9a47d76b13026 refs/tags/1.0.0\na935cfe9445cc5218fbdd7e0afb35aa1587fff61 refs/tags/1.0.1\n4b83863a9181f054bb695b11b5d663406dfd85d2 refs/tags/1.0.2\n295145fddaa4fe29c1e71a5044d968a84f9dbf69 refs/tags/1.1.0\n8f74ea4e5718436c47305b4488842e6458a13dac refs/tags/1.1.1\n4135bb291e53d615a976e997c44fb2bd9e1ad343 refs/tags/1.1.10\n8c09dbcd16612f5989065db02ea2e7a752dd2656 refs/tags/1.1.11\naedfebdb6c016431d84b0c07cf181b957a900640 refs/tags/1.1.12\n2c2e958366ef6998115740bdf110588d730e5738 refs/tags/1.1.2\nbecc77258321e6ec40d89efdddf37bafd0d07fc3 refs/tags/1.1.3\ne070d7c9853bf94c35b81cf0c0a8980c2449bb22 refs/tags/1.1.4\n12c986cbbf65e8571a486e9230808bf887e5f04f refs/tags/1.1.5\n63df8986f5b56913b02d26954fa033eeaf43714c refs/tags/1.1.6\n38e02c9c6bd728b043036fe0d1894d774cab3108 refs/tags/1.1.7\n3c3879fff16450d28ade79a6b08982bf5cefc061 refs/tags/1.1.8\ne32b811b3b2e70a1d189d7a663bc2583e9c18f96 refs/tags/1.1.9\n0c1b4277c08197ce7e7e0aa2bad91d909fcd96ac refs/tags/2.0.0\n"),
+    toData("testes", GitCurrentCommit, 0, "d9db2ad09aa38fc26625341e1b666602959e144f\n"),
+    toData("testes", GitMergeBase, 0, "d9db2ad09aa38fc26625341e1b666602959e144f\n416eec87a5ae39a1a6035552e9e9a47d76b13026\nd9db2ad09aa38fc26625341e1b666602959e144f\n"),
+    toData("testes", GitCheckout, 0, "416eec87a5ae39a1a6035552e9e9a47d76b13026"),
+
+    toData("grok", GitDiff, 0, ""),
+    toData("grok", GitTags, 0, "2ca193c31fa2377c1e991a080d60ca3215ff6cf0 refs/tags/0.0.1\n48007554b21ba2f65c726ae2fdda88d621865b4a refs/tags/0.0.2\n7092a0286421c7818cd335cca9ebc72d03d866c2 refs/tags/0.0.3\n62707b8ac684efac35d301dbde57dc750880268e refs/tags/0.0.4\n876f2504e0c2f785ffd2cf65a78e2aea474fa8aa refs/tags/0.0.5\nb7eb1f2501aa2382cb3a38353664a13af62a9888 refs/tags/0.0.6\nf5d818bfd6038884b3d8b531c58484ded20a58a4 refs/tags/0.1.0\n961eaddea49c3144d130d105195583d3f11fb6c6 refs/tags/0.2.0\n15ab8ed8d4f896232a976a9008548bd53af72a66 refs/tags/0.2.1\n426a7d7d4603f77ced658e73ad7f3f582413f6cd refs/tags/0.3.0\n83cf7a39b2fe897786fb0fe01a7a5933c3add286 refs/tags/0.3.1\n8d2e3c900edbc95fa0c036fd76f8e4f814aef2c1 refs/tags/0.3.2\n48b43372f49a3bb4dc0969d82a0fca183fb94662 refs/tags/0.3.3\n9ca947a3009ea6ba17814b20eb953272064eb2e6 refs/tags/0.4.0\n1b5643d04fba6d996a16d1ffc13d034a40003f8f refs/tags/0.5.0\n486b0eb580b1c465453d264ac758cc490c19c33e refs/tags/0.5.1\naedb0d9497390e20b9d2541cef2bb05a5cda7a71 refs/tags/0.5.2\n"),
+    toData("grok", GitCurrentCommit, 0, "4e6526a91a23eaec778184e16ce9a34d25d48bdc\n"),
+    toData("grok", GitMergeBase, 0, "4e6526a91a23eaec778184e16ce9a34d25d48bdc\n62707b8ac684efac35d301dbde57dc750880268e\n349c15fd1e03f1fcdd81a1edefba3fa6116ab911\n"),
+    toData("grok", GitCheckout, 0, "62707b8ac684efac35d301dbde57dc750880268e"),
+
+    toData("nim-bytes2human", GitDiff, 0, ""),
+    toData("nim-bytes2human", GitTags, 0, ""),
+    toData("nim-bytes2human", GitCurrentcommit, 0, "ec2c1a758cabdd4751a06c8ebf2b923f19e32731\n")
+  ]
+
+#[
+Current directory is now E:\atlastest\nim-bytes2human
+cmd git diff args [] --> ("", 0)
+cmd git show-ref --tags args [] --> ("", 1)
+cmd git log -n 1 --format=%H args [] --> (, 0)
+[Warning] (nim-bytes2human) package has no tagged releases
+nimble E:\atlastest\nim-bytes2human\bytes2human.nimble info (requires: @["nim >= 1.0.0"], srcDir: "src", tasks: @[])
+[Error] There were problems.
+Error: execution of an external program failed: 'E:\nim\tools\atlas\atlas.exe clone https://github.com/disruptek/balls'
+]#
diff --git a/tools/atlas/tests/balls.nimble b/tools/atlas/tests/balls.nimble
new file mode 100644
index 000000000..62ab4750b
--- /dev/null
+++ b/tools/atlas/tests/balls.nimble
@@ -0,0 +1,32 @@
+version = "3.4.1"
+author = "disruptek"
+description = "a unittest framework with balls 🔴🟡🟢"
+license = "MIT"
+
+# requires newTreeFrom
+requires "https://github.com/disruptek/grok >= 0.5.0 & < 1.0.0"
+requires "https://github.com/disruptek/ups < 1.0.0"
+requires "https://github.com/planetis-m/sync >= 1.0.0 & < 2.0.0"
+#requires "https://github.com/c-blake/cligen < 2.0.0"
+
+bin = @["balls"]            # build the binary for basic test running
+installExt = @["nim"]       # we need to install balls.nim also
+skipDirs = @["tests"]       # so stupid...  who doesn't want tests?
+#installFiles = @["balls.nim"] # https://github.com/nim-lang/Nim/issues/16661
+
+task test, "run tests for ci":
+  when defined(windows):
+    exec "balls.cmd"
+  else:
+    exec "balls"
+
+task demo, "produce a demo":
+  exec "nim c --define:release balls.nim"
+  when (NimMajor, NimMinor) != (1, 0):
+    echo "due to nim bug #16307, use nim-1.0"
+    quit 1
+  exec """demo docs/demo.svg "nim c --out=\$1 examples/fails.nim""""
+  exec """demo docs/clean.svg "nim c --define:danger -f --out=\$1 tests/test.nim""""
+  exec "nim c --define:release --define:ballsDry balls.nim"
+  exec """demo docs/runner.svg "balls""""
+
diff --git a/tools/atlas/tests/grok.nimble b/tools/atlas/tests/grok.nimble
new file mode 100644
index 000000000..1b6d77c08
--- /dev/null
+++ b/tools/atlas/tests/grok.nimble
@@ -0,0 +1,5 @@
+version = "0.0.4"
+author = "disruptek"
+description = "don't read too much into it"
+license = "MIT"
+requires "nim >= 1.0.0"
diff --git a/tools/atlas/tests/nim-bytes2human.nimble b/tools/atlas/tests/nim-bytes2human.nimble
new file mode 100644
index 000000000..9f3ae2479
--- /dev/null
+++ b/tools/atlas/tests/nim-bytes2human.nimble
@@ -0,0 +1,7 @@
+version     = "0.2.2"
+author      = "Juan Carlos"
+description = "Convert bytes to kilobytes, megabytes, gigabytes, etc."
+license     = "MIT"
+srcDir      = "src"
+
+requires "nim >= 1.0.0"  # https://github.com/juancarlospaco/nim-bytes2human/issues/2#issue-714338524
diff --git a/tools/atlas/tests/nim.cfg b/tools/atlas/tests/nim.cfg
new file mode 100644
index 000000000..5f568569b
--- /dev/null
+++ b/tools/atlas/tests/nim.cfg
@@ -0,0 +1,11 @@
+############# begin Atlas config section ##########
+--noNimblePath
+--path:"../balls"
+--path:"../grok"
+--path:"../ups"
+--path:"../sync"
+--path:"../npeg/src"
+--path:"../testes"
+--path:"../grok"
+--path:"../nim-bytes2human/src"
+############# end Atlas config section   ##########
diff --git a/tools/atlas/tests/npeg.nimble b/tools/atlas/tests/npeg.nimble
new file mode 100644
index 000000000..e71fc5aa5
--- /dev/null
+++ b/tools/atlas/tests/npeg.nimble
@@ -0,0 +1,48 @@
+# Package
+
+version       = "0.24.1"
+author        = "Ico Doornekamp"
+description   = "a PEG library"
+license       = "MIT"
+srcDir        = "src"
+installExt    = @["nim"]
+
+# Dependencies
+
+requires "nim >= 0.19.0"
+
+# Test
+
+task test, "Runs the test suite":
+  exec "nimble testc && nimble testcpp && nimble testarc && nimble testjs"
+
+task testc, "C tests":
+  exec "nim c -r tests/tests.nim"
+
+task testcpp, "CPP tests":
+  exec "nim cpp -r tests/tests.nim"
+
+task testjs, "JS tests":
+  exec "nim js -r tests/tests.nim"
+
+task testdanger, "Runs the test suite in danger mode":
+  exec "nim c -d:danger -r tests/tests.nim"
+
+task testwin, "Mingw tests":
+  exec "nim c -d:mingw tests/tests.nim && wine tests/tests.exe"
+
+task test32, "32 bit tests":
+  exec "nim c --cpu:i386 --passC:-m32 --passL:-m32 tests/tests.nim && tests/tests"
+
+task testall, "Test all":
+  exec "nimble test && nimble testcpp && nimble testdanger && nimble testjs && nimble testwin"
+
+when (NimMajor, NimMinor) >= (1, 1):
+  task testarc, "--gc:arc tests":
+    exec "nim c --gc:arc -r tests/tests.nim"
+else:
+  task testarc, "--gc:arc tests":
+    exec "true"
+
+task perf, "Test performance":
+  exec "nim cpp -r -d:danger tests/performance.nim"
diff --git a/tools/atlas/tests/packages/packages.json b/tools/atlas/tests/packages/packages.json
new file mode 100644
index 000000000..d054a201b
--- /dev/null
+++ b/tools/atlas/tests/packages/packages.json
@@ -0,0 +1,36 @@
+[
+  {
+    "name": "bytes2human",
+    "url": "https://github.com/juancarlospaco/nim-bytes2human",
+    "method": "git",
+    "tags": [
+      "bytes",
+      "human",
+      "minimalism",
+      "size"
+    ],
+    "description": "Convert bytes to kilobytes, megabytes, gigabytes, etc.",
+    "license": "LGPLv3",
+    "web": "https://github.com/juancarlospaco/nim-bytes2human"
+  },
+  {
+    "name": "npeg",
+    "url": "https://github.com/zevv/npeg",
+    "method": "git",
+    "tags": [
+      "PEG",
+      "parser",
+      "parsing",
+      "regexp",
+      "regular",
+      "grammar",
+      "lexer",
+      "lexing",
+      "pattern",
+      "matching"
+    ],
+    "description": "PEG (Parsing Expression Grammars) string matching library for Nim",
+    "license": "MIT",
+    "web": "https://github.com/zevv/npeg"
+  }
+]
diff --git a/tools/atlas/tests/sync.nimble b/tools/atlas/tests/sync.nimble
new file mode 100644
index 000000000..a07ae8925
--- /dev/null
+++ b/tools/atlas/tests/sync.nimble
@@ -0,0 +1,10 @@
+# Package
+
+version     = "1.4.0"
+author      = "Antonis Geralis"
+description = "Useful synchronization primitives."
+license     = "MIT"
+
+# Deps
+
+requires "nim >= 1.0.0"
diff --git a/tools/atlas/tests/testes.nimble b/tools/atlas/tests/testes.nimble
new file mode 100644
index 000000000..60fe1d508
--- /dev/null
+++ b/tools/atlas/tests/testes.nimble
@@ -0,0 +1,23 @@
+version = "1.0.0"
+author = "disruptek"
+description = "a cure for salty testes"
+license = "MIT"
+
+#requires "cligen >= 0.9.41 & <= 0.9.45"
+#requires "bump >= 1.8.18 & < 2.0.0"
+requires "https://github.com/disruptek/grok >= 0.0.4 & < 1.0.0"
+requires "https://github.com/juancarlospaco/nim-bytes2human"
+
+bin = @["testes"]           # build the binary for basic test running
+installExt = @["nim"]       # we need to install testes.nim also
+skipDirs = @["tests"]       # so stupid...  who doesn't want tests?
+
+task test, "run tests for ci":
+  exec "nim c --run testes.nim"
+
+task demo, "produce a demo":
+  when (NimMajor, NimMinor) != (1, 0):
+    echo "due to nim bug #16307, use nim-1.0"
+    quit 1
+  exec """demo docs/demo.svg "nim c --out=\$1 examples/balls.nim""""
+  exec """demo docs/clean.svg "nim c --define:danger --out=\$1 tests/testicles.nim""""
diff --git a/tools/atlas/tests/ups.nimble b/tools/atlas/tests/ups.nimble
new file mode 100644
index 000000000..d91abbe60
--- /dev/null
+++ b/tools/atlas/tests/ups.nimble
@@ -0,0 +1,13 @@
+version = "0.0.2"
+author = "disruptek"
+description = "a package handler"
+license = "MIT"
+
+requires "npeg >= 0.23.2 & < 1.0.0"
+requires "https://github.com/disruptek/testes >= 1.0.0 & < 2.0.0"
+
+task test, "run tests":
+  when defined(windows):
+    exec "testes.cmd"
+  else:
+    exec findExe"testes"

From 4eca84cff7a16aa227b1b412d84dcecacc47dad5 Mon Sep 17 00:00:00 2001
From: Miran 
Date: Tue, 27 Jul 2021 01:13:47 +0200
Subject: [PATCH 1094/2472] don't use plus instead of space (#18589)

---
 compiler/docgen.nim | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/compiler/docgen.nim b/compiler/docgen.nim
index 492668543..3bc01bbe6 100644
--- a/compiler/docgen.nim
+++ b/compiler/docgen.nim
@@ -919,9 +919,9 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
     sortName = if k in routineKinds: plainName.strip else: name
     cleanPlainSymbol = renderPlainSymbolName(nameNode)
     complexSymbol = complexName(k, n, cleanPlainSymbol)
-    plainSymbolEnc = encodeUrl(cleanPlainSymbol)
+    plainSymbolEnc = encodeUrl(cleanPlainSymbol, usePlus = false)
     symbolOrId = d.newUniquePlainSymbol(complexSymbol)
-    symbolOrIdEnc = encodeUrl(symbolOrId)
+    symbolOrIdEnc = encodeUrl(symbolOrId, usePlus = false)
     deprecationMsg = genDeprecationMsg(d, pragmaNode)
 
   nodeToHighlightedHtml(d, n, result, {renderNoBody, renderNoComments,

From 22776c410a916b29c05b798523c94c834c7371ad Mon Sep 17 00:00:00 2001
From: konsumlamm <44230978+konsumlamm@users.noreply.github.com>
Date: Tue, 27 Jul 2021 01:15:21 +0200
Subject: [PATCH 1095/2472] Update documentation for dynlib (#17356)

* Update documentation for dynlib

* Apply suggestions

* Improve error handling in example
---
 lib/pure/dynlib.nim | 78 ++++++++++++++++++++-------------------------
 1 file changed, 34 insertions(+), 44 deletions(-)

diff --git a/lib/pure/dynlib.nim b/lib/pure/dynlib.nim
index 3c1466322..48fd91b8f 100644
--- a/lib/pure/dynlib.nim
+++ b/lib/pure/dynlib.nim
@@ -17,75 +17,64 @@
 ## Loading a simple C function
 ## ---------------------------
 ##
-## The following example demonstrates loading a function called 'greet'
+## The following example demonstrates loading a function called `greet`
 ## from a library that is determined at runtime based upon a language choice.
-## If the library fails to load or the function 'greet' is not found,
+## If the library fails to load or the function `greet` is not found,
 ## it quits with a failure error code.
 ##
-## .. code-block:: Nim
-##
-##   import std/dynlib
-##
-##   type
-##     greetFunction = proc(): cstring {.gcsafe, stdcall.}
-##
-##   let lang = stdin.readLine()
-##
-##   let lib = case lang
-##   of "french":
-##     loadLib("french.dll")
-##   else:
-##     loadLib("english.dll")
-##
-##   if lib == nil:
-##     echo "Error loading library"
-##     quit(QuitFailure)
-##
-##   let greet = cast[greetFunction](lib.symAddr("greet"))
-##
-##   if greet == nil:
-##     echo "Error loading 'greet' function from library"
-##     quit(QuitFailure)
-##
-##   let greeting = greet()
-##
-##   echo greeting
-##
-##   unloadLib(lib)
-##
+runnableExamples:
+  type
+    GreetFunction = proc (): cstring {.gcsafe, stdcall.}
 
-import strutils
+  proc loadGreet(lang: string) =
+    let lib =
+      case lang
+      of "french":
+        loadLib("french.dll")
+      else:
+        loadLib("english.dll")
+    assert lib != nil, "Error loading library"
+
+    let greet = cast[GreetFunction](lib.symAddr("greet"))
+    assert greet != nil, "Error loading 'greet' function from library"
+
+    echo greet()
+
+    unloadLib(lib)
+
+
+import std/strutils
 
 type
-  LibHandle* = pointer ## a handle to a dynamically loaded library
+  LibHandle* = pointer ## A handle to a dynamically loaded library.
 
 proc loadLib*(path: string, globalSymbols = false): LibHandle {.gcsafe.}
-  ## loads a library from `path`. Returns nil if the library could not
+  ## Loads a library from `path`. Returns nil if the library could not
   ## be loaded.
 
 proc loadLib*(): LibHandle {.gcsafe.}
-  ## gets the handle from the current executable. Returns nil if the
+  ## Gets the handle from the current executable. Returns nil if the
   ## library could not be loaded.
 
 proc unloadLib*(lib: LibHandle) {.gcsafe.}
-  ## unloads the library `lib`
+  ## Unloads the library `lib`.
 
 proc raiseInvalidLibrary*(name: cstring) {.noinline, noreturn.} =
-  ## raises an `EInvalidLibrary` exception.
+  ## Raises a `LibraryError` exception.
   raise newException(LibraryError, "could not find symbol: " & $name)
 
 proc symAddr*(lib: LibHandle, name: cstring): pointer {.gcsafe.}
-  ## retrieves the address of a procedure/variable from `lib`. Returns nil
+  ## Retrieves the address of a procedure/variable from `lib`. Returns nil
   ## if the symbol could not be found.
 
 proc checkedSymAddr*(lib: LibHandle, name: cstring): pointer =
-  ## retrieves the address of a procedure/variable from `lib`. Raises
-  ## `EInvalidLibrary` if the symbol could not be found.
+  ## Retrieves the address of a procedure/variable from `lib`. Raises
+  ## `LibraryError` if the symbol could not be found.
   result = symAddr(lib, name)
   if result == nil: raiseInvalidLibrary(name)
 
 proc libCandidates*(s: string, dest: var seq[string]) =
-  ## given a library name pattern `s` write possible library names to `dest`.
+  ## Given a library name pattern `s`, write possible library names to `dest`.
   var le = strutils.find(s, '(')
   var ri = strutils.find(s, ')', le+1)
   if le >= 0 and ri > le:
@@ -97,8 +86,9 @@ proc libCandidates*(s: string, dest: var seq[string]) =
     add(dest, s)
 
 proc loadLibPattern*(pattern: string, globalSymbols = false): LibHandle =
-  ## loads a library with name matching `pattern`, similar to what `dynlib`
+  ## Loads a library with name matching `pattern`, similar to what the `dynlib`
   ## pragma does. Returns nil if the library could not be loaded.
+  ##
   ## .. warning:: this proc uses the GC and so cannot be used to load the GC.
   var candidates = newSeq[string]()
   libCandidates(pattern, candidates)

From 9cb5ab0108c62bfda30b4f9b52b2957fdd364544 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Tue, 27 Jul 2021 13:21:39 +0800
Subject: [PATCH 1096/2472] fix #18578 (#18580)

* fix #18578

* add tests

* tiny

* apply changes

* typo

* add removeStaticFile
---
 compiler/extccomp.nim         | 15 +++++++++------
 tests/compiles/mstaticlib.nim |  1 +
 tests/compiles/tstaticlib.nim | 22 ++++++++++++++++++++++
 3 files changed, 32 insertions(+), 6 deletions(-)
 create mode 100644 tests/compiles/mstaticlib.nim
 create mode 100644 tests/compiles/tstaticlib.nim

diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim
index 831acae68..a091060ff 100644
--- a/compiler/extccomp.nim
+++ b/compiler/extccomp.nim
@@ -665,9 +665,10 @@ proc addExternalFileToCompile*(conf: ConfigRef; filename: AbsoluteFile) =
   addExternalFileToCompile(conf, c)
 
 proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile,
-                objfiles: string, isDllBuild: bool): string =
+                objfiles: string, isDllBuild: bool, removeStaticFile: bool): string =
   if optGenStaticLib in conf.globalOptions:
-    removeFile output # fixes: bug #16947
+    if removeStaticFile:
+      removeFile output # fixes: bug #16947
     result = CC[conf.cCompiler].buildLib % ["libfile", quoteShell(output),
                                             "objfiles", objfiles]
   else:
@@ -750,8 +751,9 @@ proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile,
   if optCDebug in conf.globalOptions and conf.cCompiler == ccVcc:
     result.add " /Zi /FS /Od"
 
-template getLinkCmd(conf: ConfigRef; output: AbsoluteFile, objfiles: string): string =
-  getLinkCmd(conf, output, objfiles, optGenDynLib in conf.globalOptions)
+template getLinkCmd(conf: ConfigRef; output: AbsoluteFile, objfiles: string,
+                    removeStaticFile = false): string =
+  getLinkCmd(conf, output, objfiles, optGenDynLib in conf.globalOptions, removeStaticFile)
 
 template tryExceptOSErrorMessage(conf: ConfigRef; errorPrefix: string = "", body: untyped) =
   try:
@@ -891,7 +893,7 @@ proc callCCompiler*(conf: ConfigRef) =
         let objFile = conf.getObjFilePath(x)
         let buildDll = idx != mainFileIdx
         let linkTarget = conf.hcrLinkTargetName(objFile, not buildDll)
-        cmds.add(getLinkCmd(conf, linkTarget, objfiles & " " & quoteShell(objFile), buildDll))
+        cmds.add(getLinkCmd(conf, linkTarget, objfiles & " " & quoteShell(objFile), buildDll, removeStaticFile = true))
         # try to remove all .pdb files for the current binary so they don't accumulate endlessly in the nimcache
         # for more info check the comment inside of getLinkCmd() where the /PDB: MSVC flag is used
         if isVSCompatible(conf):
@@ -914,7 +916,8 @@ proc callCCompiler*(conf: ConfigRef) =
         objfiles.add(quoteShell(objFile))
       let mainOutput = if optGenScript notin conf.globalOptions: conf.prepareToWriteOutput
                        else: AbsoluteFile(conf.projectName)
-      linkCmd = getLinkCmd(conf, mainOutput, objfiles)
+
+      linkCmd = getLinkCmd(conf, mainOutput, objfiles, removeStaticFile = true)
       extraCmds = getExtraCmds(conf, mainOutput)
       if optCompileOnly notin conf.globalOptions:
         const MaxCmdLen = when defined(windows): 8_000 else: 32_000
diff --git a/tests/compiles/mstaticlib.nim b/tests/compiles/mstaticlib.nim
new file mode 100644
index 000000000..6ed593691
--- /dev/null
+++ b/tests/compiles/mstaticlib.nim
@@ -0,0 +1 @@
+echo 1234
\ No newline at end of file
diff --git a/tests/compiles/tstaticlib.nim b/tests/compiles/tstaticlib.nim
new file mode 100644
index 000000000..a18b59204
--- /dev/null
+++ b/tests/compiles/tstaticlib.nim
@@ -0,0 +1,22 @@
+import std/[os, osproc, strformat]
+
+
+const dir = "tests/compiles"
+const fileName = dir / "mstaticlib.nim"
+const nim = getCurrentCompilerExe()
+
+block: # bug #18578
+  const libName = dir / "tstaticlib1.a"
+  let (_, status) = execCmdEx(fmt"{nim} c -o:{libName} --app:staticlib {fileName}")
+  doAssert status == 0
+  doAssert fileExists(libName)
+  removeFile(libName)
+
+block: # bug #16947
+  const libName = dir / "tstaticlib2.a"
+  writeFile(libName, "echo 124")
+  doAssert fileExists(libName)
+  let (_, status) = execCmdEx(fmt"{nim} c -o:{libName} --app:staticlib {fileName}")
+  doAssert status == 0
+  doAssert fileExists(libName)
+  removeFile(libName)

From fa0209609d3bedd3466f162aa350d261907851ce Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Tue, 27 Jul 2021 09:36:19 +0200
Subject: [PATCH 1097/2472] fixes #18565 (#18593)

* fixes #18565
---
 compiler/semstmts.nim         | 22 +++++++++++-----------
 compiler/sigmatch.nim         |  4 +++-
 tests/arc/tkeys_lent.nim      | 17 +++++++++++++++++
 tests/stdlib/tgetfileinfo.nim |  1 +
 tests/stdlib/tos.nim          |  1 +
 5 files changed, 33 insertions(+), 12 deletions(-)
 create mode 100644 tests/arc/tkeys_lent.nim

diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim
index bfce91849..75323a95e 100644
--- a/compiler/semstmts.nim
+++ b/compiler/semstmts.nim
@@ -2260,20 +2260,20 @@ proc semStmtList(c: PContext, n: PNode, flags: TExprFlags): PNode =
   #                                         nkNilLit, nkEmpty}:
   #  dec last
   for i in 0..
Date: Tue, 27 Jul 2021 01:36:59 -0600
Subject: [PATCH 1098/2472] Fixed const tuples in inferred generics (#18598)

---
 compiler/semgnrc.nim                     | 33 ++++++++++++------------
 tests/tuples/tinferred_generic_const.nim | 14 ++++++++++
 2 files changed, 30 insertions(+), 17 deletions(-)
 create mode 100644 tests/tuples/tinferred_generic_const.nim

diff --git a/compiler/semgnrc.nim b/compiler/semgnrc.nim
index 4f3c50dae..3df1e0c3f 100644
--- a/compiler/semgnrc.nim
+++ b/compiler/semgnrc.nim
@@ -391,16 +391,24 @@ proc semGenericStmt(c: PContext, n: PNode,
       a[^1] = semGenericStmtScope(c, a[^1], flags, ctx)
       closeScope(c)
 
-  of nkVarSection, nkLetSection:
+  of nkVarSection, nkLetSection, nkConstSection:
+    let varKind =
+      case n.kind
+      of nkVarSection: skVar
+      of nkLetSection: skLet
+      else: skConst
     for i in 0..
Date: Tue, 27 Jul 2021 00:37:11 -0700
Subject: [PATCH 1099/2472] add comment to astalgo.debug (#18594)

---
 compiler/astalgo.nim | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/compiler/astalgo.nim b/compiler/astalgo.nim
index 3c97665f0..94fa9da93 100644
--- a/compiler/astalgo.nim
+++ b/compiler/astalgo.nim
@@ -585,6 +585,9 @@ proc value(this: var DebugPrinter; value: PNode) =
   this.openCurly
   this.key "kind"
   this.value  value.kind
+  if value.comment.len > 0:
+    this.key "comment"
+    this.value  value.comment
   when defined(useNodeIds):
     this.key "id"
     this.value value.id

From 37f5f0d7d08d6057b20d3e319b38dfe82ce1ba65 Mon Sep 17 00:00:00 2001
From: GordonBGood 
Date: Tue, 27 Jul 2021 14:38:30 +0700
Subject: [PATCH 1100/2472] Clarify use of import via pseudo dir... (#18569)

---
 doc/manual.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/manual.rst b/doc/manual.rst
index 9f614c2cd..1b65ae9ee 100644
--- a/doc/manual.rst
+++ b/doc/manual.rst
@@ -6190,6 +6190,7 @@ There are two pseudo directories:
    its semantics are: *Use the search path to look for module name but ignore the standard
    library locations*. In other words, it is the opposite of `std`.
 
+It is recommended and preferred but not currently enforced that all stdlib module imports include the std/ "pseudo directory" as part of the import name.
 
 From import statement
 ---------------------

From 493721c16c06b5681dc270679bdcbb41011614b2 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Tue, 27 Jul 2021 00:56:16 -0700
Subject: [PATCH 1101/2472] delEnv now works at CT (#18568)

---
 compiler/vmops.nim | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/compiler/vmops.nim b/compiler/vmops.nim
index bc331bbcd..6ee915bd5 100644
--- a/compiler/vmops.nim
+++ b/compiler/vmops.nim
@@ -20,7 +20,7 @@ when declared(math.copySign):
 when declared(math.signbit):
   from std/math import signbit
 
-from std/os import getEnv, existsEnv, dirExists, fileExists, putEnv, walkDir,
+from std/os import getEnv, existsEnv, delEnv, putEnv, dirExists, fileExists, walkDir,
                    getAppFilename, raiseOSError, osLastError
 
 from std/md5 import getMD5
@@ -215,6 +215,7 @@ proc registerAdditionalOps*(c: PCtx) =
     wrap2s(getEnv, osop)
     wrap1s(existsEnv, osop)
     wrap2svoid(putEnv, osop)
+    wrap1svoid(delEnv, osop)
     wrap1s(dirExists, osop)
     wrap1s(fileExists, osop)
     wrapDangerous(writeFile, ioop)

From c86f9590fb5ba6512ba33994fa0711341eb8524b Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Tue, 27 Jul 2021 12:48:27 +0200
Subject: [PATCH 1102/2472] fixes #18570 (#18599)

---
 compiler/injectdestructors.nim | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/compiler/injectdestructors.nim b/compiler/injectdestructors.nim
index cf09b3ff5..985f27878 100644
--- a/compiler/injectdestructors.nim
+++ b/compiler/injectdestructors.nim
@@ -799,7 +799,8 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode): PNode =
           result[i] = p(n[i], c, s, m)
     of nkObjConstr:
       # see also the remark about `nkTupleConstr`.
-      let isRefConstr = n.typ.skipTypes(abstractInst).kind == tyRef
+      let t = n.typ.skipTypes(abstractInst)
+      let isRefConstr = t.kind == tyRef
       let m = if isRefConstr: sinkArg
               elif mode == normal: normal
               else: sinkArg
@@ -807,7 +808,7 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode): PNode =
       result = copyTree(n)
       for i in 1..
Date: Tue, 27 Jul 2021 14:32:56 +0200
Subject: [PATCH 1103/2472] fixes #18579 (#18600)

---
 lib/system/arc.nim         |  6 ++----
 tests/arc/tunref_cycle.nim | 26 ++++++++++++++++++++++++++
 2 files changed, 28 insertions(+), 4 deletions(-)
 create mode 100644 tests/arc/tunref_cycle.nim

diff --git a/lib/system/arc.nim b/lib/system/arc.nim
index ed5e9f5ca..d66f4b997 100644
--- a/lib/system/arc.nim
+++ b/lib/system/arc.nim
@@ -207,10 +207,8 @@ proc nimDecRefIsLast(p: pointer): bool {.compilerRtl, inl.} =
 
 proc GC_unref*[T](x: ref T) =
   ## New runtime only supports this operation for 'ref T'.
-  if nimDecRefIsLast(cast[pointer](x)):
-    # XXX this does NOT work for virtual destructors!
-    `=destroy`(x[])
-    nimRawDispose(cast[pointer](x), T.alignOf)
+  var y {.cursor.} = x
+  `=destroy`(y)
 
 proc GC_ref*[T](x: ref T) =
   ## New runtime only supports this operation for 'ref T'.
diff --git a/tests/arc/tunref_cycle.nim b/tests/arc/tunref_cycle.nim
new file mode 100644
index 000000000..82551b7f7
--- /dev/null
+++ b/tests/arc/tunref_cycle.nim
@@ -0,0 +1,26 @@
+discard """
+  outputsub: '''inside closure
+hello world'''
+  cmd: "nim c --gc:orc -d:useMalloc $file"
+  valgrind: true
+"""
+
+# bug #18579
+
+var fp: proc (env: pointer) {.cdecl.}
+var env: pointer
+
+proc store(f: proc (){.closure.}) =
+  proc closeOver() =
+    echo "inside closure"
+    f()
+  (fp,env) = (cast[proc(env: pointer){.cdecl.}](rawProc closeOver), rawEnv closeOver)
+  GC_ref(cast[RootRef](env))
+
+proc run() =
+  fp(env)
+  GC_unref(cast[RootRef](env))
+
+store(proc() = echo "hello world")
+run()
+GC_fullCollect()

From 4920b06973e69c525405ff3147dc8a823a4c3c58 Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Tue, 27 Jul 2021 19:04:55 +0200
Subject: [PATCH 1104/2472] fixes #18543 (#18601)

* fixes #18543

* make tests green again
---
 changelog.md                     |   1 -
 lib/core/macros.nim              | 264 +++++++++++--------------------
 testament/important_packages.nim |   1 +
 tests/pragmas/tcustom_pragma.nim |  25 +--
 4 files changed, 107 insertions(+), 184 deletions(-)

diff --git a/changelog.md b/changelog.md
index d33e24b65..89e013cff 100644
--- a/changelog.md
+++ b/changelog.md
@@ -51,7 +51,6 @@
   implementations. Old behavior can be obtained with
   `-d:nimLegacyParseQueryStrict`. `cgi.decodeData` which uses the same
   underlying code is also updated the same way.
-- Custom pragma values have now an API for use in macros.
 
 - On POSIX systems, the default signal handlers used for Nim programs (it's
   used for printing the stacktrace on fatal signals) will now re-raise the
diff --git a/lib/core/macros.nim b/lib/core/macros.nim
index c09fae6b3..d3eaf6298 100644
--- a/lib/core/macros.nim
+++ b/lib/core/macros.nim
@@ -1068,11 +1068,11 @@ proc newEmptyNode*(): NimNode {.noSideEffect.} =
   ## Create a new empty node.
   result = newNimNode(nnkEmpty)
 
-proc newStmtList*(stmts: varargs[NimNode]): NimNode=
+proc newStmtList*(stmts: varargs[NimNode]): NimNode =
   ## Create a new statement list.
   result = newNimNode(nnkStmtList).add(stmts)
 
-proc newPar*(exprs: varargs[NimNode]): NimNode=
+proc newPar*(exprs: varargs[NimNode]): NimNode =
   ## Create a new parentheses-enclosed expression.
   newNimNode(nnkPar).add(exprs)
 
@@ -1494,148 +1494,80 @@ macro expandMacros*(body: typed): untyped =
   echo body.toStrLit
   result = body
 
-proc findPragmaNodeInRecList(arg, fieldSym: NimNode): NimNode =
-  case arg.kind
-  of nnkRecList, nnkRecCase:
-    for it in arg.children:
-      result = findPragmaNodeInRecList(it, fieldSym)
-      if result != nil: return
-  of nnkOfBranch:
-    return findPragmaNodeInRecList(arg[1], fieldSym)
-  of nnkElse:
-    return findPragmaNodeInRecList(arg[0], fieldSym)
-  of nnkIdentDefs:
-    for i in 0.. 0:
-    impl.expectKind nnkTypeDef
-    let pragmaExpr = impl[0]
-    if pragmaExpr.kind == nnkPragmaExpr:
-      result = pragmaExpr[1]
-
-proc getPragmaNodeFromType(typ: NimNode): NimNode =
-  case typ.kind
-  of nnkSym:
-    result = getPragmaNodeFromTypeSym(typ)
-  of nnkProcTy:
-    result = typ[1]
-  else: error("illegal typ kind for argument: " & $typ.kind, typ)
-
-proc getPragmaNodeFromVarLetSym(sym: NimNode): NimNode =
-  sym.expectKind nnkSym
-  if sym.symKind notin {nskVar, nskLet}: error("expected var/let sym", sym)
-  let impl = sym.getImpl
-  impl.expectKind nnkIdentDefs
-  impl.expectLen 3
-  let pragmaExpr = impl[0]
-  if pragmaExpr.kind == nnkPragmaExpr:
-    result = pragmaExpr[1]
-
-proc getPragmasByName(pragmaExpr: NimNode, name: string): seq[NimNode] =
-  if pragmaExpr.kind == nnkPragma:
-    for it in pragmaExpr:
-      if it.kind in nnkPragmaCallKinds:
-        if eqIdent(it[0], name):
-          result.add it
-      elif it.kind == nnkSym:
-        if eqIdent(it, name):
-          result.add it
-
-proc getCustomPragmaNodes(sym: NimNode, name: string): seq[NimNode] =
-  sym.expectKind nnkSym
-  case sym.symKind
-  of nskField:
-    result = getPragmaNodeFromObjFieldSym(sym).getPragmasByName(name)
-  of nskProc:
-    result = getPragmaNodeFromProcSym(sym).getPragmasByName(name)
-  of nskType:
-    result = getPragmaNodeFromTypeSym(sym).getPragmasByName(name)
-  of nskParam:
-    # When a typedesc parameter is passed to the macro, it will be of nskParam.
-    let typeInst = getTypeInst(sym)
-    if typeInst.kind == nnkBracketExpr and eqIdent(typeInst[0], "typeDesc"):
-      result = getPragmaNodeFromTypeSym(typeInst[1]).getPragmasByName(name)
+  if typ.kind == nnkBracketExpr and typ.len > 1 and typ[1].kind == nnkProcTy:
+    return typ[1][1]
+  elif typ.typeKind == ntyTypeDesc:
+    let impl = typ[1].getImpl()
+    if impl[0].kind == nnkPragmaExpr:
+      return impl[0][1]
     else:
-      error("illegal sym kind for argument: " & $sym.symKind, sym)
-  of nskVar, nskLet:
-    # This checks the type of the sym too, this is consistent with how
-    # field expressions are handled too. If this is changed, make sure to
-    # change it for fields expressions too.
-    result = getPragmaNodeFromType(sym.getTypeInst).getPragmasByName(name)
-    result.add getPragmaNodeFromVarLetSym(sym).getPragmasByName(name)
-  else:
-    error("illegal sym kind for argument: " & $sym.symKind, sym)
+      return impl[0] # handle types which don't have macro at all
 
-since (1, 5):
-  export getCustomPragmaNodes
+  if n.kind == nnkSym: # either an variable or a proc
+    let impl = n.getImpl()
+    if impl.kind in RoutineNodes:
+      return impl.pragma
+    elif impl.kind == nnkIdentDefs and impl[0].kind == nnkPragmaExpr:
+      return impl[0][1]
+    else:
+      let timpl = typ.getImpl()
+      if timpl.len>0 and timpl[0].len>1:
+        return timpl[0][1]
+      else:
+        return timpl
 
-proc hasCustomPragma*(n: NimNode, name: string): bool =
-  n.expectKind nnkSym
-  result = getCustomPragmaNodes(n, name).len > 0
+  if n.kind in {nnkDotExpr, nnkCheckedFieldExpr}:
+    let name = $(if n.kind == nnkCheckedFieldExpr: n[0][1] else: n[1])
+    let typInst = getTypeInst(if n.kind == nnkCheckedFieldExpr or n[0].kind == nnkHiddenDeref: n[0][0] else: n[0])
+    var typDef = getImpl(if typInst.kind == nnkVarTy: typInst[0] else: typInst)
+    while typDef != nil:
+      typDef.expectKind(nnkTypeDef)
+      let typ = typDef[2]
+      typ.expectKind({nnkRefTy, nnkPtrTy, nnkObjectTy})
+      let isRef = typ.kind in {nnkRefTy, nnkPtrTy}
+      if isRef and typ[0].kind in {nnkSym, nnkBracketExpr}: # defines ref type for another object(e.g. X = ref X)
+        typDef = getImpl(typ[0])
+      else: # object definition, maybe an object directly defined as a ref type
+        let
+          obj = (if isRef: typ[0] else: typ)
+        var identDefsStack = newSeq[NimNode](obj[2].len)
+        for i in 0.. 0:
+          var identDefs = identDefsStack.pop()
+          if identDefs.kind == nnkRecCase:
+            identDefsStack.add(identDefs[0])
+            for i in 1.. 0)
+  let pragmaNode = customPragmaNode(n)
+  for p in pragmaNode:
+    if (p.kind == nnkSym and p == cp) or
+        (p.kind in nnkPragmaCallKinds and p.len > 0 and p[0].kind == nnkSym and p[0] == cp):
+      return newLit(true)
+  return newLit(false)
 
-iterator iterOverFormalArgs(f: NimNode): tuple[name, typ, val: NimNode] =
-  f.expectKind nnkFormalParams
-  for i in 1.. 0 and p[0].kind == nnkSym and p[0] == cp:
+      if p.len == 2:
+        result = p[1]
+      else:
+        let def = p[0].getImpl[3]
+        result = newTree(nnkPar)
+        for i in 1 ..< def.len:
+          let key = def[i][0]
+          let val = p[i]
+          result.add newTree(nnkExprColonExpr, key, val)
+      break
+  if result.kind == nnkEmpty:
+    error(n.repr & " doesn't have a pragma named " & cp.repr()) # returning an empty node results in most cases in a cryptic error,
 
 macro unpackVarargs*(callee: untyped; args: varargs[untyped]): untyped =
   ## Calls `callee` with `args` unpacked as individual arguments.
diff --git a/testament/important_packages.nim b/testament/important_packages.nim
index afceffb2e..ab29dcd0d 100644
--- a/testament/important_packages.nim
+++ b/testament/important_packages.nim
@@ -114,6 +114,7 @@ pkg "nimsvg"
 pkg "nimterop", "nimble minitest"
 pkg "nimwc", "nim c nimwc.nim"
 pkg "nimx", "nim c --threads:on test/main.nim", allowFailure = true
+pkg "nimYAML", "nim c -r test/tserialization.nim"
 pkg "nitter", "nim c src/nitter.nim", "https://github.com/zedeus/nitter"
 pkg "norm", "nim c -r tests/sqlite/trows.nim"
 pkg "npeg", "nimble testarc"
diff --git a/tests/pragmas/tcustom_pragma.nim b/tests/pragmas/tcustom_pragma.nim
index 3ef5bdcb6..b197a7c55 100644
--- a/tests/pragmas/tcustom_pragma.nim
+++ b/tests/pragmas/tcustom_pragma.nim
@@ -384,7 +384,7 @@ block:
   discard Hello(a: 1.0, b: 12)
 
 # issue #11511
-block:
+when false:
   template myAttr {.pragma.}
 
   type TObj = object
@@ -395,23 +395,24 @@ block:
     let recList = objTy[2]
     let sym = recList[0]
     assert sym.kind == nnkSym and sym.eqIdent("a")
-    let hasAttr = sym.hasCustomPragma("myAttr")
+    let hasAttr = sym.hasCustomPragma(myAttr)
     newLit(hasAttr)
 
   doAssert hasMyAttr(TObj)
 
-# misc
-{.pragma: haha.}
-{.pragma: hoho.}
-template hehe(key, val: string, haha) {.pragma.}
+when false:
+  # misc
+  {.pragma: haha.}
+  {.pragma: hoho.}
+  template hehe(key, val: string, haha) {.pragma.}
 
-type A {.haha, hoho, haha, hehe("hi", "hu", "he").} = int
+  type A {.haha, hoho, haha, hehe("hi", "hu", "he").} = int
 
-assert A.getCustomPragmaVal(hehe) == (key: "hi", val: "hu", haha: "he")
+  assert A.getCustomPragmaVal(hehe) == (key: "hi", val: "hu", haha: "he")
 
-template hehe(key, val: int) {.pragma.}
+  template hehe(key, val: int) {.pragma.}
 
-var bb {.haha, hoho, hehe(1, 2), haha, hehe("hi", "hu", "he").} = 3
+  var bb {.haha, hoho, hehe(1, 2), haha, hehe("hi", "hu", "he").} = 3
 
-# left-to-right priority/override order for getCustomPragmaVal
-assert bb.getCustomPragmaVal(hehe) == (key: "hi", val: "hu", haha: "he")
+  # left-to-right priority/override order for getCustomPragmaVal
+  assert bb.getCustomPragmaVal(hehe) == (key: "hi", val: "hu", haha: "he")

From 8d2f6bba3a491bc93677da9590397346e63c2bb2 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Tue, 27 Jul 2021 10:50:59 -0700
Subject: [PATCH 1105/2472] support same-line doc comments in routines (#18595)

* support same-line comments in routines
* remove assert as per review comment
---
 compiler/parser.nim                           |   7 ++
 .../expected/subdir/subdir_b/utils.html       | 110 +++++++++++++++++-
 .../expected/subdir/subdir_b/utils.idx        |   9 ++
 nimdoc/testproject/expected/theindex.html     |  36 ++++++
 nimdoc/testproject/subdir/subdir_b/utils.nim  |  17 +++
 tests/stdlib/trepr.nim                        |  27 +++++
 6 files changed, 205 insertions(+), 1 deletion(-)

diff --git a/compiler/parser.nim b/compiler/parser.nim
index d00a17e9f..b089614b2 100644
--- a/compiler/parser.nim
+++ b/compiler/parser.nim
@@ -1795,6 +1795,13 @@ proc parseRoutine(p: var Parser, kind: TNodeKind): PNode =
   else:
     result.add(p.emptyNode)
   indAndComment(p, result, maybeMissEquals)
+  let body = result[^1]
+  if body.kind == nkStmtList and body.len > 0 and body[0].comment.len > 0 and body[0].kind != nkCommentStmt:
+    if result.comment.len == 0:
+      # proc fn*(a: int): int = a ## foo
+      # => moves comment `foo` to `fn`
+      swap(result.comment, body[0].comment)
+    else: discard # xxx either `assert false` or issue a warning (otherwise we'll never know of this edge case)
 
 proc newCommentStmt(p: var Parser): PNode =
   #| commentStmt = COMMENT
diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html
index 939495ad1..533cb5b9d 100644
--- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html
+++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html
@@ -115,7 +115,52 @@ window.addEventListener('DOMContentLoaded', main);
 
                      • 
   Procs
   
                          
-      
                            someType
+      
+  
+  
+  
+  
+  
+  
+  
+  
+  
                              someType
       
                            • someType(): SomeType
                              • 
 
@@ -181,6 +226,69 @@ window.addEventListener('DOMContentLoaded', main);
 
                              
 
                              Procs
                              
 
                              
+
+
                              proc fn2() {....raises: [], tags: [].}
                              
+
                              
+
+comment
+
+
                              
+
+
                              proc fn3(): auto {....raises: [], tags: [].}
                              
+
                              
+
+comment
+
+
                              
+
+
                              proc fn4(): auto {....raises: [], tags: [].}
                              
+
                              
+
+comment
+
+
                              
+
+
                              proc fn5() {....raises: [], tags: [].}
                              
+
                              
+
+comment
+
+
                              
+
+
                              proc fn6() {....raises: [], tags: [].}
                              
+
                              
+
+comment
+
+
                              
+
+
                              proc fn7() {....raises: [], tags: [].}
                              
+
                              
+
+comment
+
+
                              
+
+
                              proc fn8(): auto {....raises: [], tags: [].}
                              
+
                              
+
+comment
+
+
                              
+
+
                              func fn9(a: int): int {....raises: [], tags: [].}
                              
+
                              
+
+comment
+
+
                              
+
+
                              func fn10(a: int): int {....raises: [], tags: [].}
                              
+
                              
+
+comment
+
+
                              
 
 
                              proc someType(): SomeType {....raises: [], tags: [].}
                              
 
                              
diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.idx b/nimdoc/testproject/expected/subdir/subdir_b/utils.idx
index 6dc3953af..a87393f09 100644
--- a/nimdoc/testproject/expected/subdir/subdir_b/utils.idx
+++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.idx
@@ -3,6 +3,15 @@ enumValueB	subdir/subdir_b/utils.html#enumValueB	SomeType.enumValueB
 enumValueC	subdir/subdir_b/utils.html#enumValueC	SomeType.enumValueC	
 SomeType	subdir/subdir_b/utils.html#SomeType	utils: SomeType	
 someType	subdir/subdir_b/utils.html#someType_2	utils: someType(): SomeType	
+fn2	subdir/subdir_b/utils.html#fn2	utils: fn2()	
+fn3	subdir/subdir_b/utils.html#fn3	utils: fn3(): auto	
+fn4	subdir/subdir_b/utils.html#fn4	utils: fn4(): auto	
+fn5	subdir/subdir_b/utils.html#fn5	utils: fn5()	
+fn6	subdir/subdir_b/utils.html#fn6	utils: fn6()	
+fn7	subdir/subdir_b/utils.html#fn7	utils: fn7()	
+fn8	subdir/subdir_b/utils.html#fn8	utils: fn8(): auto	
+fn9	subdir/subdir_b/utils.html#fn9,int	utils: fn9(a: int): int	
+fn10	subdir/subdir_b/utils.html#fn10,int	utils: fn10(a: int): int	
 aEnum	subdir/subdir_b/utils.html#aEnum.t	utils: aEnum(): untyped	
 bEnum	subdir/subdir_b/utils.html#bEnum.t	utils: bEnum(): untyped	
 fromUtilsGen	subdir/subdir_b/utils.html#fromUtilsGen.t	utils: fromUtilsGen(): untyped	
diff --git a/nimdoc/testproject/expected/theindex.html b/nimdoc/testproject/expected/theindex.html
index cfaf6856d..f72f6c37e 100644
--- a/nimdoc/testproject/expected/theindex.html
+++ b/nimdoc/testproject/expected/theindex.html
@@ -171,6 +171,42 @@ window.addEventListener('DOMContentLoaded', main);
 
                            • SomeType.enumValueC
                              • 
           
                            
+
                            fn10:
                            
+
                            fn2:
                            
+
                            fn3:
                            
+
                            fn4:
                            
+
                            fn5:
                            
+
                            fn6:
                            
+
                            fn7:
                            
+
                            fn8:
                            
+
                            fn9:
                            
 
                            Foo:
                              
 
                            • testproject: Foo
                              • 
diff --git a/nimdoc/testproject/subdir/subdir_b/utils.nim b/nimdoc/testproject/subdir/subdir_b/utils.nim
index 4e3aa1f10..37c91dd3c 100644
--- a/nimdoc/testproject/subdir/subdir_b/utils.nim
+++ b/nimdoc/testproject/subdir/subdir_b/utils.nim
@@ -31,6 +31,23 @@ proc someType*(): SomeType =
   ## constructor.
   SomeType(2)
 
+
+proc fn2*() = discard ## comment
+proc fn3*(): auto = 1 ## comment
+proc fn4*(): auto = 2 * 3 + 4 ## comment
+proc fn5*() ## comment
+proc fn5*() = discard
+proc fn6*() =
+  ## comment
+proc fn7*() =
+  ## comment
+  discard
+proc fn8*(): auto =
+  ## comment
+  1+1
+func fn9*(a: int): int = 42  ## comment
+func fn10*(a: int): int = a  ## comment
+
 # bug #9235
 
 template aEnum*(): untyped =
diff --git a/tests/stdlib/trepr.nim b/tests/stdlib/trepr.nim
index 356764903..83ae7b119 100644
--- a/tests/stdlib/trepr.nim
+++ b/tests/stdlib/trepr.nim
@@ -9,6 +9,7 @@ from strutils import endsWith, contains, strip
 from std/macros import newLit
 
 macro deb(a): string = newLit a.repr.strip
+macro debTyped(a: typed): string = newLit a.repr.strip
 
 template main() =
   doAssert repr({3,5}) == "{3, 5}"
@@ -243,5 +244,31 @@ template bar(): untyped =
     echo "baz"
     4)"""
 
+  block: # one liner doc comments
+    let a1 = deb:
+      func fn1(): int = 1  ## comment
+      func fn2(): int = 1
+        ## comment
+    let a2 = debTyped:
+      func fn1(): int = 1  ## comment
+      func fn2(): int = 1
+        ## comment
+    doAssert a1 == """
+func fn1(): int =
+  ## comment
+  1
+
+func fn2(): int =
+  ## comment
+  1"""
+    doAssert a2 == """
+func fn1(): int =
+  ## comment
+  result = 1
+
+func fn2(): int =
+  ## comment
+  result = 1"""
+
 static: main()
 main()

From 4c1202972abdfe99232e5d15a6169c7b2e0f5d75 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Tue, 27 Jul 2021 21:38:19 -0700
Subject: [PATCH 1106/2472] fix #17072: add times.dateTime with ISO 8601 order
 (#18590)

* fix #17072: add times.dateTime with ISO 8601 order

* address comments
---
 changelog.md       |  6 +++++-
 lib/pure/times.nim | 29 ++++++++++++++++++-----------
 2 files changed, 23 insertions(+), 12 deletions(-)

diff --git a/changelog.md b/changelog.md
index 89e013cff..be94f92c4 100644
--- a/changelog.md
+++ b/changelog.md
@@ -331,9 +331,12 @@
 
 - Added `htmlgen.portal` for [making "SPA style" pages using HTML only](https://web.dev/hands-on-portals).
 
-- Added `ZZZ` and `ZZZZ` patterns to `times.nim` `DateTime` parsing, to match time
+- `std/times`:
+  Added `ZZZ` and `ZZZZ` patterns to `times.nim` `DateTime` parsing, to match time
   zone offsets without colons, e.g. `UTC+7 -> +0700`.
 
+  Added `dateTime` and deprecated `initDateTime`.
+
 - Added `jsconsole.dir`, `jsconsole.dirxml`, `jsconsole.timeStamp`.
 
 - Added dollar `$` and `len` for `jsre.RegExp`.
@@ -357,6 +360,7 @@
 
 - Added `dom.setInterval`, `dom.clearInterval` overloads.
 
+
 - Deprecated `sequtils.delete` and added an overload taking a `Slice` that raises a defect
   if the slice is out of bounds, likewise with `strutils.delete`.
 
diff --git a/lib/pure/times.nim b/lib/pure/times.nim
index fcdd4ec8d..b2a22250d 100644
--- a/lib/pure/times.nim
+++ b/lib/pure/times.nim
@@ -1333,14 +1333,13 @@ proc now*(): DateTime {.tags: [TimeEffect], benign.} =
   ##    `cpuTime` instead, depending on the use case.
   getTime().local
 
-proc initDateTime*(monthday: MonthdayRange, month: Month, year: int,
-                   hour: HourRange, minute: MinuteRange, second: SecondRange,
-                   nanosecond: NanosecondRange,
-                   zone: Timezone = local()): DateTime =
+proc dateTime*(year: int, month: Month, monthday: MonthdayRange, 
+               hour: HourRange = 0, minute: MinuteRange = 0, second: SecondRange = 0,
+               nanosecond: NanosecondRange = 0,
+               zone: Timezone = local()): DateTime =
   ## Create a new `DateTime <#DateTime>`_ in the specified timezone.
   runnableExamples:
-    let dt1 = initDateTime(30, mMar, 2017, 00, 00, 00, 00, utc())
-    doAssert $dt1 == "2017-03-30T00:00:00Z"
+    assert $dateTime(2017, mMar, 30, zone = utc()) == "2017-03-30T00:00:00Z"
 
   assertValidDate monthday, month, year
   let dt = DateTime(
@@ -1356,12 +1355,20 @@ proc initDateTime*(monthday: MonthdayRange, month: Month, year: int,
 
 proc initDateTime*(monthday: MonthdayRange, month: Month, year: int,
                    hour: HourRange, minute: MinuteRange, second: SecondRange,
-                   zone: Timezone = local()): DateTime =
+                   nanosecond: NanosecondRange,
+                   zone: Timezone = local()): DateTime {.deprecated: "use `dateTime`".} =
   ## Create a new `DateTime <#DateTime>`_ in the specified timezone.
-  runnableExamples:
-    let dt1 = initDateTime(30, mMar, 2017, 00, 00, 00, utc())
-    doAssert $dt1 == "2017-03-30T00:00:00Z"
-  initDateTime(monthday, month, year, hour, minute, second, 0, zone)
+  runnableExamples("--warning:deprecated:off"):
+    assert $initDateTime(30, mMar, 2017, 00, 00, 00, 00, utc()) == "2017-03-30T00:00:00Z"
+  dateTime(year, month, monthday, hour, minute, second, nanosecond, zone)
+
+proc initDateTime*(monthday: MonthdayRange, month: Month, year: int,
+                   hour: HourRange, minute: MinuteRange, second: SecondRange,
+                   zone: Timezone = local()): DateTime {.deprecated: "use `dateTime`".} =
+  ## Create a new `DateTime <#DateTime>`_ in the specified timezone.
+  runnableExamples("--warning:deprecated:off"):
+    assert $initDateTime(30, mMar, 2017, 00, 00, 00, utc()) == "2017-03-30T00:00:00Z"
+  dateTime(year, month, monthday, hour, minute, second, 0, zone)
 
 proc `+`*(dt: DateTime, dur: Duration): DateTime =
   runnableExamples:

From a273ea70e8817e3509014a1b3dcd16a360ed400b Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Wed, 28 Jul 2021 12:46:28 +0200
Subject: [PATCH 1107/2472] implements overloadable enum values; WIP (#18470)

* implements overloadable enum values
* simpler code
---
 compiler/ast.nim                   |  2 +-
 compiler/lookups.nim               |  7 +++--
 compiler/options.nim               |  3 +-
 compiler/sem.nim                   |  8 +++++
 compiler/semexprs.nim              | 36 +++++++++++++++++++++-
 compiler/semstmts.nim              | 16 +++++++++-
 compiler/semtypes.nim              | 12 ++++++--
 compiler/sigmatch.nim              |  2 +-
 tests/enum/toverloadable_enums.nim | 48 ++++++++++++++++++++++++++++++
 tests/errmsgs/t10251.nim           |  2 +-
 10 files changed, 125 insertions(+), 11 deletions(-)
 create mode 100644 tests/enum/toverloadable_enums.nim

diff --git a/compiler/ast.nim b/compiler/ast.nim
index 0eeaa8133..5b54dd8ef 100644
--- a/compiler/ast.nim
+++ b/compiler/ast.nim
@@ -993,7 +993,7 @@ type
 
 const
   OverloadableSyms* = {skProc, skFunc, skMethod, skIterator,
-    skConverter, skModule, skTemplate, skMacro}
+    skConverter, skModule, skTemplate, skMacro, skEnumField}
 
   GenericTypes*: TTypeKinds = {tyGenericInvocation, tyGenericBody,
     tyGenericParam}
diff --git a/compiler/lookups.nim b/compiler/lookups.nim
index 44a30eefa..331eef525 100644
--- a/compiler/lookups.nim
+++ b/compiler/lookups.nim
@@ -177,13 +177,16 @@ iterator allSyms*(c: PContext): (PSym, int, bool) =
 
 proc someSymFromImportTable*(c: PContext; name: PIdent; ambiguous: var bool): PSym =
   var marked = initIntSet()
+  var symSet = OverloadableSyms
+  if overloadableEnums notin c.features:
+    symSet.excl skEnumField
   result = nil
   for im in c.imports.mitems:
     for s in symbols(im, marked, name, c.graph):
       if result == nil:
         result = s
       else:
-        if s.kind notin OverloadableSyms or result.kind notin OverloadableSyms:
+        if s.kind notin symSet or result.kind notin symSet:
           ambiguous = true
 
 proc searchInScopes*(c: PContext, s: PIdent; ambiguous: var bool): PSym =
@@ -384,7 +387,7 @@ proc mergeShadowScope*(c: PContext) =
   ##
   ## Merges:
   ## shadow -> shadow: add symbols to the parent but check for redefinitions etc
-  ## shadow -> non-shadow: the above, but also handle exports and all that 
+  ## shadow -> non-shadow: the above, but also handle exports and all that
   let shadowScope = c.currentScope
   c.rawCloseScope
   for sym in shadowScope.symbols:
diff --git a/compiler/options.nim b/compiler/options.nim
index f02f02256..e37dea1b6 100644
--- a/compiler/options.nim
+++ b/compiler/options.nim
@@ -203,7 +203,8 @@ type
     vmopsDanger,
     strictFuncs,
     views,
-    strictNotNil
+    strictNotNil,
+    overloadableEnums
 
   LegacyFeature* = enum
     allowSemcheckedAstModification,
diff --git a/compiler/sem.nim b/compiler/sem.nim
index 879f6efc9..804325e56 100644
--- a/compiler/sem.nim
+++ b/compiler/sem.nim
@@ -90,6 +90,12 @@ proc fitNode(c: PContext, formal: PType, arg: PNode; info: TLineInfo): PNode =
     # error correction:
     result = copyTree(arg)
     result.typ = formal
+  elif arg.kind in nkSymChoices and formal.skipTypes(abstractInst).kind == tyEnum:
+    # Pick the right 'sym' from the sym choice by looking at 'formal' type:
+    for ch in arg:
+      if sameType(ch.typ, formal):
+        return getConstExpr(c.module, ch, c.idgen, c.graph)
+    typeMismatch(c.config, info, formal, arg.typ, arg)
   else:
     result = indexTypesMatch(c, formal, arg.typ, arg)
     if result == nil:
@@ -356,6 +362,8 @@ proc semConstExpr(c: PContext, n: PNode): PNode =
   if e == nil:
     localError(c.config, n.info, errConstExprExpected)
     return n
+  if e.kind in nkSymChoices and e[0].typ.skipTypes(abstractInst).kind == tyEnum:
+    return e
   result = getConstExpr(c.module, e, c.idgen, c.graph)
   if result == nil:
     #if e.kind == nkEmpty: globalError(n.info, errConstExprExpected)
diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim
index 42c6b465e..f2fe54fad 100644
--- a/compiler/semexprs.nim
+++ b/compiler/semexprs.nim
@@ -2707,6 +2707,34 @@ proc getNilType(c: PContext): PType =
     result.align = c.config.target.ptrSize.int16
     c.nilTypeCache = result
 
+proc enumFieldSymChoice(c: PContext, n: PNode, s: PSym): PNode =
+  var o: TOverloadIter
+  var i = 0
+  var a = initOverloadIter(o, c, n)
+  while a != nil:
+    if a.kind in OverloadableSyms-{skModule}:
+      inc(i)
+      if i > 1: break
+    a = nextOverloadIter(o, c, n)
+  let info = getCallLineInfo(n)
+  if i <= 1:
+    if sfGenSym notin s.flags:
+      result = newSymNode(s, info)
+      markUsed(c, info, s)
+      onUse(info, s)
+    else:
+      result = n
+  else:
+    result = newNodeIT(nkClosedSymChoice, info, newTypeS(tyNone, c))
+    a = initOverloadIter(o, c, n)
+    while a != nil:
+      if a.kind in OverloadableSyms-{skModule}:
+        incl(a.flags, sfUsed)
+        markOwnerModuleAsUsed(c, a)
+        result.add newSymNode(a, info)
+        onUse(info, a)
+      a = nextOverloadIter(o, c, n)
+
 proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
   when defined(nimCompilerStacktraceHints):
     setFrameMsg c.config$n.info & " " & $n.kind
@@ -2730,7 +2758,8 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
         {checkUndeclared, checkModule, checkAmbiguity, checkPureEnumFields}
     var s = qualifiedLookUp(c, n, checks)
     if c.matchedConcept == nil: semCaptureSym(s, c.p.owner)
-    if s.kind in {skProc, skFunc, skMethod, skConverter, skIterator}:
+    case s.kind
+    of skProc, skFunc, skMethod, skConverter, skIterator:
       #performProcvarCheck(c, n, s)
       result = symChoice(c, n, s, scClosed)
       if result.kind == nkSym:
@@ -2740,6 +2769,11 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
       # "procs literals" are 'owned'
       if optOwnedRefs in c.config.globalOptions:
         result.typ = makeVarType(c, result.typ, tyOwned)
+    of skEnumField:
+      if overloadableEnums in c.features:
+        result = enumFieldSymChoice(c, n, s)
+      else:
+        result = semSym(c, n, s, flags)
     else:
       result = semSym(c, n, s, flags)
   of nkSym:
diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim
index 75323a95e..cc09291c5 100644
--- a/compiler/semstmts.nim
+++ b/compiler/semstmts.nim
@@ -490,6 +490,18 @@ proc semLowerLetVarCustomPragma(c: PContext, a: PNode, n: PNode): PNode =
     ret.add result
     result = semExprNoType(c, ret)
 
+proc errorSymChoiceUseQualifier(c: PContext; n: PNode) =
+  assert n.kind in nkSymChoices
+  var err = "ambiguous identifier: '" & $n[0] & "'"
+  var i = 0
+  for child in n:
+    let candidate = child.sym
+    if i == 0: err.add " -- use one of the following:\n"
+    else: err.add "\n"
+    err.add "  " & candidate.owner.name.s & "." & candidate.name.s
+    inc i
+  localError(c.config, n.info, errGenerated, err)
+
 proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
   if n.len == 1:
     result = semLowerLetVarCustomPragma(c, n[0], n)
@@ -514,7 +526,9 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
     if a[^1].kind != nkEmpty:
       def = semExprWithType(c, a[^1], {})
 
-      if def.kind == nkSym and def.sym.kind in {skTemplate, skMacro}:
+      if def.kind in nkSymChoices and def[0].typ.skipTypes(abstractInst).kind == tyEnum:
+        errorSymChoiceUseQualifier(c, def)
+      elif def.kind == nkSym and def.sym.kind in {skTemplate, skMacro}:
         typFlags.incl taIsTemplateOrMacro
       elif def.typ.kind == tyTypeDesc and c.p.owner.kind != skMacro:
         typFlags.incl taProcContextIsNotMacro
diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim
index 484997e3a..db895cab6 100644
--- a/compiler/semtypes.nim
+++ b/compiler/semtypes.nim
@@ -144,8 +144,13 @@ proc semEnum(c: PContext, n: PNode, prev: PType): PType =
     styleCheckDef(c.config, e)
     onDef(e.info, e)
     if sfGenSym notin e.flags:
-      if not isPure: addInterfaceDecl(c, e)
-      else: declarePureEnumField(c, e)
+      if not isPure:
+        if overloadableEnums in c.features:
+          addInterfaceOverloadableSymAt(c, c.currentScope, e)
+        else:
+          addInterfaceDecl(c, e)
+      else:
+        declarePureEnumField(c, e)
     if isPure and (let conflict = strTableInclReportConflict(symbols, e); conflict != nil):
       wrongRedefinition(c, e.info, e.name.s, conflict.info)
     inc(counter)
@@ -240,7 +245,8 @@ proc semRangeAux(c: PContext, n: PNode, prev: PType): PType =
 
   if not hasUnknownTypes:
     if not sameType(rangeT[0].skipTypes({tyRange}), rangeT[1].skipTypes({tyRange})):
-      localError(c.config, n.info, "type mismatch")
+      typeMismatch(c.config, n.info, rangeT[0], rangeT[1], n)
+
     elif not isOrdinalType(rangeT[0]) and rangeT[0].kind notin {tyFloat..tyFloat128} or
         rangeT[0].kind == tyBool:
       localError(c.config, n.info, "ordinal or float type expected")
diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim
index 6bc641bd6..674dd7e2e 100644
--- a/compiler/sigmatch.nim
+++ b/compiler/sigmatch.nim
@@ -2197,7 +2197,7 @@ proc paramTypesMatch*(m: var TCandidate, f, a: PType,
     var best = -1
     for i in 0.. nkSym
+
+proc x(p: int) = discard
+proc x(p: string) = discard
+
+proc takeCallback(param: proc(p: int)) = discard
+
+takeCallback x
+
+echo ord v
diff --git a/tests/errmsgs/t10251.nim b/tests/errmsgs/t10251.nim
index 0c7fe0b3d..19adf02eb 100644
--- a/tests/errmsgs/t10251.nim
+++ b/tests/errmsgs/t10251.nim
@@ -2,7 +2,6 @@ discard """
   action:reject
   cmd: "nim check $options $file"
   nimout: '''
-t10251.nim(15, 5) Error: redefinition of 'foo'; previous declaration here: t10251.nim(13, 5)
 t10251.nim(19, 23) Error: redefinition of 'goo1'; previous declaration here: t10251.nim(19, 11)
 '''
 """
@@ -14,6 +13,7 @@ type
   Enum2 = enum
     foo, bar, baz
 
+
 type
   Enum3 {.pure.} = enum # fixed (by accident?) in https://github.com/nim-lang/Nim/pull/18263
     goo0, goo1, goo2, goo1

From e616675c419552a85bdf0b47f47586852be8b5be Mon Sep 17 00:00:00 2001
From: Miran 
Date: Wed, 28 Jul 2021 14:31:13 +0200
Subject: [PATCH 1108/2472] various small documentation improvements (#18602)

---
 lib/pure/asyncdispatch.nim          | 34 ++++++++++-----------
 lib/pure/collections/deques.nim     | 13 ++++----
 lib/pure/collections/intsets.nim    |  2 +-
 lib/pure/collections/sharedlist.nim |  2 +-
 lib/pure/collections/tables.nim     | 20 ++++++++-----
 lib/pure/net.nim                    |  8 ++---
 lib/pure/strformat.nim              | 46 ++++++++++++++---------------
 lib/pure/strscans.nim               |  2 +-
 lib/std/enumutils.nim               | 31 +++++++++++++++----
 lib/std/packedsets.nim              |  8 ++---
 lib/system/threads.nim              |  2 +-
 lib/system_overview.rst             | 42 +++++++++++++-------------
 tests/stdlib/trst.nim               | 10 +++----
 13 files changed, 123 insertions(+), 97 deletions(-)

diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim
index d98ca77df..955e66f6b 100644
--- a/lib/pure/asyncdispatch.nim
+++ b/lib/pure/asyncdispatch.nim
@@ -42,12 +42,12 @@
 ##
 ## Code to read some data from a socket may look something like this:
 ##
-##   .. code-block:: Nim
-##      var future = socket.recv(100)
-##      future.addCallback(
-##        proc () =
-##          echo(future.read)
-##      )
+## .. code-block:: Nim
+##    var future = socket.recv(100)
+##    future.addCallback(
+##      proc () =
+##        echo(future.read)
+##    )
 ##
 ## All asynchronous functions returning a `Future` will not block. They
 ## will not however return immediately. An asynchronous function will have
@@ -104,20 +104,20 @@
 ## The most reliable way to handle exceptions is to use `yield` on a future
 ## then check the future's `failed` property. For example:
 ##
-##   .. code-block:: Nim
-##     var future = sock.recv(100)
-##     yield future
-##     if future.failed:
-##       # Handle exception
+## .. code-block:: Nim
+##   var future = sock.recv(100)
+##   yield future
+##   if future.failed:
+##     # Handle exception
 ##
 ## The `async` procedures also offer limited support for the try statement.
 ##
-##    .. code-block:: Nim
-##      try:
-##        let data = await sock.recv(100)
-##        echo("Received ", data)
-##      except:
-##        # Handle exception
+## .. code-block:: Nim
+##   try:
+##     let data = await sock.recv(100)
+##     echo("Received ", data)
+##   except:
+##     # Handle exception
 ##
 ## Unfortunately the semantics of the try statement may not always be correct,
 ## and occasionally the compilation may fail altogether.
diff --git a/lib/pure/collections/deques.nim b/lib/pure/collections/deques.nim
index fe9ceca1e..ecdf199d2 100644
--- a/lib/pure/collections/deques.nim
+++ b/lib/pure/collections/deques.nim
@@ -10,8 +10,9 @@
 ## An implementation of a `deque`:idx: (double-ended queue).
 ## The underlying implementation uses a `seq`.
 ##
-## Note that none of the procs that get an individual value from the deque should be used
-## on an empty deque.
+## .. note:: None of the procs that get an individual value from the deque should be used
+##   on an empty deque.
+##
 ## If compiled with the `boundChecks` option, those procs will raise an `IndexDefect`
 ## on such access. This should not be relied upon, as `-d:danger` or `--checks:off` will
 ## disable those checks and then the procs may return garbage or crash the program.
@@ -198,7 +199,7 @@ iterator items*[T](deq: Deque[T]): lent T =
   ## Yields every element of `deq`.
   ##
   ## **See also:**
-  ## * `mitems iterator <#mitems,Deque[T]>`_
+  ## * `mitems iterator <#mitems.i,Deque[T]>`_
   runnableExamples:
     from std/sequtils import toSeq
 
@@ -214,7 +215,7 @@ iterator mitems*[T](deq: var Deque[T]): var T =
   ## Yields every element of `deq`, which can be modified.
   ##
   ## **See also:**
-  ## * `items iterator <#items,Deque[T]>`_
+  ## * `items iterator <#items.i,Deque[T]>`_
   runnableExamples:
     var a = [10, 20, 30, 40, 50].toDeque
     assert $a == "[10, 20, 30, 40, 50]"
@@ -274,7 +275,7 @@ proc addFirst*[T](deq: var Deque[T], item: sink T) =
   ## Adds an `item` to the beginning of `deq`.
   ##
   ## **See also:**
-  ## * `addLast proc <#addLast,Deque[T],T>`_
+  ## * `addLast proc <#addLast,Deque[T],sinkT>`_
   runnableExamples:
     var a = initDeque[int]()
     for i in 1 .. 5:
@@ -290,7 +291,7 @@ proc addLast*[T](deq: var Deque[T], item: sink T) =
   ## Adds an `item` to the end of `deq`.
   ##
   ## **See also:**
-  ## * `addFirst proc <#addFirst,Deque[T],T>`_
+  ## * `addFirst proc <#addFirst,Deque[T],sinkT>`_
   runnableExamples:
     var a = initDeque[int]()
     for i in 1 .. 5:
diff --git a/lib/pure/collections/intsets.nim b/lib/pure/collections/intsets.nim
index 05e5addcc..765a23e97 100644
--- a/lib/pure/collections/intsets.nim
+++ b/lib/pure/collections/intsets.nim
@@ -8,7 +8,7 @@
 #
 
 ## Specialization of the generic `packedsets module `_
-## for ordinal sparse sets.
+## (see its documentation for more examples) for ordinal sparse sets.
 
 import std/private/since
 import std/packedsets
diff --git a/lib/pure/collections/sharedlist.nim b/lib/pure/collections/sharedlist.nim
index 79f2391f6..73e147e05 100644
--- a/lib/pure/collections/sharedlist.nim
+++ b/lib/pure/collections/sharedlist.nim
@@ -35,7 +35,7 @@ template withLock(t, x: untyped) =
   release(t.lock)
 
 proc iterAndMutate*[A](x: var SharedList[A]; action: proc(x: A): bool) =
-  ## Iterates over the list. If 'action' returns true, the
+  ## Iterates over the list. If `action` returns true, the
   ## current item is removed from the list.
   ##
   ## .. warning:: It may not preserve the element order after some modifications.
diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim
index 403345755..2695f3693 100644
--- a/lib/pure/collections/tables.nim
+++ b/lib/pure/collections/tables.nim
@@ -136,6 +136,8 @@ runnableExamples:
 ## a more complex object as a key you will be greeted by a strange compiler
 ## error:
 ##
+## .. code::
+##
 ##   Error: type mismatch: got (Person)
 ##   but expected one of:
 ##   hashes.hash(x: openArray[A]): Hash
@@ -324,7 +326,7 @@ proc `[]`*[A, B](t: Table[A, B], key: A): B =
   ##   a default value (e.g. zero for int) if the key doesn't exist
   ## * `getOrDefault proc<#getOrDefault,Table[A,B],A,B>`_ to return
   ##   a custom value if the key doesn't exist
-  ## * `[]= proc<#[]=,Table[A,B],A,B>`_ for inserting a new
+  ## * `[]= proc<#[]=,Table[A,B],A,sinkB>`_ for inserting a new
   ##   (key, value) pair in the table
   ## * `hasKey proc<#hasKey,Table[A,B],A>`_ for checking if a key is in
   ##   the table
@@ -345,7 +347,7 @@ proc `[]`*[A, B](t: var Table[A, B], key: A): var B =
   ##   a default value (e.g. zero for int) if the key doesn't exist
   ## * `getOrDefault proc<#getOrDefault,Table[A,B],A,B>`_ to return
   ##   a custom value if the key doesn't exist
-  ## * `[]= proc<#[]=,Table[A,B],A,B>`_ for inserting a new
+  ## * `[]= proc<#[]=,Table[A,B],A,sinkB>`_ for inserting a new
   ##   (key, value) pair in the table
   ## * `hasKey proc<#hasKey,Table[A,B],A>`_ for checking if a key is in
   ##   the table
@@ -488,7 +490,7 @@ proc add*[A, B](t: var Table[A, B], key: A, val: sink B) {.deprecated:
   ##
   ## **This can introduce duplicate keys into the table!**
   ##
-  ## Use `[]= proc<#[]=,Table[A,B],A,B>`_ for inserting a new
+  ## Use `[]= proc<#[]=,Table[A,B],A,sinkB>`_ for inserting a new
   ## (key, value) pair in the table without introducing duplicates.
   addImpl(enlarge)
 
@@ -499,7 +501,8 @@ template tabCellHash(i)  = t.data[i].hcode
 proc del*[A, B](t: var Table[A, B], key: A) =
   ## Deletes `key` from hash table `t`. Does nothing if the key does not exist.
   ##
-  ## .. warning:: If duplicate keys were added, this may need to be called multiple times.
+  ## .. warning:: If duplicate keys were added (via the now deprecated `add` proc),
+  ##   this may need to be called multiple times.
   ##
   ## See also:
   ## * `pop proc<#pop,Table[A,B],A,B>`_
@@ -519,7 +522,8 @@ proc pop*[A, B](t: var Table[A, B], key: A, val: var B): bool =
   ## mapping of the key. Otherwise, returns `false`, and the `val` is
   ## unchanged.
   ##
-  ## .. warning:: If duplicate keys were added, this may need to be called multiple times.
+  ## .. warning:: If duplicate keys were added (via the now deprecated `add` proc),
+  ##   this may need to be called multiple times.
   ##
   ## See also:
   ## * `del proc<#del,Table[A,B],A>`_
@@ -1028,7 +1032,8 @@ proc add*[A, B](t: TableRef[A, B], key: A, val: sink B) {.deprecated:
 proc del*[A, B](t: TableRef[A, B], key: A) =
   ## Deletes `key` from hash table `t`. Does nothing if the key does not exist.
   ##
-  ## **If duplicate keys were added, this may need to be called multiple times.**
+  ## .. warning:: If duplicate keys were added (via the now deprecated `add` proc),
+  ##   this may need to be called multiple times.
   ##
   ## See also:
   ## * `pop proc<#pop,TableRef[A,B],A,B>`_
@@ -1048,7 +1053,8 @@ proc pop*[A, B](t: TableRef[A, B], key: A, val: var B): bool =
   ## mapping of the key. Otherwise, returns `false`, and the `val` is
   ## unchanged.
   ##
-  ## **If duplicate keys were added, this may need to be called multiple times.**
+  ## .. warning:: If duplicate keys were added (via the now deprecated `add` proc),
+  ##   this may need to be called multiple times.
   ##
   ## See also:
   ## * `del proc<#del,TableRef[A,B],A>`_
diff --git a/lib/pure/net.nim b/lib/pure/net.nim
index c2e4810de..c0f9e1465 100644
--- a/lib/pure/net.nim
+++ b/lib/pure/net.nim
@@ -585,10 +585,10 @@ when defineSsl:
     ##
     ## CA certificates will be loaded, in the following order, from:
     ##
-    ##  - caFile, caDir, parameters, if set
-    ##  - if `verifyMode` is set to `CVerifyPeerUseEnvVars`,
-    ##    the SSL_CERT_FILE and SSL_CERT_DIR environment variables are used
-    ##  - a set of files and directories from the `ssl_certs `_ file.
+    ## - caFile, caDir, parameters, if set
+    ## - if `verifyMode` is set to `CVerifyPeerUseEnvVars`,
+    ##   the SSL_CERT_FILE and SSL_CERT_DIR environment variables are used
+    ## - a set of files and directories from the `ssl_certs `_ file.
     ##
     ## The last two parameters specify the certificate file path and the key file
     ## path, a server socket will most likely not work without these.
diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim
index 872141383..746f01ace 100644
--- a/lib/pure/strformat.nim
+++ b/lib/pure/strformat.nim
@@ -142,12 +142,12 @@ An expression like `&"{key} is {value:arg} {{z}}"` is transformed into:
   temp
 
 Parts of the string that are enclosed in the curly braces are interpreted
-as Nim code, to escape a `{` or `}`, double it.
+as Nim code. To escape a `{` or `}`, double it.
 
-Within a curly expression,however, '{','}', must be escaped with a backslash.
+Within a curly expression, however, `{`, `}`, must be escaped with a backslash.
 
-To enable evaluating Nim expressions within curlies, inside parentheses
-colons do not need to be escaped.
+To enable evaluating Nim expressions within curlies, colons inside parentheses
+do not need to be escaped.
 ]##
 
 runnableExamples:
@@ -177,28 +177,28 @@ The general form of a standard format specifier is::
 
 The square brackets `[]` indicate an optional element.
 
-The optional 'align' flag can be one of the following:
+The optional `align` flag can be one of the following:
 
-'<'
+`<`
     Forces the field to be left-aligned within the available
     space. (This is the default for strings.)
 
-'>'
+`>`
     Forces the field to be right-aligned within the available space.
     (This is the default for numbers.)
 
-'^'
+`^`
     Forces the field to be centered within the available space.
 
 Note that unless a minimum field width is defined, the field width
 will always be the same size as the data to fill it, so that the alignment
 option has no meaning in this case.
 
-The optional 'fill' character defines the character to be used to pad
+The optional `fill` character defines the character to be used to pad
 the field to the minimum width. The fill character, if present, must be
 followed by an alignment flag.
 
-The 'sign' option is only valid for numeric types, and can be one of the following:
+The `sign` option is only valid for numeric types, and can be one of the following:
 
 =================        ====================================================
   Sign                   Meaning
@@ -211,22 +211,22 @@ The 'sign' option is only valid for numeric types, and can be one of the followi
                          positive numbers.
 =================        ====================================================
 
-If the '#' character is present, integers use the 'alternate form' for formatting.
+If the `#` character is present, integers use the 'alternate form' for formatting.
 This means that binary, octal and hexadecimal output will be prefixed
-with '0b', '0o' and '0x', respectively.
+with `0b`, `0o` and `0x`, respectively.
 
-'width' is a decimal integer defining the minimum field width. If not specified,
+`width` is a decimal integer defining the minimum field width. If not specified,
 then the field width will be determined by the content.
 
-If the width field is preceded by a zero ('0') character, this enables
+If the width field is preceded by a zero (`0`) character, this enables
 zero-padding.
 
-The 'precision' is a decimal number indicating how many digits should be displayed
+The `precision` is a decimal number indicating how many digits should be displayed
 after the decimal point in a floating point conversion. For non-numeric types the
 field indicates the maximum field size - in other words, how many characters will
 be used from the field content. The precision is ignored for integer conversions.
 
-Finally, the 'type' determines how the data should be presented.
+Finally, the `type` determines how the data should be presented.
 
 The available integer presentation types are:
 
@@ -240,7 +240,7 @@ The available integer presentation types are:
                          lower-case letters for the digits above 9.
 `X`                      Hex format. Outputs the number in base 16, using
                          uppercase letters for the digits above 9.
-(None)                   The same as 'd'.
+(None)                   The same as `d`.
 =================        ====================================================
 
 The available floating point presentation types are:
@@ -249,21 +249,21 @@ The available floating point presentation types are:
   Type                   Result
 =================        ====================================================
 `e`                      Exponent notation. Prints the number in scientific
-                         notation using the letter 'e' to indicate the
+                         notation using the letter `e` to indicate the
                          exponent.
-`E`                      Exponent notation. Same as 'e' except it converts
+`E`                      Exponent notation. Same as `e` except it converts
                          the number to uppercase.
 `f`                      Fixed point. Displays the number as a fixed-point
                          number.
-`F`                      Fixed point. Same as 'f' except it converts the
+`F`                      Fixed point. Same as `f` except it converts the
                          number to uppercase.
 `g`                      General format. This prints the number as a
                          fixed-point number, unless the number is too
-                         large, in which case it switches to 'e'
+                         large, in which case it switches to `e`
                          exponent notation.
-`G`                      General format. Same as 'g' except it switches to 'E'
+`G`                      General format. Same as `g` except it switches to `E`
                          if the number gets to large.
-(None)                   Similar to 'g', except that it prints at least one
+(None)                   Similar to `g`, except that it prints at least one
                          digit after the decimal point.
 =================        ====================================================
 
diff --git a/lib/pure/strscans.nim b/lib/pure/strscans.nim
index 73b53e3d6..291af7408 100644
--- a/lib/pure/strscans.nim
+++ b/lib/pure/strscans.nim
@@ -52,7 +52,7 @@ substrings starting with ``$``. These constructions are available:
 =================   ========================================================
 
 Even though ``$*`` and ``$+`` look similar to the regular expressions ``.*``
-and ``.+`` they work quite differently, there is no non-deterministic
+and ``.+``, they work quite differently. There is no non-deterministic
 state machine involved and the matches are non-greedy. ``[$*]``
 matches ``[xyz]`` via ``parseutils.parseUntil``.
 
diff --git a/lib/std/enumutils.nim b/lib/std/enumutils.nim
index 09cf24f51..6269950c7 100644
--- a/lib/std/enumutils.nim
+++ b/lib/std/enumutils.nim
@@ -79,8 +79,14 @@ macro enumNames(a: typed): untyped =
 iterator items*[T: HoleyEnum](E: typedesc[T]): T =
   ## Iterates over an enum with holes.
   runnableExamples:
-    type A = enum a0 = 2, a1 = 4, a2
-    type B[T] = enum b0 = 2, b1 = 4
+    type
+      A = enum
+        a0 = 2
+        a1 = 4
+        a2
+      B[T] = enum
+        b0 = 2
+        b1 = 4
     from std/sequtils import toSeq
     assert A.toSeq == [a0, a1, a2]
     assert B[float].toSeq == [B[float].b0, B[float].b1]
@@ -130,9 +136,19 @@ template symbolRank*[T: enum](a: T): int =
   ## for small enums, otherwise is `O(T.enumLen)`.
   runnableExamples:
     type
-      A = enum a0 = -3, a1 = 10, a2, a3 = (20, "f3Alt") # HoleyEnum
-      B = enum b0, b1, b2 # OrdinalEnum
-      C = enum c0 = 10, c1, c2 # OrdinalEnum
+      A = enum # HoleyEnum
+        a0 = -3
+        a1 = 10
+        a2
+        a3 = (20, "f3Alt")
+      B = enum # OrdinalEnum
+        b0
+        b1
+        b2
+      C = enum # OrdinalEnum
+        c0 = 10
+        c1
+        c2
     assert a2.symbolRank == 2
     assert b2.symbolRank == 2
     assert c2.symbolRank == 2
@@ -156,7 +172,10 @@ func symbolName*[T: enum](a: T): string =
     assert b.symbolName == "b0"
     assert $b == "kb0"
     static: assert B.high.symbolName == "b2"
-    type C = enum c0 = -3, c1 = 4, c2 = 20 # HoleyEnum
+    type C = enum # HoleyEnum
+      c0 = -3
+      c1 = 4
+      c2 = 20
     assert c1.symbolName == "c1"
   const names = enumNames(T)
   names[a.symbolRank]
diff --git a/lib/std/packedsets.nim b/lib/std/packedsets.nim
index e209d2aae..b2ee917eb 100644
--- a/lib/std/packedsets.nim
+++ b/lib/std/packedsets.nim
@@ -12,10 +12,10 @@
 ##
 ## Supports any Ordinal type.
 ##
-## **Note**: Currently the assignment operator `=` for `PackedSet[A]`
-## performs some rather meaningless shallow copy. Since Nim currently does
-## not allow the assignment operator to be overloaded, use the `assign proc
-## <#assign,PackedSet[A],PackedSet[A]>`_ to get a deep copy.
+## .. note:: Currently the assignment operator `=` for `PackedSet[A]`
+##   performs some rather meaningless shallow copy. Since Nim currently does
+##   not allow the assignment operator to be overloaded, use the `assign proc
+##   <#assign,PackedSet[A],PackedSet[A]>`_ to get a deep copy.
 ##
 ## See also
 ## ========
diff --git a/lib/system/threads.nim b/lib/system/threads.nim
index 2b5eb07fd..a9006b0c5 100644
--- a/lib/system/threads.nim
+++ b/lib/system/threads.nim
@@ -11,7 +11,7 @@
 ##
 ## **Note**: This is part of the system module. Do not import it directly.
 ## To activate thread support you need to compile
-## with the `--threads:on` command line switch.
+## with the `--threads:on`:option: command line switch.
 ##
 ## Nim's memory model for threads is quite different from other common
 ## programming languages (C, Pascal): Each thread has its own
diff --git a/lib/system_overview.rst b/lib/system_overview.rst
index d6cbe1a35..df22697e0 100644
--- a/lib/system_overview.rst
+++ b/lib/system_overview.rst
@@ -48,27 +48,27 @@ Proc                              Usage
 Seqs
 ----
 
-==============================================     ==========================================
-Proc                                               Usage
-==============================================     ==========================================
-`newSeq<#newSeq>`_                                 Create a new sequence of a given length
-`newSeqOfCap<#newSeqOfCap,Natural>`_               Create a new sequence with zero length
-                                                   and a given capacity
-`setLen<#setLen,seq[T],Natural>`_                  Set the length of a sequence
-`len<#len,seq[T]>`_                                Return the length of a sequence
-`@<#@,openArray[T]>`_                              Turn an array into a sequence
-`add<#add,seq[T],sinkT>`_                          Add an item to the sequence
-`insert<#insert,seq[T],sinkT>`_                    Insert an item at a specific position
-`delete<#delete,seq[T],Natural>`_                  Delete an item while preserving the
-                                                   order of elements (`O(n)` operation)
-`del<#del,seq[T],Natural>`_                        `O(1)` removal, doesn't preserve the order
-`pop<#pop,seq[T]>`_                                Remove and return last item of a sequence
-`x & y<#&,seq[T],seq[T]>`_                         Concatenate two sequences
-`x[a .. b]<#[],openArray[T],HSlice[U,V]>`_         Slice of a sequence (both ends included)
-`x[a .. ^b]<#[],openArray[T],HSlice[U,V]>`_        Slice of a sequence but `b` is a 
-                                                   reversed index (both ends included)
-`x[a ..\< b]<#[],openArray[T],HSlice[U,V]>`_       Slice of a sequence (excluded upper bound)
-==============================================     ==========================================
+=============================================================  ==========================================
+Proc                                                           Usage
+=============================================================  ==========================================
+`newSeq<#newSeq>`_                                             Create a new sequence of a given length
+`newSeqOfCap<#newSeqOfCap,Natural>`_                           Create a new sequence with zero length
+                                                               and a given capacity
+`setLen<#setLen,seq[T],Natural>`_                              Set the length of a sequence
+`len<#len,seq[T]>`_                                            Return the length of a sequence
+`@<#@,openArray[T]>`_                                          Turn an array into a sequence
+`add<#add,seq[T],sinkT>`_                                      Add an item to the sequence
+`insert<#insert,seq[T],sinkT>`_                                Insert an item at a specific position
+`delete<#delete,seq[T],Natural>`_                              Delete an item while preserving the
+                                                               order of elements (`O(n)` operation)
+`del<#del,seq[T],Natural>`_                                    `O(1)` removal, doesn't preserve the order
+`pop<#pop,seq[T]>`_                                            Remove and return last item of a sequence
+`x & y<#&,seq[T],seq[T]>`_                                     Concatenate two sequences
+`x[a .. b]<#[],openArray[T],HSlice[U: Ordinal,V: Ordinal]>`_   Slice of a sequence (both ends included)
+`x[a .. ^b]<#[],openArray[T],HSlice[U: Ordinal,V: Ordinal]>`_  Slice of a sequence but `b` is a 
+                                                               reversed index (both ends included)
+`x[a ..< b]<#[],openArray[T],HSlice[U: Ordinal,V: Ordinal]>`_  Slice of a sequence (excluded upper bound)
+=============================================================  ==========================================
 
 **See also:**
 * `sequtils module `_ for operations on container
diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim
index fc3ccbf4e..bc11f219c 100644
--- a/tests/stdlib/trst.nim
+++ b/tests/stdlib/trst.nim
@@ -182,9 +182,9 @@ suite "RST parsing":
   test "using .. as separator b/w directives and block quotes":
     check(dedent"""
         .. note:: someNote
-    
+
         ..
-    
+
           someBlockQuote""".toAst ==
       dedent"""
         rnInner
@@ -300,14 +300,14 @@ suite "RST indentation":
     let input1 = dedent"""
       .. code-block:: nim
           :test: "nim c $1"
-      
+
         template additive(typ: typedesc) =
           discard
       """
     let input2 = dedent"""
       .. code-block:: nim
         :test: "nim c $1"
-      
+
         template additive(typ: typedesc) =
           discard
       """
@@ -320,7 +320,7 @@ suite "RST indentation":
     let inputWrong = dedent"""
       .. code-block:: nim
        :test: "nim c $1"
-      
+
          template additive(typ: typedesc) =
            discard
       """

From 7d3c3e00efe777dcbda15f2125fa77aa7d17cad4 Mon Sep 17 00:00:00 2001
From: Vindaar 
Date: Thu, 29 Jul 2021 07:47:34 +0200
Subject: [PATCH 1109/2472] Allow `nnkAccQuoted` in `genEnumCaseStmt` (#18606)

* [enumutils] provide node kind for `Invalid node type` error

* [enumutils] add support for nnkAccQuoted in `genEnumCaseStmt`

For reasons unknown to me, when running `nim doc` on a file that uses
`parseEnum` with an enum that contains accented quotes errors at CT
with the `Invalid node for type` error. Further errors are raised,
probably because the enum parsing fails?
---
 lib/std/enumutils.nim | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/lib/std/enumutils.nim b/lib/std/enumutils.nim
index 6269950c7..81e602ad5 100644
--- a/lib/std/enumutils.nim
+++ b/lib/std/enumutils.nim
@@ -36,6 +36,10 @@ macro genEnumCaseStmt*(typ: typedesc, argSym: typed, default: typed,
     case f.kind
     of nnkEmpty: continue # skip first node of `enumTy`
     of nnkSym, nnkIdent: fStr = f.strVal
+    of nnkAccQuoted:
+      fStr = ""
+      for ch in f:
+        fStr.add ch.strVal
     of nnkEnumFieldDef:
       case f[1].kind
       of nnkStrLit: fStr = f[1].strVal
@@ -46,7 +50,7 @@ macro genEnumCaseStmt*(typ: typedesc, argSym: typed, default: typed,
         fStr = f[0].strVal
         fNum = f[1].intVal
       else: error("Invalid tuple syntax!", f[1])
-    else: error("Invalid node for enum type!", f)
+    else: error("Invalid node for enum type `" & $f.kind & "`!", f)
     # add field if string not already added
     if fNum >= userMin and fNum <= userMax:
       fStr = normalizer(fStr)

From 1bba641fed99bce61de2470c8bf73cd34480d907 Mon Sep 17 00:00:00 2001
From: konsumlamm <44230978+konsumlamm@users.noreply.github.com>
Date: Thu, 29 Jul 2021 10:50:17 +0200
Subject: [PATCH 1110/2472] Update `lib.rst` (#18605)

* Update lib.rst

* Remove "Unstable" category

Add `strbasics`

* Update doc/lib.rst

* Update doc/lib.rst

Co-authored-by: Andreas Rumpf 
---
 doc/lib.rst | 44 ++++++++++++++++++++------------------------
 1 file changed, 20 insertions(+), 24 deletions(-)

diff --git a/doc/lib.rst b/doc/lib.rst
index 988fbf6f0..54728d681 100644
--- a/doc/lib.rst
+++ b/doc/lib.rst
@@ -41,7 +41,7 @@ Automatic imports
   Basic Nim thread support. **Note:** This is part of the system module. Do not
   import it explicitly. Enabled with `--threads:on`:option:.
 
-* `channels `_
+* `channels_builtin `_
   Nim message passing support for threads. **Note:** This is part of the
   system module. Do not import it explicitly. Enabled with `--threads:on`:option:.
 
@@ -49,6 +49,9 @@ Automatic imports
 Core
 ----
 
+* `atomics `_
+  Types and operations for atomic operations and lockless algorithms.
+
 * `bitops `_
   Provides a series of low-level methods for bit manipulation.
 
@@ -66,7 +69,7 @@ Core
 
 * `macrocache `_
   Provides an API for macros to collect compile-time information across modules.
-  
+
 * `macros `_
   Contains the AST API and documentation of Nim for writing macros.
 
@@ -113,7 +116,7 @@ Collections
   The underlying implementation uses a `seq`.
 
 * `heapqueue `_
-  Implementation of a heap data structure that can be used as a priority queue.
+  Implementation of a binary heap data structure that can be used as a priority queue.
 
 * `intsets `_
   Efficient implementation of a set of ints as a sparse bit set.
@@ -129,7 +132,7 @@ Collections
   Efficient implementation of a set of ordinals as a sparse bit set.
 
 * `sets `_
-  Nim hash and bit set support.
+  Nim hash set support.
 
 * `sharedlist `_
   Nim shared linked list support. Contains a shared singly-linked list.
@@ -167,11 +170,14 @@ String handling
 * `ropes `_
   This module contains support for a *rope* data type.
   Ropes can represent very long strings efficiently;
-  especially concatenation is done in O(1) instead of O(n).
+  in particular, concatenation is done in O(1) instead of O(n).
+
+* `strbasics `_
+  This module provides some high performance string operations.
 
 * `strformat `_
   Macro based standard string interpolation/formatting. Inspired by
-  Python's `f`-strings.
+  Python's f-strings.
 
 * `strmisc `_
   This module contains uncommon string handling operations that do not
@@ -266,9 +272,6 @@ Math libraries
 * `math `_
   Mathematical operations like cosine, square root.
 
-* `mersenne `_
-  Mersenne twister random number generator.
-
 * `random `_
   Fast and tiny random number generator.
 
@@ -433,7 +436,7 @@ Hashing
 -------
 
 * `base64 `_
-  This module implements a base64 encoder and decoder.
+  This module implements a Base64 encoder and decoder.
 
 * `hashes `_
   This module implements efficient computations of hash values for diverse
@@ -446,7 +449,7 @@ Hashing
   An OID is a global ID that consists of a timestamp,
   a unique counter, and a random value. This combination should suffice to
   produce a globally distributed unique ID. This implementation was extracted
-  from the Mongodb interface and it thus binary compatible with a Mongo OID.
+  from the MongoDB interface and it thus binary compatible with a MongoDB OID.
 
 * `sha1 `_
   This module implements a sha1 encoder and decoder.
@@ -487,9 +490,8 @@ Miscellaneous
   This module implements the `with` macro for easy function chaining.
 
 
-
-Modules for JS backend
-----------------------
+Modules for the JS backend
+--------------------------
 
 * `asyncjs `_
   Types and macros for writing asynchronous procedures in JavaScript.
@@ -497,6 +499,9 @@ Modules for JS backend
 * `dom `_
   Declaration of the Document Object Model for the JS backend.
 
+* `jsbigints `_
+  Arbitrary precision integers.
+
 * `jsconsole `_
   Wrapper for the `console` object.
 
@@ -519,7 +524,6 @@ Regular expressions
   expressions. The current implementation uses PCRE.
 
 
-
 Database support
 ----------------
 
@@ -583,7 +587,7 @@ Database support
 * `mysql `_
   Contains a wrapper for the mySQL API.
 * `sqlite3 `_
-  Contains a wrapper for SQLite 3 API.
+  Contains a wrapper for the SQLite 3 API.
 * `odbcsql `_
   interface to the ODBC driver.
 
@@ -593,11 +597,3 @@ Network Programming and Internet Protocols
 
 * `openssl `_
   Wrapper for OpenSSL.
-
-
-
-Unstable
-========
-
-* `atomics `_
-  Types and operations for atomic operations and lockless algorithms.

From bbe05c15326c7a62d15c0fe47d1c6e1be3ab11a8 Mon Sep 17 00:00:00 2001
From: Andrey Makarov 
Date: Thu, 29 Jul 2021 14:30:19 +0300
Subject: [PATCH 1111/2472] docgen: draw frame around active anchors (#18607)

---
 config/nimdoc.cfg                             |   3 +-
 doc/nimdoc.css                                |   6 +
 .../expected/index.html                       |   3 +-
 nimdoc/testproject/expected/nimdoc.out.css    |   6 +
 .../expected/subdir/subdir_b/utils.html       |  42 +++--
 nimdoc/testproject/expected/testproject.html  | 177 ++++++++++++------
 6 files changed, 162 insertions(+), 75 deletions(-)

diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg
index 15f887023..162b2b4a3 100644
--- a/config/nimdoc.cfg
+++ b/config/nimdoc.cfg
@@ -46,13 +46,14 @@ doc.section.toc2 = """
 # * $seeSrc: generated HTML from doc.item.seesrc (if some switches are used).
 
 doc.item = """
-
+
                              
 
                              $header
                              
 
                              
 $deprecationMsg
 $desc
 $seeSrc
 
                              
+
                              
 """
 
 # Chunk of HTML emitted for each entry in the HTML table of contents.
diff --git a/doc/nimdoc.css b/doc/nimdoc.css
index 5c2bf2093..4abea9ce0 100644
--- a/doc/nimdoc.css
+++ b/doc/nimdoc.css
@@ -234,6 +234,12 @@ select:focus {
 }
 
 /* Docgen styles */
+
+:target {
+  border: 2px solid #B5651D;
+  border-style: dotted;
+}
+
 /* Links */
 a {
   color: var(--anchor);
diff --git a/nimdoc/test_out_index_dot_html/expected/index.html b/nimdoc/test_out_index_dot_html/expected/index.html
index fac758c9b..3499b5326 100644
--- a/nimdoc/test_out_index_dot_html/expected/index.html
+++ b/nimdoc/test_out_index_dot_html/expected/index.html
@@ -121,13 +121,14 @@ window.addEventListener('DOMContentLoaded', main);
   
                              
 
                              Procs
                              
 
                              
-
+
                              
 
                              proc foo() {....raises: [], tags: [].}
                              
 
                              
 
 I do foo
 
 
                              
+
                              
 
 
                              
 
diff --git a/nimdoc/testproject/expected/nimdoc.out.css b/nimdoc/testproject/expected/nimdoc.out.css
index 5c2bf2093..4abea9ce0 100644
--- a/nimdoc/testproject/expected/nimdoc.out.css
+++ b/nimdoc/testproject/expected/nimdoc.out.css
@@ -234,6 +234,12 @@ select:focus {
 }
 
 /* Docgen styles */
+
+:target {
+  border: 2px solid #B5651D;
+  border-style: dotted;
+}
+
 /* Links */
 a {
   color: var(--anchor);
diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html
index 533cb5b9d..47901ce0d 100644
--- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html
+++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html
@@ -213,7 +213,7 @@ window.addEventListener('DOMContentLoaded', main);
   
                              
 
                              Types
                              
 
                              
-
+
                              
 
                              SomeType = enum
   enumValueA, enumValueB, enumValueC
                              
 
                              
@@ -221,101 +221,114 @@ window.addEventListener('DOMContentLoaded', main);
 
 
 
                              
+
                              
 
 
                              
 
                              
 
                              Procs
                              
 
                              
-
+
                              
 
                              proc fn2() {....raises: [], tags: [].}
                              
 
                              
 
 comment
 
 
                              
-
+
                              
+
                              
 
                              proc fn3(): auto {....raises: [], tags: [].}
                              
 
                              
 
 comment
 
 
                              
-
+
                              
+
                              
 
                              proc fn4(): auto {....raises: [], tags: [].}
                              
 
                              
 
 comment
 
 
                              
-
+
                              
+
                              
 
                              proc fn5() {....raises: [], tags: [].}
                              
 
                              
 
 comment
 
 
                              
-
+
                              
+
                              
 
                              proc fn6() {....raises: [], tags: [].}
                              
 
                              
 
 comment
 
 
                              
-
+
                              
+
                              
 
                              proc fn7() {....raises: [], tags: [].}
                              
 
                              
 
 comment
 
 
                              
-
+
                              
+
                              
 
                              proc fn8(): auto {....raises: [], tags: [].}
                              
 
                              
 
 comment
 
 
                              
-
+
                              
+
                              
 
                              func fn9(a: int): int {....raises: [], tags: [].}
                              
 
                              
 
 comment
 
 
                              
-
+
                              
+
                              
 
                              func fn10(a: int): int {....raises: [], tags: [].}
                              
 
                              
 
 comment
 
 
                              
-
+
                              
+
                              
 
                              proc someType(): SomeType {....raises: [], tags: [].}
                              
 
                              
 
 constructor.
 
 
                              
+
                              
 
 
                              
 
                              
 
                              Templates
                              
 
                              
-
+
                              
 
                              template aEnum(): untyped
                              
 
                              
 
 
 
 
                              
-
+
                              
+
                              
 
                              template bEnum(): untyped
                              
 
                              
 
 
 
 
                              
-
+
                              
+
                              
 
                              template fromUtilsGen(): untyped
                              
 
                              
 
@@ -324,6 +337,7 @@ should be shown in utils.html only
 
                              discard "should be in utils.html only, not in module that calls fromUtilsGen"
                              ditto
 
 
                              
+
                              
 
 
                              
 
diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html
index 0f6289bbc..d98b1b00f 100644
--- a/nimdoc/testproject/expected/testproject.html
+++ b/nimdoc/testproject/expected/testproject.html
@@ -447,7 +447,7 @@ window.addEventListener('DOMContentLoaded', main);
 
                              
 
                              Types
                              
 
                              
-
+
                              
 
                              A {.inject.} = enum
   aA
                              
 
                              
@@ -455,7 +455,8 @@ window.addEventListener('DOMContentLoaded', main);
 The enum A.
 
 
                              
-
+
                              
+
                              
 
                              B {.inject.} = enum
   bB
                              
 
                              
@@ -463,7 +464,8 @@ The enum A.
 The enum B.
 
 
                              
-
+
                              
+
                              
 
                              Foo = enum
   enumValueA2
                              
 
                              
@@ -471,7 +473,8 @@ The enum B.
 
 
 
                              
-
+
                              
+
                              
 
                              FooBuzz {....deprecated: "FooBuzz msg".} = int
                              
 
                              
   
                              
@@ -481,7 +484,8 @@ The enum B.
 
 
 
                              
-
+
                              
+
                              
 
                              Shapes = enum
   Circle,                   ## A circle
   Triangle,                 ## A three-sided shape
@@ -491,64 +495,71 @@ The enum B.
 Some shapes.
 
 
                            
+
 
 
 
                            
 
                            Vars
                            
 
                            
-
+
                            
 
                            aVariable: array[1, int]
                            
 
                            
 
 
 
 
                            
-
+
                            
+
                            
 
                            someVariable: bool
                            
 
                            
 
 This should be visible.
 
 
                            
+
                            
 
 
                            
 
                            
 
                            Consts
                            
 
                            
-
+
                            
 
                            C_A = 0x7FF0000000000000'f64
                            
 
                            
 
 
 
 
                            
-
+
                            
+
                            
 
                            C_B = 0o377'i8
                            
 
                            
 
 
 
 
                            
-
+
                            
+
                            
 
                            C_C = 0o277'i8
                            
 
                            
 
 
 
 
                            
-
+
                            
+
                            
 
                            C_D = 0o177777'i16
                            
 
                            
 
 
 
 
                            
+
                            
 
 
                            
 
                            
 
                            Procs
                            
 
                            
-
+
                            
 
                            proc addfBug14485() {....raises: [], tags: [].}
                            
 
                            
 
@@ -567,14 +578,16 @@ Some proc
 ]#
    • 
 
 
-
+
+
    
 
    proc anything() {....raises: [], tags: [].}
    
 
    
 
 There is no block quote after blank lines at the beginning.
 
 
    
-
+
    
+
    
 
    proc asyncFun1(): Future[int] {....raises: [Exception, ValueError],
                                 tags: [RootEffect].}
    
 
    
@@ -582,14 +595,16 @@ There is no block quote after blank lines at the beginning.
 ok1
 
 
    
-
+
    
+
    
 
    proc asyncFun2(): owned(Future[void]) {....raises: [Exception], tags: [RootEffect].}
    
 
    
 
 
 
 
    
-
+
    
+
    
 
    proc asyncFun3(): owned(Future[void]) {....raises: [Exception], tags: [RootEffect].}
    
 
    
 
@@ -598,21 +613,24 @@ ok1
 
    discard
    ok1
 
 
    
-
+
    
+
    
 
    proc bar[T](a, b: T): T
    
 
    
 
 
 
 
    
-
+
    
+
    
 
    proc baz() {....raises: [], tags: [].}
    
 
    
 
 
 
 
    
-
+
    
+
    
 
    proc baz[T](a, b: T): T {....deprecated.}
    
 
    
   
    
@@ -622,7 +640,8 @@ ok1
 This is deprecated without message.
 
 
    
-
+
    
+
    
 
    proc buzz[T](a, b: T): T {....deprecated: "since v0.20".}
    
 
    
   
    
@@ -632,7 +651,8 @@ This is deprecated without message.
 This is deprecated with a message.
 
 
    
-
+
    
+
    
 
    proc c_nonexistent(frmt: cstring): cint {.importc: "nonexistent",
     header: "<stdio.h>", varargs, discardable.}
    
 
    
@@ -640,7 +660,8 @@ This is deprecated with a message.
 
 
 
    
-
+
    
+
    
 
    proc c_printf(frmt: cstring): cint {.importc: "printf", header: "<stdio.h>",
                                      varargs, discardable.}
    
 
    
@@ -648,7 +669,8 @@ This is deprecated with a message.
 the c printf. etc.
 
 
    
-
+
    
+
    
 
    proc fromUtils3() {....raises: [], tags: [].}
    
 
    
 
@@ -658,14 +680,16 @@ came form utils but should be shown where 
    
 
 
-
+
+
    
 
    proc isValid[T](x: T): bool
    
 
    
 
 
 
 
    
-
+
    
+
    
 
    proc low2[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect.}
    
 
    
 
@@ -678,7 +702,8 @@ came form utils but should be shown where discard "in low2"
    - + +
    proc low[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect.}
    @@ -689,7 +714,8 @@ came form utils but should be shown where low(2) # => -9223372036854775808
    - +
    +
    proc p1() {....raises: [], tags: [].}
    @@ -712,14 +738,16 @@ this is a nested doc comment # also work after
    - +
    +
    func someFunc() {....raises: [], tags: [].}
    My someFunc. Stuff in quotes here. Some link
    - +
    +
    proc tripleStrLitTest() {....raises: [], tags: [].}
    @@ -762,14 +790,16 @@ at indent 0 # should be in
    - +
    +
    proc z1(): Foo {....raises: [], tags: [].}
    cz1
    - +
    +
    proc z2() {....raises: [], tags: [].}
    @@ -778,49 +808,56 @@ cz2 
    discard "in cz2"
    - +
    +
    proc z3() {....raises: [], tags: [].}
    cz3
    - +
    +
    proc z4() {....raises: [], tags: [].}
    cz4
    - +
    +
    proc z5(): int {....raises: [], tags: [].}
    cz5
    - +
    +
    proc z6(): int {....raises: [], tags: [].}
    cz6
    - +
    +
    proc z7(): int {....raises: [], tags: [].}
    cz7
    - +
    +
    proc z8(): int {....raises: [], tags: [].}
    cz8
    - +
    +
    proc z9() {....raises: [], tags: [].}
    @@ -829,7 +866,8 @@ cz8 
    doAssert 1 + 1 == 2
    - +
    +
    proc z10() {....raises: [], tags: [].}
    @@ -838,7 +876,8 @@ cz8 
    discard 1
    cz10
    - +
    +
    proc z11() {....raises: [], tags: [].}
    @@ -847,7 +886,8 @@ cz8 
    discard 1
    - +
    +
    proc z12(): int {....raises: [], tags: [].}
    @@ -856,7 +896,8 @@ cz8 
    discard 1
    - +
    +
    proc z13() {....raises: [], tags: [].}
    @@ -865,7 +906,8 @@ cz13 
    discard
    - +
    +
    proc z17() {....raises: [], tags: [].}
    @@ -874,38 +916,42 @@ cz17 rest 
    discard 1
    rest
    +

    Methods

    - +
    method method1(self: Moo) {.base, ...raises: [], tags: [].}
    foo1
    - +
    +
    method method2(self: Moo): int {.base, ...raises: [], tags: [].}
    foo2
    - +
    +
    method method3(self: Moo): int {.base, ...raises: [], tags: [].}
    foo3
    +

    Iterators

    - +
    iterator fromUtils1(): int {....raises: [], tags: [].}
    @@ -916,14 +962,16 @@ foo3 # ok2
    - +
    +
    iterator iter1(n: int): int {....raises: [], tags: [].}
    foo1
    - +
    +
    iterator iter2(n: int): int {....raises: [], tags: [].}
    @@ -932,19 +980,21 @@ foo2 
    discard # bar
    +

    Macros

    - +
    macro bar(): untyped
    - +
    +
    macro z16()
    @@ -955,26 +1005,29 @@ foo2 
    doAssert 2 == 1 + 1
    - +
    +
    macro z18(): int
    cz18
    +

    Templates

    - +
    template foo(a, b: SomeType)
    This does nothing
    - +
    +
    template fromUtils2()
    @@ -984,7 +1037,8 @@ ok3 in module calling fromUtilsGen"""
    - +
    +
    template myfn()
    @@ -1006,7 +1060,8 @@ bar # should be inshould be still in
    - +
    +
    template testNimDocTrailingExample()
    @@ -1015,14 +1070,16 @@ bar 
    discard 2
    - +
    +
    template z6t(): int
    cz6t
    - +
    +
    template z14()
    @@ -1031,7 +1088,8 @@ cz14 
    discard
    - +
    +
    template z15()
    @@ -1048,6 +1106,7 @@ cz15 
    discard 1
    in or out?
    +
    From 6b3c77e7f4421267c551b1cd84717a19c65be2c5 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 29 Jul 2021 14:05:26 -0700 Subject: [PATCH 1112/2472] Remove tracking of environment from osenv.nim v2 (#18575) * Remove unnecessary environment tracking * try to fix windows * fix delEnv * make putEnv work on windows even with empty values; improve tests: add tests, add js, vm testing * [skip ci] fix changelog Co-authored-by: Caden Haustein --- changelog.md | 3 + lib/pure/includes/osenv.nim | 237 +++++++++----------------- lib/std/private/win_setenv.nim | 92 ++++++++++ tests/destructor/tnewruntime_misc.nim | 2 +- tests/stdlib/tos.nim | 34 +--- tests/stdlib/tosenv.nim | 58 +++++++ 6 files changed, 240 insertions(+), 186 deletions(-) create mode 100644 lib/std/private/win_setenv.nim create mode 100644 tests/stdlib/tosenv.nim diff --git a/changelog.md b/changelog.md index be94f92c4..a19697c8c 100644 --- a/changelog.md +++ b/changelog.md @@ -97,11 +97,14 @@ The downside is that these defines now have custom logic that doesn't apply for other defines. +- `std/os`: `putEnv` now raises if the 1st argument contains a `=` + - Renamed `-d:nimCompilerStackraceHints` to `-d:nimCompilerStacktraceHints`. - In `std/dom`, `Interval` is now a `ref object`, same as `Timeout`. Definitions of `setTimeout`, `clearTimeout`, `setInterval`, `clearInterval` were updated. + ## Standard library additions and changes - `strformat`: diff --git a/lib/pure/includes/osenv.nim b/lib/pure/includes/osenv.nim index db291c5fd..6aaafbfda 100644 --- a/lib/pure/includes/osenv.nim +++ b/lib/pure/includes/osenv.nim @@ -40,112 +40,15 @@ when defined(nodejs): # {.error: "requires -d:nodejs".} else: - when defined(windows): - from parseutils import skipIgnoreCase proc c_getenv(env: cstring): cstring {. importc: "getenv", header: "".} - when defined(vcc): + when defined(windows): proc c_putenv_s(envname: cstring, envval: cstring): cint {.importc: "_putenv_s", header: "".} + from std/private/win_setenv import setEnvImpl else: proc c_setenv(envname: cstring, envval: cstring, overwrite: cint): cint {.importc: "setenv", header: "".} - proc c_unsetenv(env: cstring): cint {. - importc: "unsetenv", header: "".} - - # Environment handling cannot be put into RTL, because the `envPairs` - # iterator depends on `environment`. - - var - envComputed {.threadvar.}: bool - environment {.threadvar.}: seq[string] - - when defined(nimV2): - proc unpairedEnvAllocs*(): int = - result = environment.len - if result > 0: inc result - - when defined(windows) and not defined(nimscript): - # because we support Windows GUI applications, things get really - # messy here... - when useWinUnicode: - when defined(cpp): - proc strEnd(cstr: WideCString, c = 0'i32): WideCString {. - importcpp: "(NI16*)wcschr((const wchar_t *)#, #)", header: "".} - else: - proc strEnd(cstr: WideCString, c = 0'i32): WideCString {. - importc: "wcschr", header: "".} - else: - proc strEnd(cstr: cstring, c = 0'i32): cstring {. - importc: "strchr", header: "".} - - proc getEnvVarsC() = - if not envComputed: - environment = @[] - when useWinUnicode: - var - env = getEnvironmentStringsW() - e = env - if e == nil: return # an error occurred - while true: - var eend = strEnd(e) - add(environment, $e) - e = cast[WideCString](cast[ByteAddress](eend)+2) - if eend[1].int == 0: break - discard freeEnvironmentStringsW(env) - else: - var - env = getEnvironmentStringsA() - e = env - if e == nil: return # an error occurred - while true: - var eend = strEnd(e) - add(environment, $e) - e = cast[cstring](cast[ByteAddress](eend)+1) - if eend[1] == '\0': break - discard freeEnvironmentStringsA(env) - envComputed = true - - else: - const - useNSGetEnviron = (defined(macosx) and not defined(ios) and not defined(emscripten)) or defined(nimscript) - - when useNSGetEnviron: - # From the manual: - # Shared libraries and bundles don't have direct access to environ, - # which is only available to the loader ld(1) when a complete program - # is being linked. - # The environment routines can still be used, but if direct access to - # environ is needed, the _NSGetEnviron() routine, defined in - # , can be used to retrieve the address of environ - # at runtime. - proc NSGetEnviron(): ptr cstringArray {. - importc: "_NSGetEnviron", header: "".} - elif defined(haiku): - var gEnv {.importc: "environ", header: "".}: cstringArray - else: - var gEnv {.importc: "environ".}: cstringArray - - proc getEnvVarsC() = - # retrieves the variables of char** env of C's main proc - if not envComputed: - environment = @[] - when useNSGetEnviron: - var gEnv = NSGetEnviron()[] - var i = 0 - while gEnv[i] != nil: - add environment, $gEnv[i] - inc(i) - envComputed = true - - proc findEnvVar(key: string): int = - getEnvVarsC() - var temp = key & '=' - for i in 0..high(environment): - when defined(windows): - if skipIgnoreCase(environment[i], temp) == len(temp): return i - else: - if startsWith(environment[i], temp): return i - return -1 + proc c_unsetenv(env: cstring): cint {.importc: "unsetenv", header: "".} proc getEnv*(key: string, default = ""): string {.tags: [ReadEnvEffect].} = ## Returns the value of the `environment variable`:idx: named `key`. @@ -163,16 +66,9 @@ else: assert getEnv("unknownEnv") == "" assert getEnv("unknownEnv", "doesn't exist") == "doesn't exist" - when nimvm: - discard "built into the compiler" - else: - var i = findEnvVar(key) - if i >= 0: - return substr(environment[i], find(environment[i], '=')+1) - else: - var env = c_getenv(key) - if env == nil: return default - result = $env + let env = c_getenv(key) + if env == nil: return default + result = $env proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = ## Checks whether the environment variable named `key` exists. @@ -186,11 +82,7 @@ else: runnableExamples: assert not existsEnv("unknownEnv") - when nimvm: - discard "built into the compiler" - else: - if c_getenv(key) != nil: return true - else: return findEnvVar(key) >= 0 + return c_getenv(key) != nil proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = ## Sets the value of the `environment variable`:idx: named `key` to `val`. @@ -201,33 +93,14 @@ else: ## * `existsEnv proc <#existsEnv,string>`_ ## * `delEnv proc <#delEnv,string>`_ ## * `envPairs iterator <#envPairs.i>`_ - - # Note: by storing the string in the environment sequence, - # we guarantee that we don't free the memory before the program - # ends (this is needed for POSIX compliance). It is also needed so that - # the process itself may access its modified environment variables! - when nimvm: - discard "built into the compiler" + when defined(windows): + if key.len == 0 or '=' in key: + raise newException(OSError, "invalid key, got: " & $(key, val)) + if setEnvImpl(key, val, 1'i32) != 0'i32: + raiseOSError(osLastError(), $(key, val)) else: - var indx = findEnvVar(key) - if indx >= 0: - environment[indx] = key & '=' & val - else: - add environment, (key & '=' & val) - indx = high(environment) - when defined(windows) and not defined(nimscript): - when useWinUnicode: - var k = newWideCString(key) - var v = newWideCString(val) - if setEnvironmentVariableW(k, v) == 0'i32: raiseOSError(osLastError()) - else: - if setEnvironmentVariableA(key, val) == 0'i32: raiseOSError(osLastError()) - elif defined(vcc): - if c_putenv_s(key, val) != 0'i32: - raiseOSError(osLastError()) - else: - if c_setenv(key, val, 1'i32) != 0'i32: - raiseOSError(osLastError()) + if c_setenv(key, val, 1'i32) != 0'i32: + raiseOSError(osLastError(), $(key, val)) proc delEnv*(key: string) {.tags: [WriteEnvEffect].} = ## Deletes the `environment variable`:idx: named `key`. @@ -238,21 +111,45 @@ else: ## * `existsEnv proc <#existsEnv,string>`_ ## * `putEnv proc <#putEnv,string,string>`_ ## * `envPairs iterator <#envPairs.i>`_ - when nimvm: - discard "built into the compiler" + template bail = raiseOSError(osLastError(), key) + when defined(windows): + #[ + # https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/putenv-s-wputenv-s?view=msvc-160 + > You can remove a variable from the environment by specifying an empty string (that is, "") for value_string + note that nil is not legal + ]# + if key.len == 0 or '=' in key: + raise newException(OSError, "invalid key, got: " & key) + if c_putenv_s(key, "") != 0'i32: bail else: - var indx = findEnvVar(key) - if indx < 0: return # Do nothing if the env var is not already set - when defined(windows) and not defined(nimscript): - when useWinUnicode: - var k = newWideCString(key) - if setEnvironmentVariableW(k, nil) == 0'i32: raiseOSError(osLastError()) - else: - if setEnvironmentVariableA(key, nil) == 0'i32: raiseOSError(osLastError()) + if c_unsetenv(key) != 0'i32: bail + + when defined(windows): + when useWinUnicode: + when defined(cpp): + proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.importcpp: "(NI16*)wcschr((const wchar_t *)#, #)", + header: "".} else: - if c_unsetenv(key) != 0'i32: - raiseOSError(osLastError()) - environment.delete(indx) + proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.importc: "wcschr", + header: "".} + else: + proc strEnd(cstr: cstring, c = 0'i32): cstring {.importc: "strchr", + header: "".} + elif defined(macosx) and not defined(ios) and not defined(emscripten): + # From the manual: + # Shared libraries and bundles don't have direct access to environ, + # which is only available to the loader ld(1) when a complete program + # is being linked. + # The environment routines can still be used, but if direct access to + # environ is needed, the _NSGetEnviron() routine, defined in + # , can be used to retrieve the address of environ + # at runtime. + proc NSGetEnviron(): ptr cstringArray {.importc: "_NSGetEnviron", + header: "".} + elif defined(haiku): + var gEnv {.importc: "environ", header: "".}: cstringArray + else: + var gEnv {.importc: "environ".}: cstringArray iterator envPairs*(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = ## Iterate over all `environments variables`:idx:. @@ -265,8 +162,30 @@ else: ## * `existsEnv proc <#existsEnv,string>`_ ## * `putEnv proc <#putEnv,string,string>`_ ## * `delEnv proc <#delEnv,string>`_ - getEnvVarsC() - for i in 0..high(environment): - var p = find(environment[i], '=') - yield (substr(environment[i], 0, p-1), - substr(environment[i], p+1)) + when defined(windows): + block: + template impl(get_fun, typ, size, zero, free_fun) = + let env = get_fun() + var e = env + if e == nil: break + while true: + let eend = strEnd(e) + let kv = $e + let p = find(kv, '=') + yield (substr(kv, 0, p-1), substr(kv, p+1)) + e = cast[typ](cast[ByteAddress](eend)+size) + if typeof(zero)(eend[1]) == zero: break + discard free_fun(env) + when useWinUnicode: + impl(getEnvironmentStringsW, WideCString, 2, 0, freeEnvironmentStringsW) + else: + impl(getEnvironmentStringsA, cstring, 1, '\0', freeEnvironmentStringsA) + else: + var i = 0 + when defined(macosx) and not defined(ios) and not defined(emscripten): + var gEnv = NSGetEnviron()[] + while gEnv[i] != nil: + let kv = $gEnv[i] + inc(i) + let p = find(kv, '=') + yield (substr(kv, 0, p-1), substr(kv, p+1)) diff --git a/lib/std/private/win_setenv.nim b/lib/std/private/win_setenv.nim new file mode 100644 index 000000000..067e656a3 --- /dev/null +++ b/lib/std/private/win_setenv.nim @@ -0,0 +1,92 @@ +#[ +Copyright (c) Facebook, Inc. and its affiliates. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Adapted `setenv` from https://github.com/facebook/folly/blob/master/folly/portability/Stdlib.cpp +translated from C to nim. +]# + +#[ +Introduced in https://github.com/facebook/folly/commit/5d8ca09a3f96afefb44e35808f03651a096ab9c7 + +TODO: +check errno_t vs cint +]# + +when not defined(windows): discard +else: + proc setEnvironmentVariableA*(lpName, lpValue: cstring): int32 {. + stdcall, dynlib: "kernel32", importc: "SetEnvironmentVariableA", sideEffect.} + # same as winlean.setEnvironmentVariableA + + proc c_getenv(env: cstring): cstring {.importc: "getenv", header: "".} + proc c_putenv_s(envname: cstring, envval: cstring): cint {.importc: "_putenv_s", header: "".} + proc c_wgetenv(varname: WideCString): WideCString {.importc: "_wgetenv", header: "".} + + var errno {.importc, header: "".}: cint + type wchar_t {.importc: "wchar_t".} = int16 + var gWenviron {.importc:"_wenviron".}: ptr ptr wchar_t + # xxx `ptr UncheckedArray[WideCString]` did not work + + proc mbstowcs_s(pReturnValue: ptr csize_t, wcstr: WideCString, sizeInWords: csize_t, mbstr: cstring, count: csize_t): cint {.importc: "mbstowcs_s", header: "".} + # xxx cint vs errno_t? + + proc setEnvImpl*(name: cstring, value: cstring, overwrite: cint): cint = + const EINVAL = cint(22) + const MAX_ENV = 32767 + # xxx get it from: `var MAX_ENV {.importc: "_MAX_ENV", header:"".}: cint` + if overwrite == 0 and c_getenv(name) != nil: return 0 + if value[0] != '\0': + let e = c_putenv_s(name, value) + if e != 0: + errno = e + return -1 + return 0 + #[ + We are trying to set the value to an empty string, but `_putenv_s` deletes + entries if the value is an empty string, and just calling + SetEnvironmentVariableA doesn't update `_environ`, + so we have to do these terrible things. + ]# + if c_putenv_s(name, " ") != 0: + errno = EINVAL + return -1 + # Here lies the documentation we blatently ignore to make this work. + var s = c_getenv(name) + s[0] = '\0' + #[ + This would result in a double null termination, which normally signifies the + end of the environment variable list, so we stick a completely empty + environment variable into the list instead. + ]# + s[1] = '=' + #[ + If gWenviron is null, the wide environment has not been initialized + yet, and we don't need to try to update it. We have to do this otherwise + we'd be forcing the initialization and maintenance of the wide environment + even though it's never actually used in most programs. + ]# + if gWenviron != nil: + # var buf: array[MAX_ENV + 1, WideCString] + var buf: array[MAX_ENV + 1, Utf16Char] + let buf2 = cast[WideCString](buf[0].addr) + var len: csize_t + if mbstowcs_s(len.addr, buf2, buf.len.csize_t, name, MAX_ENV) != 0: + errno = EINVAL + return -1 + c_wgetenv(buf2)[0] = '\0'.Utf16Char + c_wgetenv(buf2)[1] = '='.Utf16Char + + # And now, we have to update the outer environment to have a proper empty value. + if setEnvironmentVariableA(name, value) == 0: + errno = EINVAL + return -1 + return 0 diff --git a/tests/destructor/tnewruntime_misc.nim b/tests/destructor/tnewruntime_misc.nim index 48ea36b7c..ac061f2f7 100644 --- a/tests/destructor/tnewruntime_misc.nim +++ b/tests/destructor/tnewruntime_misc.nim @@ -7,7 +7,7 @@ axc ... destroying GenericObj[T] GenericObj[system.int] test -(allocCount: 13, deallocCount: 11) +(allocCount: 12, deallocCount: 10) 3''' """ diff --git a/tests/stdlib/tos.nim b/tests/stdlib/tos.nim index fedf2b3e3..dcb2d44f4 100644 --- a/tests/stdlib/tos.nim +++ b/tests/stdlib/tos.nim @@ -331,7 +331,7 @@ block walkDirRec: doAssert p.startsWith("walkdir_test") var s: seq[string] - for p in walkDirRec("walkdir_test", {pcFile}, {pcDir}, relative=true): + for p in walkDirRec("walkdir_test", {pcFile}, {pcDir}, relative = true): s.add(p) doAssert s.len == 2 @@ -553,19 +553,19 @@ block ospaths: doAssert joinPath("/", "") == unixToNativePath"/" doAssert joinPath("/" / "") == unixToNativePath"/" # weird test case... doAssert joinPath("/", "/a/b/c") == unixToNativePath"/a/b/c" - doAssert joinPath("foo/","") == unixToNativePath"foo/" - doAssert joinPath("foo/","abc") == unixToNativePath"foo/abc" - doAssert joinPath("foo//./","abc/.//") == unixToNativePath"foo/abc/" - doAssert joinPath("foo","abc") == unixToNativePath"foo/abc" - doAssert joinPath("","abc") == unixToNativePath"abc" + doAssert joinPath("foo/", "") == unixToNativePath"foo/" + doAssert joinPath("foo/", "abc") == unixToNativePath"foo/abc" + doAssert joinPath("foo//./", "abc/.//") == unixToNativePath"foo/abc/" + doAssert joinPath("foo", "abc") == unixToNativePath"foo/abc" + doAssert joinPath("", "abc") == unixToNativePath"abc" - doAssert joinPath("zook/.","abc") == unixToNativePath"zook/abc" + doAssert joinPath("zook/.", "abc") == unixToNativePath"zook/abc" # controversial: inconsistent with `joinPath("zook/.","abc")` # on linux, `./foo` and `foo` are treated a bit differently for executables # but not `./foo/bar` and `foo/bar` doAssert joinPath(".", "/lib") == unixToNativePath"./lib" - doAssert joinPath(".","abc") == unixToNativePath"./abc" + doAssert joinPath(".", "abc") == unixToNativePath"./abc" # cases related to issue #13455 doAssert joinPath("foo", "", "") == "foo" @@ -605,24 +605,6 @@ block getTempDir: block: # getCacheDir doAssert getCacheDir().dirExists -block osenv: - block delEnv: - const dummyEnvVar = "DUMMY_ENV_VAR" # This env var wouldn't be likely to exist to begin with - doAssert existsEnv(dummyEnvVar) == false - putEnv(dummyEnvVar, "1") - doAssert existsEnv(dummyEnvVar) == true - delEnv(dummyEnvVar) - doAssert existsEnv(dummyEnvVar) == false - delEnv(dummyEnvVar) # deleting an already deleted env var - doAssert existsEnv(dummyEnvVar) == false - block: # putEnv, bug #18502 - doAssertRaises(OSError): putEnv("DUMMY_ENV_VAR_PUT=DUMMY_VALUE", "NEW_DUMMY_VALUE") - doAssertRaises(OSError): putEnv("", "NEW_DUMMY_VALUE") - block: - doAssert getEnv("DUMMY_ENV_VAR_NONEXISTENT", "") == "" - doAssert getEnv("DUMMY_ENV_VAR_NONEXISTENT", " ") == " " - doAssert getEnv("DUMMY_ENV_VAR_NONEXISTENT", "Arrakis") == "Arrakis" - block isRelativeTo: doAssert isRelativeTo("/foo", "/") doAssert isRelativeTo("/foo/bar", "/foo") diff --git a/tests/stdlib/tosenv.nim b/tests/stdlib/tosenv.nim new file mode 100644 index 000000000..df5759d0c --- /dev/null +++ b/tests/stdlib/tosenv.nim @@ -0,0 +1,58 @@ +discard """ + matrix: "--threads" + joinable: false + targets: "c js cpp" +""" + +import std/os +from std/sequtils import toSeq +import stdtest/testutils + +template main = + block: # delEnv, existsEnv, getEnv, envPairs + for val in ["val", ""]: # ensures empty val works too + const key = "NIM_TESTS_TOSENV_KEY" + doAssert not existsEnv(key) + putEnv(key, val) + doAssert existsEnv(key) + doAssert getEnv(key) == val + when nimvm: discard + else: + doAssert (key, val) in toSeq(envPairs()) + delEnv(key) + when nimvm: discard + else: + doAssert (key, val) notin toSeq(envPairs()) + doAssert not existsEnv(key) + delEnv(key) # deleting an already deleted env var + doAssert not existsEnv(key) + + block: + doAssert getEnv("NIM_TESTS_TOSENV_NONEXISTENT", "") == "" + doAssert getEnv("NIM_TESTS_TOSENV_NONEXISTENT", " ") == " " + doAssert getEnv("NIM_TESTS_TOSENV_NONEXISTENT", "defval") == "defval" + + whenVMorJs: discard # xxx improve + do: + doAssertRaises(OSError, putEnv("NIM_TESTS_TOSENV_PUT=DUMMY_VALUE", "NEW_DUMMY_VALUE")) + doAssertRaises(OSError, putEnv("", "NEW_DUMMY_VALUE")) + doAssert not existsEnv("") + doAssert not existsEnv("NIM_TESTS_TOSENV_PUT=DUMMY_VALUE") + doAssert not existsEnv("NIM_TESTS_TOSENV_PUT") + +static: main() +main() + +when not defined(js): + block: # bug #18533 + proc c_getenv(env: cstring): cstring {.importc: "getenv", header: "".} + var thr: Thread[void] + proc threadFunc {.thread.} = putEnv("foo", "fooVal2") + + putEnv("foo", "fooVal1") + doAssert getEnv("foo") == "fooVal1" + createThread(thr, threadFunc) + joinThreads(thr) + doAssert getEnv("foo") == $c_getenv("foo") + + doAssertRaises(OSError): delEnv("foo=bar") From afc0259b125f5be032bf4c13721669c5452f9143 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 29 Jul 2021 19:28:33 -0700 Subject: [PATCH 1113/2472] fix #18385 followup, by building nimsuggest with -d:release during testing (#18581) --- koch.nim | 5 +++++ nimsuggest/tester.nim | 15 +++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/koch.nim b/koch.nim index 4296e4529..8dd897028 100644 --- a/koch.nim +++ b/koch.nim @@ -592,6 +592,11 @@ proc runCI(cmd: string) = execFold("Run rst2html tests", "nim r nimdoc/rsttester") execFold("Run nimpretty tests", "nim r nimpretty/tester.nim") when defined(posix): + # refs #18385, build with -d:release instead of -d:danger for testing + # We could also skip building nimsuggest in buildTools, or build it with -d:release + # in bundleNimsuggest depending on some environment variable when we are in CI. One advantage + # of rebuilding is this won't affect bin/nimsuggest when running runCI locally + execFold("build nimsuggest_testing", "nim c -o:bin/nimsuggest_testing -d:release nimsuggest/nimsuggest") execFold("Run nimsuggest tests", "nim r nimsuggest/tester") execFold("Run atlas tests", "nim c -r -d:atlasTests tools/atlas/atlas.nim clone https://github.com/disruptek/balls") diff --git a/nimsuggest/tester.nim b/nimsuggest/tester.nim index b62aa8783..1db33706a 100644 --- a/nimsuggest/tester.nim +++ b/nimsuggest/tester.nim @@ -6,6 +6,7 @@ # `nim r nimsuggest/tester.nim nimsuggest/tests/tsug_accquote.nim` import os, osproc, strutils, streams, re, sexp, net +from sequtils import toSeq type Test = object @@ -23,7 +24,7 @@ import std/compilesettings proc parseTest(filename: string; epcMode=false): Test = const cursorMarker = "#[!]#" - let nimsug = "bin" / addFileExt("nimsuggest", ExeExt) + let nimsug = "bin" / addFileExt("nimsuggest_testing", ExeExt) doAssert nimsug.fileExists, nimsug const libpath = querySetting(libPath) result.filename = filename @@ -272,6 +273,7 @@ proc runEpcTest(filename: string): int = let a = outp.readAll().strip() let port = parseInt(a) socket.connect("localhost", Port(port)) + for req, resp in items(s.script): if not runCmd(req, s.dest): socket.sendEpcStr(req) @@ -279,8 +281,12 @@ proc runEpcTest(filename: string): int = if not req.startsWith("mod "): let answer = sexpToAnswer(sx) doReport(filename, answer, resp, report) - finally: + socket.sendEpcStr "return arg" + # bugfix: this was in `finally` block, causing the original error to be + # potentially masked by another one in case `socket.sendEpcStr` raises + # (e.g. if socket couldn't connect in the 1st place) + finally: close(p) if report.len > 0: echo "==== EPC ========================================" @@ -336,8 +342,9 @@ proc main() = failures += runTest(xx) failures += runEpcTest(xx) else: - for x in walkFiles(tpath / "t*.nim"): - echo "Test ", x + let files = toSeq(walkFiles(tpath / "t*.nim")) + for i, x in files: + echo "$#/$# test: $#" % [$i, $files.len, x] when defined(i386): if x == "nimsuggest/tests/tmacro_highlight.nim": echo "skipping" # workaround bug #17945 From 9a26f5059be48b19d821c17fe1066c05d167bc7d Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 30 Jul 2021 00:30:22 -0700 Subject: [PATCH 1114/2472] refs #16613: check opcWrDeref for nil (#18613) --- compiler/vm.nim | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/compiler/vm.nim b/compiler/vm.nim index 9541af701..73a228b6c 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -812,11 +812,10 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = # vmgen generates opcWrDeref, which means that we must dereference # twice. # TODO: This should likely be handled differently in vmgen. - if (nfIsRef notin regs[ra].nodeAddr[].flags and - nfIsRef notin n.flags): - regs[ra].nodeAddr[][] = n[] - else: - regs[ra].nodeAddr[] = n + let nAddr = regs[ra].nodeAddr + if nAddr[] == nil: stackTrace(c, tos, pc, "opcWrDeref internal error") # refs bug #16613 + if (nfIsRef notin nAddr[].flags and nfIsRef notin n.flags): nAddr[][] = n[] + else: nAddr[] = n of rkRegisterAddr: regs[ra].regAddr[] = regs[rc] of rkNode: # xxx: also check for nkRefTy as in opcLdDeref? From c6fadb179975f79ad9914f09ce10f7fa6cbf0735 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 30 Jul 2021 15:32:00 +0800 Subject: [PATCH 1115/2472] [minor] reduce `substr` (#18611) * minor * correct * unify the type of addrLen * Update lib/packages/docutils/rstgen.nim --- lib/packages/docutils/rstgen.nim | 2 +- lib/pure/nativesockets.nim | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index 10a229479..3734392c3 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -675,7 +675,7 @@ proc readIndexDir(dir: string): var x = fileEntries[0].link let i = find(x, '#') if i > 0: - x = x.substr(0, i-1) + x.setLen(i) if i != 0: # don't add entries starting with '#' result.modules.add(x.changeFileExt("")) diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index 124afce56..13c08dd92 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -14,6 +14,7 @@ import os, options import std/private/since +import std/strbasics when hostOS == "solaris": @@ -468,7 +469,7 @@ proc getAddrString*(sockAddr: ptr SockAddr): string = if sockAddr.sa_family.cint == nativeAfInet: result = $inet_ntoa(cast[ptr Sockaddr_in](sockAddr).sin_addr) elif sockAddr.sa_family.cint == nativeAfInet6: - let addrLen = when not useWinVersion: posix.INET6_ADDRSTRLEN + let addrLen = when not useWinVersion: posix.INET6_ADDRSTRLEN.int else: 46 # it's actually 46 in both cases result = newString(addrLen) let addr6 = addr cast[ptr Sockaddr_in6](sockAddr).sin6_addr @@ -477,7 +478,7 @@ proc getAddrString*(sockAddr: ptr SockAddr): string = result.len.int32) == nil: raiseOSError(osLastError()) if posix.IN6_IS_ADDR_V4MAPPED(addr6) != 0: - result = result.substr("::ffff:".len) + result.setSlice("::ffff:".len.. Date: Fri, 30 Jul 2021 12:05:14 +0200 Subject: [PATCH 1116/2472] fixes #18371 (#18617) --- compiler/vmops.nim | 7 +++++-- lib/std/effecttraits.nim | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/compiler/vmops.nim b/compiler/vmops.nim index 6ee915bd5..a1a4996b2 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -303,12 +303,15 @@ proc registerAdditionalOps*(c: PCtx) = proc getEffectList(c: PCtx; a: VmArgs; effectIndex: int) = let fn = getNode(a, 0) + var list = newNodeI(nkBracket, fn.info) if fn.typ != nil and fn.typ.n != nil and fn.typ.n[0].len >= effectListLen and fn.typ.n[0][effectIndex] != nil: - var list = newNodeI(nkBracket, fn.info) for e in fn.typ.n[0][effectIndex]: list.add opMapTypeInstToAst(c.cache, e.typ.skipTypes({tyRef}), e.info, c.idgen) - setResult(a, list) + else: + list.add newIdentNode(getIdent(c.cache, "UncomputedEffects"), fn.info) + + setResult(a, list) registerCallback c, "stdlib.effecttraits.getRaisesListImpl", proc (a: VmArgs) = getEffectList(c, a, exceptionEffects) diff --git a/lib/std/effecttraits.nim b/lib/std/effecttraits.nim index 358280db0..635ec82d0 100644 --- a/lib/std/effecttraits.nim +++ b/lib/std/effecttraits.nim @@ -11,7 +11,7 @@ ## for Nim's macro system. ## **Since**: Version 1.4. ## -## One can test for the existance of this standard module +## One can test for the existence of this standard module ## via `defined(nimHasEffectTraitsModule)`. import macros From ecfc47b332a67bb373d433f01277bc924b75f77f Mon Sep 17 00:00:00 2001 From: Antonis Geralis <43617260+planetis-m@users.noreply.github.com> Date: Fri, 30 Jul 2021 17:36:16 +0300 Subject: [PATCH 1117/2472] Modify atlas test for short commit hashes (#18619) * test commit hashes support * Update testdata.nim * remove extra newlines --- tools/atlas/packagesjson.nim | 2 -- tools/atlas/testdata.nim | 2 +- tools/atlas/tests/balls.nimble | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tools/atlas/packagesjson.nim b/tools/atlas/packagesjson.nim index e1e23ee29..ed83ce85c 100644 --- a/tools/atlas/packagesjson.nim +++ b/tools/atlas/packagesjson.nim @@ -70,7 +70,6 @@ proc search*(pkgList: seq[Package]; terms: seq[string]) = var found = false template onFound = echo pkg - echo("") found = true break forPackage @@ -88,7 +87,6 @@ proc search*(pkgList: seq[Package]; terms: seq[string]) = onFound() else: echo(pkg) - echo(" ") if not found and terms.len > 0: echo("No package found.") diff --git a/tools/atlas/testdata.nim b/tools/atlas/testdata.nim index f960e3d08..aefaeacd2 100644 --- a/tools/atlas/testdata.nim +++ b/tools/atlas/testdata.nim @@ -25,7 +25,7 @@ const toData("ups", GitCheckout, 0, "4008f9339cd22b30e180bc87a6cca7270fd28ac1"), toData("sync", GitDiff, 0, ""), - toData("sync", GitTags, 0, "c9811af2d03dc78a920d33592d0a3daae3d9e68f refs/tags/v0.1.0\nc99d3a73344c4c02335190b489e7ea8aaa4e9584 refs/tags/v0.2.0\n6186a38c97c0f26e1a0c8016979227d26c41717b refs/tags/v0.3.0\n5506e4e09b849bd59f930fcccee3096abb089b1d refs/tags/v0.4.0\n1e4ef046ed0447aec7ca4ab3b2e2a5347ac41448 refs/tags/v0.5.0\na95f7b0b22c77b98ed63c1c4f435d550de8dbb05 refs/tags/v0.6.0\n214b868ace4ebc9140ee56734147fbd16b238c71 refs/tags/v0.7.0\n786698bf9f597834f36cd530d32faee31545a1d7 refs/tags/v0.9.0\n810bd2d75e9f6e182534ae2488670b51a9f13fc3 refs/tags/v1.0.0\na133d37991e3d4e9553c5261eabcb07d471968f7 refs/tags/v1.1.0\n0872a7cd70ea43a26deb94b8ab12cc5b89a1c212 refs/tags/v1.2.0\n19ac734438b6c04fe89711728a97fcde591324eb refs/tags/v1.3.0\nde5c7337ebc22422190e8aeca37d05651735f440 refs/tags/v1.4.0\n"), + toData("sync", GitRevParse, 0, "810bd2d75e9f6e182534ae2488670b51a9f13fc3\n"), toData("sync", GitCurrentCommit, 0, "de5c7337ebc22422190e8aeca37d05651735f440\n"), toData("sync", GitMergeBase, 0, "de5c7337ebc22422190e8aeca37d05651735f440\n810bd2d75e9f6e182534ae2488670b51a9f13fc3\n810bd2d75e9f6e182534ae2488670b51a9f13fc3\n"), diff --git a/tools/atlas/tests/balls.nimble b/tools/atlas/tests/balls.nimble index 62ab4750b..143e757e9 100644 --- a/tools/atlas/tests/balls.nimble +++ b/tools/atlas/tests/balls.nimble @@ -6,7 +6,7 @@ license = "MIT" # requires newTreeFrom requires "https://github.com/disruptek/grok >= 0.5.0 & < 1.0.0" requires "https://github.com/disruptek/ups < 1.0.0" -requires "https://github.com/planetis-m/sync >= 1.0.0 & < 2.0.0" +requires "https://github.com/planetis-m/sync#810bd2d" #requires "https://github.com/c-blake/cligen < 2.0.0" bin = @["balls"] # build the binary for basic test running From 52e276c82daf2b8a33f4e4304eac3074bb37b70d Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 31 Jul 2021 23:54:08 -0700 Subject: [PATCH 1118/2472] exportC => exportc (#18625) --- tests/template/template_pragmas.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/template/template_pragmas.nim b/tests/template/template_pragmas.nim index d6c99080d..da532b010 100644 --- a/tests/template/template_pragmas.nim +++ b/tests/template/template_pragmas.nim @@ -3,7 +3,7 @@ proc output_x:string {.compileTime.} = "x" template t = const x = output_x() let - bar {.exportC:"bar" & x.} = 100 + bar {.exportc:"bar" & x.} = 100 static: doAssert(compiles (t())) From 916d0c21af6153338a8064ff953b579ff87c6ba6 Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 1 Aug 2021 17:19:43 +0800 Subject: [PATCH 1119/2472] fix #18620 (#18624) * fix #18620 * add testcase --- lib/system/excpt.nim | 5 +---- tests/exception/t18620.nim | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 tests/exception/t18620.nim diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index 9f65db2fe..4814b77b4 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -524,10 +524,7 @@ proc getStackTrace(e: ref Exception): string = proc getStackTraceEntries*(e: ref Exception): seq[StackTraceEntry] = ## Returns the attached stack trace to the exception `e` as ## a `seq`. This is not yet available for the JS backend. - when not defined(nimSeqsV2): - shallowCopy(result, e.trace) - else: - result = move(e.trace) + shallowCopy(result, e.trace) proc getStackTraceEntries*(): seq[StackTraceEntry] = ## Returns the stack trace entries for the current stack trace. diff --git a/tests/exception/t18620.nim b/tests/exception/t18620.nim new file mode 100644 index 000000000..ee23f8bac --- /dev/null +++ b/tests/exception/t18620.nim @@ -0,0 +1,17 @@ +discard """ + matrix: "--gc:arc; --gc:refc" +""" + +proc hello() = + raise newException(ValueError, "You are wrong") + +var flag = false + +try: + hello() +except ValueError as e: + flag = true + doAssert len(getStackTraceEntries(e)) > 0 + doAssert len(getStackTraceEntries(e)) > 0 + +doAssert flag From 2bc07554ccd64d7405d0f386f17f7b1faa95bd6f Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 1 Aug 2021 19:52:24 +0800 Subject: [PATCH 1120/2472] make proc names consistent (#18626) * rename `endswith` to `endsWith` * rename --- compiler/semtypes.nim | 10 +++++----- tests/arc/tasyncawait.nim | 2 +- tests/async/tasyncawait.nim | 2 +- tests/async/tasyncnetudp.nim | 2 +- tests/async/tasyncssl.nim | 2 +- tests/async/tnewasyncudp.nim | 2 +- tests/async/twinasyncrw.nim | 2 +- tools/atlas/osutils.nim | 2 +- tools/trimcc.nim | 2 +- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index db895cab6..cb7f847a9 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -1700,14 +1700,14 @@ proc semStaticType(c: PContext, childNode: PNode, prev: PType): PType = result.rawAddSon(base) result.flags.incl tfHasStatic -proc semTypeof(c: PContext; n: PNode; prev: PType): PType = +proc semTypeOf(c: PContext; n: PNode; prev: PType): PType = openScope(c) let t = semExprWithType(c, n, {efInTypeof}) closeScope(c) fixupTypeOf(c, prev, t) result = t.typ -proc semTypeof2(c: PContext; n: PNode; prev: PType): PType = +proc semTypeOf2(c: PContext; n: PNode; prev: PType): PType = openScope(c) var m = BiggestInt 1 # typeOfIter if n.len == 3: @@ -1731,7 +1731,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = of nkTypeOfExpr: # for ``typeof(countup(1,3))``, see ``tests/ttoseq``. checkSonsLen(n, 1, c.config) - result = semTypeof(c, n[0], prev) + result = semTypeOf(c, n[0], prev) if result.kind == tyTypeDesc: result.flags.incl tfExplicit of nkPar: if n.len == 1: result = semTypeNode(c, n[0], prev) @@ -1833,9 +1833,9 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = result = semAnyRef(c, n, tyRef, prev) elif op.id == ord(wType): checkSonsLen(n, 2, c.config) - result = semTypeof(c, n[1], prev) + result = semTypeOf(c, n[1], prev) elif op.s == "typeof" and n[0].kind == nkSym and n[0].sym.magic == mTypeOf: - result = semTypeof2(c, n, prev) + result = semTypeOf2(c, n, prev) elif op.s == "owned" and optOwnedRefs notin c.config.globalOptions and n.len == 2: result = semTypeExpr(c, n[1], prev) else: diff --git a/tests/arc/tasyncawait.nim b/tests/arc/tasyncawait.nim index 75d9bc9b5..91a7453cd 100644 --- a/tests/arc/tasyncawait.nim +++ b/tests/arc/tasyncawait.nim @@ -40,7 +40,7 @@ proc readMessages(client: AsyncFD) {.async.} = clientCount.inc break else: - if line.startswith("Message "): + if line.startsWith("Message "): msgCount.inc else: doAssert false diff --git a/tests/async/tasyncawait.nim b/tests/async/tasyncawait.nim index f658a15ed..e86542b2d 100644 --- a/tests/async/tasyncawait.nim +++ b/tests/async/tasyncawait.nim @@ -34,7 +34,7 @@ proc readMessages(client: AsyncFD) {.async.} = clientCount.inc break else: - if line.startswith("Message "): + if line.startsWith("Message "): msgCount.inc else: doAssert false diff --git a/tests/async/tasyncnetudp.nim b/tests/async/tasyncnetudp.nim index ef6dfc5e1..dade96fb2 100644 --- a/tests/async/tasyncnetudp.nim +++ b/tests/async/tasyncnetudp.nim @@ -59,7 +59,7 @@ proc readMessages(server: AsyncSocket) {.async.} = while i < maxResponses: let (data, fromIp, fromPort) = await recvFrom(server, 16384) - if data.startswith("Message ") and fromIp == "127.0.0.1": + if data.startsWith("Message ") and fromIp == "127.0.0.1": await sendTo(server, fromIp, fromPort, data) inc(msgCount) diff --git a/tests/async/tasyncssl.nim b/tests/async/tasyncssl.nim index a582818eb..222aaa3a1 100644 --- a/tests/async/tasyncssl.nim +++ b/tests/async/tasyncssl.nim @@ -36,7 +36,7 @@ when defined(ssl): inc(clientCount) break else: - if line.startswith("Message "): + if line.startsWith("Message "): inc(msgCount) else: doAssert false diff --git a/tests/async/tnewasyncudp.nim b/tests/async/tnewasyncudp.nim index 76462c21d..ba1e4134e 100644 --- a/tests/async/tnewasyncudp.nim +++ b/tests/async/tnewasyncudp.nim @@ -81,7 +81,7 @@ proc readMessages(server: AsyncFD) {.async.} = addr(slen)) size = 0 var grammString = $cstring(addr buffer) - if grammString.startswith("Message ") and + if grammString.startsWith("Message ") and saddr.sin_addr.s_addr == nativesockets.ntohl(INADDR_LOOPBACK.uint32): await sendTo(server, addr grammString[0], len(grammString), cast[ptr SockAddr](addr saddr), slen) diff --git a/tests/async/twinasyncrw.nim b/tests/async/twinasyncrw.nim index 69b092607..ad27ca38f 100644 --- a/tests/async/twinasyncrw.nim +++ b/tests/async/twinasyncrw.nim @@ -220,7 +220,7 @@ when defined(windows): clientCount.inc break else: - if line.startswith("Message "): + if line.startsWith("Message "): msgCount.inc else: doAssert false diff --git a/tools/atlas/osutils.nim b/tools/atlas/osutils.nim index 1bb414c03..6134830b5 100644 --- a/tools/atlas/osutils.nim +++ b/tools/atlas/osutils.nim @@ -17,7 +17,7 @@ proc cloneUrl*(url, dest: string; cloneUsingHttps: bool): string = # github + https + trailing url slash causes a # checkout/ls-remote to fail with Repository not found var isGithub = false - if modUrl.contains("github.com") and modUrl.endswith("/"): + if modUrl.contains("github.com") and modUrl.endsWith("/"): modUrl = modUrl[0 .. ^2] isGithub = true diff --git a/tools/trimcc.nim b/tools/trimcc.nim index 959eaf310..41795e1aa 100644 --- a/tools/trimcc.nim +++ b/tools/trimcc.nim @@ -31,7 +31,7 @@ proc processIncludes(dir: string, whitelist: StringTableRef) = if ('.' notin name and "include" in path) or ("c++" in path): let n = whitelist.getOrDefault(name) if n != "processed": whitelist[name] = "found" - if name.endswith(".h"): + if name.endsWith(".h"): let n = whitelist.getOrDefault(name) if n == "found": includes(path, name, whitelist) of pcDir: processIncludes(path, whitelist) From 499d4690fbbb59b6baf1698b1ed1574dde6ea7c5 Mon Sep 17 00:00:00 2001 From: Kyle Brown Date: Mon, 2 Aug 2021 00:38:15 -0700 Subject: [PATCH 1121/2472] Add Artix to distro list (#18629) --- lib/pure/distros.nim | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/pure/distros.nim b/lib/pure/distros.nim index c1bc44eeb..7a9910c1e 100644 --- a/lib/pure/distros.nim +++ b/lib/pure/distros.nim @@ -52,6 +52,7 @@ type CentOS Deepin ArchLinux + Artix Antergos PCLinuxOS Mageia @@ -133,7 +134,7 @@ type const - LacksDevPackages* = {Distribution.Gentoo, Distribution.Slackware, Distribution.ArchLinux} + LacksDevPackages* = {Distribution.Gentoo, Distribution.Slackware, Distribution.ArchLinux, Distribution.Artix} # we cache the result of the 'cmdRelease' # execution for faster platform detections. @@ -180,6 +181,8 @@ proc detectOsImpl(d: Distribution): bool = result = "rhel" in osReleaseID() of Distribution.ArchLinux: result = "arch" in osReleaseID() + of Distribution.Artix: + result = "artix" in osReleaseID() of Distribution.NixOS: # Check if this is a Nix build or NixOS environment result = existsEnv("NIX_BUILD_TOP") or existsEnv("__NIXOS_SET_ENVIRONMENT_DONE") @@ -249,7 +252,7 @@ proc foreignDepInstallCmd*(foreignPackageName: string): (string, bool) = result = ("pkg_add " & p, true) elif detectOs(PCLinuxOS): result = ("rpm -ivh " & p, true) - elif detectOs(ArchLinux) or detectOs(Manjaro): + elif detectOs(ArchLinux) or detectOs(Manjaro) or detectOs(Artix): result = ("pacman -S " & p, true) else: result = (" install " & p, true) From c0abdaa2b4587526f235ef41186e84c4f87813e1 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 2 Aug 2021 22:22:47 -0700 Subject: [PATCH 1122/2472] improve runnableExamples and docs for std/nre (#18634) * improve runnableExamples and docs for std/nre * avoid too long lines in example --- lib/impure/nre.nim | 117 +++++++++++++++++++-------------------------- 1 file changed, 49 insertions(+), 68 deletions(-) diff --git a/lib/impure/nre.nim b/lib/impure/nre.nim index 7b2d7d3ee..01ef77427 100644 --- a/lib/impure/nre.nim +++ b/lib/impure/nre.nim @@ -23,12 +23,12 @@ when defined(js): ## **Note**: If you love `sequtils.toSeq` we have bad news for you. This ## library doesn't work with it due to documented compiler limitations. As ## a workaround, use this: -## -## .. code-block:: nim -## -## import std/nre except toSeq -## -## +runnableExamples: + # either `import std/nre except toSeq` or fully qualify `sequtils.toSeq`: + import std/sequtils + iterator iota(n: int): int = + for i in 0..\w)").get.captures["letter"] == "a"` - ## - `"abc".match(re"(\w)\w").get.captures[-1] == "ab"` + ## `nil` is returned. See examples for `match`. ## ## `captureBounds[]: HSlice[int, int]` ## gets the bounds of the given capture according to the same rules as ## the above. If the capture is not filled, then `None` is returned. - ## The bounds are both inclusive. - ## - ## - `"abc".match(re"(\w)").get.captureBounds[0] == 0 .. 0` - ## - `0 in "abc".match(re"(\w)").get.captureBounds == true` - ## - `"abc".match(re"").get.captureBounds[-1] == 0 .. -1` - ## - `"abc".match(re"abc").get.captureBounds[-1] == 0 .. 2` + ## The bounds are both inclusive. See examples for `match`. ## ## `match: string` ## the full text of the match. @@ -198,7 +184,6 @@ type pattern*: Regex ## The regex doing the matching. ## Not nil. str*: string ## The string that was matched against. - ## Not nil. pcreMatchBounds: seq[HSlice[cint, cint]] ## First item is the bounds of the match ## Other items are the captures ## `a` is inclusive start, `b` is exclusive end @@ -226,18 +211,6 @@ type ## for whatever reason. The message contains the error ## code. -runnableExamples: - # This MUST be kept in sync with the examples in RegexMatch - doAssert "abc".match(re"(\w)").get.captures[0] == "a" - doAssert "abc".match(re"(?\w)").get.captures["letter"] == "a" - doAssert "abc".match(re"(\w)\w").get.captures[-1] == "ab" - - doAssert "abc".match(re"(\w)").get.captureBounds[0] == 0 .. 0 - doAssert 0 in "abc".match(re"(\w)").get.captureBounds == true - doAssert "abc".match(re"").get.captureBounds[-1] == 0 .. -1 - doAssert "abc".match(re"abc").get.captureBounds[-1] == 0 .. 2 - - proc destroyRegex(pattern: Regex) = pcre.free_substring(cast[cstring](pattern.pcreObj)) if pattern.pcreExtra != nil: @@ -540,18 +513,27 @@ proc matchImpl(str: string, pattern: Regex, start, endpos: int, flags: int): Opt proc match*(str: string, pattern: Regex, start = 0, endpos = int.high): Option[RegexMatch] = ## Like `find(...)<#find,string,Regex,int>`_, but anchored to the start of the ## string. - ## runnableExamples: - doAssert "foo".match(re"f").isSome - doAssert "foo".match(re"o").isNone + assert "foo".match(re"f").isSome + assert "foo".match(re"o").isNone + assert "abc".match(re"(\w)").get.captures[0] == "a" + assert "abc".match(re"(?\w)").get.captures["letter"] == "a" + assert "abc".match(re"(\w)\w").get.captures[-1] == "ab" + + assert "abc".match(re"(\w)").get.captureBounds[0] == 0 .. 0 + assert 0 in "abc".match(re"(\w)").get.captureBounds + assert "abc".match(re"").get.captureBounds[-1] == 0 .. -1 + assert "abc".match(re"abc").get.captureBounds[-1] == 0 .. 2 return str.matchImpl(pattern, start, endpos, pcre.ANCHORED) iterator findIter*(str: string, pattern: Regex, start = 0, endpos = int.high): RegexMatch = ## Works the same as `find(...)<#find,string,Regex,int>`_, but finds every - ## non-overlapping match. `"2222".find(re"22")` is `"22", "22"`, not - ## `"22", "22", "22"`. - ## + ## non-overlapping match: + runnableExamples: + import std/sugar + assert collect(for a in "2222".findIter(re"22"): a.match) == @["22", "22"] + # not @["22", "22", "22"] ## Arguments are the same as `find(...)<#find,string,Regex,int>`_ ## ## Variants: @@ -619,11 +601,10 @@ proc contains*(str: string, pattern: Regex, start = 0, endpos = int.high): bool ## Determine if the string contains the given pattern between the end and ## start positions: ## This function is equivalent to `isSome(str.find(pattern, start, endpos))`. - ## runnableExamples: - doAssert "abc".contains(re"bc") - doAssert not "abc".contains(re"cd") - doAssert not "abc".contains(re"a", start = 1) + assert "abc".contains(re"bc") + assert not "abc".contains(re"cd") + assert not "abc".contains(re"a", start = 1) return isSome(str.find(pattern, start, endpos)) @@ -635,16 +616,16 @@ proc split*(str: string, pattern: Regex, maxSplit = -1, start = 0): seq[string] ## runnableExamples: # - If the match is zero-width, then the string is still split: - doAssert "123".split(re"") == @["1", "2", "3"] + assert "123".split(re"") == @["1", "2", "3"] # - If the pattern has a capture in it, it is added after the string # split: - doAssert "12".split(re"(\d)") == @["", "1", "", "2", ""] + assert "12".split(re"(\d)") == @["", "1", "", "2", ""] # - If `maxsplit != -1`, then the string will only be split # `maxsplit - 1` times. This means that there will be `maxsplit` # strings in the output seq. - doAssert "1.2.3".split(re"\.", maxsplit = 2) == @["1", "2.3"] + assert "1.2.3".split(re"\.", maxsplit = 2) == @["1", "2.3"] result = @[] var lastIdx = start @@ -747,9 +728,9 @@ proc escapeRe*(str: string): string {.gcsafe.} = ## ## Escaped char: `\ + * ? [ ^ ] $ ( ) { } = ! < > | : -` runnableExamples: - doAssert escapeRe("fly+wind") == "fly\\+wind" - doAssert escapeRe("!") == "\\!" - doAssert escapeRe("nim*") == "nim\\*" + assert escapeRe("fly+wind") == "fly\\+wind" + assert escapeRe("!") == "\\!" + assert escapeRe("nim*") == "nim\\*" #([\\+*?[^\]$(){}=!<>|:-]) const SpecialCharMatcher = {'\\', '+', '*', '?', '[', '^', ']', '$', '(', From 6563a685c1076cb5342f65d192b8f2a983220dba Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Tue, 3 Aug 2021 13:16:33 -0300 Subject: [PATCH 1123/2472] Documentation only, add 1 example (#18633) --- doc/manual_experimental.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/doc/manual_experimental.rst b/doc/manual_experimental.rst index 22db184a8..3272a4d0d 100644 --- a/doc/manual_experimental.rst +++ b/doc/manual_experimental.rst @@ -1157,6 +1157,24 @@ the ordinary AST predicates: a = b inc a, c +Another example: + +.. code-block:: nim + proc somefunc(s: string) = assert s == "variable" + proc somefunc(s: string{nkStrLit}) = assert s == "literal" + proc somefunc(s: string{nkRStrLit}) = assert s == r"raw" + proc somefunc(s: string{nkTripleStrLit}) = assert s == """triple""" + proc somefunc(s: static[string]) = assert s == "constant" + + # Use parameter constraints to provide overloads based on both the input parameter type and form. + var variable = "variable" + somefunc(variable) + const constant = "constant" + somefunc(constant) + somefunc("literal") + somefunc(r"raw") + somefunc("""triple""") + Pattern operators ----------------- From 2cddf7fc96a01bcd0f6029f38de005314333daca Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Sun, 8 Aug 2021 14:26:34 -0300 Subject: [PATCH 1124/2472] Documentation only, add 1 example (#18621) * ReSync with Devel * ReSync * Documentation only, add 1 example to For loop macro * Flip it * Update doc/manual.rst Co-authored-by: Andreas Rumpf Co-authored-by: Andreas Rumpf --- doc/manual.rst | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/doc/manual.rst b/doc/manual.rst index 1b65ae9ee..41ed49715 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -5830,6 +5830,27 @@ For loop macro A macro that takes as its only input parameter an expression of the special type `system.ForLoopStmt` can rewrite the entirety of a `for` loop: +.. code-block:: nim + :test: "nim c $1" + + import std/macros + + macro example(loop: ForLoopStmt) = + result = newTree(nnkForStmt) # Create a new For loop. + result.add loop[^3] # This is "item". + result.add loop[^2][^1] # This is "[1, 2, 3]". + result.add newCall(bindSym"echo", loop[0]) + + for item in example([1, 2, 3]): discard + +Expands to: + +.. code-block:: nim + for item in items([1, 2, 3]): + echo item + +Another example: + .. code-block:: nim :test: "nim c $1" @@ -5866,7 +5887,6 @@ type `system.ForLoopStmt` can rewrite the entirety of a `for` loop: echo a, " ", b - Special Types ============= From 9269a72f6f6f19e4bcffce82ad47c68755e7a696 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 9 Aug 2021 01:27:43 +0800 Subject: [PATCH 1125/2472] [nre]fix #17129 (#18632) * fix #17129 * correct * give reference implementaion links * add comment * typo * I'm conservative * change --- lib/impure/nre.nim | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/impure/nre.nim b/lib/impure/nre.nim index 01ef77427..f346d84cb 100644 --- a/lib/impure/nre.nim +++ b/lib/impure/nre.nim @@ -539,7 +539,7 @@ iterator findIter*(str: string, pattern: Regex, start = 0, endpos = int.high): R ## Variants: ## ## - `proc findAll(...)` returns a `seq[string]` - # see pcredemo for explanation + # see pcredemo for explanation => https://www.pcre.org/original/doc/html/pcredemo.html let matchesCrLf = pattern.matchesCrLf() let unicode = uint32(getinfo[culong](pattern, pcre.INFO_OPTIONS) and pcre.UTF8) > 0u32 @@ -560,7 +560,7 @@ iterator findIter*(str: string, pattern: Regex, start = 0, endpos = int.high): R # either the end of the input or the string # cannot be split here - we also need to bail # if we've never matched and we've already tried to... - if offset >= strlen or neverMatched: + if flags == 0 or offset >= strlen or neverMatched: # All matches found break if matchesCrLf and offset < (str.len - 1) and @@ -576,7 +576,6 @@ iterator findIter*(str: string, pattern: Regex, start = 0, endpos = int.high): R else: neverMatched = false offset = match.get.matchBounds.b + 1 - yield match.get proc find*(str: string, pattern: Regex, start = 0, endpos = int.high): Option[RegexMatch] = From 24445d31b394a120a8571c0d9dc4ab07b7672cd7 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 8 Aug 2021 10:28:49 -0700 Subject: [PATCH 1126/2472] improve several tests in testament (#18635) * silence error output from template_various.nim * any => auto in tests * avoid showing failed for parseSpec since this is expected behavior in 2 cases: tincludefile.nim, tnav1.nim * enforce InheritFromException * fixup --- testament/categories.nim | 2 +- tests/config.nims | 1 + tests/generics/mdotlookup.nim | 4 +- tests/iter/tyieldintry.nim | 60 +++++++++++++--------------- tests/overload/toverload_various.nim | 2 +- tests/template/template_various.nim | 6 ++- tests/template/tparams_gensymed.nim | 2 +- 7 files changed, 38 insertions(+), 39 deletions(-) diff --git a/testament/categories.nim b/testament/categories.nim index b5e8a5454..6375105ee 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -588,7 +588,7 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string, options: st except ValueError: # e.g. for `tests/navigator/tincludefile.nim` which have multiple # specs; this will be handled elsewhere - echo "parseSpec failed for: '$1', assuming this will be handled outside of megatest" % file + echo "parseSpec raised ValueError for: '$1', assuming this will be handled outside of megatest" % file continue if isJoinableSpec(spec): specs.add spec diff --git a/tests/config.nims b/tests/config.nims index f876cb612..d7f6ae7f9 100644 --- a/tests/config.nims +++ b/tests/config.nims @@ -28,6 +28,7 @@ hint("Processing", off) # switch("define", "nimTestsEnableFlaky") # switch("hint", "ConvFromXtoItselfNotNeeded") +# switch("warningAsError", "InheritFromException") # would require fixing a few tests # experimental API's are enabled in testament, refs https://github.com/timotheecour/Nim/issues/575 # sync with `kochdocs.docDefines` or refactor. diff --git a/tests/generics/mdotlookup.nim b/tests/generics/mdotlookup.nim index 470fa0a51..3112c133f 100644 --- a/tests/generics/mdotlookup.nim +++ b/tests/generics/mdotlookup.nim @@ -1,9 +1,9 @@ -proc baz(o: any): int = 5 # if bar is exported, it works +proc baz(o: auto): int = 5 # if bar is exported, it works type MyObj = object x: int -proc foo*(b: any) = +proc foo*(b: auto) = var o: MyObj echo b.baz, " ", o.x.baz, " ", b.baz() diff --git a/tests/iter/tyieldintry.nim b/tests/iter/tyieldintry.nim index 3d5260800..9862fe1de 100644 --- a/tests/iter/tyieldintry.nim +++ b/tests/iter/tyieldintry.nim @@ -1,17 +1,15 @@ discard """ -targets: "c" -output: "ok" + targets: "c cpp" """ -var closureIterResult = newSeq[int]() -# XXX Investigate why this fails now for 'nim cpp' +var closureIterResult = newSeq[int]() proc checkpoint(arg: int) = closureIterResult.add(arg) type - TestException = object of Exception - AnotherException = object of Exception + TestError = object of CatchableError + AnotherError = object of CatchableError proc testClosureIterAux(it: iterator(): int, exceptionExpected: bool, expectedResults: varargs[int]) = closureIterResult.setLen(0) @@ -21,7 +19,7 @@ proc testClosureIterAux(it: iterator(): int, exceptionExpected: bool, expectedRe try: for i in it(): closureIterResult.add(i) - except TestException: + except TestError: exceptionCaught = true if closureIterResult != @expectedResults or exceptionCaught != exceptionExpected: @@ -39,8 +37,8 @@ proc test(it: iterator(): int, expectedResults: varargs[int]) = proc testExc(it: iterator(): int, expectedResults: varargs[int]) = testClosureIterAux(it, true, expectedResults) -proc raiseException() = - raise newException(TestException, "Test exception!") +proc raiseTestError() = + raise newException(TestError, "Test exception!") block: iterator it(): int {.closure.} = @@ -58,8 +56,8 @@ block: yield 0 try: checkpoint(1) - raiseException() - except TestException: + raiseTestError() + except TestError: checkpoint(2) yield 3 checkpoint(4) @@ -89,7 +87,7 @@ block: yield 0 try: yield 1 - raiseException() + raiseTestError() yield 2 finally: checkpoint(3) @@ -103,8 +101,8 @@ block: iterator it(): int {.closure.} = try: try: - raiseException() - except AnotherException: + raiseTestError() + except AnotherError: yield 123 finally: checkpoint(3) @@ -117,8 +115,8 @@ block: iterator it(): int {.closure.} = try: yield 1 - raiseException() - except AnotherException: + raiseTestError() + except AnotherError: checkpoint(123) finally: checkpoint(2) @@ -134,12 +132,12 @@ block: yield 1 try: yield 2 - raiseException() - except AnotherException: + raiseTestError() + except AnotherError: yield 123 finally: yield 3 - except AnotherException: + except AnotherError: yield 124 finally: yield 4 @@ -170,10 +168,10 @@ block: try: try: yield 0 - raiseException() + raiseTestError() finally: checkpoint(1) - except TestException: + except TestError: yield 2 return finally: @@ -188,10 +186,10 @@ block: try: try: yield 0 - raiseException() + raiseTestError() finally: return # Return in finally should stop exception propagation - except AnotherException: + except AnotherError: yield 2 return finally: @@ -228,9 +226,9 @@ block: var foo = 123 let i = try: yield 0 - raiseException() + raiseTestError() 1 - except TestException as e: + except TestError as e: assert(e.msg == "Test exception!") case foo of 1: @@ -262,9 +260,9 @@ block: template tryexcept: int = try: yield 1 - raiseException() + raiseTestError() 123 - except TestException: + except TestError: yield 2 checkpoint(3) 4 @@ -323,11 +321,11 @@ block: for i in 0 .. 1: try: yield 1 - raiseException() - except TestException as e: + raiseTestError() + except TestError as e: doAssert(e.msg == "Test exception!") yield 2 - except AnotherException: + except AnotherError: yield 123 except: yield 1234 @@ -497,5 +495,3 @@ block: #17849 - yield in case subject yield 5 test(it, 1, 2, 13, 5) - -echo "ok" diff --git a/tests/overload/toverload_various.nim b/tests/overload/toverload_various.nim index f913ce3ee..55a8d17e6 100644 --- a/tests/overload/toverload_various.nim +++ b/tests/overload/toverload_various.nim @@ -395,7 +395,7 @@ block: template bar2[F,T](x: FooUn[F,T]): int = 1 template bar2[F,T1,T2](x: FooBi[F,T1,T2]): int = 2 - proc test(x: any, n: int) = + proc test(x: auto, n: int) = doAssert(foo1(x) == n) doAssert(foo2(x) == n) doAssert(bar1(x) == n) diff --git a/tests/template/template_various.nim b/tests/template/template_various.nim index 75226ea2d..e7a2be748 100644 --- a/tests/template/template_various.nim +++ b/tests/template/template_various.nim @@ -94,7 +94,7 @@ block generic_templates: var i3: int = t1[int]("xx") - +from strutils import contains block tgetast_typeliar: proc error(s: string) = quit s @@ -111,7 +111,9 @@ block tgetast_typeliar: var message : NimNode = newLit(condition.repr) # echo message result = getAst assertOrReturn2(condition, message) - echo result.repr + # echo result.repr + let s = result.repr + doAssert """error("Assertion failed:""" in s proc point(size: int16): tuple[x, y: int16] = # returns random point in square area with given `size` diff --git a/tests/template/tparams_gensymed.nim b/tests/template/tparams_gensymed.nim index 1444667f4..b68d7e253 100644 --- a/tests/template/tparams_gensymed.nim +++ b/tests/template/tparams_gensymed.nim @@ -72,7 +72,7 @@ proc concreteProc(x: int) = forStatic i, 0..3: echo i -proc genericProc(x: any) = +proc genericProc(x: auto) = forStatic i, 0..3: echo i From eb19db6595846c92e5eb999d96605fe4bb0934f7 Mon Sep 17 00:00:00 2001 From: Kyle Brown Date: Sun, 8 Aug 2021 10:30:51 -0700 Subject: [PATCH 1127/2472] Nim pretty and add void (#18652) * pretty and add void * distro fixup * Update lib/pure/distros.nim Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * tab to space * requested change to be cleaner Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- lib/pure/distros.nim | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/lib/pure/distros.nim b/lib/pure/distros.nim index 7a9910c1e..ddf0fd30c 100644 --- a/lib/pure/distros.nim +++ b/lib/pure/distros.nim @@ -109,7 +109,7 @@ type Clonezilla SteamOS Absolute - NixOS ## NixOS or a Nix build environment + NixOS ## NixOS or a Nix build environment AUSTRUMI Arya Porteus @@ -124,6 +124,7 @@ type ExTiX Rockstor GoboLinux + Void BSD FreeBSD @@ -134,7 +135,9 @@ type const - LacksDevPackages* = {Distribution.Gentoo, Distribution.Slackware, Distribution.ArchLinux, Distribution.Artix} + LacksDevPackages* = {Distribution.Gentoo, Distribution.Slackware, + Distribution.ArchLinux, Distribution.Artix, Distribution.Antergos, + Distribution.BlackArch, Distribution.ArchBang} # we cache the result of the 'cmdRelease' # execution for faster platform detections. @@ -153,7 +156,8 @@ template hostnamectl(): untyped = cmdRelease("hostnamectl", hostnamectlRes) proc detectOsWithAllCmd(d: Distribution): bool = let dd = toLowerAscii($d) result = dd in toLowerAscii(osReleaseID()) or dd in toLowerAscii(release()) or - dd in toLowerAscii(uname()) or ("operating system: " & dd) in toLowerAscii(hostnamectl()) + dd in toLowerAscii(uname()) or ("operating system: " & dd) in + toLowerAscii(hostnamectl()) proc detectOsImpl(d: Distribution): bool = case d @@ -173,9 +177,9 @@ proc detectOsImpl(d: Distribution): bool = case d of Distribution.Gentoo: result = ("-" & $d & " ") in uname() - of Distribution.Elementary, Distribution.Ubuntu, Distribution.Debian, Distribution.Fedora, - Distribution.OpenMandriva, Distribution.CentOS, Distribution.Alpine, - Distribution.Mageia, Distribution.Zorin: + of Distribution.Elementary, Distribution.Ubuntu, Distribution.Debian, + Distribution.Fedora, Distribution.OpenMandriva, Distribution.CentOS, + Distribution.Alpine, Distribution.Mageia, Distribution.Zorin, Distribution.Void: result = toLowerAscii($d) in osReleaseID() of Distribution.RedHat: result = "rhel" in osReleaseID() @@ -254,6 +258,8 @@ proc foreignDepInstallCmd*(foreignPackageName: string): (string, bool) = result = ("rpm -ivh " & p, true) elif detectOs(ArchLinux) or detectOs(Manjaro) or detectOs(Artix): result = ("pacman -S " & p, true) + elif detectOs(Void): + result = ("xbps-install " & p, true) else: result = (" install " & p, true) elif defined(haiku): From 0d3af5454b66dfe4a8a7017030708a890207c658 Mon Sep 17 00:00:00 2001 From: Christian Ulrich Date: Sun, 8 Aug 2021 20:11:07 +0200 Subject: [PATCH 1128/2472] Only allow IPv4 literals in strict form (#18656) * Only allow IPv4 literals in strict form The strict form as defined in RFC 6943, section 3.1.1 only allows the dotted form ddd.ddd.ddd.ddd of IPv4 literals, where ddd is a one to three digit decimal number between 0 and 255. Until now octal numbers (with a leading zero) were interpreted as decimal numbers which has security implications, see CVE-2021-29922 and CVE-2021-29923. * Update lib/pure/net.nim Co-authored-by: Dominik Picheta --- lib/pure/net.nim | 25 +++++++++-- tests/stdlib/tnet.nim | 35 +++++++++++++++ tests/stdlib/tparseipv6.nim | 90 ++++++++++++++++++++----------------- 3 files changed, 106 insertions(+), 44 deletions(-) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index c0f9e1465..09cdc8afb 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -281,14 +281,20 @@ proc parseIPv4Address(addressStr: string): IpAddress = byteCount = 0 currentByte: uint16 = 0 separatorValid = false + leadingZero = false result = IpAddress(family: IpAddressFamily.IPv4) for i in 0 .. high(addressStr): if addressStr[i] in strutils.Digits: # Character is a number + if leadingZero: + raise newException(ValueError, + "Invalid IP address. Octal numbers are not allowed") currentByte = currentByte * 10 + cast[uint16](ord(addressStr[i]) - ord('0')) - if currentByte > 255'u16: + if currentByte == 0'u16: + leadingZero = true + elif currentByte > 255'u16: raise newException(ValueError, "Invalid IP Address. Value is out of range") separatorValid = true @@ -300,6 +306,7 @@ proc parseIPv4Address(addressStr: string): IpAddress = currentByte = 0 byteCount.inc separatorValid = false + leadingZero = false else: raise newException(ValueError, "Invalid IP Address. Address contains an invalid character") @@ -388,10 +395,16 @@ proc parseIPv6Address(addressStr: string): IpAddress = result.address_v6[groupCount*2+1] = cast[uint8](currentShort and 0xFF) groupCount.inc() else: # Must parse IPv4 address + var leadingZero = false for i, c in addressStr[v4StartPos..high(addressStr)]: if c in strutils.Digits: # Character is a number + if leadingZero: + raise newException(ValueError, + "Invalid IP address. Octal numbers not allowed") currentShort = currentShort * 10 + cast[uint32](ord(c) - ord('0')) - if currentShort > 255'u32: + if currentShort == 0'u32: + leadingZero = true + elif currentShort > 255'u32: raise newException(ValueError, "Invalid IP Address. Value is out of range") separatorValid = true @@ -402,6 +415,7 @@ proc parseIPv6Address(addressStr: string): IpAddress = currentShort = 0 byteCount.inc() separatorValid = false + leadingZero = false else: # Invalid character raise newException(ValueError, "Invalid IP Address. Address contains an invalid character") @@ -431,7 +445,12 @@ proc parseIPv6Address(addressStr: string): IpAddress = proc parseIpAddress*(addressStr: string): IpAddress = ## Parses an IP address - ## Raises ValueError on error + ## + ## Raises ValueError on error. + ## + ## For IPv4 addresses, only the strict form as + ## defined in RFC 6943 is considered valid, see + ## https://datatracker.ietf.org/doc/html/rfc6943#section-3.1.1. if addressStr.len == 0: raise newException(ValueError, "IP Address string is empty") if addressStr.contains(':'): diff --git a/tests/stdlib/tnet.nim b/tests/stdlib/tnet.nim index b19d31f6c..671e0f7bc 100644 --- a/tests/stdlib/tnet.nim +++ b/tests/stdlib/tnet.nim @@ -50,6 +50,41 @@ block: # parseIpAddress tests expect(ValueError): discard parseIpAddress("gggg:cdba:0000:0000:0000:0000:3257:9652") + block: # ipv4-compatible ipv6 address (embedded ipv4 address) + check parseIpAddress("::ffff:10.0.0.23") == parseIpAddress("::ffff:0a00:0017") + + block: # octal number in ipv4 address + expect(ValueError): + discard parseIpAddress("010.8.8.8") + expect(ValueError): + discard parseIpAddress("8.010.8.8") + + block: # hexadecimal number in ipv4 address + expect(ValueError): + discard parseIpAddress("0xc0.168.0.1") + expect(ValueError): + discard parseIpAddress("192.0xa8.0.1") + + block: # less than 4 numbers in ipv4 address + expect(ValueError): + discard parseIpAddress("127.0.1") + + block: # octal number in embedded ipv4 address + expect(ValueError): + discard parseIpAddress("::ffff:010.8.8.8") + expect(ValueError): + discard parseIpAddress("::ffff:8.010.8.8") + + block: # hexadecimal number in embedded ipv4 address + expect(ValueError): + discard parseIpAddress("::ffff:0xc0.168.0.1") + expect(ValueError): + discard parseIpAddress("::ffff:192.0xa8.0.1") + + block: # less than 4 numbers in embedded ipv4 address + expect(ValueError): + discard parseIpAddress("::ffff:127.0.1") + block: # "IpAddress/Sockaddr conversion" proc test(ipaddrstr: string) = var ipaddr_1 = parseIpAddress(ipaddrstr) diff --git a/tests/stdlib/tparseipv6.nim b/tests/stdlib/tparseipv6.nim index 3e1c23e58..dd9abc511 100644 --- a/tests/stdlib/tparseipv6.nim +++ b/tests/stdlib/tparseipv6.nim @@ -8,23 +8,23 @@ const positives = [ "::f:8:a8f:218.17.235.229", "::b:228.19.241.2", - "::8:c:a:f:8.35.8.096", - "::3:e:a:bc:04.19.2.9", + "::8:c:a:f:8.35.8.96", + "::3:e:a:bc:4.19.2.9", "::2:212.242.248.19", "::df:a5f:3.250.208.9", "::8:c:5:e63:250.208.249.0", "::b:f:181.12.9.98", - "::a:f8:77.08.243.232", - "::a:b:85:e4d9:252.9.229.056", + "::a:f8:77.8.243.232", + "::a:b:85:e4d9:252.9.229.56", "941:c:8a:c:e::917", "e8:7a:e:ad:88a:8:203.235.225.46", - "139c:9e::f8:254.08.21.249", + "139c:9e::f8:254.8.21.249", "b38:f0:e::f9:89.6.12.18", "ef::8", "5::ab", "a::8:255.247.96.253", "b:c0::c:254.248.95.254", - "::8c:2:99.251.024.3", + "::8c:2:99.251.24.3", "98::c:247.240.249.57", "9::9", "628::f1ed:f", @@ -106,68 +106,68 @@ const "::315", "::a:a", "::aed3:a", - "f0eb:0:e8:b:c:a:254.098.233.17", + "f0eb:0:e8:b:c:a:254.98.233.17", "bfa:7fc:c66d:15:e9a:ded:254.119.9.9", - "d:ffa8:9:a:879:3:202.39.08.245", + "d:ffa8:9:a:879:3:202.39.8.245", "8e:2:8:fa8a:f1d1:1aa8:252.254.245.81", - "5:d4:a:e9:8:8:06.38.98.253", - "9c5:4:a5c:f:a6:8c9d:05.250.8.2", + "5:d4:a:e9:8:8:6.38.98.253", + "9c5:4:a5c:f:a6:8c9d:5.250.8.2", "d19a:2:f808:be:f:c:98.86.197.249", - "8:26ac:8:8:cb:f:242.00.254.85", + "8:26ac:8:8:cb:f:242.0.254.85", "38:e:1:0b88:f:0:8.89.248.92", - "e7:ff96:a:f:f:b:253.91.052.195", + "e7:ff96:a:f:f:b:253.91.52.195", "d:8:2:5:894:5:254.0.240.199", "2:98:9:8aa:9c8f:fa:252.98.248.17", - "e9:d4f:890:ccbe:5:8:088.200.228.216", + "e9:d4f:890:ccbe:5:8:88.200.228.216", "3:3:9:5:6a:df5:255.251.8.12", - "0280:3:8:8:4:9:255.000.251.249", + "0280:3:8:8:4:9:255.0.251.249", "8:af7:db:aa:0:9:238.248.250.255", - "ff:ee:9a:9252:a:289:059.083.18.255", + "ff:ee:9a:9252:a:289:59.83.18.255", "9f6:5:fc9:b:a89:a:142.1.250.254", "e:981a:da:bf94:9:f8:254.242.18.95", "3c:1:4:f2:89:f:8.91.255.14", - "e::9a2:c:9.050.80.8", + "e::9a2:c:9.50.80.8", "9::4a:07:fb:211.241.254.228", "9be::2:e:215.189.48.188", - "f::f:d:069.148.99.168", + "f::f:d:69.148.99.168", "f::a:97.18.240.47", "c::a98e:1:251.253.252.254", "668::82:214.87.208.9", "9c0::cf0:ecb:253.208.238.255", - "a::0:f1:210.240.238.049", - "8::a:1:251.238.34.09", + "a::0:f1:210.240.238.49", + "8::a:1:251.238.34.9", "81:dfe::b8:8.255.249.248", - "d3::7:b:9:83.189.08.244", - "8::9:8:8:00.7.11.252", + "d3::7:b:9:83.189.8.244", + "8::9:8:8:0.7.11.252", "2:8::c:a8:250.221.9.249", "2::f:99.8.249.247", "c:22f5::5:2c:243.15.79.89", "e:8e::da:251.243.255.2", "f15f:9::a:255.70.247.218", - "f:b::9f38:31.220.94.022", - "9::9a48:03.98.249.119", + "f:b::9f38:31.220.94.22", + "9::9a48:3.98.249.119", "d:d:9b87::2d:a:249.253.38.8", "d86d:99b::a9b:5:242.236.8.244", "eb:3::f:9cf:1.253.1.228", "b::ba2:255.247.114.64", - "2f:ec:bcb::9:219.254.250.094", + "2f:ec:bcb::9:219.254.250.94", "da8a:f6::a:e0:19.251.241.251", - "5e:c1::a:021.250.8.254", + "5e:c1::a:21.250.8.254", "c:9::8c9b:248.219.212.252", "2:a::8d4a:216.255.198.223", - "1f::66:255.30.08.150", + "1f::66:255.30.8.150", "bc2b:8f::2ff9:6.245.99.230", "a:8::a8:9.251.246.255", - "f:7:7::98:06.14.1.208", + "f:7:7::98:6.14.1.208", "e:2::9:218.249.255.254", "79:f::6:250.255.98.246", - "47:9:fb9f::9:038.136.17.251", + "47:9:fb9f::9:38.136.17.251", "ed::a:247.9.23.239", "6f::f1:88.254.119.9", "a::d:218.199.236.0", - "fc88::9:203.196.04.95", - "::8.048.255.85", - "::253.07.255.36", + "fc88::9:203.196.4.95", + "::8.48.255.85", + "::253.7.255.36", "9:d::253.7.178.229", "::250.84.158.253", "::8.55.204.248", @@ -175,31 +175,39 @@ const "df9:88ca::248.255.108.17", "8e9b::250.206.0.82", "::209.8.254.209", - "::247.088.8.8", + "::247.88.8.8", "::cb:f:ba41:250.208.19.249", "::fe:0e8:243.240.229.5", "::c:223.251.5.226", - "::8:08.03.8.250", + "::8:8.3.8.250", "::f:8.88.11.255", - "::fda:48:aa:05.189.07.2", - "::8:c3f:f:240.06.212.255", + "::fda:48:aa:5.189.7.2", + "::8:c3f:f:240.6.212.255", "::f:0aa:244.123.99.16", - "::c9b5:c:034.8.090.196", + "::c9b5:c:34.8.90.196", "::98:c9:254.14.241.81" ] negatives = ["foo.bar", "::::::::::::", "yet another failure", - "de:6:c:ab5:6a::9:252.06.06.249", - "f9:5f7:fa38:9:b::b6:09.255.248.252", + "de:6:c:ab5:6a::9:252.6.6.249", + "f9:5f7:fa38:9:b::b6:9.255.248.252", "97:c:5b:81:8a::f5dd:144.252.250.9", - "9:8:cd:8:a9::f:247.255.09.255", + "9:8:cd:8:a9::f:247.255.9.255", "18:1:8c:2:3::9:8.254.252.139", - "e:c298:3:e:a::bb12:254.246.05.250", + "e:c298:3:e:a::bb12:254.246.5.250", "e:e:c:8e:fd::8:253.8.49.231", "9:97f:f:e929:8a::c9:0.8.252.10", - "0df:b24:7:89:c::2b:16.249.240.092", + "0df:b24:7:89:c::2b:16.249.240.92", "b:8f5f:485:c:9a::84c:178.7.249.34", + "::3:e:a:bc:091.19.2.9", + "::a:f8:77.08.243.232", + "::8c:2:99.251.029.3", + "::8:c:a:f:8.35.8.096", + "d:ffa8:9:a:879:3:0202.39.8.245", + "139c:9e::f8:254.07.21.249", + "f0eb:0:e8:b:c:a:254.233.043.17", + "::a:b:85:e4d9:252.9.229.056", ] proc ok(pos: openarray[string]) = From a7e622267e1049a454f4f522c0dbce5eb4017495 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 9 Aug 2021 02:44:14 +0800 Subject: [PATCH 1129/2472] use lent (#18638) --- lib/system/excpt.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index 4814b77b4..3e520570f 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -521,10 +521,10 @@ proc getStackTrace(e: ref Exception): string = else: result = "" -proc getStackTraceEntries*(e: ref Exception): seq[StackTraceEntry] = +proc getStackTraceEntries*(e: ref Exception): lent seq[StackTraceEntry] = ## Returns the attached stack trace to the exception `e` as ## a `seq`. This is not yet available for the JS backend. - shallowCopy(result, e.trace) + e.trace proc getStackTraceEntries*(): seq[StackTraceEntry] = ## Returns the stack trace entries for the current stack trace. From 988c02b7cbef634523b87af04097d06ed453dd61 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 8 Aug 2021 19:24:45 -0700 Subject: [PATCH 1130/2472] renamed: lib/std/private/vmutils.nim -> lib/std/private/bitops_utils.nim to avoid confusion with unrelated std/vmutils (#18660) --- lib/pure/bitops.nim | 4 +--- lib/std/private/{vmutils.nim => bitops_utils.nim} | 0 lib/system/countbits_impl.nim | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) rename lib/std/private/{vmutils.nim => bitops_utils.nim} (100%) diff --git a/lib/pure/bitops.nim b/lib/pure/bitops.nim index 6d3865e5d..1b3838621 100644 --- a/lib/pure/bitops.nim +++ b/lib/pure/bitops.nim @@ -27,9 +27,7 @@ import macros import std/private/since -from std/private/vmutils import forwardImpl, toUnsigned - - +from std/private/bitops_utils import forwardImpl, toUnsigned func bitnot*[T: SomeInteger](x: T): T {.magic: "BitnotI".} ## Computes the `bitwise complement` of the integer `x`. diff --git a/lib/std/private/vmutils.nim b/lib/std/private/bitops_utils.nim similarity index 100% rename from lib/std/private/vmutils.nim rename to lib/std/private/bitops_utils.nim diff --git a/lib/system/countbits_impl.nim b/lib/system/countbits_impl.nim index 184f97bd4..020423e01 100644 --- a/lib/system/countbits_impl.nim +++ b/lib/system/countbits_impl.nim @@ -9,7 +9,7 @@ ## Contains the used algorithms for counting bits. -from std/private/vmutils import forwardImpl, toUnsigned +from std/private/bitops_utils import forwardImpl, toUnsigned const useBuiltins* = not defined(noIntrinsicsBitOpts) From 8ce782d463b0bca091be5a34db190dc441c23ce4 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 8 Aug 2021 23:51:39 -0700 Subject: [PATCH 1131/2472] Add optional recursive arg to distinctBase (v2) (#18659) * Add optional recursive arg to distinctBase * Add docs and examples Co-authored-by: ALANVF --- compiler/semmagic.nim | 5 +++-- lib/pure/typetraits.nim | 17 ++++++++++++----- lib/std/jsonutils.nim | 4 ++-- lib/system/repr_v2.nim | 2 +- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim index 04f4c4729..87f7da815 100644 --- a/compiler/semmagic.nim +++ b/compiler/semmagic.nim @@ -190,9 +190,10 @@ proc evalTypeTrait(c: PContext; traitCall: PNode, operand: PType, context: PSym) result = newIntNodeT(toInt128(operand.len), traitCall, c.idgen, c.graph) of "distinctBase": var arg = operand.skipTypes({tyGenericInst}) + let rec = semConstExpr(c, traitCall[2]).intVal != 0 while arg.kind == tyDistinct: - arg = arg.base - arg = arg.skipTypes(skippedTypes + {tyGenericInst}) + arg = arg.base.skipTypes(skippedTypes + {tyGenericInst}) + if not rec: break result = getTypeDescNode(c, arg, operand.owner, traitCall.info) else: localError(c.config, traitCall.info, "unknown trait: " & s) diff --git a/lib/pure/typetraits.nim b/lib/pure/typetraits.nim index e63585f63..8dc1b8cf2 100644 --- a/lib/pure/typetraits.nim +++ b/lib/pure/typetraits.nim @@ -110,25 +110,32 @@ template pointerBase*[T](_: typedesc[ptr T | ref T]): typedesc = assert (var s = "abc"; s[0].addr).typeof.pointerBase is char T -proc distinctBase*(T: typedesc): typedesc {.magic: "TypeTrait".} = +proc distinctBase*(T: typedesc, recursive: static bool = true): typedesc {.magic: "TypeTrait".} = ## Returns the base type for distinct types, or the type itself otherwise. + ## If `recursive` is false, only the immediate distinct base will be returned. ## ## **See also:** - ## * `distinctBase template <#distinctBase.t,T>`_ + ## * `distinctBase template <#distinctBase.t,T,static[bool]>`_ runnableExamples: type MyInt = distinct int + type MyOtherInt = distinct MyInt doAssert distinctBase(MyInt) is int + doAssert distinctBase(MyOtherInt) is int + doAssert distinctBase(MyOtherInt, false) is MyInt doAssert distinctBase(int) is int since (1, 1): - template distinctBase*[T](a: T): untyped = - ## Overload of `distinctBase <#distinctBase,typedesc>`_ for values. + template distinctBase*[T](a: T, recursive: static bool = true): untyped = + ## Overload of `distinctBase <#distinctBase,typedesc,static[bool]>`_ for values. runnableExamples: type MyInt = distinct int + type MyOtherInt = distinct MyInt doAssert 12.MyInt.distinctBase == 12 + doAssert 12.MyOtherInt.distinctBase == 12 + doAssert 12.MyOtherInt.distinctBase(false) is MyInt doAssert 12.distinctBase == 12 when T is distinct: - distinctBase(typeof(a))(a) + distinctBase(typeof(a), recursive)(a) else: # avoids hint ConvFromXtoItselfNotNeeded a diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim index c141985d3..727a75288 100644 --- a/lib/std/jsonutils.nim +++ b/lib/std/jsonutils.nim @@ -78,8 +78,8 @@ proc initToJsonOptions*(): ToJsonOptions = ## initializes `ToJsonOptions` with sane options. ToJsonOptions(enumMode: joptEnumOrd, jsonNodeMode: joptJsonNodeAsRef) -proc distinctBase(T: typedesc): typedesc {.magic: "TypeTrait".} -template distinctBase[T](a: T): untyped = distinctBase(typeof(a))(a) +proc distinctBase(T: typedesc, recursive: static bool = true): typedesc {.magic: "TypeTrait".} +template distinctBase[T](a: T, recursive: static bool = true): untyped = distinctBase(typeof(a), recursive)(a) macro getDiscriminants(a: typedesc): seq[string] = ## return the discriminant keys diff --git a/lib/system/repr_v2.nim b/lib/system/repr_v2.nim index 8471ea148..f99f09799 100644 --- a/lib/system/repr_v2.nim +++ b/lib/system/repr_v2.nim @@ -1,7 +1,7 @@ proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} ## imported from typetraits -proc distinctBase(T: typedesc): typedesc {.magic: "TypeTrait".} +proc distinctBase(T: typedesc, recursive: static bool = true): typedesc {.magic: "TypeTrait".} ## imported from typetraits proc repr*(x: NimNode): string {.magic: "Repr", noSideEffect.} From 31fc0f97187cef472cf113629a90c7aa126d3693 Mon Sep 17 00:00:00 2001 From: Kyle Brown Date: Tue, 10 Aug 2021 16:26:42 -0700 Subject: [PATCH 1132/2472] Remove unused imports, and deprecated function usage (#18663) * clean up imports and slice to remove delete * revert buggy code * Replace "delete" with setlen to remove depreciation warning --- compiler/docgen.nim | 4 ++-- compiler/jsgen.nim | 2 +- tools/atlas/atlas.nim | 2 +- tools/atlas/packagesjson.nim | 1 - 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 3bc01bbe6..d39a20aae 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -835,7 +835,7 @@ proc docstringSummary(rstText: string): string = result = rstText.substr(2).strip var pos = result.find('\L') if pos > 0: - result.delete(pos, result.len - 1) + result.setLen(pos - 1) result.add("…") if pos < maxDocstringChars: return @@ -843,7 +843,7 @@ proc docstringSummary(rstText: string): string = pos = result.find({'.', ',', ':'}) let last = result.len - 1 if pos > 0 and pos < last: - result.delete(pos, last) + result.setLen(pos - 1) result.add("…") proc genDeprecationMsg(d: PDoc, n: PNode): string = diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 85947535e..49a57b4a6 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -29,7 +29,7 @@ implements the required case distinction. import - ast, astalgo, trees, magicsys, options, + ast, trees, magicsys, options, nversion, msgs, idents, types, ropes, passes, ccgutils, wordrecg, renderer, cgmeth, lowerings, sighashes, modulegraphs, lineinfos, rodutils, diff --git a/tools/atlas/atlas.nim b/tools/atlas/atlas.nim index d0990c1f7..761ca5db8 100644 --- a/tools/atlas/atlas.nim +++ b/tools/atlas/atlas.nim @@ -9,7 +9,7 @@ ## Simple tool to automate frequent workflows: Can "clone" ## a Nimble dependency and its dependencies recursively. -import std/[parseopt, strutils, os, osproc, sequtils, unicode, tables, sets] +import std/[parseopt, strutils, os, osproc, unicode, tables, sets] import parse_requires, osutils, packagesjson const diff --git a/tools/atlas/packagesjson.nim b/tools/atlas/packagesjson.nim index ed83ce85c..0b8599769 100644 --- a/tools/atlas/packagesjson.nim +++ b/tools/atlas/packagesjson.nim @@ -1,6 +1,5 @@ import std / [json, os, sets, strutils] -import osutils type Package* = ref object From d0dd923acf8c2ddfa5e0ea7574c8d5200416e9f5 Mon Sep 17 00:00:00 2001 From: RSDuck Date: Wed, 11 Aug 2021 08:54:49 +0200 Subject: [PATCH 1133/2472] Switch maintanance (#18668) * Fix and improve Nintendo Switch support * Document the necessity for nimAllocPagesViaMalloc * update changelog * Use --gc:orc in examples --- changelog.md | 3 + doc/nimc.rst | 21 ++--- lib/nintendoswitch/switch_memory.nim | 36 --------- lib/pure/os.nim | 9 +-- lib/system/osalloc.nim | 111 --------------------------- 5 files changed, 15 insertions(+), 165 deletions(-) delete mode 100644 lib/nintendoswitch/switch_memory.nim diff --git a/changelog.md b/changelog.md index a19697c8c..0e3c54d77 100644 --- a/changelog.md +++ b/changelog.md @@ -104,6 +104,8 @@ - In `std/dom`, `Interval` is now a `ref object`, same as `Timeout`. Definitions of `setTimeout`, `clearTimeout`, `setInterval`, `clearInterval` were updated. +- The allocator for Nintendo Switch, which was nonfunctional because + of breaking changes in libnx, was removed, in favour of the new `-d:nimAllocPagesViaMalloc` option. ## Standard library additions and changes @@ -363,6 +365,7 @@ - Added `dom.setInterval`, `dom.clearInterval` overloads. +- Allow reading parameters when compiling for Nintendo Switch. - Deprecated `sequtils.delete` and added an overload taking a `Slice` that raises a defect if the slice is out of bounds, likewise with `strutils.delete`. diff --git a/doc/nimc.rst b/doc/nimc.rst index ed9077912..ad5ecaa76 100644 --- a/doc/nimc.rst +++ b/doc/nimc.rst @@ -406,16 +406,18 @@ to your usual `nim c`:cmd: or `nim cpp`:cmd: command and set the `passC`:option: and `passL`:option: command line switches to something like: .. code-block:: cmd - nim c ... --passC="-I$DEVKITPRO/libnx/include" ... + nim c ... --d:nimAllocPagesViaMalloc --gc:orc --passC="-I$DEVKITPRO/libnx/include" ... --passL="-specs=$DEVKITPRO/libnx/switch.specs -L$DEVKITPRO/libnx/lib -lnx" or setup a ``nim.cfg`` file like so:: #nim.cfg + --gc:orc + --d:nimAllocPagesViaMalloc --passC="-I$DEVKITPRO/libnx/include" --passL="-specs=$DEVKITPRO/libnx/switch.specs -L$DEVKITPRO/libnx/lib -lnx" -The DevkitPro setup must be the same as the default with their new installer +The devkitPro setup must be the same as the default with their new installer `here for Mac/Linux `_ or `here for Windows `_. @@ -426,20 +428,19 @@ For example, with the above-mentioned config: nim c --os:nintendoswitch switchhomebrew.nim This will generate a file called ``switchhomebrew.elf`` which can then be turned into -an nro file with the `elf2nro`:cmd: tool in the DevkitPro release. Examples can be found at +an nro file with the `elf2nro`:cmd: tool in the devkitPro release. Examples can be found at `the nim-libnx github repo `_. -There are a few things that don't work because the DevkitPro libraries don't support them. +There are a few things that don't work because the devkitPro libraries don't support them. They are: 1. Waiting for a subprocess to finish. A subprocess can be started, but right now it can't be waited on, which sort of makes subprocesses a bit hard to use -2. Dynamic calls. DevkitPro libraries have no dlopen/dlclose functions. -3. Command line parameters. It doesn't make sense to have these for a console - anyways, so no big deal here. -4. mqueue. Sadly there are no mqueue headers. -5. ucontext. No headers for these either. No coroutines for now :( -6. nl_types. No headers for this. +2. Dynamic calls. Switch OS (Horizon) doesn't support dynamic libraries, so dlopen/dlclose are not available. +3. mqueue. Sadly there are no mqueue headers. +4. ucontext. No headers for these either. No coroutines for now :( +5. nl_types. No headers for this. +6. As mmap is not supported, the nimAllocPagesViaMalloc option has to be used. DLL generation ============== diff --git a/lib/nintendoswitch/switch_memory.nim b/lib/nintendoswitch/switch_memory.nim deleted file mode 100644 index f34bd363a..000000000 --- a/lib/nintendoswitch/switch_memory.nim +++ /dev/null @@ -1,36 +0,0 @@ -## All of these library headers and source can be found in the github repo -## https://github.com/switchbrew/libnx. - -const virtMemHeader = "" -const svcHeader = "" -const mallocHeader = "" - -## Aligns a block of memory with request `size` to `bytes` size. For -## example, a request of memalign(0x1000, 0x1001) == 0x2000 bytes allocated -proc memalign*(bytes: csize, size: csize): pointer {.importc: "memalign", - header: mallocHeader.} - -# Should be required, but not needed now because of how -# svcUnmapMemory frees all memory -#proc free*(address: pointer) {.importc: "free", -# header: mallocHeader.} - -## Maps a memaligned block of memory from `src_addr` to `dst_addr`. The -## Nintendo Switch requires this call in order to make use of memory, otherwise -## an invalid memory access occurs. -proc svcMapMemory*(dst_addr: pointer; src_addr: pointer; size: uint64): uint32 {. - importc: "svcMapMemory", header: svcHeader.} - -## Unmaps (frees) all memory from both `dst_addr` and `src_addr`. **Must** be called -## whenever svcMapMemory is used. The Switch will expect all memory to be allocated -## before gfxExit() calls () -proc svcUnmapMemory*(dst_addr: pointer; src_addr: pointer; size: uint64): uint32 {. - importc: "svcUnmapMemory", header: svcHeader.} - -proc virtmemReserveMap*(size: csize): pointer {.importc: "virtmemReserveMap", - header: virtMemHeader.} - -# Should be required, but not needed now because of how -# svcUnmapMemory frees all memory -#proc virtmemFreeMap*(address: pointer; size: csize) {.importc: "virtmemFreeMap", -# header: virtMemHeader.} diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 73e3fcb31..fd618e93c 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -2910,13 +2910,6 @@ elif defined(nodejs): result = $argv[i] else: raise newException(IndexDefect, formatErrorIndexBound(i - 1, argv.len - 2)) -elif defined(nintendoswitch): - proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = - raise newException(OSError, "paramStr is not implemented on Nintendo Switch") - - proc paramCount*(): int {.tags: [ReadIOEffect].} = - raise newException(OSError, "paramCount is not implemented on Nintendo Switch") - elif defined(windows): # Since we support GUI applications with Nim, we sometimes generate # a WinMain entry proc. But a WinMain proc has no access to the parsed @@ -3190,7 +3183,7 @@ proc getAppFilename*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noW result = getApplAux("/proc/self/exe") elif defined(solaris): result = getApplAux("/proc/" & $getpid() & "/path/a.out") - elif defined(genode) or defined(nintendoswitch): + elif defined(genode): raiseOSError(OSErrorCode(-1), "POSIX command line not supported") elif defined(freebsd) or defined(dragonfly) or defined(netbsd): result = getApplFreebsd() diff --git a/lib/system/osalloc.nim b/lib/system/osalloc.nim index 2830adb48..39bf65d6c 100644 --- a/lib/system/osalloc.nim +++ b/lib/system/osalloc.nim @@ -103,117 +103,6 @@ elif defined(emscripten) and not defined(StandaloneHeapSize): elif defined(genode) and not defined(StandaloneHeapSize): include genode/alloc # osAllocPages, osTryAllocPages, osDeallocPages -elif defined(nintendoswitch) and not defined(StandaloneHeapSize): - - import nintendoswitch/switch_memory - - type - PSwitchBlock = ptr NSwitchBlock - ## This will hold the heap pointer data in a separate - ## block of memory that is PageSize bytes above - ## the requested memory. It's the only good way - ## to pass around data with heap allocations - NSwitchBlock {.pure, inheritable.} = object - realSize: int - heap: pointer # pointer to main heap alloc - heapMirror: pointer # pointer to virtmem mapped heap - - proc alignSize(size: int): int {.inline.} = - ## Align a size integer to be in multiples of PageSize - ## The nintendo switch will not allocate memory that is not - ## aligned to 0x1000 bytes and will just crash. - (size + (PageSize - 1)) and not (PageSize - 1) - - proc deallocate(heapMirror: pointer, heap: pointer, size: int) = - # Unmap the allocated memory - discard svcUnmapMemory(heapMirror, heap, size.uint64) - # These should be called (theoretically), but referencing them crashes the switch. - # The above call seems to free all heap memory, so these are not needed. - # virtmemFreeMap(nswitchBlock.heapMirror, nswitchBlock.realSize.csize) - # free(nswitchBlock.heap) - - proc freeMem(p: pointer) = - # Retrieve the switch block data from the pointer we set before - # The data is located just sizeof(NSwitchBlock) bytes below - # the top of the pointer to the heap - let - nswitchDescrPos = cast[ByteAddress](p) -% sizeof(NSwitchBlock) - nswitchBlock = cast[PSwitchBlock](nswitchDescrPos) - - deallocate( - nswitchBlock.heapMirror, nswitchBlock.heap, nswitchBlock.realSize - ) - - proc storeHeapData(address, heapMirror, heap: pointer, size: int) {.inline.} = - ## Store data in the heap for deallocation purposes later - - # the position of our heap pointer data. Since we allocated PageSize extra - # bytes, we should have a buffer on top of the requested size of at least - # PageSize bytes, which is much larger than sizeof(NSwitchBlock). So we - # decrement the address by sizeof(NSwitchBlock) and use that address - # to store our pointer data - let nswitchBlockPos = cast[ByteAddress](address) -% sizeof(NSwitchBlock) - - # We need to store this in a pointer obj (PSwitchBlock) so that the data sticks - # at the address we've chosen. If NSwitchBlock is used here, the data will - # be all 0 when we try to retrieve it later. - var nswitchBlock = cast[PSwitchBlock](nswitchBlockPos) - nswitchBlock.realSize = size - nswitchBlock.heap = heap - nswitchBlock.heapMirror = heapMirror - - proc getOriginalHeapPosition(address: pointer, difference: int): pointer {.inline.} = - ## This function sets the heap back to the originally requested - ## size - let - pos = cast[int](address) - newPos = cast[ByteAddress](pos) +% difference - - return cast[pointer](newPos) - - template allocPages(size: int, outOfMemoryStmt: untyped): untyped = - # This is to ensure we get a block of memory the requested - # size, as well as space to store our structure - let realSize = alignSize(size + sizeof(NSwitchBlock)) - - let heap = memalign(PageSize, realSize) - - if heap.isNil: - outOfMemoryStmt - - let heapMirror = virtmemReserveMap(realSize.csize) - result = heapMirror - - let rc = svcMapMemory(heapMirror, heap, realSize.uint64) - # Any return code not equal 0 means an error in libnx - if rc.uint32 != 0: - deallocate(heapMirror, heap, realSize) - outOfMemoryStmt - - # set result to be the original size requirement - result = getOriginalHeapPosition(result, realSize - size) - - storeHeapData(result, heapMirror, heap, realSize) - - proc osAllocPages(size: int): pointer {.inline.} = - allocPages(size): - raiseOutOfMem() - - proc osTryAllocPages(size: int): pointer = - allocPages(size): - return nil - - proc osDeallocPages(p: pointer, size: int) = - # Note that in order for the Switch not to crash, a call to - # deallocHeap(runFinalizers = true, allowGcAfterwards = false) - # must be run before gfxExit(). The Switch requires all memory - # to be deallocated before the graphics application has exited. - # - # gfxExit() can be found in in the github - # repo https://github.com/switchbrew/libnx - when reallyOsDealloc: - freeMem(p) - elif defined(posix) and not defined(StandaloneHeapSize): const PROT_READ = 1 # page can be read From 854006575498704c42b5ac465fdc74c3efdb5b97 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 11 Aug 2021 02:42:01 -0700 Subject: [PATCH 1134/2472] --cc:env now works correctly to assign linker executable, allowing to cross-compile/run for windows on osx via wine (#18672) * --cc:env now works correctly to assign linker executable, allowing using wine on osx * fixup --- compiler/extccomp.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index a091060ff..e7e1bcd51 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -252,7 +252,7 @@ compiler envcc: buildGui: "", buildDll: " -shared ", buildLib: "", # XXX: not supported yet - linkerExe: "cc", + linkerExe: "", linkTmpl: "-o $exefile $buildgui $builddll $objfiles $options", includeCmd: " -I", linkDirCmd: "", # XXX: not supported yet From 6c1bd4bb1cb7a6af38b5929fe02b069b03e39db4 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 11 Aug 2021 03:17:17 -0700 Subject: [PATCH 1135/2472] fix: `var a{.foo.} = expr` inside templates (refs #15920) (except when `foo` is overloaded) (#13869) * fix: `var a{.foo.} = expr` inside templates * add test * improve tdecls test * improve tests * add failing test * PRTEMP * fixup --- compiler/semstmts.nim | 35 ++++++++++++---------- compiler/semtempl.nim | 15 ++++++++-- tests/pragmas/tpragmas_misc.nim | 52 +++++++++++++++++++++++++++++++++ tests/stdlib/tdecls.nim | 23 ++++++++++++++- tests/stdlib/tsince.nim | 2 +- 5 files changed, 107 insertions(+), 20 deletions(-) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index cc09291c5..14dc89781 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -453,23 +453,29 @@ proc semLowerLetVarCustomPragma(c: PContext, a: PNode, n: PNode): PNode = return nil let nodePragma = b[1][0] # see: `singlePragma` - if nodePragma.kind notin {nkIdent, nkAccQuoted}: - return nil - let ident = considerQuotedIdent(c, nodePragma) - var userPragma = strTableGet(c.userPragmas, ident) - if userPragma != nil: return nil - - let w = nodePragma.whichPragma - if n.kind == nkVarSection and w in varPragmas or - n.kind == nkLetSection and w in letPragmas or - n.kind == nkConstSection and w in constPragmas: - return nil var amb = false - let sym = searchInScopes(c, ident, amb) - # XXX what if amb is true? - if sym == nil or sfCustomPragma in sym.flags: return nil + var sym: PSym = nil + case nodePragma.kind + of nkIdent, nkAccQuoted: + let ident = considerQuotedIdent(c, nodePragma) + var userPragma = strTableGet(c.userPragmas, ident) + if userPragma != nil: return nil + let w = nodePragma.whichPragma + if n.kind == nkVarSection and w in varPragmas or + n.kind == nkLetSection and w in letPragmas or + n.kind == nkConstSection and w in constPragmas: + return nil + sym = searchInScopes(c, ident, amb) + # XXX what if amb is true? + # CHECKME: should that test also apply to `nkSym` case? + if sym == nil or sfCustomPragma in sym.flags: return nil + of nkSym: + sym = nodePragma.sym + else: + return nil # skip if not in scope; skip `template myAttr() {.pragma.}` + let lhs = b[0] let clash = strTableGet(c.currentScope.symbols, lhs.ident) if clash != nil: @@ -477,7 +483,6 @@ proc semLowerLetVarCustomPragma(c: PContext, a: PNode, n: PNode): PNode = wrongRedefinition(c, lhs.info, lhs.ident.s, clash.info) result = newTree(nkCall) - doAssert nodePragma.kind in {nkIdent, nkAccQuoted}, $nodePragma.kind result.add nodePragma result.add lhs if a[1].kind != nkEmpty: diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim index a2b0f99ba..2636784af 100644 --- a/compiler/semtempl.nim +++ b/compiler/semtempl.nim @@ -208,9 +208,18 @@ proc addLocalDecl(c: var TemplCtx, n: var PNode, k: TSymKind) = if (n.kind == nkPragmaExpr and n.len >= 2 and n[1].kind == nkPragma): let pragmaNode = n[1] for i in 0..= version: + body + when false: # bug + template fun3(): int {.since3: (1, 3).} = 12 + + block: # ditto, w + # case with overload + template since2(version: (int, int), body: untyped) {.dirty.} = + when (NimMajor, NimMinor) >= version: + body + template since2(version: (int, int, int), body: untyped) {.dirty.} = + when (NimMajor, NimMinor, NimPatch) >= version: + body + when false: # bug + template fun3(): int {.since2: (1, 3).} = 12 + +when true: # D20210801T100514:here + from macros import genSym + block: + template fn() = + var ret {.gensym.}: int # must special case template pragmas so it doesn't get confused + discard ret + fn() + static: discard genSym() diff --git a/tests/stdlib/tdecls.nim b/tests/stdlib/tdecls.nim index 3567639e0..53e070bee 100644 --- a/tests/stdlib/tdecls.nim +++ b/tests/stdlib/tdecls.nim @@ -1,6 +1,10 @@ +discard """ + targets: "c cpp js" +""" + import std/decls -block: +template fun() = var s = @[10,11,12] var a {.byaddr.} = s[0] a+=100 @@ -34,6 +38,13 @@ block: doAssert compiles(block: var b2 {.byaddr.}: int = s[2]) +proc fun2() = fun() +fun() +fun2() +static: fun2() +when false: # pending bug #13887 + static: fun() + ## We can define custom pragmas in user code template byUnsafeAddr(lhs, typ, expr) = when typ is type(nil): @@ -68,3 +79,13 @@ block: # nkAccQuoted let a {.`cast`.} = s[0] doAssert a == "foo" doAssert a[0].unsafeAddr == s[0][0].unsafeAddr + +block: # bug #15920 + template foo(lhs, typ, expr) = + let lhs = expr + proc fun1()= + let a {.foo.} = 1 + template fun2()= + let a {.foo.} = 1 + fun1() # ok + fun2() # BUG diff --git a/tests/stdlib/tsince.nim b/tests/stdlib/tsince.nim index 14dd09c15..d0320ff12 100644 --- a/tests/stdlib/tsince.nim +++ b/tests/stdlib/tsince.nim @@ -27,6 +27,6 @@ since (99, 3): doAssert false when false: - # pending https://github.com/timotheecour/Nim/issues/129 + # pending bug #15920 # Error: cannot attach a custom pragma to 'fun3' template fun3(): int {.since: (1, 3).} = 12 From bc14b7735929f764b09446073375174236fab161 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 11 Aug 2021 17:19:23 +0200 Subject: [PATCH 1136/2472] fixes #18665 DFA generator bug (#18676) --- compiler/dfa.nim | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/compiler/dfa.nim b/compiler/dfa.nim index 402945042..65c483e9c 100644 --- a/compiler/dfa.nim +++ b/compiler/dfa.nim @@ -769,6 +769,12 @@ proc gen(c: var Con; n: PNode) = of nkCharLit..nkNilLit: discard of nkAsgn, nkFastAsgn: gen(c, n[1]) + + if n[0].kind in PathKinds0: + let a = c.skipTrivials(n[0]) + if a.kind in nkCallKinds: + gen(c, a) + # watch out: 'obj[i].f2 = value' sets 'f2' but # "uses" 'i'. But we are only talking about builtin array indexing so # it doesn't matter and 'x = 34' is NOT a usage of 'x'. From c94933acb7a8fe08e1ef479d02bd20732ea1ea23 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 11 Aug 2021 13:30:51 -0700 Subject: [PATCH 1137/2472] fix #18674 --nimcache now works better with --os:windows (#18675) --- compiler/commands.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compiler/commands.nim b/compiler/commands.nim index 1c3de29ba..82e908152 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -537,6 +537,10 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; conf.lazyPaths.keepItIf(it != path) of "nimcache": expectArg(conf, switch, arg, pass, info) + var arg = arg + # refs bug #18674, otherwise `--os:windows` messes up with `--nimcache` set + # in config nims files, e.g. via: `import os; switch("nimcache", "/tmp/somedir")` + if conf.target.targetOS == osWindows and DirSep == '/': arg = arg.replace('\\', '/') conf.nimcacheDir = processPath(conf, arg, info, notRelativeToProj=true) of "out", "o": expectArg(conf, switch, arg, pass, info) From 018465a2345b2d11f7d1d711010d97e636af98d0 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 12 Aug 2021 08:25:11 +0200 Subject: [PATCH 1138/2472] fixes #18643 [backport:1.0] (#18678) --- compiler/ccgexprs.nim | 2 +- tests/array/tarray.nim | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 79c73b602..acf83f966 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -920,7 +920,7 @@ proc genArrayElem(p: BProc, n, x, y: PNode, d: var TLoc) = if optBoundsCheck in p.options and ty.kind != tyUncheckedArray: if not isConstExpr(y): # semantic pass has already checked for const index expressions - if firstOrd(p.config, ty) == 0: + if firstOrd(p.config, ty) == 0 and lastOrd(p.config, ty) >= 0: if (firstOrd(p.config, b.t) < firstOrd(p.config, ty)) or (lastOrd(p.config, b.t) > lastOrd(p.config, ty)): linefmt(p, cpsStmts, "if ((NU)($1) > (NU)($2)){ #raiseIndexError2($1, $2); $3}$n", [rdCharLoc(b), intLiteral(lastOrd(p.config, ty)), raiseInstr(p)]) diff --git a/tests/array/tarray.nim b/tests/array/tarray.nim index 2765ad06d..d10011ef2 100644 --- a/tests/array/tarray.nim +++ b/tests/array/tarray.nim @@ -594,3 +594,14 @@ block t17705: a = int(a) var b = array[0, int].high b = int(b) + +block t18643: + # https://github.com/nim-lang/Nim/issues/18643 + let a: array[0, int] = [] + var caught = false + let b = 9999999 + try: + echo a[b] + except IndexDefect: + caught = true + doAssert caught, "IndexDefect not caught!" From 5c1304a4181596e764b05679cd8a0044ab3398dd Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 12 Aug 2021 07:50:08 -0700 Subject: [PATCH 1139/2472] fix #18670 quoteShellCommand, quoteShell, quoteShellWindows on windows (#18671) --- lib/pure/os.nim | 11 ++++++----- tests/stdlib/tos.nim | 13 ++++++++++++- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index fd618e93c..cfdacdb3c 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -1045,15 +1045,12 @@ proc expandTilde*(path: string): string {. # TODO: handle `~bob` and `~bob/` which means home of bob result = path -# TODO: consider whether quoteShellPosix, quoteShellWindows, quoteShell, quoteShellCommand -# belong in `strutils` instead; they are not specific to paths proc quoteShellWindows*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = ## Quote `s`, so it can be safely passed to Windows API. ## ## Based on Python's `subprocess.list2cmdline`. ## See `this link `_ ## for more details. - let needQuote = {' ', '\t'} in s or s.len == 0 result = "" var backslashBuff = "" @@ -1064,8 +1061,8 @@ proc quoteShellWindows*(s: string): string {.noSideEffect, rtl, extern: "nosp$1" if c == '\\': backslashBuff.add(c) elif c == '\"': - result.add(backslashBuff) - result.add(backslashBuff) + for i in 0.. 0: + result.add(backslashBuff) if needQuote: + result.add(backslashBuff) result.add("\"") + proc quoteShellPosix*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = ## Quote ``s``, so it can be safely passed to POSIX shell. const safeUnixChars = {'%', '+', '-', '.', '/', '_', ':', '=', '@', diff --git a/tests/stdlib/tos.nim b/tests/stdlib/tos.nim index dcb2d44f4..b7816fd41 100644 --- a/tests/stdlib/tos.nim +++ b/tests/stdlib/tos.nim @@ -622,7 +622,18 @@ block: # quoteShellWindows doAssert quoteShellWindows("aaa\"") == "aaa\\\"" doAssert quoteShellWindows("") == "\"\"" -block: # quoteShellWindows +block: # quoteShellCommand + when defined(windows): + doAssert quoteShellCommand(["a b c", "d", "e"]) == """"a b c" d e""" + doAssert quoteShellCommand(["""ab"c""", r"\", "d"]) == """ab\"c \ d""" + doAssert quoteShellCommand(["""ab"c""", """ \""", "d"]) == """ab\"c " \\" d""" + doAssert quoteShellCommand(["""a\\\b""", """de fg""", "h"]) == """a\\\b "de fg" h""" + doAssert quoteShellCommand(["""a\"b""", "c", "d"]) == """a\\\"b c d""" + doAssert quoteShellCommand(["""a\\b c""", "d", "e"]) == """"a\\b c" d e""" + doAssert quoteShellCommand(["""a\\b\ c""", "d", "e"]) == """"a\\b\ c" d e""" + doAssert quoteShellCommand(["ab", ""]) == """ab """"" + +block: # quoteShellPosix doAssert quoteShellPosix("aaa") == "aaa" doAssert quoteShellPosix("aaa a") == "'aaa a'" doAssert quoteShellPosix("") == "''" From f559319a682fb2abbf2945fd3390585a54f044d5 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 13 Aug 2021 00:21:01 +0800 Subject: [PATCH 1140/2472] fix a sqlite bug (#18669) --- lib/impure/db_sqlite.nim | 15 ++++----------- lib/std/private/dbutils.nim | 15 +++++++++++++++ tests/stdlib/tsqlitebindatas.nim | 28 ++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 11 deletions(-) create mode 100644 lib/std/private/dbutils.nim diff --git a/lib/impure/db_sqlite.nim b/lib/impure/db_sqlite.nim index 7bd807a12..b600576df 100644 --- a/lib/impure/db_sqlite.nim +++ b/lib/impure/db_sqlite.nim @@ -171,7 +171,7 @@ import sqlite3, macros import db_common export db_common -import std/private/since +import std/private/[since, dbutils] type DbConn* = PSqlite3 ## Encapsulates a database connection. @@ -211,14 +211,7 @@ proc dbQuote*(s: string): string = add(result, '\'') proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string = - result = "" - var a = 0 - for c in items(string(formatstr)): - if c == '?': - add(result, dbQuote(args[a])) - inc(a) - else: - add(result, c) + dbFormatImpl(formatstr, dbQuote, args) proc prepare*(db: DbConn; q: string): SqlPrepared {.since: (1, 3).} = ## Creates a new `SqlPrepared` statement. @@ -642,7 +635,7 @@ proc getValue*(db: DbConn, stmtName: SqlPrepared): string proc tryInsertID*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): int64 - {.tags: [WriteDbEffect], raises: [].} = + {.tags: [WriteDbEffect], raises: [DbError].} = ## Executes the query (typically "INSERT") and returns the ## generated ID for the row or -1 in case of an error. ## @@ -699,7 +692,7 @@ proc insertID*(db: DbConn, query: SqlQuery, proc tryInsert*(db: DbConn, query: SqlQuery, pkName: string, args: varargs[string, `$`]): int64 - {.tags: [WriteDbEffect], raises: [], since: (1, 3).} = + {.tags: [WriteDbEffect], raises: [DbError], since: (1, 3).} = ## same as tryInsertID tryInsertID(db, query, args) diff --git a/lib/std/private/dbutils.nim b/lib/std/private/dbutils.nim new file mode 100644 index 000000000..0ae3b3702 --- /dev/null +++ b/lib/std/private/dbutils.nim @@ -0,0 +1,15 @@ +import db_common + + +template dbFormatImpl*(formatstr: SqlQuery, dbQuote: proc (s: string): string, args: varargs[string]): string = + var res = "" + var a = 0 + for c in items(string(formatstr)): + if c == '?': + if a == args.len: + dbError("""The number of "?" given exceeds the number of parameters present in the query.""") + add(res, dbQuote(args[a])) + inc(a) + else: + add(res, c) + res diff --git a/tests/stdlib/tsqlitebindatas.nim b/tests/stdlib/tsqlitebindatas.nim index 643f1e2e6..754c80ae1 100644 --- a/tests/stdlib/tsqlitebindatas.nim +++ b/tests/stdlib/tsqlitebindatas.nim @@ -48,3 +48,31 @@ block tsqlitebindatas: ## db_sqlite binary data db.close() doAssert tryRemoveFile(dbName) + + +block: + block: + const dbName = buildDir / "db.sqlite3" + var db = db_sqlite.open(dbName, "", "", "") + var witness = false + try: + db.exec(sql("CREATE TABLE table1 (url TEXT, other_field INT);")) + db.exec(sql("REPLACE INTO table (url, another_field) VALUES (?, '123');")) + except DbError as e: + witness = true + doAssert e.msg == "The number of \"?\" given exceeds the number of parameters present in the query." + finally: + db.close() + removeFile(dbName) + + doAssert witness + + block: + const dbName = buildDir / "db.sqlite3" + var db = db_sqlite.open(dbName, "", "", "") + try: + db.exec(sql("CREATE TABLE table1 (url TEXT, other_field INT);")) + db.exec(sql("INSERT INTO table1 (url, other_field) VALUES (?, ?);"), "http://domain.com/test?param=1", 123) + finally: + db.close() + removeFile(dbName) From 883c04d444e943ba8e3d071a4dfcaca68ab300f9 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 13 Aug 2021 08:57:12 +0800 Subject: [PATCH 1141/2472] don't use space after proc names (#18681) --- lib/pure/strutils.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index ac078df4e..3179c73d2 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -2726,7 +2726,7 @@ func addf*(s: var string, formatstr: string, a: varargs[string, `$`]) {.rtl, add s, formatstr[i] inc(i) -func `%` *(formatstr: string, a: openArray[string]): string {.rtl, +func `%`*(formatstr: string, a: openArray[string]): string {.rtl, extern: "nsuFormatOpenArray".} = ## Interpolates a format string with the values from `a`. ## @@ -2774,7 +2774,7 @@ func `%` *(formatstr: string, a: openArray[string]): string {.rtl, result = newStringOfCap(formatstr.len + a.len shl 4) addf(result, formatstr, a) -func `%` *(formatstr, a: string): string {.rtl, +func `%`*(formatstr, a: string): string {.rtl, extern: "nsuFormatSingleElem".} = ## This is the same as `formatstr % [a]` (see ## `% func<#%25,string,openArray[string]>`_). From 3a1109a53be172a92c63ebc5bdb69c45e78e9ca6 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 12 Aug 2021 23:57:29 -0700 Subject: [PATCH 1142/2472] initial fix for compiling Nim on Zephyr RTOS (issue #18684) (#18685) * initial fix for compiling Nim on Zephyr RTOS (issue #18684) Co-authored-by: Jaremy J. Creechley --- lib/nimbase.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/nimbase.h b/lib/nimbase.h index e480c55fd..a83bd3006 100644 --- a/lib/nimbase.h +++ b/lib/nimbase.h @@ -75,7 +75,8 @@ __AVR__ #endif /* ------------------------------------------------------------------------- */ -#if defined(__GNUC__) +#if defined(__GNUC__) && !defined(__ZEPHYR__) +/* Zephyr does some magic in it's headers that override the GCC stdlib. This breaks that. */ # define _GNU_SOURCE 1 #endif From b24812df5f952427c1266e48c40b7956bdf3d999 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 13 Aug 2021 00:35:48 -0700 Subject: [PATCH 1143/2472] properly fix #10053 ; `FieldDefect` msg now shows discriminant value + lineinfo, in all backends (c,vm,js) (#11955) * fix #10053 FieldError for vm * fixup * FieldError now also shows runtime value of discriminant * fix field error reporting in vm * also report culprit line info in err msg * fix errors for newruntime 2 * fix for js * fixup * PRTEMP4 * works * works * works perfect * refactor * std/private/repr_impl * suppport --gc:arc * cleanup * refactor * simplify * simplify * simplify * fixup * move out compiler.vmgen.genCustom * fixup * fixup * add tests * revert compiler/debugutils.nim * simplify reprDiscriminant * fixup * lib/std/private/repr_impl.nim -> lib/system/repr_impl.nim * try to fix D20210812T165220 * honor --declaredlocs * control toFileLineCol via --declaredlocs --- compiler/astmsgs.nim | 11 ++++++++++ compiler/ccgexprs.nim | 41 +++++++++++++++++++++++++----------- compiler/ccgtypes.nim | 6 ++++++ compiler/cgen.nim | 2 +- compiler/jsgen.nim | 11 +++++----- compiler/pragmas.nim | 2 +- compiler/renderer.nim | 12 ----------- compiler/vm.nim | 5 ++++- compiler/vmgen.nim | 14 ++++++------ lib/system/chcks.nim | 10 +++++++++ lib/system/indexerrors.nim | 4 ++++ lib/system/jssys.nim | 4 ++-- lib/system/repr.nim | 2 ++ lib/system/repr_impl.nim | 15 +++++++++++++ lib/system/repr_v2.nim | 7 ++++++ lib/system/reprjs.nim | 2 ++ tests/misc/mfield_defect.nim | 30 ++++++++++++++++++++++++++ tests/misc/trunner.nim | 16 +++++++++++--- 18 files changed, 151 insertions(+), 43 deletions(-) create mode 100644 lib/system/repr_impl.nim create mode 100644 tests/misc/mfield_defect.nim diff --git a/compiler/astmsgs.nim b/compiler/astmsgs.nim index d9105b761..a9027126a 100644 --- a/compiler/astmsgs.nim +++ b/compiler/astmsgs.nim @@ -26,3 +26,14 @@ proc addDeclaredLoc*(result: var string, conf: ConfigRef; typ: PType) = proc addDeclaredLocMaybe*(result: var string, conf: ConfigRef; typ: PType) = if optDeclaredLocs in conf.globalOptions: addDeclaredLoc(result, conf, typ) + +template quoteExpr*(a: string): untyped = + ## can be used for quoting expressions in error msgs. + "'" & a & "'" + +proc genFieldDefect*(conf: ConfigRef, field: string, disc: PSym): string = + let obj = disc.owner.name.s # `types.typeToString` might be better, eg for generics + result = "field '$#' is not accessible for type '$#'" % [field, obj] + if optDeclaredLocs in conf.globalOptions: + result.add " [discriminant declared in $#]" % toFileLineCol(conf, disc.info) + result.add " using '$# = " % disc.name.s diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index acf83f966..7eb4a7a1d 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -18,6 +18,8 @@ proc getNullValueAuxT(p: BProc; orig, t: PType; obj, constOrNil: PNode, # -------------------------- constant expressions ------------------------ +proc rdSetElemLoc(conf: ConfigRef; a: TLoc, typ: PType): Rope + proc int64Literal(i: BiggestInt): Rope = if i > low(int64): result = "IL64($1)" % [rope(i)] @@ -873,16 +875,34 @@ proc genFieldCheck(p: BProc, e: PNode, obj: Rope, field: PSym) = v.r.add(".") v.r.add(disc.sym.loc.r) genInExprAux(p, it, u, v, test) - let msg = genFieldDefect(field, disc.sym) + var msg = "" + if optDeclaredLocs in p.config.globalOptions: + # xxx this should be controlled by a separate flag, and + # used for other similar defects so that location information is shown + # even without the expensive `--stacktrace`; binary size could be optimized + # by encoding the file names separately from `file(line:col)`, essentially + # passing around `TLineInfo` + the set of files in the project. + msg.add toFileLineCol(p.config, e.info) & " " + msg.add genFieldDefect(p.config, field.name.s, disc.sym) let strLit = genStringLiteral(p.module, newStrNode(nkStrLit, msg)) - if op.magic == mNot: - linefmt(p, cpsStmts, - "if ($1){ #raiseFieldError($2); $3}$n", - [rdLoc(test), strLit, raiseInstr(p)]) + + ## discriminant check + template fun(code) = linefmt(p, cpsStmts, code, [rdLoc(test)]) + if op.magic == mNot: fun("if ($1) ") else: fun("if (!($1)) ") + + ## call raiseFieldError2 on failure + let discIndex = rdSetElemLoc(p.config, v, u.t) + if optTinyRtti in p.config.globalOptions: + # not sure how to use `genEnumToStr` here + const code = "{ #raiseFieldError2($1, (NI)$3); $2} $n" + linefmt(p, cpsStmts, code, [strLit, raiseInstr(p), discIndex]) else: - linefmt(p, cpsStmts, - "if (!($1)){ #raiseFieldError($2); $3}$n", - [rdLoc(test), strLit, raiseInstr(p)]) + # complication needed for signed types + let first = p.config.firstOrd(disc.sym.typ) + let firstLit = int64Literal(cast[int](first)) + let discName = genTypeInfo(p.config, p.module, disc.sym.typ, e.info) + const code = "{ #raiseFieldError2($1, #reprDiscriminant(((NI)$3) + (NI)$4, $5)); $2} $n" + linefmt(p, cpsStmts, code, [strLit, raiseInstr(p), discIndex, firstLit, discName]) proc genCheckedRecordField(p: BProc, e: PNode, d: var TLoc) = assert e[0].kind == nkDotExpr @@ -1565,10 +1585,7 @@ proc genNewFinalize(p: BProc, e: PNode) = initLocExpr(p, e[1], a) initLocExpr(p, e[2], f) initLoc(b, locExpr, a.lode, OnHeap) - if optTinyRtti in p.config.globalOptions: - ti = genTypeInfoV2(p.module, refType, e.info) - else: - ti = genTypeInfoV1(p.module, refType, e.info) + ti = genTypeInfo(p.config, p.module, refType, e.info) p.module.s[cfsTypeInit3].addf("$1->finalizer = (void*)$2;$n", [ti, rdLoc(f)]) b.r = ropecg(p.module, "($1) #newObj($2, sizeof($3))", [ getTypeDesc(p.module, refType), diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 67e701804..d05c4cd8b 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -1514,3 +1514,9 @@ proc genTypeInfoV1(m: BModule, t: PType; info: TLineInfo): Rope = proc genTypeSection(m: BModule, n: PNode) = discard + +proc genTypeInfo*(config: ConfigRef, m: BModule, t: PType; info: TLineInfo): Rope = + if optTinyRtti in config.globalOptions: + result = genTypeInfoV2(m, t, info) + else: + result = genTypeInfoV1(m, t, info) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 3b004d399..f55da7f65 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -15,7 +15,7 @@ import ccgutils, os, ropes, math, passes, wordrecg, treetab, cgmeth, rodutils, renderer, cgendata, aliases, lowerings, tables, sets, ndi, lineinfos, pathutils, transf, - injectdestructors + injectdestructors, astmsgs when not defined(leanCompiler): import spawn, semparallel diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 49a57b4a6..3aa5c474e 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -33,7 +33,7 @@ import nversion, msgs, idents, types, ropes, passes, ccgutils, wordrecg, renderer, cgmeth, lowerings, sighashes, modulegraphs, lineinfos, rodutils, - transf, injectdestructors, sourcemap + transf, injectdestructors, sourcemap, astmsgs import json, sets, math, tables, intsets, strutils @@ -1208,12 +1208,13 @@ proc genCheckedFieldOp(p: PProc, n: PNode, addrTyp: PType, r: var TCompRes) = let tmp = p.getTemp() lineF(p, "var $1 = $2;$n", tmp, obj.res) - useMagic(p, "raiseFieldError") + useMagic(p, "raiseFieldError2") useMagic(p, "makeNimstrLit") - let msg = genFieldDefect(field, disc) - lineF(p, "if ($1[$2.$3]$4undefined) { raiseFieldError(makeNimstrLit($5)); }$n", + useMagic(p, "reprDiscriminant") # no need to offset by firstOrd unlike for cgen + let msg = genFieldDefect(p.config, field.name.s, disc) + lineF(p, "if ($1[$2.$3]$4undefined) { raiseFieldError2(makeNimstrLit($5), reprDiscriminant($2.$3, $6)); }$n", setx.res, tmp, disc.loc.r, if negCheck: ~"!==" else: ~"===", - makeJSString(msg)) + makeJSString(msg), genTypeInfo(p, disc.typ)) if addrTyp != nil and mapType(p, addrTyp) == etyBaseIndex: r.typ = etyBaseIndex diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 8ae931d84..6637a8673 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -699,7 +699,7 @@ proc typeBorrow(c: PContext; sym: PSym, n: PNode) = incl(sym.typ.flags, tfBorrowDot) proc markCompilerProc(c: PContext; s: PSym) = - # minor hack ahead: FlowVar is the only generic .compilerProc type which + # minor hack ahead: FlowVar is the only generic .compilerproc type which # should not have an external name set: if s.kind != skType or s.name.s != "FlowVar": makeExternExport(c, s, "$1", s.info) diff --git a/compiler/renderer.nim b/compiler/renderer.nim index 3346e629c..393f35289 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -1758,15 +1758,3 @@ proc getTokSym*(r: TSrcGen): PSym = result = r.tokens[r.idx-1].sym else: result = nil - -proc quoteExpr*(a: string): string {.inline.} = - ## can be used for quoting expressions in error msgs. - "'" & a & "'" - -proc genFieldDefect*(field: PSym, disc: PSym): string = - ## this needs to be in a module accessible by jsgen, ccgexprs, and vm to - ## provide this error msg FieldDefect; msgs would be better but it does not - ## import ast - result = field.name.s.quoteExpr & " is not accessible using discriminant " & - disc.name.s.quoteExpr & " of type " & - disc.owner.name.s.quoteExpr diff --git a/compiler/vm.nim b/compiler/vm.nim index 73a228b6c..9fa25efbe 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -1472,7 +1472,10 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = else: return TFullReg(kind: rkNone) of opcInvalidField: - stackTrace(c, tos, pc, errFieldXNotFound & regs[ra].node.strVal) + let msg = regs[ra].node.strVal + let disc = regs[instr.regB].regToNode + let msg2 = formatFieldDefect(msg, $disc) + stackTrace(c, tos, pc, msg2) of opcSetLenStr: decodeB(rkNode) #createStrKeepNode regs[ra] diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 4957a3339..725afdd3f 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -31,7 +31,7 @@ import tables import strutils, ast, types, msgs, renderer, vmdef, - intsets, magicsys, options, lowerings, lineinfos, transf + intsets, magicsys, options, lowerings, lineinfos, transf, astmsgs from modulegraphs import getBody @@ -1742,12 +1742,14 @@ proc genCheckedObjAccessAux(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags let lab1 = c.xjmp(n, if negCheck: opcFJmp else: opcTJmp, rs) c.freeTemp(rs) let strType = getSysType(c.graph, n.info, tyString) - var fieldNameRegister: TDest = c.getTemp(strType) - let strLit = newStrNode($accessExpr[1], accessExpr[1].info) + var msgReg: TDest = c.getTemp(strType) + let fieldName = $accessExpr[1] + let msg = genFieldDefect(c.config, fieldName, disc.sym) + let strLit = newStrNode(msg, accessExpr[1].info) strLit.typ = strType - c.genLit(strLit, fieldNameRegister) - c.gABC(n, opcInvalidField, fieldNameRegister) - c.freeTemp(fieldNameRegister) + c.genLit(strLit, msgReg) + c.gABC(n, opcInvalidField, msgReg, discVal) + c.freeTemp(msgReg) c.patch(lab1) proc genCheckedObjAccess(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) = diff --git a/lib/system/chcks.nim b/lib/system/chcks.nim index 255d97cb2..8c7c95795 100644 --- a/lib/system/chcks.nim +++ b/lib/system/chcks.nim @@ -26,8 +26,18 @@ proc raiseIndexError() {.compilerproc, noinline.} = sysFatal(IndexDefect, "index out of bounds") proc raiseFieldError(f: string) {.compilerproc, noinline.} = + ## remove after bootstrap > 1.5.1 sysFatal(FieldDefect, f) +when defined(gcdestructors): + proc raiseFieldError2(f: string, discVal: int) {.compilerproc, noinline.} = + ## raised when field is inaccessible given runtime value of discriminant + sysFatal(FieldError, f & $discVal & "'") +else: + proc raiseFieldError2(f: string, discVal: string) {.compilerproc, noinline.} = + ## raised when field is inaccessible given runtime value of discriminant + sysFatal(FieldError, formatFieldDefect(f, discVal)) + proc raiseRangeErrorI(i, a, b: BiggestInt) {.compilerproc, noinline.} = when defined(standalone): sysFatal(RangeDefect, "value out of range") diff --git a/lib/system/indexerrors.nim b/lib/system/indexerrors.nim index 1b91789bd..6a8cb8a0a 100644 --- a/lib/system/indexerrors.nim +++ b/lib/system/indexerrors.nim @@ -1,4 +1,5 @@ # imported by other modules, unlike helpers.nim which is included +# xxx this is now included instead of imported, we should import instead template formatErrorIndexBound*[T](i, a, b: T): string = when defined(standalone): @@ -9,3 +10,6 @@ template formatErrorIndexBound*[T](i, a, b: T): string = template formatErrorIndexBound*[T](i, n: T): string = formatErrorIndexBound(i, 0, n) + +template formatFieldDefect*(f, discVal): string = + f & discVal & "'" diff --git a/lib/system/jssys.nim b/lib/system/jssys.nim index b42dc3a20..63e7b3d9d 100644 --- a/lib/system/jssys.nim +++ b/lib/system/jssys.nim @@ -171,8 +171,8 @@ proc raiseRangeError() {.compilerproc, noreturn.} = proc raiseIndexError(i, a, b: int) {.compilerproc, noreturn.} = raise newException(IndexDefect, formatErrorIndexBound(int(i), int(a), int(b))) -proc raiseFieldError(f: string) {.compilerproc, noreturn.} = - raise newException(FieldDefect, f) +proc raiseFieldError2(f: string, discVal: string) {.compilerproc, noreturn.} = + raise newException(FieldDefect, formatFieldDefect(f, discVal)) proc setConstr() {.varargs, asmNoStackFrame, compilerproc.} = asm """ diff --git a/lib/system/repr.nim b/lib/system/repr.nim index 020c2281f..e049d18fa 100644 --- a/lib/system/repr.nim +++ b/lib/system/repr.nim @@ -72,6 +72,8 @@ proc reprEnum(e: int, typ: PNimType): string {.compilerRtl.} = result = $e & " (invalid data!)" +include system/repr_impl + type PByteArray = ptr UncheckedArray[byte] # array[0xffff, byte] diff --git a/lib/system/repr_impl.nim b/lib/system/repr_impl.nim new file mode 100644 index 000000000..b9ec1890f --- /dev/null +++ b/lib/system/repr_impl.nim @@ -0,0 +1,15 @@ +#[ +other APIs common to system/repr and system/reprjs could be refactored here, eg: +* reprChar +* reprBool +* reprStr + +Another possibility in future work would be to have a single include file instead +of system/repr and system/reprjs, and use `when defined(js)` inside it. +]# + +proc reprDiscriminant*(e: int, typ: PNimType): string {.compilerRtl.} = + case typ.kind + of tyEnum: reprEnum(e, typ) + of tyBool: $(e != 0) + else: $e diff --git a/lib/system/repr_v2.nim b/lib/system/repr_v2.nim index f99f09799..ba94b881d 100644 --- a/lib/system/repr_v2.nim +++ b/lib/system/repr_v2.nim @@ -1,3 +1,5 @@ +include system/inclrtl + proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} ## imported from typetraits @@ -66,6 +68,11 @@ proc repr*[Enum: enum](x: Enum): string {.magic: "EnumToStr", noSideEffect.} ## If a `repr` operator for a concrete enumeration is provided, this is ## used instead. (In other words: *Overwriting* is possible.) +proc reprDiscriminant*(e: int): string {.compilerproc.} = + # repr and reprjs can use `PNimType` to symbolize `e`; making this work here + # would require a way to pass the set of enum stringified values to cgen. + $e + proc repr*(p: pointer): string = ## repr of pointer as its hexadecimal value if p == nil: diff --git a/lib/system/reprjs.nim b/lib/system/reprjs.nim index cdc3403f8..28935a4ef 100644 --- a/lib/system/reprjs.nim +++ b/lib/system/reprjs.nim @@ -30,6 +30,8 @@ proc reprEnum(e: int, typ: PNimType): string {.compilerRtl.} = else: result = $e & " (invalid data!)" +include system/repr_impl + proc reprChar(x: char): string {.compilerRtl.} = result = "\'" case x diff --git a/tests/misc/mfield_defect.nim b/tests/misc/mfield_defect.nim new file mode 100644 index 000000000..53bfba40e --- /dev/null +++ b/tests/misc/mfield_defect.nim @@ -0,0 +1,30 @@ +#[ +ran from trunner +]# + + + + + + +# line 10 +type Kind = enum k0, k1, k2, k3, k4 + +type Foo = object + case kind: Kind + of k0: f0: int + of k1: f1: int + of k2: f2: int + of k3: f3: int + of k4: f4: int + +proc main()= + var foo = Foo(kind: k3, f3: 3) + let s1 = foo.f3 + doAssert s1 == 3 + let s2 = foo.f2 + +when defined case1: + static: main() +when defined case2: + main() diff --git a/tests/misc/trunner.nim b/tests/misc/trunner.nim index 8426e9aee..f874d38d9 100644 --- a/tests/misc/trunner.nim +++ b/tests/misc/trunner.nim @@ -37,9 +37,9 @@ proc runNimCmd(file, options = "", rtarg = ""): auto = echo cmd echo result[0] & "\n" & $result[1] -proc runNimCmdChk(file, options = "", rtarg = ""): string = - let (ret, status) = runNimCmd(file, options, rtarg = rtarg) - doAssert status == 0, $(file, options) & "\n" & ret +proc runNimCmdChk(file, options = "", rtarg = "", status = 0): string = + let (ret, status2) = runNimCmd(file, options, rtarg = rtarg) + doAssert status2 == status, $(file, options, status, status2) & "\n" & ret ret proc genShellCmd(filename: string): string = @@ -376,5 +376,15 @@ mused3.nim(13, 8) Warning: imported and not used: 'mused3b' [UnusedImport] mused3.nim(75, 10) Hint: duplicate import of 'mused3a'; previous import here: mused3.nim(74, 10) [DuplicateModuleImport] """ + block: # FieldDefect + proc fn(opt: string, expected: string) = + let output = runNimCmdChk("misc/mfield_defect.nim", fmt"-r --warning:all:off --declaredlocs {opt}", status = 1) + doAssert expected in output, opt & "\noutput:\n" & output & "expected:\n" & expected + fn("-d:case1"): """mfield_defect.nim(25, 15) Error: field 'f2' is not accessible for type 'Foo' [discriminant declared in mfield_defect.nim(14, 8)] using 'kind = k3'""" + fn("-d:case2 --gc:refc"): """mfield_defect.nim(25, 15) field 'f2' is not accessible for type 'Foo' [discriminant declared in mfield_defect.nim(14, 8)] using 'kind = k3'""" + fn("-d:case1 -b:js"): """mfield_defect.nim(25, 15) Error: field 'f2' is not accessible for type 'Foo' [discriminant declared in mfield_defect.nim(14, 8)] using 'kind = k3'""" + fn("-d:case2 -b:js"): """field 'f2' is not accessible for type 'Foo' [discriminant declared in mfield_defect.nim(14, 8)] using 'kind = k3'""" + # 3 instead of k3, because of lack of RTTI + fn("-d:case2 --gc:arc"): """mfield_defect.nim(25, 15) field 'f2' is not accessible for type 'Foo' [discriminant declared in mfield_defect.nim(14, 8)] using 'kind = 3'""" else: discard # only during debugging, tests added here will run with `-d:nimTestsTrunnerDebugging` enabled From 4463a3c3d75daabfef52a139221f729e8af9e81e Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 13 Aug 2021 12:34:28 +0200 Subject: [PATCH 1144/2472] manual: minor improvements (#18687) --- doc/manual.rst | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/doc/manual.rst b/doc/manual.rst index 41ed49715..9c7c219e9 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -31,7 +31,7 @@ the `destructors `_ document. This document describes the lexis, the syntax, and the semantics of the Nim language. To learn how to compile Nim programs and generate documentation see -`Compiler User Guide `_ and `DocGen Tools Guide `_. +the `Compiler User Guide `_ and the `DocGen Tools Guide `_. The language constructs are explained using an extended BNF, in which `(a)*` means 0 or more `a`'s, `a+` means 1 or more `a`'s, and `(a)?` means an @@ -583,12 +583,11 @@ In the following examples, `-1` is parsed as two separate tokens "abc"-1 -There exists a literal for each numerical type that is -defined. The suffix starting with an apostrophe ('\'') is called a +The suffix starting with an apostrophe ('\'') is called a `type suffix`:idx:. Literals without a type suffix are of an integer type unless the literal contains a dot or `E|e` in which case it is of type `float`. This integer type is `int` if the literal is in the range -`low(i32)..high(i32)`, otherwise it is `int64`. +`low(int32)..high(int32)`, otherwise it is `int64`. For notational convenience, the apostrophe of a type suffix is optional if it is not ambiguous (only hexadecimal floating-point literals with a type suffix can be ambiguous). @@ -619,10 +618,9 @@ notation: `0B0_10001110100_0000101001000111101011101111111011000101001101001001'f64` is approximately 1.72826e35 according to the IEEE floating-point standard. -Literals are bounds checked so that they fit the datatype. Non-base-10 -literals are used mainly for flags and bit pattern representations, therefore -bounds checking is done on bit width, not value range. If the literal fits in -the bit width of the datatype, it is accepted. +Literals must match the datatype, for example, `333'i8` is an invalid literal. +Non-base-10 literals are used mainly for flags and bit pattern representations, +therefore the checking is done on bit width and not on value range. Hence: 0b10000000'u8 == 0x80'u8 == 128, but, 0b10000000'i8 == 0x80'i8 == -1 instead of causing an overflow error. @@ -1980,8 +1978,7 @@ details like this when mixing garbage-collected data with unmanaged memory. Procedural type --------------- A procedural type is internally a pointer to a procedure. `nil` is -an allowed value for variables of a procedural type. Nim uses procedural -types to achieve `functional`:idx: programming techniques. +an allowed value for a variable of a procedural type. Examples: @@ -2494,8 +2491,8 @@ An expression `b` can be assigned to an expression `a` iff `a` is an `l-value` and `isImplicitlyConvertible(b.typ, a.typ)` holds. -Overloading resolution -====================== +Overload resolution +=================== In a call `p(args)` the routine `p` that matches best is selected. If multiple routines match equally well, the ambiguity is reported during From 52cc0a82cad856fa58ebbb16a9e7ad8c7012e8cf Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 14 Aug 2021 02:13:30 -0700 Subject: [PATCH 1145/2472] fix #18692 AsyncHttpServer was hanging because client.close was not called (#18693) --- lib/pure/asynchttpserver.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index fd0cad5ca..f23c3d3f9 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -362,7 +362,9 @@ proc processClient(server: AsyncHttpServer, client: AsyncSocket, address: string let retry = await processRequest( server, request, client, address, lineFut, callback ) - if not retry: break + if not retry: + client.close() + break const nimMaxDescriptorsFallback* {.intdefine.} = 16_000 ## fallback value for \ From b3e077863aa6985eebab0c37785ba127b3192c80 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Sat, 14 Aug 2021 21:51:41 -0300 Subject: [PATCH 1146/2472] Documentation only, Terminal (#18696) Add simple progress bar example to terminal module --- lib/pure/terminal.nim | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index ce5660557..25b1a4cdd 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -15,6 +15,21 @@ ## code `exitprocs.addExitProc(resetAttributes)` to restore the defaults. ## Similarly, if you hide the cursor, make sure to unhide it with ## `showCursor` before quitting. +## +## Progress bar +## ============ +## +## Basic progress bar example: +runnableExamples("-r:off"): + import std/[os, strutils] + + for i in 0..100: + stdout.styledWriteLine(fgRed, "0% ", fgWhite, '#'.repeat i, if i > 50: fgGreen else: fgYellow, "\t", $i , "%") + sleep 42 + cursorUp 1 + eraseLine() + + stdout.resetAttributes() ##[ ## Playing with colorful and styled text @@ -43,7 +58,6 @@ runnableExamples("-r:off"): stdout.styledWriteLine(fgRed, "red text ", styleBright, "bold red", fgDefault, " bold text") - import macros import strformat from strutils import toLowerAscii, `%` From 1acba63cb725a0eee8f8b02f585057e030ce6295 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 16 Aug 2021 00:32:12 -0700 Subject: [PATCH 1147/2472] cross compilation targetting windows now supports `nim r`: `nim r -d:mingw main` (#18682) * cross compilation targetting windows now supports `nim r`: `nim r -d:mingw main` * quoteShell * address comment: remove `conf.getConfigVar("nimrun.exe")` --- changelog.md | 3 +++ compiler/nim.nim | 20 +++++++++++++++----- doc/nimc.rst | 2 ++ 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/changelog.md b/changelog.md index 0e3c54d77..40c6c9ace 100644 --- a/changelog.md +++ b/changelog.md @@ -482,6 +482,9 @@ nim r main # uses cached binary nim r main arg1 arg2 # ditto (runtime arguments are irrelevant) +- `nim r` now supports cross compilation from unix to windows when specifying `-d:mingw` by using wine, + e.g.: `nim r --eval:'import os; echo "a" / "b"'` prints `a\b` + - The style checking of the compiler now supports a `--styleCheck:usages` switch. This switch enforces that every symbol is written as it was declared, not enforcing the official Nim style guide. To be enabled, this has to be combined either diff --git a/compiler/nim.nim b/compiler/nim.nim index 91b1bc2db..b8256d576 100644 --- a/compiler/nim.nim +++ b/compiler/nim.nim @@ -70,6 +70,13 @@ proc processCmdLine(pass: TCmdLinePass, cmd: string; config: ConfigRef) = config.arguments.len > 0 and config.cmd notin {cmdTcc, cmdNimscript, cmdCrun}: rawMessage(config, errGenerated, errArgsNeedRunOption) +proc getNimRunExe(conf: ConfigRef): string = + # xxx consider defining `conf.getConfigVar("nimrun.exe")` to allow users to + # customize the binary to run the command with, e.g. for custom `nodejs` or `wine`. + if conf.isDefined("mingw"): + if conf.isDefined("i386"): result = "wine" + elif conf.isDefined("amd64"): result = "wine64" + proc handleCmdLine(cache: IdentCache; conf: ConfigRef) = let self = NimProg( supportsStdinFile: true, @@ -95,18 +102,21 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) = let output = conf.absOutFile case conf.cmd of cmdBackends, cmdTcc: - var cmdPrefix = "" + let nimRunExe = getNimRunExe(conf) + var cmdPrefix: string + if nimRunExe.len > 0: cmdPrefix.add nimRunExe.quoteShell case conf.backend of backendC, backendCpp, backendObjc: discard of backendJs: # D20210217T215950:here this flag is needed for node < v15.0.0, otherwise # tasyncjs_fail` would fail, refs https://nodejs.org/api/cli.html#cli_unhandled_rejections_mode - cmdPrefix = findNodeJs() & " --unhandled-rejections=strict " + if cmdPrefix.len == 0: cmdPrefix = findNodeJs().quoteShell + cmdPrefix.add " --unhandled-rejections=strict" else: doAssert false, $conf.backend - # No space before command otherwise on windows you'd get a cryptic: - # `The parameter is incorrect` + if cmdPrefix.len > 0: cmdPrefix.add " " + # without the `cmdPrefix.len > 0` check, on windows you'd get a cryptic: + # `The parameter is incorrect` execExternalProgram(conf, cmdPrefix & output.quoteShell & ' ' & conf.arguments) - # execExternalProgram(conf, cmdPrefix & ' ' & output.quoteShell & ' ' & conf.arguments) of cmdDocLike, cmdRst2html, cmdRst2tex: # bugfix(cmdRst2tex was missing) if conf.arguments.len > 0: # reserved for future use diff --git a/doc/nimc.rst b/doc/nimc.rst index ad5ecaa76..98b754729 100644 --- a/doc/nimc.rst +++ b/doc/nimc.rst @@ -317,6 +317,8 @@ To cross-compile for Windows from Linux or macOS using the MinGW-w64 toolchain: .. code:: cmd nim c -d:mingw myproject.nim + # `nim r` also works, running the binary via `wine` or `wine64`: + nim r -d:mingw --eval:'import os; echo "a" / "b"' Use `--cpu:i386`:option: or `--cpu:amd64`:option: to switch the CPU architecture. From 901c5ded527bb06c52bfc1fd38c9a5fadee0f49a Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 16 Aug 2021 21:14:35 +0800 Subject: [PATCH 1148/2472] fix #18702(fix `parseutils.parseFloat`) (#18703) [backport:1.0] * fix #18702 * Apply suggestions from code review --- lib/system/strmantle.nim | 9 +++++---- tests/stdlib/tparseutils.nim | 13 +++++++++++-- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/lib/system/strmantle.nim b/lib/system/strmantle.nim index 81ac5c751..f55168c01 100644 --- a/lib/system/strmantle.nim +++ b/lib/system/strmantle.nim @@ -211,15 +211,16 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat, var ti = 0 let maxlen = t.high - "e+000".len # reserve enough space for exponent - result = i - start + let endPos = i + result = endPos - start i = start # re-parse without error checking, any error should be handled by the code above. - if i < s.len and s[i] == '.': i.inc - while i < s.len and s[i] in {'0'..'9','+','-'}: + if i < endPos and s[i] == '.': i.inc + while i < endPos and s[i] in {'0'..'9','+','-'}: if ti < maxlen: t[ti] = s[i]; inc(ti) inc(i) - while i < s.len and s[i] in {'.', '_'}: # skip underscore and decimal point + while i < endPos and s[i] in {'.', '_'}: # skip underscore and decimal point inc(i) # insert exponent diff --git a/tests/stdlib/tparseutils.nim b/tests/stdlib/tparseutils.nim index 3bc54dff1..db7a0ac8d 100644 --- a/tests/stdlib/tparseutils.nim +++ b/tests/stdlib/tparseutils.nim @@ -1,5 +1,5 @@ -import parseutils -import sequtils +import std/[parseutils, sequtils, sugar] + let input = "$test{} $this is ${an{ example}} " let expected = @[(ikVar, "test"), (ikStr, "{} "), (ikVar, "this"), @@ -41,3 +41,12 @@ doAssert value == 1_000_000 var i64Value: int64 discard parseBiggestInt("9223372036854775807", i64Value) doAssert i64Value == 9223372036854775807 + +block: + var f: float + let res = collect: + for x in ["9.123456789012345+","11.123456789012345+","9.123456789012345-","8.123456789012345+","9.12345678901234-","9.123456789012345"]: + (parseFloat(x, f, 0), $f) + doAssert res == @[(17, "9.123456789012344"), (18, "11.123456789012344"), + (17, "9.123456789012344"), (17, "8.123456789012344"), + (16, "9.12345678901234"), (17, "9.123456789012344")] From 5fd150b7f792754b0431f2cc2e9830fb23ce15f6 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Wed, 18 Aug 2021 14:25:25 +0200 Subject: [PATCH 1149/2472] asynchttpserver: ipv6 support (#18706) Fix #18705 --- lib/pure/asynchttpserver.nim | 13 +++++++------ tests/errmsgs/tgcsafety.nim | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index f23c3d3f9..d7daacd03 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -41,7 +41,7 @@ runnableExamples("-r:off"): import asyncnet, asyncdispatch, parseutils, uri, strutils import httpcore -from nativesockets import getLocalAddr, AF_INET +from nativesockets import getLocalAddr, Domain, AF_INET, AF_INET6 import std/private/since export httpcore except parseHeader @@ -82,7 +82,7 @@ proc getPort*(self: AsyncHttpServer): Port {.since: (1, 5, 1).} = server.listen(Port(0)) assert server.getPort.uint16 > 0 server.close() - result = getLocalAddr(self.socket.getFd, AF_INET)[1] + result = getLocalAddr(self.socket)[1] proc newAsyncHttpServer*(reuseAddr = true, reusePort = false, maxBody = 8388608): AsyncHttpServer = @@ -372,13 +372,13 @@ const ## This can be set on the command line during compilation ## via `-d:nimMaxDescriptorsFallback=N` -proc listen*(server: AsyncHttpServer; port: Port; address = "") = +proc listen*(server: AsyncHttpServer; port: Port; address = ""; domain = AF_INET) = ## Listen to the given port and address. when declared(maxDescriptors): server.maxFDs = try: maxDescriptors() except: nimMaxDescriptorsFallback else: server.maxFDs = nimMaxDescriptorsFallback - server.socket = newAsyncSocket() + server.socket = newAsyncSocket(domain) if server.reuseAddr: server.socket.setSockOpt(OptReuseAddr, true) if server.reusePort: @@ -404,7 +404,8 @@ proc acceptRequest*(server: AsyncHttpServer, proc serve*(server: AsyncHttpServer, port: Port, callback: proc (request: Request): Future[void] {.closure, gcsafe.}, address = ""; - assumedDescriptorsPerRequest = -1) {.async.} = + assumedDescriptorsPerRequest = -1; + domain = AF_INET) {.async.} = ## Starts the process of listening for incoming HTTP connections on the ## specified address and port. ## @@ -417,7 +418,7 @@ proc serve*(server: AsyncHttpServer, port: Port, ## ## You should prefer to call `acceptRequest` instead with a custom server ## loop so that you're in control over the error handling and logging. - listen server, port, address + listen server, port, address, domain while true: if shouldAcceptRequest(server, assumedDescriptorsPerRequest): var (address, client) = await server.socket.acceptAddr() diff --git a/tests/errmsgs/tgcsafety.nim b/tests/errmsgs/tgcsafety.nim index 6021d19e1..701adeebf 100644 --- a/tests/errmsgs/tgcsafety.nim +++ b/tests/errmsgs/tgcsafety.nim @@ -6,7 +6,7 @@ tgcsafety.nim(31, 18) Error: type mismatch: got Date: Wed, 18 Aug 2021 21:53:52 +0800 Subject: [PATCH 1150/2472] [wip]better hint message for JS (#18704) * better hint message for JS * both --- compiler/msgs.nim | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 20a55daee..56531eb68 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -669,17 +669,28 @@ proc genSuccessX*(conf: ConfigRef) = else: formatSize(getTotalMem()) & " totmem" let loc = $conf.linesCompiled var build = "" + var flags = "" + const debugModeHints = "none (DEBUG BUILD, `-d:release` generates faster code)" if conf.cmd in cmdBackends: - build.add "gc: $#; " % $conf.selectedGC - if optThreads in conf.globalOptions: build.add "threads: on; " - build.add "opt: " - if optOptimizeSpeed in conf.options: build.add "speed" - elif optOptimizeSize in conf.options: build.add "size" - else: build.add "none (DEBUG BUILD, `-d:release` generates faster code)" - # pending https://github.com/timotheecour/Nim/issues/752, point to optimization.html - var flags = "" - if isDefined(conf, "danger"): flags.add " -d:danger" - elif isDefined(conf, "release"): flags.add " -d:release" + if conf.backend != backendJs: + build.add "gc: $#; " % $conf.selectedGC + if optThreads in conf.globalOptions: build.add "threads: on; " + build.add "opt: " + if optOptimizeSpeed in conf.options: build.add "speed" + elif optOptimizeSize in conf.options: build.add "size" + else: build.add debugModeHints + # pending https://github.com/timotheecour/Nim/issues/752, point to optimization.html + if isDefined(conf, "danger"): flags.add " -d:danger" + elif isDefined(conf, "release"): flags.add " -d:release" + else: + build.add "opt: " + if isDefined(conf, "danger"): + build.add "speed" + flags.add " -d:danger" + elif isDefined(conf, "release"): + build.add "speed" + flags.add " -d:release" + else: build.add debugModeHints if flags.len > 0: build.add "; options:" & flags let sec = formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3) let project = if conf.filenameOption == foAbs: $conf.projectFull else: $conf.projectName From 7b58dc2de0f606b757a558dfdda9d930ae63f41a Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 18 Aug 2021 10:33:11 -0700 Subject: [PATCH 1151/2472] allow building 1.4.0 from devel (#18708) * allow building 1.4.0 from devel * changelog --- changelog.md | 3 +++ compiler/ast.nim | 8 ++++++-- compiler/ccgexprs.nim | 21 +++++++++++++++++---- compiler/jsgen.nim | 1 + compiler/semfold.nim | 1 + compiler/vmgen.nim | 2 +- 6 files changed, 29 insertions(+), 7 deletions(-) diff --git a/changelog.md b/changelog.md index 40c6c9ace..77a77f2f5 100644 --- a/changelog.md +++ b/changelog.md @@ -485,6 +485,9 @@ - `nim r` now supports cross compilation from unix to windows when specifying `-d:mingw` by using wine, e.g.: `nim r --eval:'import os; echo "a" / "b"'` prints `a\b` +- `nim` can compile version 1.4.0 as follows: `nim c --lib:lib --stylecheck:off -d:nimVersion140 compiler/nim`. + `-d:nimVersion140` is not needed for bootstrapping, only for building 1.4.0 from devel. + - The style checking of the compiler now supports a `--styleCheck:usages` switch. This switch enforces that every symbol is written as it was declared, not enforcing the official Nim style guide. To be enabled, this has to be combined either diff --git a/compiler/ast.nim b/compiler/ast.nim index 5b54dd8ef..08fbc70e1 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -652,7 +652,9 @@ type mUnaryMinusI, mUnaryMinusI64, mAbsI, mNot, mUnaryPlusI, mBitnotI, mUnaryPlusF64, mUnaryMinusF64, - mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mCStrToStr, + mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, + mFloatToStr, # for -d:nimVersion140 + mCStrToStr, mStrToStr, mEnumToStr, mAnd, mOr, mImplies, mIff, mExists, mForall, mOld, @@ -720,7 +722,9 @@ const mEqRef, mEqProc, mLePtr, mLtPtr, mEqCString, mXor, mUnaryMinusI, mUnaryMinusI64, mAbsI, mNot, mUnaryPlusI, mBitnotI, mUnaryPlusF64, mUnaryMinusF64, - mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mCStrToStr, + mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, + mFloatToStr, + mCStrToStr, mStrToStr, mEnumToStr, mAnd, mOr, mEqStr, mLeStr, mLtStr, diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 7eb4a7a1d..374876367 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -894,15 +894,23 @@ proc genFieldCheck(p: BProc, e: PNode, obj: Rope, field: PSym) = let discIndex = rdSetElemLoc(p.config, v, u.t) if optTinyRtti in p.config.globalOptions: # not sure how to use `genEnumToStr` here - const code = "{ #raiseFieldError2($1, (NI)$3); $2} $n" - linefmt(p, cpsStmts, code, [strLit, raiseInstr(p), discIndex]) + if p.config.isDefined("nimVersion140"): + const code = "{ #raiseFieldError($1); $2} $n" + linefmt(p, cpsStmts, code, [strLit, raiseInstr(p)]) + else: + const code = "{ #raiseFieldError2($1, (NI)$3); $2} $n" + linefmt(p, cpsStmts, code, [strLit, raiseInstr(p), discIndex]) else: # complication needed for signed types let first = p.config.firstOrd(disc.sym.typ) let firstLit = int64Literal(cast[int](first)) let discName = genTypeInfo(p.config, p.module, disc.sym.typ, e.info) - const code = "{ #raiseFieldError2($1, #reprDiscriminant(((NI)$3) + (NI)$4, $5)); $2} $n" - linefmt(p, cpsStmts, code, [strLit, raiseInstr(p), discIndex, firstLit, discName]) + if p.config.isDefined("nimVersion140"): + const code = "{ #raiseFieldError($1); $2} $n" + linefmt(p, cpsStmts, code, [strLit, raiseInstr(p)]) + else: + const code = "{ #raiseFieldError2($1, #reprDiscriminant(((NI)$3) + (NI)$4, $5)); $2} $n" + linefmt(p, cpsStmts, code, [strLit, raiseInstr(p), discIndex, firstLit, discName]) proc genCheckedRecordField(p: BProc, e: PNode, d: var TLoc) = assert e[0].kind == nkDotExpr @@ -2320,6 +2328,11 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = of mInt64ToStr: genDollar(p, e, d, "#nimInt64ToStr($1)") of mBoolToStr: genDollar(p, e, d, "#nimBoolToStr($1)") of mCharToStr: genDollar(p, e, d, "#nimCharToStr($1)") + of mFloatToStr: + if e[1].typ.skipTypes(abstractInst).kind == tyFloat32: + genDollar(p, e, d, "#nimFloat32ToStr($1)") + else: + genDollar(p, e, d, "#nimFloatToStr($1)") of mCStrToStr: genDollar(p, e, d, "#cstrToNimstr($1)") of mStrToStr, mUnown: expr(p, e[1], d) of mIsolate: genCall(p, e, d) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 3aa5c474e..931e56475 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -434,6 +434,7 @@ const # magic checked op; magic unchecked op; mBoolToStr: ["nimBoolToStr", "nimBoolToStr"], mIntToStr: ["cstrToNimstr", "cstrToNimstr"], mInt64ToStr: ["cstrToNimstr", "cstrToNimstr"], + mFloatToStr: ["cstrToNimstr", "cstrToNimstr"], mCStrToStr: ["cstrToNimstr", "cstrToNimstr"], mStrToStr: ["", ""]] diff --git a/compiler/semfold.nim b/compiler/semfold.nim index e5f0643bc..04ed73209 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -290,6 +290,7 @@ proc evalOp(m: TMagic, n, a, b, c: PNode; idgen: IdGenerator; g: ModuleGraph): P of mBoolToStr: if getOrdValue(a) == 0: result = newStrNodeT("false", n, g) else: result = newStrNodeT("true", n, g) + of mFloatToStr: result = newStrNodeT($getFloat(a), n, g) of mCStrToStr, mCharToStr: if a.kind == nkBracket: var s = "" diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 725afdd3f..f2758ff37 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1127,7 +1127,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = let t = skipTypes(n.typ, abstractVar-{tyTypeDesc}) if t.kind in {tyUInt8..tyUInt32} or (t.kind == tyUInt and t.size < 8): c.gABC(n, opcNarrowU, dest, TRegister(t.size*8)) - of mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mCStrToStr, mStrToStr, mEnumToStr: + of mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mFloatToStr, mCStrToStr, mStrToStr, mEnumToStr: genConv(c, n, n[1], dest) of mEqStr, mEqCString: genBinaryABC(c, n, dest, opcEqStr) of mLeStr: genBinaryABC(c, n, dest, opcLeStr) From 394f4ac7bb92fe5aaf902495c6b43b3451667602 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 19 Aug 2021 02:33:52 -0700 Subject: [PATCH 1152/2472] improvements to `addInt` and `$` for integer types (#18592) * improvements to $(SomeInteger) and addInt * remove mIntToStr, mInt64ToStr * improvements * fix tests/pragmas/tinjectstmt.nim; the diff is harmless, cgen code is identical with -d:danger or debug mode * rm tests/system/tstrmantle.nim * revert compiler/jsgen.nim for -d:nimVersion140 --- compiler/ast.nim | 8 +-- lib/std/private/digitsutils.nim | 72 +++++++++++++++----- lib/std/private/miscdollars.nim | 9 ++- lib/system.nim | 9 ++- lib/system/assertions.nim | 5 +- lib/system/dollars.nim | 56 ++++++---------- lib/system/repr_v2.nim | 17 +++-- lib/system/strmantle.nim | 30 --------- nimsuggest/tests/tqualified_highlight.nim | 2 +- tests/pragmas/tinjectstmt.nim | 13 ++-- tests/system/tdollars.nim | 81 +++++++++++++++-------- tests/system/tstrmantle.nim | 46 ------------- 12 files changed, 162 insertions(+), 186 deletions(-) delete mode 100644 tests/system/tstrmantle.nim diff --git a/compiler/ast.nim b/compiler/ast.nim index 08fbc70e1..0bdcc0e97 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -652,8 +652,8 @@ type mUnaryMinusI, mUnaryMinusI64, mAbsI, mNot, mUnaryPlusI, mBitnotI, mUnaryPlusF64, mUnaryMinusF64, - mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, - mFloatToStr, # for -d:nimVersion140 + mCharToStr, mBoolToStr, + mIntToStr, mInt64ToStr, mFloatToStr, # for -d:nimVersion140 mCStrToStr, mStrToStr, mEnumToStr, mAnd, mOr, @@ -722,8 +722,8 @@ const mEqRef, mEqProc, mLePtr, mLtPtr, mEqCString, mXor, mUnaryMinusI, mUnaryMinusI64, mAbsI, mNot, mUnaryPlusI, mBitnotI, mUnaryPlusF64, mUnaryMinusF64, - mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, - mFloatToStr, + mCharToStr, mBoolToStr, + mIntToStr, mInt64ToStr, mFloatToStr, mCStrToStr, mStrToStr, mEnumToStr, mAnd, mOr, diff --git a/lib/std/private/digitsutils.nim b/lib/std/private/digitsutils.nim index 7aefc36bc..268a3ba7d 100644 --- a/lib/std/private/digitsutils.nim +++ b/lib/std/private/digitsutils.nim @@ -29,18 +29,34 @@ const # doAssert res == digits100 proc utoa2Digits*(buf: var openArray[char]; pos: int; digits: uint32) {.inline.} = - assert(digits <= 99) buf[pos] = digits100[2 * digits] buf[pos+1] = digits100[2 * digits + 1] #copyMem(buf, unsafeAddr(digits100[2 * digits]), 2 * sizeof((char))) proc trailingZeros2Digits*(digits: uint32): int32 {.inline.} = - assert(digits <= 99) return trailingZeros100[digits] -func addIntImpl*(result: var string, origin: uint64) = +when defined(js): + proc numToString(a: SomeInteger): cstring {.importjs: "((#) + \"\")".} + +func addChars[T](result: var string, x: T, start: int, n: int) {.inline.} = + let old = result.len + result.setLen old + n + template impl = + for i in 0.. 0: result.add "(" - # simplify this after moving moving `include strmantle` above import assertions` - when declared(addInt): result.addInt line - else: result.add $line + addInt(result, line) if col > 0: result.add ", " - when declared(addInt): result.addInt col - else: result.add $col + addInt(result, col) result.add ")" diff --git a/lib/system.nim b/lib/system.nim index e6a2439df..740c4419f 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2463,14 +2463,13 @@ when notJSnotNims: else: {.error: "Only closure iterator is allowed!".} +from std/private/digitsutils import addInt +export addInt + when defined(js): include "system/jssys" include "system/reprjs" -when defined(js) or defined(nimscript): - proc addInt*(result: var string; x: int64) = - result.add $x - proc quit*(errormsg: string, errorcode = QuitFailure) {.noreturn.} = ## A shorthand for `echo(errormsg); quit(errorcode)`. when defined(nimscript) or defined(js) or (hostOS == "standalone"): @@ -2918,7 +2917,7 @@ proc addQuoted*[T](s: var string, x: T) = s.addEscapedChar(x) s.add("'") # prevent temporary string allocation - elif T is SomeSignedInt: + elif T is SomeInteger: s.addInt(x) elif T is SomeFloat: s.addFloat(x) diff --git a/lib/system/assertions.nim b/lib/system/assertions.nim index cd789845b..6f64a55b7 100644 --- a/lib/system/assertions.nim +++ b/lib/system/assertions.nim @@ -12,7 +12,6 @@ import std/private/miscdollars type InstantiationInfo = tuple[filename: string, line: int, column: int] -proc `$`(x: int): string {.magic: "IntToStr", noSideEffect.} proc `$`(info: InstantiationInfo): string = # The +1 is needed here # instead of overriding `$` (and changing its meaning), consider explicit name. @@ -108,7 +107,9 @@ template doAssertRaises*(exception: typedesc, code: untyped) = wrong = true except exception: discard - except Exception as e: raiseAssert(begin & " raised '" & $e.name & "'" & msgEnd) + except Exception as e: + mixin `$` # alternatively, we could define $cstring in this module + raiseAssert(begin & " raised '" & $e.name & "'" & msgEnd) except: raisedForeign() if wrong: raiseAssert(begin & " nothing was raised" & msgEnd) diff --git a/lib/system/dollars.nim b/lib/system/dollars.nim index 8634db382..46085d2aa 100644 --- a/lib/system/dollars.nim +++ b/lib/system/dollars.nim @@ -1,45 +1,31 @@ +## `$` is Nim's general way of spelling `toString`:idx:. +runnableExamples: + assert $0.1 == "0.1" + assert $(-2*3) == "-6" + import std/private/digitsutils import system/formatfloat export addFloat -proc `$`*(x: int): string {.magic: "IntToStr", noSideEffect.} - ## The stringify operator for an integer argument. Returns `x` - ## converted to a decimal string. `$` is Nim's general way of - ## spelling `toString`:idx:. +proc `$`*(x: int): string {.raises: [].} = + ## Outplace version of `addInt`. + result.addInt(x) -template dollarImpl(x: uint | uint64, result: var string) = - addIntImpl(result, x) +proc `$`*(x: int64): string {.raises: [].} = + ## Outplace version of `addInt`. + result.addInt(x) -when defined(js): - import std/private/since - since (1, 3): - proc `$`*(x: uint): string = - ## Caveat: currently implemented as $(cast[int](x)), tied to current - ## semantics of js' Number type. - # for c, see strmantle.`$` - when nimvm: - dollarImpl(x, result) - else: - result = $(int(x)) +proc `$`*(x: uint64): string {.raises: [].} = + ## Outplace version of `addInt`. + addInt(result, x) - proc `$`*(x: uint64): string = - ## Compatibility note: - ## the results may change in future releases if/when js target implements - ## 64bit ints. - # pending https://github.com/nim-lang/RFCs/issues/187 - when nimvm: - dollarImpl(x, result) - else: - result = $(cast[int](x)) -else: - proc `$`*(x: uint64): string {.noSideEffect, raises: [].} = - ## The stringify operator for an unsigned integer argument. Returns `x` - ## converted to a decimal string. - dollarImpl(x, result) - -proc `$`*(x: int64): string {.magic: "Int64ToStr", noSideEffect.} - ## The stringify operator for an integer argument. Returns `x` - ## converted to a decimal string. +# same as old `ctfeWhitelist` behavior, whether or not this is a good idea. +template gen(T) = + # xxx simplify this by supporting this in compiler: int{lit} | uint64{lit} | int64{lit} + func `$`*(x: T{lit}): string {.compileTime.} = result.addInt(x) +gen(int) +gen(uint64) +gen(int64) func `$`*(x: float | float32): string = ## Outplace version of `addFloat`. diff --git a/lib/system/repr_v2.nim b/lib/system/repr_v2.nim index ba94b881d..6ab5f3c3f 100644 --- a/lib/system/repr_v2.nim +++ b/lib/system/repr_v2.nim @@ -8,18 +8,17 @@ proc distinctBase(T: typedesc, recursive: static bool = true): typedesc {.magic: proc repr*(x: NimNode): string {.magic: "Repr", noSideEffect.} -proc repr*(x: int): string {.magic: "IntToStr", noSideEffect.} - ## repr for an integer argument. Returns `x` - ## converted to a decimal string. +proc repr*(x: int): string = + ## Same as $x + $x -proc repr*(x: int64): string {.magic: "Int64ToStr", noSideEffect.} - ## repr for an integer argument. Returns `x` - ## converted to a decimal string. +proc repr*(x: int64): string = + ## Same as $x + $x proc repr*(x: uint64): string {.noSideEffect.} = - ## repr for an unsigned integer argument. Returns `x` - ## converted to a decimal string. - $x #Calls `$` from system/strmantle.nim + ## Same as $x + $x proc repr*(x: float): string = ## Same as $x diff --git a/lib/system/strmantle.nim b/lib/system/strmantle.nim index f55168c01..9cf4f9e55 100644 --- a/lib/system/strmantle.nim +++ b/lib/system/strmantle.nim @@ -43,32 +43,6 @@ proc hashString(s: string): int {.compilerproc.} = h = h + h shl 15 result = cast[int](h) -proc addInt*(result: var string; x: int64) = - ## Converts integer to its string representation and appends it to `result`. - ## - ## .. code-block:: Nim - ## var - ## a = "123" - ## b = 45 - ## a.addInt(b) # a <- "12345" - var num: uint64 - - if x < 0: - if x == low(int64): - num = uint64(x) - else: - num = uint64(-x) - let base = result.len - setLen(result, base + 1) - result[base] = '-' - else: - num = uint64(x) - addIntImpl(result, num) - -proc nimIntToStr(x: int): string {.compilerRtl.} = - result = newStringOfCap(sizeof(x)*4) - result.addInt x - proc c_strtod(buf: cstring, endptr: ptr cstring): float64 {. importc: "strtod", header: "", noSideEffect.} @@ -240,10 +214,6 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat, when defined(nimHasInvariant): {.pop.} # staticBoundChecks -proc nimInt64ToStr(x: int64): string {.compilerRtl.} = - result = newStringOfCap(sizeof(x)*4) - result.addInt x - proc nimBoolToStr(x: bool): string {.compilerRtl.} = return if x: "true" else: "false" diff --git a/nimsuggest/tests/tqualified_highlight.nim b/nimsuggest/tests/tqualified_highlight.nim index 3b521ecc1..b83669e72 100644 --- a/nimsuggest/tests/tqualified_highlight.nim +++ b/nimsuggest/tests/tqualified_highlight.nim @@ -10,5 +10,5 @@ highlight;;skProc;;1;;7;;4 highlight;;skTemplate;;2;;7;;4 highlight;;skTemplate;;2;;7;;4 highlight;;skTemplate;;2;;7;;4 -highlight;;skProc;;3;;8;;1 +highlight;;skFunc;;3;;8;;1 """ diff --git a/tests/pragmas/tinjectstmt.nim b/tests/pragmas/tinjectstmt.nim index bca041e46..c6256bda6 100644 --- a/tests/pragmas/tinjectstmt.nim +++ b/tests/pragmas/tinjectstmt.nim @@ -7,15 +7,18 @@ ok0 ok1 onInject: 3 onInject: 4 -0 onInject: 5 +0 onInject: 6 -1 onInject: 7 onInject: 8 +1 +onInject: 9 +onInject: 10 +onInject: 11 2 ok2 -onInject: 9 +onInject: 12 ''' """ @@ -25,7 +28,7 @@ onInject: 9 {.injectStmt.} pragma can be used to inject a statement before every other statement in the current module. It's now undocumented and may be removed in the future and replaced with something more general and without its limitations. -e.g. (e.g. doesn't work in VM or js backends). +(e.g. doesn't work in VM or js backends). ]# from system/ansi_c import c_printf @@ -44,5 +47,5 @@ proc main()= echo a echo "ok2" -static: main() # xxx injectStmt not honred in VM +static: main() # xxx injectStmt not honored in VM main() diff --git a/tests/system/tdollars.nim b/tests/system/tdollars.nim index 34801d9ee..17d195e76 100644 --- a/tests/system/tdollars.nim +++ b/tests/system/tdollars.nim @@ -102,40 +102,41 @@ block: # #14350, #16674, #16686 for JS doAssert nil2 == cstring("") block: - block: - let x = -1'i8 - let y = uint32(x) - - doAssert $y == "4294967295" - - block: - let x = -1'i16 - let y = uint32(x) - - doAssert $y == "4294967295" - - block: - let x = -1'i32 - let y = uint32(x) - - doAssert $y == "4294967295" + when defined(js): # bug #18591 + let a1 = -1'i8 + let a2 = uint8(a1) + # if `uint8(a1)` changes meaning to `cast[uint8](a1)` in future, update this test; + # until then, this is the correct semantics. + let a3 = $a2 + doAssert a2 < 3 + doAssert a3 == "-1" + proc intToStr(a: uint8): cstring {.importjs: "(# + \"\")".} + doAssert $intToStr(a2) == "-1" + else: + block: + let x = -1'i8 + let y = uint32(x) + doAssert $y == "4294967295" + block: + let x = -1'i16 + let y = uint32(x) + doAssert $y == "4294967295" + block: + let x = -1'i32 + let y = uint32(x) + doAssert $y == "4294967295" + block: + proc foo1(arg: int): string = + let x = uint32(arg) + $x + doAssert $foo1(-1) == "4294967295" block: let x = 4294967295'u32 doAssert $x == "4294967295" - block: doAssert $(4294967295'u32) == "4294967295" - - block: - proc foo1(arg: int): string = - let x = uint32(arg) - $x - - doAssert $foo1(-1) == "4294967295" - - proc main()= block: let a = -0.0 @@ -159,6 +160,32 @@ proc main()= doAssert $uint32.high == "4294967295" + block: # addInt + var res = newStringOfCap(24) + template test2(a, b) = + res.setLen(0) + res.addInt a + doAssert res == b + + for i in 0 .. 9: + res.addInt int64(i) + doAssert res == "0123456789" + + res.setLen(0) + for i in -9 .. 0: + res.addInt int64(i) + doAssert res == "-9-8-7-6-5-4-3-2-10" + + when not defined(js): + test2 high(int64), "9223372036854775807" + test2 low(int64), "-9223372036854775808" + + test2 high(int32), "2147483647" + test2 low(int32), "-2147483648" + test2 high(int16), "32767" + test2 low(int16), "-32768" + test2 high(int8), "127" + test2 low(int8), "-128" static: main() main() diff --git a/tests/system/tstrmantle.nim b/tests/system/tstrmantle.nim deleted file mode 100644 index 1f195adde..000000000 --- a/tests/system/tstrmantle.nim +++ /dev/null @@ -1,46 +0,0 @@ -var res = newStringOfCap(24) - -for i in 0 .. 9: - res.addInt int64(i) - -doAssert res == "0123456789" - -res.setLen(0) - -for i in -9 .. 0: - res.addInt int64(i) - -doAssert res == "-9-8-7-6-5-4-3-2-10" - -res.setLen(0) -res.addInt high(int64) -doAssert res == "9223372036854775807" - -res.setLen(0) -res.addInt low(int64) -doAssert res == "-9223372036854775808" - -res.setLen(0) -res.addInt high(int32) -doAssert res == "2147483647" - -res.setLen(0) -res.addInt low(int32) -doAssert res == "-2147483648" - -res.setLen(0) -res.addInt high(int16) -doAssert res == "32767" - -res.setLen(0) -res.addInt low(int16) -doAssert res == "-32768" - - -res.setLen(0) -res.addInt high(int8) -doAssert res == "127" - -res.setLen(0) -res.addInt low(int8) -doAssert res == "-128" From 373bbd9bb4cbcb06fbfd80ab0e968127dc098873 Mon Sep 17 00:00:00 2001 From: Tomohiro Date: Thu, 19 Aug 2021 18:35:40 +0900 Subject: [PATCH 1153/2472] Add ceilDiv to math (#18596) * Use assert in runnableExamples and improve boundary check * Add more tests for ceilDiv * Fix comment in ceilDiv * Calling ceilDiv with int type T such like sizeof(T) > 8 is error --- changelog.md | 2 ++ lib/pure/math.nim | 52 ++++++++++++++++++++++++++++++++++++++++++ tests/stdlib/tmath.nim | 40 ++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+) diff --git a/changelog.md b/changelog.md index 77a77f2f5..447401cd0 100644 --- a/changelog.md +++ b/changelog.md @@ -148,6 +148,8 @@ - Added `clamp` in `math` which allows using a `Slice` to clamp to a value. +- Added `ceilDiv` in `math` for round up integer division. + - The JSON module can now handle integer literals and floating point literals of arbitrary length and precision. Numbers that do not fit the underlying `BiggestInt` or `BiggestFloat` fields are diff --git a/lib/pure/math.nim b/lib/pure/math.nim index 410660d4a..66f1a2e2c 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -941,6 +941,58 @@ func euclMod*[T: SomeNumber](x, y: T): T {.since: (1, 5, 1).} = if result < 0: result += abs(y) +func ceilDiv*[T: SomeInteger](x, y: T): T {.inline, since: (1, 5, 1).} = + ## Ceil division is conceptually defined as `ceil(x / y)`. + ## + ## Assumes `x >= 0` and `y > 0` (and `x + y - 1 <= high(T)` if T is SomeUnsignedInt). + ## + ## This is different from the `system.div `_ + ## operator, which works like `trunc(x / y)`. + ## That is, `div` rounds towards `0` and `ceilDiv` rounds up. + ## + ## This function has the above input limitation, because that allows the + ## compiler to generate faster code and it is rarely used with + ## negative values or unsigned integers close to `high(T)/2`. + ## If you need a `ceilDiv` that works with any input, see: + ## https://github.com/demotomohiro/divmath. + ## + ## **See also:** + ## * `system.div proc `_ for integer division + ## * `floorDiv func <#floorDiv,T,T>`_ for integer division which rounds down. + runnableExamples: + assert ceilDiv(12, 3) == 4 + assert ceilDiv(13, 3) == 5 + + when sizeof(T) == 8: + type UT = uint64 + elif sizeof(T) == 4: + type UT = uint32 + elif sizeof(T) == 2: + type UT = uint16 + elif sizeof(T) == 1: + type UT = uint8 + else: + {.fatal: "Unsupported int type".} + + assert x >= 0 and y > 0 + when T is SomeUnsignedInt: + assert x + y - 1 >= x + + # If the divisor is const, the backend C/C++ compiler generates code without a `div` + # instruction, as it is slow on most CPUs. + # If the divisor is a power of 2 and a const unsigned integer type, the + # compiler generates faster code. + # If the divisor is const and a signed integer, generated code becomes slower + # than the code with unsigned integers, because division with signed integers + # need to works for both positive and negative value without `idiv`/`sdiv`. + # That is why this code convert parameters to unsigned. + # This post contains a comparison of the performance of signed/unsigned integers: + # https://github.com/nim-lang/Nim/pull/18596#issuecomment-894420984. + # If signed integer arguments were not converted to unsigned integers, + # `ceilDiv` wouldn't work for any positive signed integer value, because + # `x + (y - 1)` can overflow. + ((x.UT + (y.UT - 1.UT)) div y.UT).T + func frexp*[T: float32|float64](x: T): tuple[frac: T, exp: int] {.inline.} = ## Splits `x` into a normalized fraction `frac` and an integral power of 2 `exp`, ## such that `abs(frac) in 0.5..<1` and `x == frac * 2 ^ exp`, except for special diff --git a/tests/stdlib/tmath.nim b/tests/stdlib/tmath.nim index 2cf544ddb..9bc5b994f 100644 --- a/tests/stdlib/tmath.nim +++ b/tests/stdlib/tmath.nim @@ -106,6 +106,46 @@ template main() = doAssert euclDiv(-9, -3) == 3 doAssert euclMod(-9, -3) == 0 + block: # ceilDiv + doAssert ceilDiv(8, 3) == 3 + doAssert ceilDiv(8, 4) == 2 + doAssert ceilDiv(8, 5) == 2 + doAssert ceilDiv(11, 3) == 4 + doAssert ceilDiv(12, 3) == 4 + doAssert ceilDiv(13, 3) == 5 + doAssert ceilDiv(41, 7) == 6 + doAssert ceilDiv(0, 1) == 0 + doAssert ceilDiv(1, 1) == 1 + doAssert ceilDiv(1, 2) == 1 + doAssert ceilDiv(2, 1) == 2 + doAssert ceilDiv(2, 2) == 1 + doAssert ceilDiv(0, high(int)) == 0 + doAssert ceilDiv(1, high(int)) == 1 + doAssert ceilDiv(0, high(int) - 1) == 0 + doAssert ceilDiv(1, high(int) - 1) == 1 + doAssert ceilDiv(high(int) div 2, high(int) div 2 + 1) == 1 + doAssert ceilDiv(high(int) div 2, high(int) div 2 + 2) == 1 + doAssert ceilDiv(high(int) div 2 + 1, high(int) div 2) == 2 + doAssert ceilDiv(high(int) div 2 + 2, high(int) div 2) == 2 + doAssert ceilDiv(high(int) div 2 + 1, high(int) div 2 + 1) == 1 + doAssert ceilDiv(high(int), 1) == high(int) + doAssert ceilDiv(high(int) - 1, 1) == high(int) - 1 + doAssert ceilDiv(high(int) - 1, 2) == high(int) div 2 + doAssert ceilDiv(high(int) - 1, high(int)) == 1 + doAssert ceilDiv(high(int) - 1, high(int) - 1) == 1 + doAssert ceilDiv(high(int) - 1, high(int) - 2) == 2 + doAssert ceilDiv(high(int), high(int)) == 1 + doAssert ceilDiv(high(int), high(int) - 1) == 2 + doAssert ceilDiv(255'u8, 1'u8) == 255'u8 + doAssert ceilDiv(254'u8, 2'u8) == 127'u8 + when not defined(danger): + doAssertRaises(AssertionDefect): discard ceilDiv(41, 0) + doAssertRaises(AssertionDefect): discard ceilDiv(41, -1) + doAssertRaises(AssertionDefect): discard ceilDiv(-1, 1) + doAssertRaises(AssertionDefect): discard ceilDiv(-1, -1) + doAssertRaises(AssertionDefect): discard ceilDiv(254'u8, 3'u8) + doAssertRaises(AssertionDefect): discard ceilDiv(255'u8, 2'u8) + block: # splitDecimal() tests doAssert splitDecimal(54.674).intpart == 54.0 doAssert splitDecimal(54.674).floatpart ==~ 0.674 From 8fa0decf6b4917e5559b9fd84a18825849dc11df Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 19 Aug 2021 21:54:45 +0800 Subject: [PATCH 1154/2472] fix a typo (#18715) --- compiler/semcall.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/semcall.nim b/compiler/semcall.nim index d8dba3e6c..2478420f7 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -280,7 +280,7 @@ const proc notFoundError*(c: PContext, n: PNode, errors: CandidateErrors) = # Gives a detailed error message; this is separated from semOverloadedCall, - # as semOverlodedCall is already pretty slow (and we need this information + # as semOverloadedCall is already pretty slow (and we need this information # only in case of an error). if c.config.m.errorOutputs == {}: # fail fast: From 13b97291835046eb1368ff4cd240cdcc7d0a2899 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 20 Aug 2021 01:56:52 +0800 Subject: [PATCH 1155/2472] fix #18627(Program segfaults with ARC when using openArray[string]) (#18713) * fix #18627 * add testcase * rename * another * remove tyVarargs --- compiler/ast.nim | 2 +- tests/arc/topenarray.nim | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 tests/arc/topenarray.nim diff --git a/compiler/ast.nim b/compiler/ast.nim index 0bdcc0e97..e7507c9c6 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -1595,7 +1595,7 @@ proc propagateToOwner*(owner, elem: PType; propagateHasAsgn = true) = if mask != {} and propagateHasAsgn: let o2 = owner.skipTypes({tyGenericInst, tyAlias, tySink}) if o2.kind in {tyTuple, tyObject, tyArray, - tySequence, tySet, tyDistinct, tyOpenArray, tyVarargs}: + tySequence, tySet, tyDistinct}: o2.flags.incl mask owner.flags.incl mask diff --git a/tests/arc/topenarray.nim b/tests/arc/topenarray.nim new file mode 100644 index 000000000..03ec7adf2 --- /dev/null +++ b/tests/arc/topenarray.nim @@ -0,0 +1,24 @@ +discard """ + input: "hi" + output: ''' +hi +Nim +''' + matrix: "--gc:arc -d:useMalloc; --gc:arc" +""" +block: # bug 18627 + proc setPosition(params: openArray[string]) = + for i in params.toOpenArray(0, params.len - 1): + echo i + + proc uciLoop() = + let params = @[readLine(stdin)] + setPosition(params) + + uciLoop() + + proc uciLoop2() = + let params = @["Nim"] + for i in params.toOpenArray(0, params.len - 1): + echo i + uciLoop2() From f2910077ac41a614154f3851f5bffc02b06124a2 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 20 Aug 2021 01:13:03 -0700 Subject: [PATCH 1156/2472] sigmatch: support optional params with last block arg(s) (#18631) * sigmatch: support optional params with last block arg * add tests * works with multiple block args * cleanup * address comment --- compiler/sigmatch.nim | 14 +++++++- tests/misc/trfc405.nim | 81 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 tests/misc/trfc405.nim diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 674dd7e2e..465a9a51d 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -2305,6 +2305,17 @@ proc incrIndexType(t: PType) = template isVarargsUntyped(x): untyped = x.kind == tyVarargs and x[0].kind == tyUntyped +proc findFirstArgBlock(m: var TCandidate, n: PNode): int = + # see https://github.com/nim-lang/RFCs/issues/405 + result = int.high + for a2 in countdown(n.len-1, 0): + # checking `nfBlockArg in n[a2].flags` wouldn't work inside templates + if n[a2].kind != nkStmtList: break + let formalLast = m.callee.n[m.callee.n.len - (n.len - a2)] + if formalLast.kind == nkSym and formalLast.sym.ast == nil: + result = a2 + else: break + proc matchesAux(c: PContext, n, nOrig: PNode, m: var TCandidate, marker: var IntSet) = template noMatch() = @@ -2345,7 +2356,7 @@ proc matchesAux(c: PContext, n, nOrig: PNode, m: var TCandidate, marker: var Int formalLen = m.callee.n.len formal = if formalLen > 1: m.callee.n[1].sym else: nil # current routine parameter container: PNode = nil # constructed container - + let firstArgBlock = findFirstArgBlock(m, n) while a < n.len: c.openShadowScope @@ -2441,6 +2452,7 @@ proc matchesAux(c: PContext, n, nOrig: PNode, m: var TCandidate, marker: var Int if m.callee.n[f].kind != nkSym: internalError(c.config, n[a].info, "matches") noMatch() + if a >= firstArgBlock: f = max(f, m.callee.n.len - (n.len - a)) formal = m.callee.n[f].sym m.firstMismatch.kind = kTypeMismatch if containsOrIncl(marker, formal.position) and container.isNil: diff --git a/tests/misc/trfc405.nim b/tests/misc/trfc405.nim new file mode 100644 index 000000000..adc39b1df --- /dev/null +++ b/tests/misc/trfc405.nim @@ -0,0 +1,81 @@ +# https://github.com/nim-lang/RFCs/issues/405 + +template main = + template fn1(a = 1, b = 2, body): auto = (a, b, astToStr(body)) + let a1 = fn1(10, 20): + foo + doAssert a1 == (10, 20, "\nfoo") + + template fn2(a = 1, b = 2, body): auto = (a, b, astToStr(body)) + let a2 = fn2(a = 10): foo + doAssert a2 == (10, 2, "\nfoo") + let a2b = fn2(b = 20): foo + doAssert a2b == (1, 20, "\nfoo") + + template fn3(x: int, a = 1, b = 2, body): auto = (a, b, astToStr(body)) + let a3 = fn3(3, 10, 20): foo + doAssert a3 == (10, 20, "\nfoo") + let a3b = fn3(3, a = 10): foo + doAssert a3b == (10, 2, "\nfoo") + + template fn4(x: int, y: int, body): auto = (x, y, astToStr(body)) + let a4 = fn4(1, 2): foo + doAssert a4 == (1, 2, "\nfoo") + + template fn5(x = 1, y = 2, body: untyped = 3): auto = (x, y, astToStr(body)) + doAssert compiles(fn5(1, 2, foo)) + doAssert not compiles(fn5(1, foo)) + + block: + # with an overload + var witness = 0 + template fn6() = discard + template fn6(procname: string, body: untyped): untyped = witness.inc + fn6("abc"): discard + assert witness == 1 + + block: + # with overloads + var witness = 0 + template fn6() = discard + template fn6(a: int) = discard + template fn6(procname: string, body: untyped): untyped = witness.inc + fn6("abc"): discard + assert witness == 1 + + template fn6(b = 1.5, body: untyped): untyped = witness.inc + fn6(1.3): discard + assert witness == 2 + + block: + var witness = 0 + template fn6(a: int) = discard + template fn6(a: string) = discard + template fn6(ignore: string, b = 1.5, body: untyped): untyped = witness.inc + fn6(""): + foobar1 + foobar2 + doAssert witness == 1 + fn6(""): discard + doAssert witness == 2 + + block: # multi block args + template fn8(a = 1, b = 2, body1: untyped, body2: untyped): auto = (a, b, astToStr(body1), astToStr(body2)) + let a1 = fn8(): + foobar1 + foobar2 + do: + foobar3 + foobar4 + doAssert a1 == (1, 2, "\nfoobar1\nfoobar2", "\nfoobar3\nfoobar4") + + let a2 = fn8(b = 20): + foobar1 + foobar2 + do: + foobar3 + foobar4 + doAssert a2 == (1, 20, "\nfoobar1\nfoobar2", "\nfoobar3\nfoobar4") + +static: main() +main() From e52221e2967d7511f26dc020cae5fe62dcae5696 Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 21 Aug 2021 02:00:04 +0800 Subject: [PATCH 1157/2472] [docs]fix an item missing (#18717) --- doc/astspec.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/astspec.txt b/doc/astspec.txt index 92212bf8c..6d44f7f7e 100644 --- a/doc/astspec.txt +++ b/doc/astspec.txt @@ -364,6 +364,7 @@ AST: nnkExprColonExpr(nnkIdent("a"), nnkIntLit(1)), nnkExprColonExpr(nnkIdent("b"), nnkIntLit(2)), nnkExprColonExpr(nnkIdent("c"), nnkIntLit(3))) + nnkTupleConstr() Since the one tuple would be syntactically identical to parentheses with an expression in them, the parser expects a trailing comma for From f0c6593412c117c7f8e19421016303416d6b8bd3 Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 21 Aug 2021 14:21:06 +0800 Subject: [PATCH 1158/2472] add testcase for overloadable_enums (#18722) * add testcase for overloadable_enums * link --- tests/enum/toverloadable_enums.nim | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/enum/toverloadable_enums.nim b/tests/enum/toverloadable_enums.nim index 6f3aac3e2..2152d7fff 100644 --- a/tests/enum/toverloadable_enums.nim +++ b/tests/enum/toverloadable_enums.nim @@ -46,3 +46,32 @@ proc takeCallback(param: proc(p: int)) = discard takeCallback x echo ord v + +block: # https://github.com/nim-lang/RFCs/issues/8 + type + Enum1 = enum + A, B, C + Enum2 = enum + A, Z + + proc f(e: Enum1): int = ord(e) + proc g(e: Enum2): int = ord(e) + + proc h(e: Enum1): int = ord(e) + proc h(e: Enum2): int = ord(e) + + let fA = f(A) # Type of A is well defined + let gA = g(A) # Same as above + + let hA1 = h(Enum1.A) # A requires disambiguation + let hA2 = h(Enum2.A) # Similarly + let hA3 = h(B) + let hA4 = h(B) + let x = ord(Enum1.A) # Also + doAssert fA == 0 + doAssert gA == 0 + doAssert hA1 == 0 + doAssert hA2 == 0 + doAssert x == 0 + doAssert hA3 == 1 + doAssert hA4 == 1 From 5b26f2bd81d6fc7d48befbfb4fa3317f713af787 Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 21 Aug 2021 14:22:00 +0800 Subject: [PATCH 1159/2472] fix deprecated example (#18721) --- doc/astspec.txt | 16 ++++++++++++---- tests/astspec/tastspec.nim | 22 +++++++++++++++------- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/doc/astspec.txt b/doc/astspec.txt index 6d44f7f7e..6d3fa1f8c 100644 --- a/doc/astspec.txt +++ b/doc/astspec.txt @@ -1082,7 +1082,7 @@ its entirety to see some of the complexities. Concrete syntax: .. code-block:: nim - type Obj[T] = object {.inheritable.} + type Obj[T] {.inheritable.} = object name: string case isFat: bool of true: @@ -1094,10 +1094,18 @@ AST: .. code-block:: nim # ... + nnkPragmaExpr( + nnkIdent("Obj"), + nnkPragma(nnkIdent("inheritable")) + ), + nnkGenericParams( + nnkIdentDefs( + nnkIdent("T"), + nnkEmpty(), + nnkEmpty()) + ), nnkObjectTy( - nnkPragma( - nnkIdent("inheritable") - ), + nnkEmpty(), nnkEmpty(), nnkRecList( # list of object parameters nnkIdentDefs( diff --git a/tests/astspec/tastspec.nim b/tests/astspec/tastspec.nim index 33a245b1b..c99d8ec79 100644 --- a/tests/astspec/tastspec.nim +++ b/tests/astspec/tastspec.nim @@ -871,11 +871,20 @@ static: scope: macro testRecCase(ast: untyped): untyped = - ast.peelOff({nnkStmtList, nnkTypeSection})[2].matchAst: - of nnkObjectTy( - nnkPragma( - ident"inheritable" + ast.peelOff({nnkStmtList, nnkTypeSection}).matchAst: + of nnkTypeDef( + nnkPragmaExpr( + ident"Obj", + nnkPragma(ident"inheritable") ), + nnkGenericParams( + nnkIdentDefs( + ident"T", + nnkEmpty(), + nnkEmpty()) + ), + nnkObjectTy( + nnkEmpty(), nnkEmpty(), nnkRecList( # list of object parameters nnkIdentDefs( @@ -914,6 +923,7 @@ static: ident"T" ), nnkEmpty() + ) ) ) ) @@ -922,10 +932,8 @@ static: ): echo "ok" - - testRecCase: - type Obj[T] = object {.inheritable.} + type Obj[T] {.inheritable.} = object name: string case isFat: bool of true: From 061a9183f7b49289b05cf134893d703760f467a7 Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 22 Aug 2021 12:21:53 +0800 Subject: [PATCH 1160/2472] replace wrt with proper word (#18724) * what does wrt mean? * clarify --- compiler/ast.nim | 2 +- compiler/docgen.nim | 2 +- compiler/sighashes.nim | 2 +- lib/core/macros.nim | 6 ++++-- tests/iter/titer3.nim | 2 +- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index e7507c9c6..446c00f49 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -860,7 +860,7 @@ type case kind*: TSymKind of routineKinds: #procInstCache*: seq[PInstantiation] - gcUnsafetyReason*: PSym # for better error messages wrt gcsafe + gcUnsafetyReason*: PSym # for better error messages regarding gcsafe transformedBody*: PNode # cached body after transf pass of skLet, skVar, skField, skForVar: guard*: PSym diff --git a/compiler/docgen.nim b/compiler/docgen.nim index d39a20aae..706b54bcd 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -172,7 +172,7 @@ proc presentationPath*(conf: ConfigRef, file: AbsoluteFile): RelativeFile = if result.isEmpty: bail() elif conf.docRoot.len > 0: # we're (currently) requiring `isAbsolute` to avoid confusion when passing - # a relative path (would it be relative wrt $PWD or to projectfile) + # a relative path (would it be relative with regard to $PWD or to projectfile) conf.globalAssert conf.docRoot.isAbsolute, arg=conf.docRoot conf.globalAssert conf.docRoot.dirExists, arg=conf.docRoot # needed because `canonicalizePath` called on `file` diff --git a/compiler/sighashes.nim b/compiler/sighashes.nim index 0ff97017f..8534f4c0c 100644 --- a/compiler/sighashes.nim +++ b/compiler/sighashes.nim @@ -390,7 +390,7 @@ proc idOrSig*(s: PSym, currentModule: string, sigCollisions: var CountTable[SigHash]): Rope = if s.kind in routineKinds and s.typ != nil: # signatures for exported routines are reliable enough to - # produce a unique name and this means produced C++ is more stable wrt + # produce a unique name and this means produced C++ is more stable regarding # Nim changes: let sig = hashProc(s) result = rope($sig) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index d3eaf6298..645f5067a 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -442,9 +442,11 @@ proc ident*(name: string): NimNode {.magic: "StrToIdent", noSideEffect.} ## Create a new ident node from a string. type - BindSymRule* = enum ## specifies how `bindSym` behaves + BindSymRule* = enum ## Specifies how `bindSym` behaves. The difference + ## between open and closed symbols can be found in + ## ``_ brClosed, ## only the symbols in current scope are bound - brOpen, ## open wrt overloaded symbols, but may be a single + brOpen, ## open for overloaded symbols, but may be a single ## symbol if not ambiguous (the rules match that of ## binding in generics) brForceOpen ## same as brOpen, but it will always be open even diff --git a/tests/iter/titer3.nim b/tests/iter/titer3.nim index 9dcfd7be5..defd56c98 100644 --- a/tests/iter/titer3.nim +++ b/tests/iter/titer3.nim @@ -26,7 +26,7 @@ var x = [[1, 2, 3], [4, 5, 6]] for y in iter1(x[0]): write(stdout, $y) writeLine(stdout, "") -# ensure closure and inline iterators have the same behaviour wrt +# ensure closure and inline iterators have the same behaviour regarding # parameter passing iterator clo(a: int): int {.closure.} = From cc08d5c2c641e9390e9e05012defc9616f5d6ba8 Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 22 Aug 2021 13:40:20 +0800 Subject: [PATCH 1161/2472] fix #17898(randomPathName called twice in a row can return the same string on windows) (#18729) * close #17898 * no need to consider js --- lib/std/tempfiles.nim | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/lib/std/tempfiles.nim b/lib/std/tempfiles.nim index c97db67b6..f1cfd5885 100644 --- a/lib/std/tempfiles.nim +++ b/lib/std/tempfiles.nim @@ -17,7 +17,7 @@ See also: * `mkstemp` (posix), refs https://man7.org/linux/man-pages/man3/mkstemp.3.html ]# -import os, random +import os, random, std/monotimes const @@ -96,11 +96,25 @@ proc safeOpen(filename: string): File = discard posix.close(fileHandle) # TODO handles failure when closing file raiseOSError(osLastError(), filename) + +type + NimTempPathState = object + state: Rand + isInit: bool + +var nimTempPathState {.threadvar.}: NimTempPathState + template randomPathName(length: Natural): string = var res = newString(length) - var state = initRand() + if not nimTempPathState.isInit: + var time = getMonoTime().ticks + when compileOption("threads"): + time = time xor int64(getThreadId()) + nimTempPathState.isInit = true + nimTempPathState.state = initRand(time) + for i in 0 ..< length: - res[i] = state.sample(letters) + res[i] = nimTempPathState.state.sample(letters) res proc getTempDirImpl(dir: string): string {.inline.} = From e65dc36ddd6bbb1a87a607ee8c8491732730fa95 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Sun, 22 Aug 2021 16:32:55 -0300 Subject: [PATCH 1162/2472] Implement RFC-391 (#18585) --- lib/std/jsfetch.nim | 177 ++++++++++++++++++++++---------------------- tests/config.nims | 1 - tools/kochdocs.nim | 2 +- 3 files changed, 89 insertions(+), 91 deletions(-) diff --git a/lib/std/jsfetch.nim b/lib/std/jsfetch.nim index 1e7bb4d29..a25fe7ca8 100644 --- a/lib/std/jsfetch.nim +++ b/lib/std/jsfetch.nim @@ -1,121 +1,120 @@ ## - Fetch for the JavaScript target: https://developer.mozilla.org/docs/Web/API/Fetch_API -## .. Note:: jsfetch is Experimental. jsfetch module requires `-d:nimExperimentalJsfetch` +## .. Note:: jsfetch is Experimental. when not defined(js): {.fatal: "Module jsfetch is designed to be used with the JavaScript backend.".} -when defined(nimExperimentalJsfetch): - import std/[asyncjs, jsheaders, jsformdata] - from std/httpcore import HttpMethod - from std/jsffi import JsObject +import std/[asyncjs, jsheaders, jsformdata] +from std/httpcore import HttpMethod +from std/jsffi import JsObject - type - FetchOptions* = ref object of JsRoot ## Options for Fetch API. - keepalive*: bool - metod* {.importjs: "method".}: cstring - body*, integrity*, referrer*, mode*, credentials*, cache*, redirect*, referrerPolicy*: cstring +type + FetchOptions* = ref object of JsRoot ## Options for Fetch API. + keepalive*: bool + metod* {.importjs: "method".}: cstring + body*, integrity*, referrer*, mode*, credentials*, cache*, redirect*, referrerPolicy*: cstring - FetchModes* = enum ## Mode options. - fmCors = "cors" - fmNoCors = "no-cors" - fmSameOrigin = "same-origin" + FetchModes* = enum ## Mode options. + fmCors = "cors" + fmNoCors = "no-cors" + fmSameOrigin = "same-origin" - FetchCredentials* = enum ## Credential options. See https://developer.mozilla.org/en-US/docs/Web/API/Request/credentials - fcInclude = "include" - fcSameOrigin = "same-origin" - fcOmit = "omit" + FetchCredentials* = enum ## Credential options. See https://developer.mozilla.org/en-US/docs/Web/API/Request/credentials + fcInclude = "include" + fcSameOrigin = "same-origin" + fcOmit = "omit" - FetchCaches* = enum ## https://developer.mozilla.org/docs/Web/API/Request/cache - fchDefault = "default" - fchNoStore = "no-store" - fchReload = "reload" - fchNoCache = "no-cache" - fchForceCache = "force-cache" + FetchCaches* = enum ## https://developer.mozilla.org/docs/Web/API/Request/cache + fchDefault = "default" + fchNoStore = "no-store" + fchReload = "reload" + fchNoCache = "no-cache" + fchForceCache = "force-cache" - FetchRedirects* = enum ## Redirects options. - frFollow = "follow" - frError = "error" - frManual = "manual" + FetchRedirects* = enum ## Redirects options. + frFollow = "follow" + frError = "error" + frManual = "manual" - FetchReferrerPolicies* = enum ## Referrer Policy options. - frpNoReferrer = "no-referrer" - frpNoReferrerWhenDowngrade = "no-referrer-when-downgrade" - frpOrigin = "origin" - frpOriginWhenCrossOrigin = "origin-when-cross-origin" - frpUnsafeUrl = "unsafe-url" + FetchReferrerPolicies* = enum ## Referrer Policy options. + frpNoReferrer = "no-referrer" + frpNoReferrerWhenDowngrade = "no-referrer-when-downgrade" + frpOrigin = "origin" + frpOriginWhenCrossOrigin = "origin-when-cross-origin" + frpUnsafeUrl = "unsafe-url" - Body* = ref object of JsRoot ## https://developer.mozilla.org/en-US/docs/Web/API/Body - bodyUsed*: bool + Body* = ref object of JsRoot ## https://developer.mozilla.org/en-US/docs/Web/API/Body + bodyUsed*: bool - Response* = ref object of JsRoot ## https://developer.mozilla.org/en-US/docs/Web/API/Response - bodyUsed*, ok*, redirected*: bool - typ* {.importjs: "type".}: cstring - url*, statusText*: cstring - status*: cint - headers*: Headers - body*: Body + Response* = ref object of JsRoot ## https://developer.mozilla.org/en-US/docs/Web/API/Response + bodyUsed*, ok*, redirected*: bool + typ* {.importjs: "type".}: cstring + url*, statusText*: cstring + status*: cint + headers*: Headers + body*: Body - Request* = ref object of JsRoot ## https://developer.mozilla.org/en-US/docs/Web/API/Request - bodyUsed*, ok*, redirected*: bool - typ* {.importjs: "type".}: cstring - url*, statusText*: cstring - status*: cint - headers*: Headers - body*: Body + Request* = ref object of JsRoot ## https://developer.mozilla.org/en-US/docs/Web/API/Request + bodyUsed*, ok*, redirected*: bool + typ* {.importjs: "type".}: cstring + url*, statusText*: cstring + status*: cint + headers*: Headers + body*: Body - func newResponse*(body: cstring | FormData): Response {.importjs: "(new Response(#))".} - ## Constructor for `Response`. This does *not* call `fetch()`. Same as `new Response()`. +func newResponse*(body: cstring | FormData): Response {.importjs: "(new Response(#))".} + ## Constructor for `Response`. This does *not* call `fetch()`. Same as `new Response()`. - func newRequest*(url: cstring): Request {.importjs: "(new Request(#))".} - ## Constructor for `Request`. This does *not* call `fetch()`. Same as `new Request()`. +func newRequest*(url: cstring): Request {.importjs: "(new Request(#))".} + ## Constructor for `Request`. This does *not* call `fetch()`. Same as `new Request()`. - func clone*(self: Response | Request): Response {.importjs: "#.$1()".} - ## https://developer.mozilla.org/en-US/docs/Web/API/Response/clone +func clone*(self: Response | Request): Response {.importjs: "#.$1()".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Response/clone - proc text*(self: Response): Future[cstring] {.importjs: "#.$1()".} - ## https://developer.mozilla.org/en-US/docs/Web/API/Body/text +proc text*(self: Response): Future[cstring] {.importjs: "#.$1()".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Body/text - proc json*(self: Response): Future[JsObject] {.importjs: "#.$1()".} - ## https://developer.mozilla.org/en-US/docs/Web/API/Body/json +proc json*(self: Response): Future[JsObject] {.importjs: "#.$1()".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Body/json - proc formData*(self: Body): Future[FormData] {.importjs: "#.$1()".} - ## https://developer.mozilla.org/en-US/docs/Web/API/Body/formData +proc formData*(self: Body): Future[FormData] {.importjs: "#.$1()".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Body/formData - proc unsafeNewFetchOptions*(metod, body, mode, credentials, cache, referrerPolicy: cstring; - keepalive: bool; redirect = "follow".cstring; referrer = "client".cstring; integrity = "".cstring): FetchOptions {.importjs: - "{method: #, body: #, mode: #, credentials: #, cache: #, referrerPolicy: #, keepalive: #, redirect: #, referrer: #, integrity: #}".} - ## .. warning:: Unsafe `newfetchOptions`. +proc unsafeNewFetchOptions*(metod, body, mode, credentials, cache, referrerPolicy: cstring; + keepalive: bool; redirect = "follow".cstring; referrer = "client".cstring; integrity = "".cstring): FetchOptions {.importjs: + "{method: #, body: #, mode: #, credentials: #, cache: #, referrerPolicy: #, keepalive: #, redirect: #, referrer: #, integrity: #}".} + ## .. warning:: Unsafe `newfetchOptions`. - func newfetchOptions*(metod: HttpMethod; body: cstring; - mode: FetchModes; credentials: FetchCredentials; cache: FetchCaches; referrerPolicy: FetchReferrerPolicies; - keepalive: bool; redirect = frFollow; referrer = "client".cstring; integrity = "".cstring): FetchOptions = - ## Constructor for `FetchOptions`. - result = FetchOptions( - body: body, mode: $mode, credentials: $credentials, cache: $cache, referrerPolicy: $referrerPolicy, - keepalive: keepalive, redirect: $redirect, referrer: referrer, integrity: integrity, - metod: (case metod - of HttpHead: "HEAD".cstring - of HttpGet: "GET".cstring - of HttpPost: "POST".cstring - of HttpPut: "PUT".cstring - of HttpDelete: "DELETE".cstring - of HttpPatch: "PATCH".cstring - else: "GET".cstring - ) +func newfetchOptions*(metod: HttpMethod; body: cstring; + mode: FetchModes; credentials: FetchCredentials; cache: FetchCaches; referrerPolicy: FetchReferrerPolicies; + keepalive: bool; redirect = frFollow; referrer = "client".cstring; integrity = "".cstring): FetchOptions = + ## Constructor for `FetchOptions`. + result = FetchOptions( + body: body, mode: $mode, credentials: $credentials, cache: $cache, referrerPolicy: $referrerPolicy, + keepalive: keepalive, redirect: $redirect, referrer: referrer, integrity: integrity, + metod: (case metod + of HttpHead: "HEAD".cstring + of HttpGet: "GET".cstring + of HttpPost: "POST".cstring + of HttpPut: "PUT".cstring + of HttpDelete: "DELETE".cstring + of HttpPatch: "PATCH".cstring + else: "GET".cstring ) + ) - proc fetch*(url: cstring | Request): Future[Response] {.importjs: "$1(#)".} - ## `fetch()` API, simple `GET` only, returns a `Future[Response]`. +proc fetch*(url: cstring | Request): Future[Response] {.importjs: "$1(#)".} + ## `fetch()` API, simple `GET` only, returns a `Future[Response]`. - proc fetch*(url: cstring | Request; options: FetchOptions): Future[Response] {.importjs: "$1(#, #)".} - ## `fetch()` API that takes a `FetchOptions`, returns a `Future[Response]`. +proc fetch*(url: cstring | Request; options: FetchOptions): Future[Response] {.importjs: "$1(#, #)".} + ## `fetch()` API that takes a `FetchOptions`, returns a `Future[Response]`. - func toCstring*(self: Request | Response | Body | FetchOptions): cstring {.importjs: "JSON.stringify(#)".} +func toCstring*(self: Request | Response | Body | FetchOptions): cstring {.importjs: "JSON.stringify(#)".} - func `$`*(self: Request | Response | Body | FetchOptions): string = $toCstring(self) +func `$`*(self: Request | Response | Body | FetchOptions): string = $toCstring(self) -runnableExamples("-d:nimExperimentalJsfetch -r:off"): +runnableExamples("-r:off"): import std/[asyncjs, jsconsole, jsheaders, jsformdata] from std/httpcore import HttpMethod from std/jsffi import JsObject diff --git a/tests/config.nims b/tests/config.nims index d7f6ae7f9..0327f0b76 100644 --- a/tests/config.nims +++ b/tests/config.nims @@ -33,7 +33,6 @@ hint("Processing", off) # experimental API's are enabled in testament, refs https://github.com/timotheecour/Nim/issues/575 # sync with `kochdocs.docDefines` or refactor. switch("define", "nimExperimentalAsyncjsThen") -switch("define", "nimExperimentalJsfetch") switch("define", "nimExperimentalLinenoiseExtra") switch("define", "nimPreviewFloatRoundtrip") diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 8e8085f73..fe3335c0f 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -8,7 +8,7 @@ const gaCode* = " --doc.googleAnalytics:UA-48159761-1" # errormax: subsequent errors are probably consequences of 1st one; a simple # bug could cause unlimited number of errors otherwise, hard to debug in CI. - docDefines = "-d:nimExperimentalAsyncjsThen -d:nimExperimentalJsfetch -d:nimExperimentalLinenoiseExtra" + docDefines = "-d:nimExperimentalAsyncjsThen -d:nimExperimentalLinenoiseExtra" nimArgs = "--errormax:3 --hint:Conf:off --hint:Path:off --hint:Processing:off --hint:XDeclaredButNotUsed:off --warning:UnusedImport:off -d:boot --putenv:nimversion=$# $#" % [system.NimVersion, docDefines] gitUrl = "https://github.com/nim-lang/Nim" docHtmlOutput = "doc/html" From 2d84f6e7c836bf18c0ecbe07fb413166861196ac Mon Sep 17 00:00:00 2001 From: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Date: Mon, 23 Aug 2021 00:13:39 +0200 Subject: [PATCH 1163/2472] Fix #18718 (#18731) Update tests --- lib/pure/stats.nim | 13 +++--- tests/stdlib/tstats.nim | 93 ++++++++++++++++++++++------------------- 2 files changed, 59 insertions(+), 47 deletions(-) diff --git a/lib/pure/stats.nim b/lib/pure/stats.nim index e46dff212..58014e503 100644 --- a/lib/pure/stats.nim +++ b/lib/pure/stats.nim @@ -42,7 +42,7 @@ runnableExamples: template `~=`(a, b: float): bool = almostEqual(a, b) - var statistics: RunningStat ## Must be var + var statistics: RunningStat # must be var statistics.push(@[1.0, 2.0, 1.0, 4.0, 1.0, 4.0, 1.0, 2.0]) doAssert statistics.n == 8 doAssert statistics.mean() ~= 2.0 @@ -76,7 +76,7 @@ type proc clear*(s: var RunningStat) = ## Resets `s`. s.n = 0 - s.min = toBiggestFloat(int.high) + s.min = 0.0 s.max = 0.0 s.sum = 0.0 s.mom1 = 0.0 @@ -86,11 +86,14 @@ proc clear*(s: var RunningStat) = proc push*(s: var RunningStat, x: float) = ## Pushes a value `x` for processing. - if s.n == 0: s.min = x + if s.n == 0: + s.min = x + s.max = x + else: + if s.min > x: s.min = x + if s.max < x: s.max = x inc(s.n) # See Knuth TAOCP vol 2, 3rd edition, page 232 - if s.min > x: s.min = x - if s.max < x: s.max = x s.sum += x let n = toFloat(s.n) let delta = x - s.mom1 diff --git a/tests/stdlib/tstats.nim b/tests/stdlib/tstats.nim index 37240c884..92a2ed8b8 100644 --- a/tests/stdlib/tstats.nim +++ b/tests/stdlib/tstats.nim @@ -1,46 +1,55 @@ -include stats +import std/stats -proc clean(x: float): float = - result = round(1.0e8*x).float * 1.0e-8 +proc `~=`(x, y: float): bool = + abs(x - y) < 10e-8 -var rs: RunningStat -rs.push(@[1.0, 2.0, 1.0, 4.0, 1.0, 4.0, 1.0, 2.0]) -doAssert(rs.n == 8) -doAssert(clean(rs.mean) == 2.0) -doAssert(clean(rs.variance()) == 1.5) -doAssert(clean(rs.varianceS()) == 1.71428571) -doAssert(clean(rs.skewness()) == 0.81649658) -doAssert(clean(rs.skewnessS()) == 1.01835015) -doAssert(clean(rs.kurtosis()) == -1.0) -doAssert(clean(rs.kurtosisS()) == -0.7000000000000001) +template main() = + var rs: RunningStat + rs.push(@[1.0, 2.0, 1.0, 4.0, 1.0, 4.0, 1.0, 2.0]) + doAssert(rs.n == 8) + doAssert rs.mean ~= 2.0 + doAssert rs.variance() ~= 1.5 + doAssert rs.varianceS() ~= 1.71428571 + doAssert rs.skewness() ~= 0.81649658 + doAssert rs.skewnessS() ~= 1.01835015 + doAssert rs.kurtosis() ~= -1.0 + doAssert rs.kurtosisS() ~= -0.7000000000000001 -var rs1, rs2: RunningStat -rs1.push(@[1.0, 2.0, 1.0, 4.0]) -rs2.push(@[1.0, 4.0, 1.0, 2.0]) -let rs3 = rs1 + rs2 -doAssert(clean(rs3.mom2) == clean(rs.mom2)) -doAssert(clean(rs3.mom3) == clean(rs.mom3)) -doAssert(clean(rs3.mom4) == clean(rs.mom4)) -rs1 += rs2 -doAssert(clean(rs1.mom2) == clean(rs.mom2)) -doAssert(clean(rs1.mom3) == clean(rs.mom3)) -doAssert(clean(rs1.mom4) == clean(rs.mom4)) -rs1.clear() -rs1.push(@[1.0, 2.2, 1.4, 4.9]) -doAssert(rs1.sum == 9.5) -doAssert(rs1.mean() == 2.375) + var rs1, rs2: RunningStat + rs1.push(@[1.0, 2.0, 1.0, 4.0]) + rs2.push(@[1.0, 4.0, 1.0, 2.0]) + let rs3 = rs1 + rs2 + doAssert rs3.variance ~= rs.variance + doAssert rs3.skewness ~= rs.skewness + doAssert rs3.kurtosis ~= rs.kurtosis + rs1 += rs2 + doAssert rs1.variance ~= rs.variance + doAssert rs1.skewness ~= rs.skewness + doAssert rs1.kurtosis ~= rs.kurtosis + rs1.clear() + rs1.push(@[1.0, 2.2, 1.4, 4.9]) + doAssert(rs1.sum == 9.5) + doAssert(rs1.mean() == 2.375) -when not defined(cpu32): - # XXX For some reason on 32bit CPUs these results differ - var rr: RunningRegress - rr.push(@[0.0, 1.0, 2.8, 3.0, 4.0], @[0.0, 1.0, 2.3, 3.0, 4.0]) - doAssert(rr.slope() == 0.9695585996955861) - doAssert(rr.intercept() == -0.03424657534246611) - doAssert(rr.correlation() == 0.9905100362239381) - var rr1, rr2: RunningRegress - rr1.push(@[0.0, 1.0], @[0.0, 1.0]) - rr2.push(@[2.8, 3.0, 4.0], @[2.3, 3.0, 4.0]) - let rr3 = rr1 + rr2 - doAssert(rr3.correlation() == rr.correlation()) - doAssert(clean(rr3.slope()) == clean(rr.slope())) - doAssert(clean(rr3.intercept()) == clean(rr.intercept())) \ No newline at end of file + when not defined(cpu32): + # XXX For some reason on 32bit CPUs these results differ + var rr: RunningRegress + rr.push(@[0.0, 1.0, 2.8, 3.0, 4.0], @[0.0, 1.0, 2.3, 3.0, 4.0]) + doAssert(rr.slope() == 0.9695585996955861) + doAssert(rr.intercept() == -0.03424657534246611) + doAssert(rr.correlation() == 0.9905100362239381) + var rr1, rr2: RunningRegress + rr1.push(@[0.0, 1.0], @[0.0, 1.0]) + rr2.push(@[2.8, 3.0, 4.0], @[2.3, 3.0, 4.0]) + let rr3 = rr1 + rr2 + doAssert(rr3.correlation() == rr.correlation()) + doAssert rr3.slope() ~= rr.slope() + doAssert rr3.intercept() ~= rr.intercept() + + block: # bug #18718 + var rs: RunningStat + rs.push(-1.0) + doAssert rs.max == -1.0 + +static: main() +main() From 8a8267e2bc36ac8070fa2842fecd9cfe53e0a9ef Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 23 Aug 2021 21:05:22 +0800 Subject: [PATCH 1164/2472] fix a typo (#18736) --- compiler/sighashes.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/sighashes.nim b/compiler/sighashes.nim index 8534f4c0c..01a0fa784 100644 --- a/compiler/sighashes.nim +++ b/compiler/sighashes.nim @@ -166,7 +166,7 @@ proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]) = if t.n.len > 0: let oldFlags = symWithFlags.flags # Hack to prevent endless recursion - # xxx intead, use a hash table to indicate we've already visited a type, which + # xxx instead, use a hash table to indicate we've already visited a type, which # would also be more efficient. symWithFlags.flags.excl {sfAnon, sfGenSym} hashTree(c, t.n, flags + {CoHashTypeInsideNode}) From 9674ff4361468137afdbedb86196b7082aaa6efd Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 24 Aug 2021 02:13:10 +0800 Subject: [PATCH 1165/2472] make deprecated message better (#18737) --- lib/pure/mersenne.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/mersenne.nim b/lib/pure/mersenne.nim index 6ec0d66b4..37c5085b1 100644 --- a/lib/pure/mersenne.nim +++ b/lib/pure/mersenne.nim @@ -8,7 +8,7 @@ ## random number generator. ## .. note:: The procs in this module work at compile-time. -{.deprecated: "use `std/random` instead.".} +{.deprecated: "use `std/random` instead".} runnableExamples: var rand = newMersenneTwister(uint32.high) ## must be "var" From 24178bf8aa37c179caab9e16682940e8340f2bfc Mon Sep 17 00:00:00 2001 From: Miran Date: Tue, 24 Aug 2021 17:15:26 +0200 Subject: [PATCH 1166/2472] get rid of the warnings during bootstrapping (#18741) * fix bootstrapping hints and warnings * revert removals in ccgtypes --- compiler/nimlexbase.nim | 4 ++-- compiler/semfold.nim | 2 +- compiler/sighashes.nim | 2 +- compiler/suggest.nim | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/compiler/nimlexbase.nim b/compiler/nimlexbase.nim index 99d07d093..b8cb9f78a 100644 --- a/compiler/nimlexbase.nim +++ b/compiler/nimlexbase.nim @@ -104,7 +104,7 @@ proc fillBuffer(L: var TBaseLexer) = oldBufLen = L.bufLen L.bufLen = L.bufLen * 2 L.bufStorage.setLen(L.bufLen) - L.buf = L.bufStorage + L.buf = L.bufStorage.cstring assert(L.bufLen - oldBufLen == oldBufLen) charsRead = llStreamRead(L.stream, addr(L.buf[oldBufLen]), oldBufLen) @@ -147,7 +147,7 @@ proc openBaseLexer(L: var TBaseLexer, inputstream: PLLStream, bufLen = 8192) = L.bufpos = 0 L.offsetBase = 0 L.bufStorage = newString(bufLen) - L.buf = L.bufStorage + L.buf = L.bufStorage.cstring L.bufLen = bufLen L.sentinel = bufLen - 1 L.lineStart = 0 diff --git a/compiler/semfold.nim b/compiler/semfold.nim index 04ed73209..1f8efc4de 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -144,7 +144,7 @@ proc evalOp(m: TMagic, n, a, b, c: PNode; idgen: IdGenerator; g: ModuleGraph): P if a.typ.kind == tyString: result = newIntNodeT(toInt128(a.strVal.len), n, idgen, g) elif a.typ.kind == tyCstring: - result = newIntNodeT(toInt128(nimCStrLen(a.strVal)), n, idgen, g) + result = newIntNodeT(toInt128(nimCStrLen(a.strVal.cstring)), n, idgen, g) else: result = newIntNodeT(toInt128(a.len), n, idgen, g) of mUnaryPlusI, mUnaryPlusF64: result = a # throw `+` away diff --git a/compiler/sighashes.nim b/compiler/sighashes.nim index 01a0fa784..d02728627 100644 --- a/compiler/sighashes.nim +++ b/compiler/sighashes.nim @@ -18,7 +18,7 @@ proc `&=`(c: var MD5Context, ch: char) = # XXX suspicious code here; relies on ch being zero terminated? md5Update(c, unsafeAddr ch, 1) proc `&=`(c: var MD5Context, r: Rope) = - for l in leaves(r): md5Update(c, l, l.len) + for l in leaves(r): md5Update(c, l.cstring, l.len) proc `&=`(c: var MD5Context, i: BiggestInt) = md5Update(c, cast[cstring](unsafeAddr i), sizeof(i)) proc `&=`(c: var MD5Context, f: BiggestFloat) = diff --git a/compiler/suggest.nim b/compiler/suggest.nim index eaa30040a..84c94d793 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -32,7 +32,7 @@ # included from sigmatch.nim -import algorithm, sets, prefixmatches, lineinfos, parseutils, linter, tables +import algorithm, sets, prefixmatches, parseutils, tables from wordrecg import wDeprecated, wError, wAddr, wYield when defined(nimsuggest): From 3d1bba04ab1092630983695734d6984ddff4688c Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 24 Aug 2021 23:16:12 +0800 Subject: [PATCH 1167/2472] [minor]use `addInt` and `addFloat` (#18733) * unify int and uint * back * minor --- compiler/ic/ic.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim index bd3be27f0..7387d165b 100644 --- a/compiler/ic/ic.nim +++ b/compiler/ic/ic.nim @@ -94,10 +94,10 @@ proc toString*(tree: PackedTree; n: NodePos; m: PackedModule; nesting: int; result.addInt m.numbers[LitId tree.nodes[pos].operand] of externUIntLit: result.add " " - result.add $cast[uint64](m.numbers[LitId tree.nodes[pos].operand]) + result.addInt cast[uint64](m.numbers[LitId tree.nodes[pos].operand]) of nkFloatLit..nkFloat128Lit: result.add " " - result.add $cast[BiggestFloat](m.numbers[LitId tree.nodes[pos].operand]) + result.addFloat cast[BiggestFloat](m.numbers[LitId tree.nodes[pos].operand]) else: result.add "(\n" for i in 1..(nesting+1)*2: result.add ' ' From 3aa16c1de00c723d48e48fe3fdf07a276d1b4b6a Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 24 Aug 2021 21:50:40 -0700 Subject: [PATCH 1168/2472] fix RFC #341: dot-like operators are now parsed with same precedence as `.` (#18711) * fix RFC #341: dot-like operators are now parsed with same precedence as `.` * fixup * [skip ci] address comment in changelog * address comment * update grammmar * add manual entry * fixup * -d:nimPreviewDotLikeOps * address comment to unblock PR: move nimPreviewDotLikeOps out of config/config.nims --- changelog.md | 4 ++++ compiler/lineinfos.nim | 2 ++ compiler/parser.nim | 34 +++++++++++++++++++++++++++++----- doc/grammar.txt | 2 ++ doc/manual.rst | 9 +++++++++ tests/config.nims | 1 + tests/misc/trfc_341.nim | 27 +++++++++++++++++++++++++++ 7 files changed, 74 insertions(+), 5 deletions(-) create mode 100644 tests/misc/trfc_341.nim diff --git a/changelog.md b/changelog.md index 447401cd0..3ba008cf3 100644 --- a/changelog.md +++ b/changelog.md @@ -104,6 +104,10 @@ - In `std/dom`, `Interval` is now a `ref object`, same as `Timeout`. Definitions of `setTimeout`, `clearTimeout`, `setInterval`, `clearInterval` were updated. +- With `-d:nimPreviewDotLikeOps` (default in devel), dot-like operators (operators starting with `.`, but not with `..`) + now have the same precedence as `.`, so that `a.?b.c` is now parsed as `(a.?b).c` instead of `a.?(b.c)`. + A warning is generated when a dot-like operator is used without `-d:nimPreviewDotLikeOps`. + - The allocator for Nintendo Switch, which was nonfunctional because of breaking changes in libnx, was removed, in favour of the new `-d:nimAllocPagesViaMalloc` option. diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index c64d7b9dd..fd7ce0c04 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -46,6 +46,7 @@ type warnCannotOpenFile = "CannotOpenFile", warnOctalEscape = "OctalEscape", warnXIsNeverRead = "XIsNeverRead", warnXmightNotBeenInit = "XmightNotBeenInit", warnDeprecated = "Deprecated", warnConfigDeprecated = "ConfigDeprecated", + warnDotLikeOps = "DotLikeOps", warnSmallLshouldNotBeUsed = "SmallLshouldNotBeUsed", warnUnknownMagic = "UnknownMagic", warnRstRedefinitionOfLabel = "RedefinitionOfLabel", warnRstUnknownSubstitutionX = "UnknownSubstitutionX", @@ -116,6 +117,7 @@ const warnXmightNotBeenInit: "'$1' might not have been initialized", warnDeprecated: "$1", warnConfigDeprecated: "config file '$1' is deprecated", + warnDotLikeOps: "$1", warnSmallLshouldNotBeUsed: "'l' should not be used as an identifier; may look like '1' (one)", warnUnknownMagic: "unknown magic '$1' might crash the compiler", warnRstRedefinitionOfLabel: "redefinition of label '$1'", diff --git a/compiler/parser.nim b/compiler/parser.nim index b089614b2..30180e545 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -466,6 +466,17 @@ proc dotExpr(p: var Parser, a: PNode): PNode = exprColonEqExprListAux(p, tkParRi, y) result = y +proc dotLikeExpr(p: var Parser, a: PNode): PNode = + #| dotLikeExpr = expr DOTLIKEOP optInd symbol + var info = p.parLineInfo + result = newNodeI(nkInfix, info) + optInd(p, result) + var opNode = newIdentNodeP(p.tok.ident, p) + getTok(p) + result.add(opNode) + result.add(a) + result.add(parseSymbol(p, smAfterDot)) + proc qualifiedIdent(p: var Parser): PNode = #| qualifiedIdent = symbol ('.' optInd symbol)? result = parseSymbol(p) @@ -788,10 +799,15 @@ proc commandExpr(p: var Parser; r: PNode; mode: PrimaryMode): PNode = p.hasProgress = false result.add commandParam(p, isFirstParam, mode) +proc isDotLike(tok: Token): bool = + result = tok.tokType == tkOpr and tok.ident.s.len > 1 and + tok.ident.s[0] == '.' and tok.ident.s[1] != '.' + proc primarySuffix(p: var Parser, r: PNode, baseIndent: int, mode: PrimaryMode): PNode = #| primarySuffix = '(' (exprColonEqExpr comma?)* ')' #| | '.' optInd symbol generalizedLit? + #| | DOTLIKEOP optInd symbol generalizedLit? #| | '[' optInd exprColonEqExprList optPar ']' #| | '{' optInd exprColonEqExprList optPar '}' #| | &( '`'|IDENT|literal|'cast'|'addr'|'type') expr # command syntax @@ -840,11 +856,19 @@ proc primarySuffix(p: var Parser, r: PNode, # `foo ref` or `foo ptr`. Unfortunately, these two are also # used as infix operators for the memory regions feature and # the current parsing rules don't play well here. - if p.inPragma == 0 and (isUnary(p.tok) or p.tok.tokType notin {tkOpr, tkDotDot}): - # actually parsing {.push hints:off.} as {.push(hints:off).} is a sweet - # solution, but pragmas.nim can't handle that - result = commandExpr(p, result, mode) - break + let isDotLike2 = p.tok.isDotLike + if isDotLike2 and p.lex.config.isDefined("nimPreviewDotLikeOps"): + # synchronize with `tkDot` branch + result = dotLikeExpr(p, result) + result = parseGStrLit(p, result) + else: + if isDotLike2: + parMessage(p, warnDotLikeOps, "dot-like operators will be parsed differently with `-d:nimPreviewDotLikeOps`") + if p.inPragma == 0 and (isUnary(p.tok) or p.tok.tokType notin {tkOpr, tkDotDot}): + # actually parsing {.push hints:off.} as {.push(hints:off).} is a sweet + # solution, but pragmas.nim can't handle that + result = commandExpr(p, result, mode) + break else: break diff --git a/doc/grammar.txt b/doc/grammar.txt index d4f4a0515..f58621b97 100644 --- a/doc/grammar.txt +++ b/doc/grammar.txt @@ -29,6 +29,7 @@ exprList = expr ^+ comma exprColonEqExprList = exprColonEqExpr (comma exprColonEqExpr)* (comma)? dotExpr = expr '.' optInd (symbol | '[:' exprList ']') explicitGenericInstantiation = '[:' exprList ']' ( '(' exprColonEqExpr ')' )? +dotLikeExpr = expr DOTLIKEOP optInd symbol qualifiedIdent = symbol ('.' optInd symbol)? setOrTableConstr = '{' ((exprColonEqExpr comma)* | ':' ) '}' castExpr = 'cast' ('[' optInd typeDesc optPar ']' '(' optInd expr optPar ')') / @@ -56,6 +57,7 @@ tupleConstr = '(' optInd (exprColonEqExpr comma?)* optPar ')' arrayConstr = '[' optInd (exprColonEqExpr comma?)* optPar ']' primarySuffix = '(' (exprColonEqExpr comma?)* ')' | '.' optInd symbol generalizedLit? + | DOTLIKEOP optInd symbol generalizedLit? | '[' optInd exprColonEqExprList optPar ']' | '{' optInd exprColonEqExprList optPar '}' | &( '`'|IDENT|literal|'cast'|'addr'|'type') expr # command syntax diff --git a/doc/manual.rst b/doc/manual.rst index 9c7c219e9..72d50a901 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -748,6 +748,7 @@ has the second-lowest precedence. Otherwise, precedence is determined by the first character. + ================ ======================================================= ================== =============== Precedence level Operators First character Terminal symbol ================ ======================================================= ================== =============== @@ -783,6 +784,14 @@ of a call or whether it is parsed as a tuple constructor: .. code-block:: nim echo (1, 2) # pass the tuple (1, 2) to echo +Dot-like operators +------------------ + +Terminal symbol in the grammar: `DOTLIKEOP`. + +Dot-like operators are operators starting with `.`, but not with `..`, for e.g. `.?`; +they have the same precedence as `.`, so that `a.?b.c` is parsed as `(a.?b).c` instead of `a.?(b.c)`. + Grammar ------- diff --git a/tests/config.nims b/tests/config.nims index 0327f0b76..86787db1d 100644 --- a/tests/config.nims +++ b/tests/config.nims @@ -36,3 +36,4 @@ switch("define", "nimExperimentalAsyncjsThen") switch("define", "nimExperimentalLinenoiseExtra") switch("define", "nimPreviewFloatRoundtrip") +switch("define", "nimPreviewDotLikeOps") diff --git a/tests/misc/trfc_341.nim b/tests/misc/trfc_341.nim new file mode 100644 index 000000000..37cf675c6 --- /dev/null +++ b/tests/misc/trfc_341.nim @@ -0,0 +1,27 @@ +# test for https://github.com/nim-lang/RFCs/issues/341 +import std/json +import std/jsonutils +import std/macros + +macro fn1(a: untyped): string = newLit a.lispRepr + +doAssert fn1(a.?b.c) == """(DotExpr (Infix (Ident ".?") (Ident "a") (Ident "b")) (Ident "c"))""" + +template `.?`(a: JsonNode, b: untyped{ident}): JsonNode = + a[astToStr(b)] + +proc identity[T](a: T): T = a +proc timesTwo[T](a: T): T = a * 2 + +template main = + let a = (a1: 1, a2: "abc", a3: (a4: 2.5)) + let j = a.toJson + doAssert j.?a1.getInt == 1 + doAssert j.?a3.?a4.getFloat == 2.5 + doAssert j.?a3.?a4.getFloat.timesTwo == 5.0 + doAssert j.?a3.identity.?a4.getFloat.timesTwo == 5.0 + doAssert j.identity.?a3.identity.?a4.identity.getFloat.timesTwo == 5.0 + doAssert j.identity.?a3.?a4.identity.getFloat.timesTwo == 5.0 + +static: main() +main() From c70e4040bd69e1ee32b6c676dbfc763c7cf4df37 Mon Sep 17 00:00:00 2001 From: Dankr4d Date: Wed, 25 Aug 2021 19:27:00 +0200 Subject: [PATCH 1169/2472] fixes #14511 [backport:1.4] (#18732) * fixes #14511 [backport:1.4] Signed-off-by: Dankr4d * Replaced fix with code from alaviss, for better readability, with small changes. Signed-off-by: Dankr4d * - Specified output in test. Signed-off-by: Dankr4d * Replaced case in nnkRecCase with a simpler version, which just adds the last son. Signed-off-by: Dankr4d * Update tests/macros/t14511.nim * Update tests/macros/t14511.nim Co-authored-by: Andreas Rumpf --- lib/core/macros.nim | 21 ++++++++-------- tests/macros/t14511.nim | 54 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 11 deletions(-) create mode 100644 tests/macros/t14511.nim diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 645f5067a..806ea7e31 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -1541,18 +1541,17 @@ proc customPragmaNode(n: NimNode): NimNode = for i in 0.. 0: var identDefs = identDefsStack.pop() - if identDefs.kind == nnkRecCase: - identDefsStack.add(identDefs[0]) - for i in 1.. Date: Wed, 25 Aug 2021 22:04:18 -0700 Subject: [PATCH 1170/2472] followup #18711 grammar_nanny now reports unused non-terminals (#18746) * followup #18711 cleanup unused grammar rules * make tools/grammar_nanny.nim report unused terminals * revert removal of some grammar comments --- tools/grammar_nanny.nim | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/grammar_nanny.nim b/tools/grammar_nanny.nim index 397041559..5120ef976 100644 --- a/tools/grammar_nanny.nim +++ b/tools/grammar_nanny.nim @@ -4,7 +4,7 @@ import std / [strutils, sets] import ".." / compiler / [ - llstream, ast, lexer, options, msgs, idents, + llstream, lexer, options, msgs, idents, lineinfos, pathutils] proc checkGrammarFileImpl(cache: IdentCache, config: ConfigRef) = @@ -35,6 +35,10 @@ proc checkGrammarFileImpl(cache: IdentCache, config: ConfigRef) = usedSyms.incl word else: rawGetTok(L, tok) + for u in declaredSyms: + if u notin usedSyms: + echo "Unused non-terminal: ", u + for u in usedSyms: if u notin declaredSyms: echo "Undeclared non-terminal: ", u From 041edaa1df96fc7963f01bf5048f2bee86b6b363 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Thu, 26 Aug 2021 18:07:54 -0300 Subject: [PATCH 1171/2472] Implement RFC-413 (#18749) * Implement RFC 413 * Implement RFC 413 * Implement RFC 413 * Implement RFC 413 * https://github.com/nim-lang/Nim/pull/18749#discussion_r696320995 * Update lib/js/dom_extensions.nim Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Update lib/js/dom_extensions.nim * https://github.com/nim-lang/Nim/pull/18749#discussion_r696913310 Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Co-authored-by: Andreas Rumpf --- changelog.md | 3 +++ lib/js/dom.nim | 3 +++ lib/js/dom_extensions.nim | 6 ++---- tests/effects/tstrict_funcs_imports_js.nim | 1 - 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/changelog.md b/changelog.md index 3ba008cf3..65ae61833 100644 --- a/changelog.md +++ b/changelog.md @@ -371,6 +371,9 @@ - Added `dom.setInterval`, `dom.clearInterval` overloads. +- Merged `dom_extensions` module into `dom` module, + it was a module with a single line, see https://github.com/nim-lang/RFCs/issues/413 + - Allow reading parameters when compiling for Nintendo Switch. - Deprecated `sequtils.delete` and added an overload taking a `Slice` that raises a defect diff --git a/lib/js/dom.nim b/lib/js/dom.nim index 3f0b9f630..aca79faca 100644 --- a/lib/js/dom.nim +++ b/lib/js/dom.nim @@ -1771,3 +1771,6 @@ since (1, 3): ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readAsDataURL proc readAsText*(f: FileReader, b: Blob|File, encoding = cstring"UTF-8") {.importcpp: "#.readAsText(#, #)".} ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readAsText + +since (1, 5): + proc elementsFromPoint*(n: DocumentOrShadowRoot; x, y: float): seq[Element] {.importcpp.} diff --git a/lib/js/dom_extensions.nim b/lib/js/dom_extensions.nim index a1ceff5b4..c15a03195 100644 --- a/lib/js/dom_extensions.nim +++ b/lib/js/dom_extensions.nim @@ -1,5 +1,3 @@ import std/dom - -{.push importcpp.} -proc elementsFromPoint*(n: DocumentOrShadowRoot; x, y: float): seq[Element] -{.pop.} +export elementsFromPoint +{.deprecated: "use `std/dom` instead".} diff --git a/tests/effects/tstrict_funcs_imports_js.nim b/tests/effects/tstrict_funcs_imports_js.nim index b7fcd343a..667887ff0 100644 --- a/tests/effects/tstrict_funcs_imports_js.nim +++ b/tests/effects/tstrict_funcs_imports_js.nim @@ -7,7 +7,6 @@ discard """ import asyncjs, dom, - dom_extensions, jsconsole, jsffi, jsre From 5eba3725707df0f62dcf9c380eaa518635b0f0c0 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 27 Aug 2021 17:50:06 +0800 Subject: [PATCH 1172/2472] [minor] break loops if it is ambiguous (#18745) * [minor] break loops if it is ambiguous * Update compiler/lookups.nim Co-authored-by: Timothee Cour Co-authored-by: Timothee Cour --- compiler/lookups.nim | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/compiler/lookups.nim b/compiler/lookups.nim index 331eef525..4c072eed8 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -181,13 +181,14 @@ proc someSymFromImportTable*(c: PContext; name: PIdent; ambiguous: var bool): PS if overloadableEnums notin c.features: symSet.excl skEnumField result = nil - for im in c.imports.mitems: - for s in symbols(im, marked, name, c.graph): - if result == nil: - result = s - else: - if s.kind notin symSet or result.kind notin symSet: + block outer: + for im in c.imports.mitems: + for s in symbols(im, marked, name, c.graph): + if result == nil: + result = s + elif s.kind notin symSet or result.kind notin symSet: ambiguous = true + break outer proc searchInScopes*(c: PContext, s: PIdent; ambiguous: var bool): PSym = for scope in allScopes(c.currentScope): From 719718df2f98f2bd9aa4286ff8aeac3f02e4d46c Mon Sep 17 00:00:00 2001 From: Miran Date: Sat, 28 Aug 2021 08:27:56 +0200 Subject: [PATCH 1173/2472] more minor bootstrapping cleanups (#18759) --- compiler/ccgtypes.nim | 3 --- compiler/cgen.nim | 1 - compiler/vmops.nim | 1 - 3 files changed, 5 deletions(-) diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index d05c4cd8b..492e1f181 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -12,7 +12,6 @@ # ------------------------- Name Mangling -------------------------------- import sighashes, modulegraphs -from lowerings import createObj proc genProcHeader(m: BModule, prc: PSym, asPtr: bool = false): Rope @@ -256,8 +255,6 @@ proc addAbiCheck(m: BModule, t: PType, name: Rope) = m.s[cfsTypeInfo].addf("NIM_STATIC_ASSERT(sizeof($1) == $2, $3);$n", [name, rope(size), msg2.rope]) # see `testCodegenABICheck` for example error message it generates -from ccgutils import ccgIntroducedPtr -export ccgIntroducedPtr proc fillResult(conf: ConfigRef; param: PNode) = fillLoc(param.sym.loc, locParam, param, ~"Result", diff --git a/compiler/cgen.nim b/compiler/cgen.nim index f55da7f65..a06a87172 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -23,7 +23,6 @@ when not defined(leanCompiler): import strutils except `%` # collides with ropes.`%` from ic / ic import ModuleBackendFlag -from modulegraphs import ModuleGraph, PPassContext import dynlib when not declared(dynlib.libCandidates): diff --git a/compiler/vmops.nim b/compiler/vmops.nim index a1a4996b2..08225b065 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -29,7 +29,6 @@ from std/hashes import hash from std/osproc import nil from system/formatfloat import addFloatRoundtrip, addFloatSprintf -from sighashes import symBodyDigest # There are some useful procs in vmconv. import vmconv From 014edde36a2f61b0a232ceb4800681a2ce261759 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 28 Aug 2021 16:34:27 -0700 Subject: [PATCH 1174/2472] followup #18759: make compiler build cleanly with 0 warnings/hints (#18764) --- compiler/semtypes.nim | 2 -- compiler/vmops.nim | 6 ++++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index cb7f847a9..9f7a17a53 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -10,8 +10,6 @@ # this module does the semantic checking of type declarations # included from sem.nim -import math - const errStringOrIdentNodeExpected = "string or ident node expected" errStringLiteralExpected = "string literal expected" diff --git a/compiler/vmops.nim b/compiler/vmops.nim index 08225b065..e287dd41d 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -15,10 +15,12 @@ from std/math import sqrt, ln, log10, log2, exp, round, arccos, arcsin, lgamma when declared(math.copySign): - from std/math import copySign + # pending bug #18762, avoid renaming math + from std/math as math2 import copySign when declared(math.signbit): - from std/math import signbit + # ditto + from std/math as math3 import signbit from std/os import getEnv, existsEnv, delEnv, putEnv, dirExists, fileExists, walkDir, getAppFilename, raiseOSError, osLastError From c07d8da7b9d491500096dbd4c224f5762cbf89ac Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 29 Aug 2021 15:18:54 +0800 Subject: [PATCH 1175/2472] add missing docs (#18758) --- lib/nimrtl.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/nimrtl.nim b/lib/nimrtl.nim index 301ffb43b..93349b287 100644 --- a/lib/nimrtl.nim +++ b/lib/nimrtl.nim @@ -26,6 +26,7 @@ batchable: false ## * unicode ## * pegs ## * ropes +## * cstrutils ## when system.appType != "lib": From 0ddd9519c0007b705d70b5a75b634831396dd904 Mon Sep 17 00:00:00 2001 From: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Date: Sun, 29 Aug 2021 10:42:52 +0200 Subject: [PATCH 1176/2472] Remove `Covariance` section from the experimental manual (#18688) * Remove `Covariance` section * Add blank lines after `.. code-block::` * Fix CI? --- doc/manual_experimental.rst | 164 ++++++++++++++---------------------- 1 file changed, 63 insertions(+), 101 deletions(-) diff --git a/doc/manual_experimental.rst b/doc/manual_experimental.rst index 3272a4d0d..458512292 100644 --- a/doc/manual_experimental.rst +++ b/doc/manual_experimental.rst @@ -70,6 +70,7 @@ type `void` are treated as non-existent, `void` as a return type means that the procedure does not return a value: .. code-block:: nim + proc nothing(x, y: void): void = echo "ha" @@ -78,6 +79,7 @@ the procedure does not return a value: The `void` type is particularly useful for generic code: .. code-block:: nim + proc callProc[T](p: proc (x: T), x: T) = when T is void: p() @@ -93,6 +95,7 @@ The `void` type is particularly useful for generic code: However, a `void` type cannot be inferred in generic code: .. code-block:: nim + callProc(emptyProc) # Error: type mismatch: got (proc ()) # but expected one of: @@ -102,106 +105,6 @@ The `void` type is only valid for parameters and return types; other symbols cannot have the type `void`. - -Covariance -========== - -Covariance in Nim can be introduced only through pointer-like types such -as `ptr` and `ref`. Sequence, Array and OpenArray types, instantiated -with pointer-like types will be considered covariant if and only if they -are also immutable. The introduction of a `var` modifier or additional -`ptr` or `ref` indirections would result in invariant treatment of -these types. - -`proc` types are currently always invariant, but future versions of Nim -may relax this rule. - -User-defined generic types may also be covariant with respect to some of -their parameters. By default, all generic params are considered invariant, -but you may choose the apply the prefix modifier `in` to a parameter to -make it contravariant or `out` to make it covariant: - -.. code-block:: nim - type - AnnotatedPtr[out T] = - metadata: MyTypeInfo - p: ref T - - RingBuffer[out T] = - startPos: int - data: seq[T] - - Action {.importcpp: "std::function".} [in T] = object - -When the designated generic parameter is used to instantiate a pointer-like -type as in the case of `AnnotatedPtr` above, the resulting generic type will -also have pointer-like covariance: - -.. code-block:: nim - type - GuiWidget = object of RootObj - Button = object of GuiWidget - ComboBox = object of GuiWidget - - var - widgetPtr: AnnotatedPtr[GuiWidget] - buttonPtr: AnnotatedPtr[Button] - - ... - - proc drawWidget[T](x: AnnotatedPtr[GuiWidget]) = ... - - # you can call procs expecting base types by supplying a derived type - drawWidget(buttonPtr) - - # and you can convert more-specific pointer types to more general ones - widgetPtr = buttonPtr - -Just like with regular pointers, covariance will be enabled only for immutable -values: - -.. code-block:: nim - proc makeComboBox[T](x: var AnnotatedPtr[GuiWidget]) = - x.p = new(ComboBox) - - makeComboBox(buttonPtr) # Error, AnnotatedPtr[Button] cannot be modified - # to point to a ComboBox - -On the other hand, in the `RingBuffer` example above, the designated generic -param is used to instantiate the non-pointer `seq` type, which means that -the resulting generic type will have covariance that mimics an array or -sequence (i.e. it will be covariant only when instantiated with `ptr` and -`ref` types): - -.. code-block:: nim - - type - Base = object of RootObj - Derived = object of Base - - proc consumeBaseValues(b: RingBuffer[Base]) = ... - - var derivedValues: RingBuffer[Derived] - - consumeBaseValues(derivedValues) # Error, Base and Derived values may differ - # in size - - proc consumeBasePointers(b: RingBuffer[ptr Base]) = ... - - var derivedPointers: RingBuffer[ptr Derived] - - consumeBaseValues(derivedPointers) # This is legal - -Please note that Nim will treat the user-defined pointer-like types as -proper alternatives to the built-in pointer types. That is, types such -as `seq[AnnotatedPtr[T]]` or `RingBuffer[AnnotatedPtr[T]]` will also be -considered covariant and you can create new pointer-like types by instantiating -other user-defined pointer-like types. - -The contravariant parameters introduced with the `in` modifier are currently -useful only when interfacing with imported types having such semantics. - - Automatic dereferencing ======================= @@ -209,6 +112,7 @@ Automatic dereferencing is performed for the first argument of a routine call. This feature has to be enabled via `{.experimental: "implicitDeref".}`: .. code-block:: nim + {.experimental: "implicitDeref".} proc depth(x: NodeObj): int = ... @@ -219,6 +123,7 @@ This feature has to be enabled via `{.experimental: "implicitDeref".}`: echo n.depth # no need to write n[].depth either + Code reordering =============== @@ -281,6 +186,7 @@ statement) can have their entire initialization statement reordered. Be wary of what code is executed at the top level: .. code-block:: nim + {.experimental: "codeReordering".} proc a() = @@ -300,6 +206,7 @@ what code is executed at the top level: code is the same as it would be with code reordering disabled. .. code-block:: nim + {.experimental: "codeReordering".} proc x() = @@ -313,6 +220,7 @@ It is important to note that reordering *only* works for symbols at top level scope. Therefore, the following will *fail to compile:* .. code-block:: nim + {.experimental: "codeReordering".} proc a() = @@ -331,6 +239,7 @@ has different names. This does not need an `experimental` switch, but is an unstable feature. .. code-block:: Nim + proc foo(x: int) = echo "Using x: ", x proc foo(y: int) = @@ -349,6 +258,7 @@ As a special more convenient notation, proc expressions involved in procedure calls can use the `do` keyword: .. code-block:: nim + sort(cities) do (x,y: string) -> int: cmp(x.len, y.len) @@ -371,6 +281,7 @@ parentheses is just a block of code. The `do` notation can be used to pass multiple blocks to a macro: .. code-block:: nim + macro performWithUndo(task, undo: untyped) = ... performWithUndo do: @@ -380,7 +291,6 @@ pass multiple blocks to a macro: # code to undo it - Special Operators ================= @@ -404,6 +314,7 @@ the expression, where the unknown field or proc name is passed to an `untyped` parameter: .. code-block:: nim + a.b # becomes `.`(a, b) a.b(c, d) # becomes `.`(a, b, c, d) @@ -411,6 +322,7 @@ The matched dot operators can be symbols of any callable kind (procs, templates and macros), depending on the desired effect: .. code-block:: nim + template `.`(js: PJsonNode, field: untyped): JSON = js[astToStr(field)] var js = parseJson("{ x: 1, y: 2}") @@ -435,6 +347,7 @@ operator `.=` This operator will be matched against assignments to missing fields. .. code-block:: nim + a.b = c # becomes `.=`(a, b, c) Call operator @@ -445,6 +358,7 @@ for existing routines. The experimental `callOperator` switch must be enabled to use this operator. .. code-block:: nim + {.experimental: "callOperator".} template `()`(a: int, b: float): untyped = $(a, b) @@ -480,6 +394,7 @@ All types for which `nil` is a valid value can be annotated with the `not nil` annotation to exclude `nil` as a valid value: .. code-block:: nim + {.experimental: "notnil".} type @@ -502,6 +417,7 @@ here. .. include:: manual_experimental_strictnotnil.rst + Concepts ======== @@ -511,6 +427,7 @@ arbitrary set of requirements that the matched type must satisfy. Concepts are written in the following form: .. code-block:: nim + type Comparable = concept x, y (x < y) is bool @@ -536,6 +453,7 @@ instance. You can also apply the `type` modifier to create a named instance of the type itself: .. code-block:: nim + type MyConcept = concept x, var v, ref r, ptr p, static s, type T ... @@ -545,6 +463,7 @@ and parameters are expected. This provides a more convenient way to check for the presence of callable symbols with specific signatures: .. code-block:: nim + type OutputStream = concept var s s.write(string) @@ -555,6 +474,7 @@ type, following the `concept` keyword is also considered to have the explicit modifier and will be matched only as a type. .. code-block:: nim + type # Let's imagine a user-defined casting framework with operators # such as `val.to(string)` and `val.to(JSonValue)`. We can test @@ -592,6 +512,7 @@ concept and, as a result, a wrong overload is selected, you can apply the `explain` pragma to either the concept body or a particular call-site. .. code-block:: nim + type MyConcept {.explain.} = concept ... @@ -607,6 +528,7 @@ Generic concepts and type binding rules The concept types can be parametric just like the regular generic types: .. code-block:: nim + ### matrixalgo.nim import std/typetraits @@ -680,6 +602,7 @@ Unbound static params will be inferred from expressions involving the `==` operator and also when types dependent on them are being matched: .. code-block:: nim + type MatrixReducer[M, N: static int; T] = concept x x.reduce(SquareMatrix[N, T]) is array[M, int] @@ -694,6 +617,7 @@ matched without permanently inferring it. This may be useful when you need to match several procs accepting the same wide class of types: .. code-block:: nim + type Enumerable[T] = concept e for v in e: @@ -716,6 +640,7 @@ types used in multiple signatures, without actually requiring any concrete types, thus allowing you to encode implementation-defined types: .. code-block:: nim + type MyConcept = concept x type T1 = auto @@ -776,6 +701,7 @@ accessible through the dot operator in procs where the concept was successfully matched to a concrete type: .. code-block:: nim + type DateTime = concept t1, t2, type T const Min = T.MinDate @@ -810,6 +736,7 @@ one. As an alternative way of defining concept refinements, you can use the object inheritance syntax involving the `of` keyword: .. code-block:: nim + type Graph = concept g, type G of EquallyComparable, Copyable type @@ -853,6 +780,7 @@ object inheritance syntax involving the `of` keyword: the concept body: .. code-block:: nim + type Stringable = concept x $x is string @@ -904,6 +832,7 @@ object inheritance syntax involving the `of` keyword: types to the corresponding VTable type. .. code-block:: nim + type IntEnumerable = vtref Enumerable[int] @@ -970,6 +899,7 @@ language may weaken this restriction.) The signature has to be: .. code-block:: nim + proc `=deepCopy`(x: T): T This mechanism will be used by most data structures that support shared memory @@ -1037,6 +967,7 @@ phase of the compiler: This means they provide an easy way to enhance the compilation pipeline with user defined optimizations: .. code-block:: nim + template optMul{`*`(a, 2)}(a: int): int = a+a let x = 3 @@ -1059,6 +990,7 @@ Unfortunately optimizations are hard to get right and even the tiny example is **wrong**: .. code-block:: nim + template optMul{`*`(a, 2)}(a: int): int = a+a proc f(): int = @@ -1071,6 +1003,7 @@ We cannot duplicate 'a' if it denotes an expression that has a side effect! Fortunately Nim supports side effect analysis: .. code-block:: nim + template optMul{`*`(a, 2)}(a: int{noSideEffect}): int = a+a proc f(): int = @@ -1088,11 +1021,13 @@ cannot really do that however as the following code only swaps arguments blindly: .. code-block:: nim + template mulIsCommutative{`*`(a, b)}(a, b: int): int = b*a What optimizers really need to do is a *canonicalization*: .. code-block:: nim + template canonMul{`*`(a, b)}(a: int{lit}, b: int): int = b*a The `int{lit}` parameter pattern matches against an expression of @@ -1152,6 +1087,7 @@ but also to every other bound parameter; syntactically they need to occur after the ordinary AST predicates: .. code-block:: nim + template ex{a = b + c}(a: int{noalias}, b, c: int) = # this transformation is only valid if 'b' and 'c' do not alias 'a': a = b @@ -1160,6 +1096,7 @@ the ordinary AST predicates: Another example: .. code-block:: nim + proc somefunc(s: string) = assert s == "variable" proc somefunc(s: string{nkStrLit}) = assert s == "literal" proc somefunc(s: string{nkRStrLit}) = assert s == r"raw" @@ -1189,6 +1126,7 @@ The `|` operator The `|` operator if used as infix operator creates an ordered choice: .. code-block:: nim + template t{0|1}(): untyped = 3 let a = 1 # outputs 3: @@ -1198,6 +1136,7 @@ The matching is performed after the compiler performed some optimizations like constant folding, so the following does not work: .. code-block:: nim + template t{0|1}(): untyped = 3 # outputs 1: echo 1 @@ -1215,6 +1154,7 @@ A pattern expression can be bound to a pattern parameter via the `expr{param}` notation: .. code-block:: nim + template t{(0|1|2){x}}(x: untyped): untyped = x+1 let a = 1 # outputs 2: @@ -1227,6 +1167,7 @@ The `~` operator The `~` operator is the **not** operator in patterns: .. code-block:: nim + template t{x = (~x){y} and (~x){z}}(x, y, z: bool) = x = y if x: x = z @@ -1246,6 +1187,7 @@ The `*` operator can *flatten* a nested binary expression like `a & b & c` to `&(a, b, c)`: .. code-block:: nim + var calls = 0 @@ -1270,6 +1212,7 @@ which is flattened into a call expression; thus the invocation of `optConc` produces: .. code-block:: nim + `&&`("my", space & "awe", "some ", "concat") @@ -1280,6 +1223,7 @@ The `**` is much like the `*` operator, except that it gathers not only all the arguments, but also the matched operators in reverse polish notation: .. code-block:: nim + import std/macros type @@ -1324,6 +1268,7 @@ parameter is of the type `varargs` it is treated specially and it can match 0 or more arguments in the AST to be matched against: .. code-block:: nim + template optWrite{ write(f, x) ((write|writeLine){w})(f, y) @@ -1339,6 +1284,7 @@ The following example shows how some simple partial evaluation can be implemented with term rewriting: .. code-block:: nim + proc p(x, y: int; cond: bool): int = result = if cond: x + y else: x - y @@ -1352,6 +1298,7 @@ Example: Hoisting The following example shows how some form of hoisting can be implemented: .. code-block:: nim + import std/pegs template optPeg{peg(pattern)}(pattern: string{lit}): Peg = @@ -1375,6 +1322,7 @@ Parameter constraints can also be used for ordinary routine parameters; these constraints affect ordinary overloading resolution then: .. code-block:: nim + proc optLit(a: string{lit|`const`}) = echo "string literal" proc optLit(a: string) = @@ -1426,6 +1374,7 @@ Spawn statement `spawn`:idx: can be used to pass a task to the thread pool: .. code-block:: nim + import std/threadpool proc processLine(line: string) = @@ -1458,6 +1407,7 @@ with the `^` operator is **blocking**. However, one can use `blockUntilAny` to wait on multiple flow variables at the same time: .. code-block:: nim + import std/threadpool, ... # wait until 2 out of 3 servers received the update: @@ -1552,6 +1502,7 @@ Protecting global variables Object fields and global variables can be annotated via a `guard` pragma: .. code-block:: nim + var glock: TLock var gdata {.guard: glock.}: int @@ -1559,6 +1510,7 @@ The compiler then ensures that every access of `gdata` is within a `locks` section: .. code-block:: nim + proc invalid = # invalid: unguarded access: echo gdata @@ -1577,6 +1529,7 @@ semantics and should not be used directly! It should only be used in templates that also implement some form of locking at runtime: .. code-block:: nim + template lock(a: TLock; body: untyped) = pthread_mutex_lock(a) {.locks: [a].}: @@ -1590,6 +1543,7 @@ The guard does not need to be of any particular type. It is flexible enough to model low level lockfree mechanisms: .. code-block:: nim + var dummyLock {.compileTime.}: int var atomicCounter {.guard: dummyLock.}: int @@ -1617,6 +1571,7 @@ Since objects can reside on the heap or on the stack this greatly enhances the expressivity of the language: .. code-block:: nim + type ProtectedCounter = object v {.guard: L.}: int @@ -1631,6 +1586,7 @@ The access to field `x.v` is allowed since its guard `x.L` is active. After template expansion, this amounts to: .. code-block:: nim + proc incCounters(counters: var openArray[ProtectedCounter]) = for i in 0..counters.high: pthread_mutex_lock(counters[i].L) @@ -1651,6 +1607,7 @@ Two paths are considered equivalent if they are syntactically the same. This means the following compiles (for now) even though it really should not: .. code-block:: nim + {.locks: [a[i].L].}: inc i access a[i].v @@ -1671,6 +1628,7 @@ of the same level can only be acquired *at the same time* within a single `locks` section: .. code-block:: nim + var a, b: TLock[2] var x: TLock[1] # invalid locking order: TLock[1] cannot be acquired before TLock[2]: @@ -1697,6 +1655,7 @@ the runtime check is required to ensure a global ordering for two locks `a` and `b` of the same lock level: .. code-block:: nim + template multilock(a, b: ptr TLock; body: untyped) = if cast[ByteAddress](a) < cast[ByteAddress](b): pthread_mutex_lock(a) @@ -1717,6 +1676,7 @@ level. This then means that the routine may acquire locks of up to this level. This is essential so that procs can be called within a `locks` section: .. code-block:: nim + proc p() {.locks: 3.} = discard var a: TLock[4] @@ -1739,6 +1699,7 @@ cannot be inferred statically, leading to compiler warnings. By using having unknown lock level as well: .. code-block:: nim + type SomeBase* = ref object of RootObj type SomeDerived* = ref object of SomeBase memberProc*: proc () @@ -1761,6 +1722,7 @@ e.g. when rewriting term to same term plus extra content. e.g. with given example `echo("ab")` will be rewritten just once: .. code-block:: nim + template pwnEcho{echo(x)}(x: untyped) = {.noRewrite.}: echo("pwned!") From fa7c1aa8647aa97a4a204751db727c3cbeded749 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 29 Aug 2021 06:54:07 -0700 Subject: [PATCH 1177/2472] move PNode.comment to a side channel, reducing memory usage during compilation by a factor 1.25x (#18760) * move PNode.comment so a side channel, reducing memory usage * fix a bug * fixup * use sfHasComment to speedup comment lookups * fix for IC * Update compiler/parser.nim Co-authored-by: Andreas Rumpf --- compiler/ast.nim | 77 +++++++++++++++++++++++++++++++------------ compiler/commands.nim | 3 +- compiler/parser.nim | 14 +++++--- 3 files changed, 67 insertions(+), 27 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 446c00f49..5893a5671 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -10,7 +10,7 @@ # abstract syntax tree + symbol table import - lineinfos, hashes, options, ropes, idents, int128 + lineinfos, hashes, options, ropes, idents, int128, tables from strutils import toLowerAscii export int128 @@ -497,6 +497,7 @@ type nfExecuteOnReload # A top-level statement that will be executed during reloads nfLastRead # this node is a last read nfFirstWrite# this node is a first write + nfHasComment # node has a comment TNodeFlags* = set[TNodeFlag] TTypeFlag* = enum # keep below 32 for efficiency reasons (now: 43) @@ -774,7 +775,6 @@ type ident*: PIdent else: sons*: TNodeSeq - comment*: string TStrTable* = object # a table[PIdent] of PSym counter*: int @@ -991,6 +991,38 @@ type TImplication* = enum impUnknown, impNo, impYes +template nodeId(n: PNode): int = cast[int](n) + +type Gconfig = object + # we put comments in a side channel to avoid increasing `sizeof(TNode)`, which + # reduces memory usage given that `PNode` is the most allocated type by far. + comments: Table[int, string] # nodeId => comment + useIc*: bool + +var gconfig {.threadvar.}: Gconfig + +proc setUseIc*(useIc: bool) = gconfig.useIc = useIc + +proc comment*(n: PNode): string = + if nfHasComment in n.flags and not gconfig.useIc: + # IC doesn't track comments, see `packed_ast`, so this could fail + result = gconfig.comments[n.nodeId] + +proc `comment=`*(n: PNode, a: string) = + let id = n.nodeId + if a.len > 0: + # if needed, we could periodically cleanup gconfig.comments when its size increases, + # to ensure only live nodes (and with nfHasComment) have an entry in gconfig.comments; + # for compiling compiler, the waste is very small: + # num calls to newNodeImpl: 14984160 (num of PNode allocations) + # size of gconfig.comments: 33585 + # num of nodes with comments that were deleted and hence wasted: 3081 + n.flags.incl nfHasComment + gconfig.comments[id] = a + elif nfHasComment in n.flags: + n.flags.excl nfHasComment + gconfig.comments.del(id) + # BUGFIX: a module is overloadable so that a proc can have the # same name as an imported module. This is necessary because of # the poor naming choices in the standard library. @@ -1207,9 +1239,17 @@ when defined(useNodeIds): const nodeIdToDebug* = -1 # 2322968 var gNodeId: int -proc newNode*(kind: TNodeKind): PNode = - ## new node with unknown line info, no type, and no children - result = PNode(kind: kind, info: unknownLineInfo) +template newNodeImpl(info2) = + result = PNode(kind: kind, info: info2) + when false: + # this would add overhead, so we skip it; it results in a small amount of leaked entries + # for old PNode that gets re-allocated at the same address as a PNode that + # has `nfHasComment` set (and an entry in that table). Only `nfHasComment` + # should be used to test whether a PNode has a comment; gconfig.comments + # can contain extra entries for deleted PNode's with comments. + gconfig.comments.del(cast[int](result)) + +template setIdMaybe() = when defined(useNodeIds): result.id = gNodeId if result.id == nodeIdToDebug: @@ -1217,27 +1257,22 @@ proc newNode*(kind: TNodeKind): PNode = writeStackTrace() inc gNodeId +proc newNode*(kind: TNodeKind): PNode = + ## new node with unknown line info, no type, and no children + newNodeImpl(unknownLineInfo) + setIdMaybe() + proc newNodeI*(kind: TNodeKind, info: TLineInfo): PNode = ## new node with line info, no type, and no children - result = PNode(kind: kind, info: info) - when defined(useNodeIds): - result.id = gNodeId - if result.id == nodeIdToDebug: - echo "KIND ", result.kind - writeStackTrace() - inc gNodeId + newNodeImpl(info) + setIdMaybe() proc newNodeI*(kind: TNodeKind, info: TLineInfo, children: int): PNode = ## new node with line info, type, and children - result = PNode(kind: kind, info: info) + newNodeImpl(info) if children > 0: newSeq(result.sons, children) - when defined(useNodeIds): - result.id = gNodeId - if result.id == nodeIdToDebug: - echo "KIND ", result.kind - writeStackTrace() - inc gNodeId + setIdMaybe() proc newNodeIT*(kind: TNodeKind, info: TLineInfo, typ: PType): PNode = ## new node with line info, type, and no children @@ -1644,8 +1679,8 @@ proc copyNode*(src: PNode): PNode = template transitionNodeKindCommon(k: TNodeKind) = let obj {.inject.} = n[] - n[] = TNode(kind: k, typ: obj.typ, info: obj.info, flags: obj.flags, - comment: obj.comment) + n[] = TNode(kind: k, typ: obj.typ, info: obj.info, flags: obj.flags) + # n.comment = obj.comment # shouldn't be needed, the address doesnt' change when defined(useNodeIds): n.id = obj.id diff --git a/compiler/commands.nim b/compiler/commands.nim index 82e908152..079717862 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -8,7 +8,7 @@ # # This module handles the parsing of command line arguments. - +from ast import setUseIc # We do this here before the 'import' statement so 'defined' does not get # confused with 'TGCMode.gcMarkAndSweep' etc. @@ -862,6 +862,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; of "v2": conf.symbolFiles = v2Sf of "stress": conf.symbolFiles = stressTest else: localError(conf, info, "invalid option for --incremental: " & arg) + setUseIc(conf.symbolFiles != disabledSf) of "skipcfg": processOnOffSwitchG(conf, {optSkipSystemConfigFile}, arg, pass, info) of "skipprojcfg": diff --git a/compiler/parser.nim b/compiler/parser.nim index 30180e545..225493902 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -170,13 +170,15 @@ proc validInd(p: var Parser): bool {.inline.} = proc rawSkipComment(p: var Parser, node: PNode) = if p.tok.tokType == tkComment: if node != nil: + var rhs = node.comment when defined(nimpretty): if p.tok.commentOffsetB > p.tok.commentOffsetA: - node.comment.add fileSection(p.lex.config, p.lex.fileIdx, p.tok.commentOffsetA, p.tok.commentOffsetB) + rhs.add fileSection(p.lex.config, p.lex.fileIdx, p.tok.commentOffsetA, p.tok.commentOffsetB) else: - node.comment.add p.tok.literal + rhs.add p.tok.literal else: - node.comment.add p.tok.literal + rhs.add p.tok.literal + node.comment = move rhs else: parMessage(p, errInternal, "skipComment") getTok(p) @@ -1824,8 +1826,10 @@ proc parseRoutine(p: var Parser, kind: TNodeKind): PNode = if result.comment.len == 0: # proc fn*(a: int): int = a ## foo # => moves comment `foo` to `fn` - swap(result.comment, body[0].comment) - else: discard # xxx either `assert false` or issue a warning (otherwise we'll never know of this edge case) + result.comment = body[0].comment + body[0].comment = "" + else: + assert false, p.lex.config$body.info # avoids hard to track bugs, fail early. proc newCommentStmt(p: var Parser): PNode = #| commentStmt = COMMENT From f02de25ca1b660b0b6e1ae2d4981106744814fe3 Mon Sep 17 00:00:00 2001 From: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Date: Tue, 31 Aug 2021 08:14:05 +0200 Subject: [PATCH 1178/2472] Fix #15150 (#18730) --- lib/pure/asyncfile.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pure/asyncfile.nim b/lib/pure/asyncfile.nim index c51b338e3..222a89b97 100644 --- a/lib/pure/asyncfile.nim +++ b/lib/pure/asyncfile.nim @@ -202,10 +202,11 @@ proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] = proc read*(f: AsyncFile, size: int): Future[string] = ## Read `size` bytes from the specified file asynchronously starting at - ## the current position of the file pointer. + ## the current position of the file pointer. `size` should be greater than zero. ## ## If the file pointer is past the end of the file then an empty string is ## returned. + assert size > 0 var retFuture = newFuture[string]("asyncfile.read") when defined(windows) or defined(nimdoc): From 8f4bdb3596bb7650f2688b8243dc3164abfc21c9 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 31 Aug 2021 19:32:37 +0800 Subject: [PATCH 1179/2472] [minor]break loops after a candidate is added to seqs (#18770) * [minor]break loops when added * Update compiler/lookups.nim Co-authored-by: Clyybber Co-authored-by: Clyybber --- compiler/lookups.nim | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/compiler/lookups.nim b/compiler/lookups.nim index 4c072eed8..fc30408e5 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -211,14 +211,16 @@ proc debugScopes*(c: PContext; limit=0, max = int.high) {.deprecated.} = proc searchInScopesFilterBy*(c: PContext, s: PIdent, filter: TSymKinds): seq[PSym] = result = @[] - for scope in allScopes(c.currentScope): - var ti: TIdentIter - var candidate = initIdentIter(ti, scope.symbols, s) - while candidate != nil: - if candidate.kind in filter: - if result.len == 0: + block outer: + for scope in allScopes(c.currentScope): + var ti: TIdentIter + var candidate = initIdentIter(ti, scope.symbols, s) + while candidate != nil: + if candidate.kind in filter: result.add candidate - candidate = nextIdentIter(ti, scope.symbols) + # Break here, because further symbols encountered would be shadowed + break outer + candidate = nextIdentIter(ti, scope.symbols) if result.len == 0: var marked = initIntSet() From 3469f3a39320cf35d02d11b78c7db0be79e992f8 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 31 Aug 2021 20:05:45 +0800 Subject: [PATCH 1180/2472] add Apple Silicon to supporting platforms (#18772) * add Apple Silicon to supporting platforms * Update readme.md Co-authored-by: flywind Co-authored-by: Andreas Rumpf --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 7c6e28be3..b28c25ec1 100644 --- a/readme.md +++ b/readme.md @@ -35,7 +35,7 @@ architecture combinations: * Windows (Windows XP or greater) - x86 and x86_64 * Linux (most, if not all, distributions) - x86, x86_64, ppc64 and armv6l - * Mac OS X (10.04 or greater) - x86, x86_64 and ppc64 + * Mac OS X (10.04 or greater) - x86, x86_64, ppc64 and Apple Silicon (based on the ARM64 architecture) More platforms are supported, however, they are not tested regularly and they may not be as stable as the above-listed platforms. From 72fa5833adac590e3c78e2b774cd33f28827594b Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 2 Sep 2021 13:41:14 +0800 Subject: [PATCH 1181/2472] fix #9778 (pairs iterator calling a helper proc with tuple return type will cut the iterator yield into half) (#18767) * test someSymFromImportTable * Update compiler/lookups.nim * test nkTupleConstr * use isConstExpr * add tests * add comments and todo * use todo --- compiler/transf.nim | 55 +++++++-- tests/stdlib/tyield.nim | 256 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 300 insertions(+), 11 deletions(-) create mode 100644 tests/stdlib/tyield.nim diff --git a/compiler/transf.nim b/compiler/transf.nim index fb59887b0..80794b581 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -364,7 +364,7 @@ proc transformYield(c: PTransf, n: PNode): PNode = if e.typ.isNil: return result # can happen in nimsuggest for unknown reasons if c.transCon.forStmt.len != 3: e = skipConv(e) - if e.kind in {nkPar, nkTupleConstr}: + if e.kind == nkTupleConstr: for i in 0.. 0: - var changeNode = result[0] - changeNode.info = c.transCon.forStmt.info - for i, child in changeNode: - child.info = changeNode.info + for idx in 0 ..< result.len: + var changeNode = result[idx] + changeNode.info = c.transCon.forStmt.info + for i, child in changeNode: + child.info = changeNode.info proc transformAddrDeref(c: PTransf, n: PNode, a, b: TNodeKind): PNode = result = transformSons(c, n) diff --git a/tests/stdlib/tyield.nim b/tests/stdlib/tyield.nim new file mode 100644 index 000000000..85be97365 --- /dev/null +++ b/tests/stdlib/tyield.nim @@ -0,0 +1,256 @@ +discard """ + targets: "c cpp js" +""" + +import std/[sugar, algorithm] + +block: + var x = @[(6.0, 6, '6'), + (5.0, 5, '5'), + (4.0, 4, '4'), + (3.0, 3, '3'), + (2.0, 2, '2'), + (1.0, 1, '1')] + + let y = x.reversed + + block: + let res = collect: + for (f, i, c) in x: + (f, i, c) + + doAssert res == x + + iterator popAscending[T](q: var seq[T]): T = + while q.len > 0: yield q.pop + + block: + var res = collect: + for f, i, c in popAscending(x): + (f, i, c) + + doAssert res == y + + let z = reversed(res) + let res2 = collect: + for (f, i, c) in popAscending(res): + (f, i, c) + + doAssert res2 == z + + +block: + var visits = 0 + block: + proc bar(): (int, int) = + inc visits + (visits, visits) + + iterator foo(): (int, int) = + yield bar() + + for a, b in foo(): + doAssert a == b + + doAssert visits == 1 + + block: + proc iterAux(a: seq[int], i: var int): (int, string) = + result = (a[i], $a[i]) + inc i + + iterator pairs(a: seq[int]): (int, string) = + var i = 0 + while i < a.len: + yield iterAux(a, i) + + var x = newSeq[int](10) + for i in 0 ..< x.len: + x[i] = i + + let res = collect: + for k, v in x: + (k, v) + + let expected = collect: + for i in 0 ..< x.len: + (i, $i) + + doAssert res == expected + + block: + proc bar(): (int, int, int) = + inc visits + (visits, visits, visits) + + iterator foo(): (int, int, int) = + yield bar() + + for a, b, c in foo(): + doAssert a == b + + doAssert visits == 2 + + + block: + + proc bar(): int = + inc visits + visits + + proc car(): int = + inc visits + visits + + iterator foo(): (int, int) = + yield (bar(), car()) + yield (bar(), car()) + + for a, b in foo(): + doAssert b == a + 1 + + doAssert visits == 6 + + + block: + proc bar(): (int, int) = + inc visits + (visits, visits) + + proc t2(): int = 99 + + iterator foo(): (int, int) = + yield (12, t2()) + yield bar() + + let res = collect: + for (a, b) in foo(): + (a, b) + + doAssert res == @[(12, 99), (7, 7)] + doAssert visits == 7 + + block: + proc bar(): (int, int) = + inc visits + (visits, visits) + + proc t2(): int = 99 + + iterator foo(): (int, int) = + yield ((12, t2())) + yield (bar()) + + let res = collect: + for (a, b) in foo(): + (a, b) + + doAssert res == @[(12, 99), (8, 8)] + doAssert visits == 8 + + block: + proc bar(): (int, int) = + inc visits + (visits, visits) + + proc t1(): int = 99 + proc t2(): int = 99 + + iterator foo(): (int, int) = + yield (t1(), t2()) + yield bar() + + let res = collect: + for a, b in foo(): + (a, b) + + doAssert res == @[(99, 99), (9, 9)] + doAssert visits == 9 + + + block: + proc bar(): ((int, int), string) = + inc visits + ((visits, visits), $visits) + + proc t2(): int = 99 + + iterator foo(): ((int, int), string) = + yield ((1, 2), $t2()) + yield bar() + + let res = collect: + for a, b in foo(): + (a, b) + + doAssert res == @[((1, 2), "99"), ((10, 10), "10")] + doAssert visits == 10 + + + block: + proc bar(): (int, int) = + inc visits + (visits, visits) + + iterator foo(): (int, int) = + yield (for i in 0 ..< 10: discard bar(); bar()) + yield (bar()) + + let res = collect: + for (a, b) in foo(): + (a, b) + + doAssert res == @[(21, 21), (22, 22)] + + block: + proc bar(): (int, int) = + inc visits + (visits, visits) + + proc t2(): int = 99 + + iterator foo(): (int, int) = + yield if true: bar() else: (t2(), t2()) + yield (bar()) + + let res = collect: + for a, b in foo(): + (a, b) + + doAssert res == @[(23, 23), (24, 24)] + + +block: + iterator foo(): (int, int, int) = + var time = 777 + yield (1, time, 3) + + let res = collect: + for a, b, c in foo(): + (a, b, c) + + doAssert res == @[(1, 777, 3)] + +block: + iterator foo(): (int, int, int) = + var time = 777 + yield (1, time, 3) + + let res = collect: + for t in foo(): + (t[0], t[1], t[2]) + + doAssert res == @[(1, 777, 3)] + + +block: + proc bar(): (int, int, int) = + (1, 2, 3) + iterator foo(): (int, int, int) = + yield bar() + + let res = collect: + for a, b, c in foo(): + (a, b, c) + + doAssert res == @[(1, 2, 3)] From e0ef859130f429df1891e31a85955daa753346b4 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 2 Sep 2021 12:10:14 +0200 Subject: [PATCH 1182/2472] strict effects (#18777) * fixes #17369 * megatest is green for --cpu:arm64 * docgen output includes more tags/raises * implemented 'effectsOf' * algorithm.nim: uses new effectsOf annotation * closes #18376 * closes #17475 * closes #13905 * allow effectsOf: [a, b] * added a test case * parameters that are not ours cannot be declared as .effectsOf * documentation * manual: added the 'sort' example * bootstrap with the new better options --- changelog.md | 17 ++ compiler/ast.nim | 6 +- compiler/ccgcalls.nim | 4 +- compiler/commands.nim | 3 +- compiler/condsyms.nim | 1 + compiler/docgen.nim | 7 +- compiler/lineinfos.nim | 2 + compiler/nim.cfg | 5 + compiler/options.nim | 3 +- compiler/pragmas.nim | 30 ++- compiler/sem.nim | 7 +- compiler/semcall.nim | 2 + compiler/semdata.nim | 4 +- compiler/sempass2.nim | 116 +++++--- compiler/semstmts.nim | 9 +- compiler/types.nim | 17 +- compiler/wordrecg.nim | 2 +- doc/manual.rst | 252 ++++++++++-------- lib/pure/algorithm.nim | 23 +- lib/pure/times.nim | 2 +- lib/std/private/globs.nim | 7 +- lib/system/io.nim | 2 +- lib/system/nimscript.nim | 6 +- .../expected/index.html | 2 +- nimdoc/testproject/expected/testproject.html | 10 +- tests/effects/teffects1.nim | 3 +- tests/effects/teffects2.nim | 3 +- tests/effects/teffects7.nim | 4 +- tests/effects/teffects8.nim | 3 +- tests/effects/tstrict_effects.nim | 27 ++ tests/effects/tstrict_effects2.nim | 28 ++ tests/effects/tstrict_effects3.nim | 17 ++ tests/effects/tstrict_effects_sort.nim | 27 ++ tests/pragmas/tuserpragma2.nim | 3 +- tests/stdlib/tstdlib_various.nim | 31 +-- 35 files changed, 482 insertions(+), 203 deletions(-) create mode 100644 tests/effects/tstrict_effects.nim create mode 100644 tests/effects/tstrict_effects2.nim create mode 100644 tests/effects/tstrict_effects3.nim create mode 100644 tests/effects/tstrict_effects_sort.nim diff --git a/changelog.md b/changelog.md index 65ae61833..10824cfc6 100644 --- a/changelog.md +++ b/changelog.md @@ -427,6 +427,23 @@ - On embedded devices `malloc` can now be used instead of `mmap` via `-d:nimAllocPagesViaMalloc`. This is only supported for `--gc:orc` or `--gc:arc`. +- The effect system was refined and there is a new `.effectsOf` annotation that does + explicitly what was previously done implicitly. See the manual for details. + To write code that is portable with older Nim versions, use this idiom: + +```nim + +when defined(nimHasEffectsOf): + {.experimental: "strictEffects".} +else: + {.pragma: effectsOf.} + +proc mysort(s: seq; cmp: proc(a, b: T): int) {.effectsOf: cmp.} + +``` + + To enable the new effect system, use --experimental:strictEffects. + ## Compiler changes diff --git a/compiler/ast.nim b/compiler/ast.nim index 5893a5671..50d048edd 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -229,7 +229,7 @@ type TNodeKinds* = set[TNodeKind] type - TSymFlag* = enum # 47 flags! + TSymFlag* = enum # 48 flags! sfUsed, # read access of sym (for warnings) or simply used sfExported, # symbol is exported from module sfFromGeneric, # symbol is instantiation of a generic; this is needed @@ -299,6 +299,7 @@ type sfUsedInFinallyOrExcept # symbol is used inside an 'except' or 'finally' sfSingleUsedTemp # For temporaries that we know will only be used once sfNoalias # 'noalias' annotation, means C's 'restrict' + sfEffectsDelayed # an 'effectsDelayed' parameter TSymFlags* = set[TSymFlag] @@ -568,6 +569,7 @@ type # sizeof, alignof, offsetof at CT tfExplicitCallConv tfIsConstructor + tfEffectSystemWorkaround TTypeFlags* = set[TTypeFlag] @@ -1781,7 +1783,7 @@ proc containsNode*(n: PNode, kinds: TNodeKinds): bool = proc hasSubnodeWith*(n: PNode, kind: TNodeKind): bool = case n.kind - of nkEmpty..nkNilLit: result = n.kind == kind + of nkEmpty..nkNilLit, nkFormalParams: result = n.kind == kind else: for i in 0.. pragma: arg result.transitionSonsKind(n.kind) +proc processEffectsOf(c: PContext, n: PNode; owner: PSym) = + proc processParam(c: PContext; n: PNode) = + let r = c.semExpr(c, n) + if r.kind == nkSym and r.sym.kind == skParam: + if r.sym.owner == owner: + incl r.sym.flags, sfEffectsDelayed + else: + localError(c.config, n.info, errGenerated, "parameter cannot be declared as .effectsOf") + else: + localError(c.config, n.info, errGenerated, "parameter name expected") + + if n.kind notin nkPragmaCallKinds or n.len != 2: + localError(c.config, n.info, errGenerated, "parameter name expected") + else: + let it = n[1] + if it.kind in {nkCurly, nkBracket}: + for x in items(it): processParam(c, x) + else: + processParam(c, it) + proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, validPragmas: TSpecialWords, comesFromPush, isStatement: bool): bool = @@ -895,6 +915,8 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, of wNoalias: noVal(c, it) incl(sym.flags, sfNoalias) + of wEffectsOf: + processEffectsOf(c, it, sym) of wThreadVar: noVal(c, it) incl(sym.flags, {sfThread, sfGlobal}) diff --git a/compiler/sem.nim b/compiler/sem.nim index 804325e56..24709cf21 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -400,9 +400,10 @@ when not defined(nimHasSinkInference): include hlo, seminst, semcall proc resetSemFlag(n: PNode) = - excl n.flags, nfSem - for i in 0.. 0: rr = rr.lastSon - localError(g.config, r.info, errGenerated, renderTree(rr) & " " & msg & typeToString(r.typ)) + message(g.config, r.info, if emitWarnings: warnEffect else: errGenerated, + renderTree(rr) & " " & msg & typeToString(r.typ)) popInfoContext(g.config) # hint about unnecessarily listed exception types: if hints: @@ -1258,11 +1293,11 @@ proc checkMethodEffects*(g: ModuleGraph; disp, branch: PSym) = let p = disp.ast[pragmasPos] let raisesSpec = effectSpec(p, wRaises) if not isNil(raisesSpec): - checkRaisesSpec(g, raisesSpec, actual[exceptionEffects], + checkRaisesSpec(g, false, raisesSpec, actual[exceptionEffects], "can raise an unlisted exception: ", hints=off, subtypeRelation) let tagsSpec = effectSpec(p, wTags) if not isNil(tagsSpec): - checkRaisesSpec(g, tagsSpec, actual[tagEffects], + checkRaisesSpec(g, false, tagsSpec, actual[tagEffects], "can have an unlisted effect: ", hints=off, subtypeRelation) if sfThread in disp.flags and notGcSafe(branch.typ): localError(g.config, branch.info, "base method is GC-safe, but '$1' is not" % @@ -1283,7 +1318,7 @@ proc checkMethodEffects*(g: ModuleGraph; disp, branch: PSym) = "base method has lock level $1, but dispatcher has $2" % [$branch.typ.lockLevel, $disp.typ.lockLevel]) -proc setEffectsForProcType*(g: ModuleGraph; t: PType, n: PNode) = +proc setEffectsForProcType*(g: ModuleGraph; t: PType, n: PNode; s: PSym = nil) = var effects = t.n[0] if t.kind != tyProc or effects.kind != nkEffectList: return if n.kind != nkEmpty: @@ -1292,9 +1327,14 @@ proc setEffectsForProcType*(g: ModuleGraph; t: PType, n: PNode) = let raisesSpec = effectSpec(n, wRaises) if not isNil(raisesSpec): effects[exceptionEffects] = raisesSpec + elif s != nil and (s.magic != mNone or {sfImportc, sfExportc} * s.flags == {sfImportc}): + effects[exceptionEffects] = newNodeI(nkArgList, effects.info) + let tagsSpec = effectSpec(n, wTags) if not isNil(tagsSpec): effects[tagEffects] = tagsSpec + elif s != nil and (s.magic != mNone or {sfImportc, sfExportc} * s.flags == {sfImportc}): + effects[tagEffects] = newNodeI(nkArgList, effects.info) let requiresSpec = propSpec(n, wRequires) if not isNil(requiresSpec): @@ -1304,15 +1344,21 @@ proc setEffectsForProcType*(g: ModuleGraph; t: PType, n: PNode) = effects[ensuresEffects] = ensuresSpec effects[pragmasEffects] = n + if s != nil and s.magic != mNone: + if s.magic != mEcho: + t.flags.incl tfNoSideEffect -proc initEffects(g: ModuleGraph; effects: PNode; s: PSym; t: var TEffects; c: PContext) = +proc rawInitEffects(g: ModuleGraph; effects: PNode) = newSeq(effects.sons, effectListLen) - effects[exceptionEffects] = newNodeI(nkArgList, s.info) - effects[tagEffects] = newNodeI(nkArgList, s.info) + effects[exceptionEffects] = newNodeI(nkArgList, effects.info) + effects[tagEffects] = newNodeI(nkArgList, effects.info) effects[requiresEffects] = g.emptyNode effects[ensuresEffects] = g.emptyNode effects[pragmasEffects] = g.emptyNode +proc initEffects(g: ModuleGraph; effects: PNode; s: PSym; t: var TEffects; c: PContext) = + rawInitEffects(g, effects) + t.exc = effects[exceptionEffects] t.tags = effects[tagEffects] t.owner = s @@ -1341,10 +1387,14 @@ proc trackProc*(c: PContext; s: PSym, body: PNode) = if effects.kind != nkEffectList: return # effects already computed? if not s.hasRealBody: return - if effects.len == effectListLen: return + let emitWarnings = tfEffectSystemWorkaround in s.typ.flags + if effects.len == effectListLen and not emitWarnings: return + + var inferredEffects = newNodeI(nkEffectList, s.info) var t: TEffects - initEffects(g, effects, s, t, c) + initEffects(g, inferredEffects, s, t, c) + rawInitEffects g, effects track(t, body) if s.kind != skMacro: @@ -1369,17 +1419,21 @@ proc trackProc*(c: PContext; s: PSym, body: PNode) = let p = s.ast[pragmasPos] let raisesSpec = effectSpec(p, wRaises) if not isNil(raisesSpec): - checkRaisesSpec(g, raisesSpec, t.exc, "can raise an unlisted exception: ", + checkRaisesSpec(g, emitWarnings, raisesSpec, t.exc, "can raise an unlisted exception: ", hints=on, subtypeRelation, hintsArg=s.ast[0]) # after the check, use the formal spec: effects[exceptionEffects] = raisesSpec + else: + effects[exceptionEffects] = t.exc let tagsSpec = effectSpec(p, wTags) if not isNil(tagsSpec): - checkRaisesSpec(g, tagsSpec, t.tags, "can have an unlisted effect: ", + checkRaisesSpec(g, emitWarnings, tagsSpec, t.tags, "can have an unlisted effect: ", hints=off, subtypeRelation) # after the check, use the formal spec: effects[tagEffects] = tagsSpec + else: + effects[tagEffects] = t.tags let requiresSpec = propSpec(p, wRequires) if not isNil(requiresSpec): diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 14dc89781..df0e2778b 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -1953,6 +1953,10 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, if not hasProto: implicitPragmas(c, s, n.info, validPragmas) + if n[pragmasPos].kind != nkEmpty: + setEffectsForProcType(c.graph, s.typ, n[pragmasPos], s) + s.typ.flags.incl tfEffectSystemWorkaround + # To ease macro generation that produce forwarded .async procs we now # allow a bit redundancy in the pragma declarations. The rule is # a prototype's pragma list must be a superset of the current pragma @@ -2211,8 +2215,9 @@ proc evalInclude(c: PContext, n: PNode): PNode = incMod(c, n, it, result) proc setLine(n: PNode, info: TLineInfo) = - for i in 0..`_ pretends to be free of side effects +so that it can be used for debugging routines marked as `noSideEffect`. + +`func` is syntactic sugar for a proc with no side effects: + +.. code-block:: nim + func `+` (x, y: int): int + + +To override the compiler's side effect analysis a `{.noSideEffect.}` +`cast` pragma block can be used: + +.. code-block:: nim + + func f() = + {.cast(noSideEffect).}: + echo "test" + +**Side effects are usually inferred. The inference for side effects is +analogous to the inference for exception tracking.** + + +GC safety effect +---------------- + +We call a proc `p` `GC safe`:idx: when it doesn't access any global variable +that contains GC'ed memory (`string`, `seq`, `ref` or a closure) either +directly or indirectly through a call to a GC unsafe proc. + +**The GC safety property is usually inferred. The inference for GC safety is +analogous to the inference for exception tracking.** + +The `gcsafe`:idx: annotation can be used to mark a proc to be gcsafe, +otherwise this property is inferred by the compiler. Note that `noSideEffect` +implies `gcsafe`. + +Routines that are imported from C are always assumed to be `gcsafe`. + +To override the compiler's gcsafety analysis a `{.cast(gcsafe).}` pragma block can +be used: + +.. code-block:: nim + + var + someGlobal: string = "some string here" + perThread {.threadvar.}: string + + proc setPerThread() = + {.cast(gcsafe).}: + deepCopy(perThread, someGlobal) + + +See also: + +- `Shared heap memory management `_. + + Effects pragma -------------- @@ -6366,55 +6486,6 @@ This pragma can also take in an optional warning string to relay to developers. proc thing(x: bool) {.deprecated: "use thong instead".} -noSideEffect pragma -------------------- - -The `noSideEffect` pragma is used to mark a proc/iterator that can have only -side effects through parameters. This means that the proc/iterator only changes locations that are -reachable from its parameters and the return value only depends on the -parameters. If none of its parameters have the type `var`, `ref`, `ptr`, `cstring`, or `proc`, -then no locations are modified. - -It is a static error to mark a proc/iterator to have no side effect if the compiler cannot verify this. - -As a special semantic rule, the built-in `debugEcho -`_ pretends to be free of side effects -so that it can be used for debugging routines marked as `noSideEffect`. - -`func` is syntactic sugar for a proc with no side effects: - -.. code-block:: nim - func `+` (x, y: int): int - - -To override the compiler's side effect analysis a `{.noSideEffect.}` -`cast` pragma block can be used: - -.. code-block:: nim - - func f() = - {.cast(noSideEffect).}: - echo "test" - -When a `noSideEffect` proc has proc params `bar`, whether it can be used inside a `noSideEffect` context -depends on what the compiler knows about `bar`: - -.. code-block:: nim - :test: "nim c $1" - - func foo(bar: proc(): int): int = bar() - var count = 0 - proc fn1(): int = 1 - proc fn2(): int = (count.inc; count) - func fun1() = discard foo(fn1) # ok because fn1 is inferred as `func` - # func fun2() = discard foo(fn2) # would give: Error: 'fun2' can have side effects - - # with callbacks, the compiler is conservative, ie that bar will have side effects - var foo2: type(foo) = foo - func main() = - discard foo(fn1) # ok - # discard foo2(fn1) # now this errors - compileTime pragma ------------------ @@ -7861,6 +7932,10 @@ collected) heap, and sharing of memory is restricted to global variables. This helps to prevent race conditions. GC efficiency is improved quite a lot, because the GC never has to stop other threads and see what they reference. +The only way to create a thread is via `spawn` or +`createThread`. The invoked proc must not use `var` parameters nor must +any of its parameters contain a `ref` or `closure` type. This enforces +the *no heap sharing restriction*. Thread pragma ------------- @@ -7875,43 +7950,6 @@ A thread proc is passed to `createThread` or `spawn` and invoked indirectly; so the `thread` pragma implies `procvar`. -GC safety ---------- - -We call a proc `p` `GC safe`:idx: when it doesn't access any global variable -that contains GC'ed memory (`string`, `seq`, `ref` or a closure) either -directly or indirectly through a call to a GC unsafe proc. - -The `gcsafe`:idx: annotation can be used to mark a proc to be gcsafe, -otherwise this property is inferred by the compiler. Note that `noSideEffect` -implies `gcsafe`. The only way to create a thread is via `spawn` or -`createThread`. The invoked proc must not use `var` parameters nor must -any of its parameters contain a `ref` or `closure` type. This enforces -the *no heap sharing restriction*. - -Routines that are imported from C are always assumed to be `gcsafe`. -To disable the GC-safety checking the `--threadAnalysis:off`:option: command-line -switch can be used. This is a temporary workaround to ease the porting effort -from old code to the new threading model. - -To override the compiler's gcsafety analysis a `{.cast(gcsafe).}` pragma block can -be used: - -.. code-block:: nim - - var - someGlobal: string = "some string here" - perThread {.threadvar.}: string - - proc setPerThread() = - {.cast(gcsafe).}: - deepCopy(perThread, someGlobal) - - -See also: - -- `Shared heap memory management `_. - Threadvar pragma ---------------- diff --git a/lib/pure/algorithm.nim b/lib/pure/algorithm.nim index c96f599e8..1ddcc9843 100644 --- a/lib/pure/algorithm.nim +++ b/lib/pure/algorithm.nim @@ -147,8 +147,13 @@ proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T] {.inline, deprecated: "use: `reversed(toOpenArray(a, first, last))`".} = reversed(toOpenArray(a, first, last)) +when defined(nimHasEffectsOf): + {.experimental: "strictEffects".} +else: + {.pragma: effectsOf.} + proc binarySearch*[T, K](a: openArray[T], key: K, - cmp: proc (x: T, y: K): int {.closure.}): int = + cmp: proc (x: T, y: K): int {.closure.}): int {.effectsOf: cmp.} = ## Binary search for `key` in `a`. Return the index of `key` or -1 if not found. ## Assumes that `a` is sorted according to `cmp`. ## @@ -210,7 +215,7 @@ const onlySafeCode = true proc lowerBound*[T, K](a: openArray[T], key: K, - cmp: proc(x: T, k: K): int {.closure.}): int = + cmp: proc(x: T, k: K): int {.closure.}): int {.effectsOf: cmp.} = ## Returns the index of the first element in `a` that is not less than ## (i.e. greater or equal to) `key`, or last if no such element is found. ## In other words if you have a sorted sequence and you call @@ -260,7 +265,7 @@ proc lowerBound*[T](a: openArray[T], key: T): int = lowerBound(a, key, cmp[T]) ## * `upperBound proc<#upperBound,openArray[T],T>`_ proc upperBound*[T, K](a: openArray[T], key: K, - cmp: proc(x: T, k: K): int {.closure.}): int = + cmp: proc(x: T, k: K): int {.closure.}): int {.effectsOf: cmp.} = ## Returns the index of the first element in `a` that is greater than ## `key`, or last if no such element is found. ## In other words if you have a sorted sequence and you call @@ -318,7 +323,7 @@ template `<-`(a, b) = copyMem(addr(a), addr(b), sizeof(T)) proc mergeAlt[T](a, b: var openArray[T], lo, m, hi: int, - cmp: proc (x, y: T): int {.closure.}, order: SortOrder) = + cmp: proc (x, y: T): int {.closure.}, order: SortOrder) {.effectsOf: cmp.} = # Optimization: If max(left) <= min(right) there is nothing to do! # 1 2 3 4 ## 5 6 7 8 # -> O(n) for sorted arrays. @@ -358,7 +363,7 @@ proc mergeAlt[T](a, b: var openArray[T], lo, m, hi: int, func sort*[T](a: var openArray[T], cmp: proc (x, y: T): int {.closure.}, - order = SortOrder.Ascending) = + order = SortOrder.Ascending) {.effectsOf: cmp.} = ## Default Nim sort (an implementation of merge sort). The sorting ## is guaranteed to be stable (that is, equal elements stay in the same order) ## and the worst case is guaranteed to be O(n log n). @@ -420,7 +425,7 @@ proc sort*[T](a: var openArray[T], order = SortOrder.Ascending) = sort[T](a, ## * `sortedByIt template<#sortedByIt.t,untyped,untyped>`_ proc sorted*[T](a: openArray[T], cmp: proc(x, y: T): int {.closure.}, - order = SortOrder.Ascending): seq[T] = + order = SortOrder.Ascending): seq[T] {.effectsOf: cmp.} = ## Returns `a` sorted by `cmp` in the specified `order`. ## ## **See also:** @@ -497,7 +502,7 @@ template sortedByIt*(seq1, op: untyped): untyped = func isSorted*[T](a: openArray[T], cmp: proc(x, y: T): int {.closure.}, - order = SortOrder.Ascending): bool = + order = SortOrder.Ascending): bool {.effectsOf: cmp.} = ## Checks to see whether `a` is already sorted in `order` ## using `cmp` for the comparison. The parameters are identical ## to `sort`. Requires O(n) time. @@ -545,7 +550,7 @@ proc isSorted*[T](a: openArray[T], order = SortOrder.Ascending): bool = proc merge*[T]( result: var seq[T], x, y: openArray[T], cmp: proc(x, y: T): int {.closure.} -) {.since: (1, 5, 1).} = +) {.since: (1, 5, 1), effectsOf: cmp.} = ## Merges two sorted `openArray`. `x` and `y` are assumed to be sorted. ## If you do not wish to provide your own `cmp`, ## you may use `system.cmp` or instead call the overloaded @@ -638,7 +643,7 @@ proc product*[T](x: openArray[seq[T]]): seq[seq[T]] = ## Produces the Cartesian product of the array. ## Every element of the result is a combination of one element from each seq in `x`, ## with the ith element coming from `x[i]`. - ## + ## ## .. warning:: complexity may explode. runnableExamples: assert product(@[@[1], @[2]]) == @[@[1, 2]] diff --git a/lib/pure/times.nim b/lib/pure/times.nim index b2a22250d..b41bb28b5 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -1333,7 +1333,7 @@ proc now*(): DateTime {.tags: [TimeEffect], benign.} = ## `cpuTime` instead, depending on the use case. getTime().local -proc dateTime*(year: int, month: Month, monthday: MonthdayRange, +proc dateTime*(year: int, month: Month, monthday: MonthdayRange, hour: HourRange = 0, minute: MinuteRange = 0, second: SecondRange = 0, nanosecond: NanosecondRange = 0, zone: Timezone = local()): DateTime = diff --git a/lib/std/private/globs.nim b/lib/std/private/globs.nim index b3726c9c3..190316f93 100644 --- a/lib/std/private/globs.nim +++ b/lib/std/private/globs.nim @@ -8,13 +8,18 @@ import os when defined(windows): from strutils import replace +when defined(nimHasEffectsOf): + {.experimental: "strictEffects".} +else: + {.pragma: effectsOf.} + type PathEntry* = object kind*: PathComponent path*: string iterator walkDirRecFilter*(dir: string, follow: proc(entry: PathEntry): bool = nil, - relative = false, checkDir = true): PathEntry {.tags: [ReadDirEffect].} = + relative = false, checkDir = true): PathEntry {.tags: [ReadDirEffect], effectsOf: follow.} = ## Improved `os.walkDirRec`. #[ note: a yieldFilter isn't needed because caller can filter at call site, without diff --git a/lib/system/io.nim b/lib/system/io.nim index d8b2ac741..59f070f73 100644 --- a/lib/system/io.nim +++ b/lib/system/io.nim @@ -78,7 +78,7 @@ proc c_fputs(c: cstring, f: File): cint {. proc c_fgets(c: cstring, n: cint, f: File): cstring {. importc: "fgets", header: "", tags: [ReadIOEffect].} proc c_fgetc(stream: File): cint {. - importc: "fgetc", header: "", tags: [ReadIOEffect].} + importc: "fgetc", header: "", tags: [].} proc c_ungetc(c: cint, f: File): cint {. importc: "ungetc", header: "", tags: [].} proc c_putc(c: cint, stream: File): cint {. diff --git a/lib/system/nimscript.nim b/lib/system/nimscript.nim index 9ece10e3e..b4554d778 100644 --- a/lib/system/nimscript.nim +++ b/lib/system/nimscript.nim @@ -261,7 +261,7 @@ proc cpDir*(`from`, to: string) {.raises: [OSError].} = checkOsError() proc exec*(command: string) {. - raises: [OSError], tags: [ExecIOEffect].} = + raises: [OSError], tags: [ExecIOEffect, WriteIOEffect].} = ## Executes an external process. If the external process terminates with ## a non-zero exit code, an OSError exception is raised. ## @@ -274,7 +274,7 @@ proc exec*(command: string) {. checkOsError() proc exec*(command: string, input: string, cache = "") {. - raises: [OSError], tags: [ExecIOEffect].} = + raises: [OSError], tags: [ExecIOEffect, WriteIOEffect].} = ## Executes an external process. If the external process terminates with ## a non-zero exit code, an OSError exception is raised. log "exec: " & command: @@ -284,7 +284,7 @@ proc exec*(command: string, input: string, cache = "") {. echo output proc selfExec*(command: string) {. - raises: [OSError], tags: [ExecIOEffect].} = + raises: [OSError], tags: [ExecIOEffect, WriteIOEffect].} = ## Executes an external command with the current nim/nimble executable. ## `Command` must not contain the "nim " part. let c = selfExe() & " " & command diff --git a/nimdoc/test_out_index_dot_html/expected/index.html b/nimdoc/test_out_index_dot_html/expected/index.html index 3499b5326..458a0394f 100644 --- a/nimdoc/test_out_index_dot_html/expected/index.html +++ b/nimdoc/test_out_index_dot_html/expected/index.html @@ -122,7 +122,7 @@ window.addEventListener('DOMContentLoaded', main);

    Procs

    -
    proc foo() {....raises: [], tags: [].}
    +
    proc foo() {....raises: [], tags: [WriteIOEffect].}
    I do foo diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html index d98b1b00f..dbacb57a7 100644 --- a/nimdoc/testproject/expected/testproject.html +++ b/nimdoc/testproject/expected/testproject.html @@ -654,7 +654,7 @@ This is deprecated with a message. 
    proc c_nonexistent(frmt: cstring): cint {.importc: "nonexistent",
-    header: "<stdio.h>", varargs, discardable.}
    + header: "<stdio.h>", varargs, discardable, ...raises: [], tags: [].}
    @@ -663,7 +663,7 @@ This is deprecated with a message. 
    proc c_printf(frmt: cstring): cint {.importc: "printf", header: "<stdio.h>",
-                                     varargs, discardable.}
    + varargs, discardable, ...raises: [], tags: [].}
    the c printf. etc. @@ -690,7 +690,8 @@ came form utils but should be shown where
    -
    proc low2[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect.}
    +
    proc low2[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect,
+    ...raises: [], tags: [].}

    Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.

    @@ -704,7 +705,8 @@ came form utils but should be shown where
    -
    proc low[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect.}
    +
    proc low[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect,
+    ...raises: [], tags: [].}

    Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.

    diff --git a/tests/effects/teffects1.nim b/tests/effects/teffects1.nim index 66d6a3518..82efefe77 100644 --- a/tests/effects/teffects1.nim +++ b/tests/effects/teffects1.nim @@ -6,7 +6,7 @@ teffects1.nim(22, 29) Hint: 'lier' cannot raise 'IO2Error' [XCannotRaiseY] teffects1.nim(38, 21) Error: type mismatch: got but expected 'MyProcType = proc (x: int): string{.closure.}' .raise effects differ''' """ - +{.push warningAsError[Effect]: on.} type TObj {.pure, inheritable.} = object TObjB = object of TObj @@ -41,3 +41,4 @@ type mismatch: got but ]# {.pop.} +{.pop.} diff --git a/tests/effects/teffects2.nim b/tests/effects/teffects2.nim index e4b50aba5..ec3c0a887 100644 --- a/tests/effects/teffects2.nim +++ b/tests/effects/teffects2.nim @@ -2,7 +2,7 @@ discard """ errormsg: "can raise an unlisted exception: ref IOError" line: 19 """ - +{.push warningAsError[Effect]: on.} type TObj = object {.pure, inheritable.} TObjB = object of TObj @@ -17,3 +17,4 @@ proc lier(): int {.raises: [IOError].} = proc forw: int = raise newException(IOError, "arg") +{.pop.} diff --git a/tests/effects/teffects7.nim b/tests/effects/teffects7.nim index 73865b18d..9b7fbf5f0 100644 --- a/tests/effects/teffects7.nim +++ b/tests/effects/teffects7.nim @@ -2,7 +2,7 @@ discard """ errormsg: "can raise an unlisted exception: ref ValueError" line: 10 """ - +{.push warningAsError[Effect]: on.} proc foo() {.raises: [].} = try: discard @@ -12,3 +12,5 @@ proc foo() {.raises: [].} = discard foo() + +{.pop.} diff --git a/tests/effects/teffects8.nim b/tests/effects/teffects8.nim index fb3c088d6..359b3a1df 100644 --- a/tests/effects/teffects8.nim +++ b/tests/effects/teffects8.nim @@ -2,7 +2,7 @@ discard """ errormsg: "can raise an unlisted exception: Exception" line: 10 """ - +{.push warningAsError[Effect]: on.} proc foo() {.raises: [].} = try: discard @@ -10,3 +10,4 @@ proc foo() {.raises: [].} = raise foo() +{.pop.} diff --git a/tests/effects/tstrict_effects.nim b/tests/effects/tstrict_effects.nim new file mode 100644 index 000000000..eee8fb71a --- /dev/null +++ b/tests/effects/tstrict_effects.nim @@ -0,0 +1,27 @@ +discard """ + errormsg: "s1 can raise an unlisted exception: CatchableError" + line: 27 +""" + +{.push warningAsError[Effect]: on.} +{.experimental: "strictEffects".} + +# bug #18376 + +{.push raises: [Defect].} +type Call = proc (x: int): int {.gcsafe, raises: [Defect, CatchableError].} + +type Bar* = object + foo*: Call + +proc passOn*(x: Call) = discard + +proc barCal(b: var Bar, s: string, s1: Call) = + #compiler complains that his line can throw CatchableError + passOn s1 + + +proc passOnB*(x: Call) {.effectsOf: x.} = discard + +proc barCal2(b: var Bar, s: string, s1: Call) = + passOnB s1 diff --git a/tests/effects/tstrict_effects2.nim b/tests/effects/tstrict_effects2.nim new file mode 100644 index 000000000..acc0a0540 --- /dev/null +++ b/tests/effects/tstrict_effects2.nim @@ -0,0 +1,28 @@ +discard """ + errormsg: "can raise an unlisted exception: Exception" + line: 23 +""" + +{.push warningAsError[Effect]: on.} +{.experimental: "strictEffects".} + +# bug #13905 + +proc atoi(v: cstring): cint {.importc: "atoi", cdecl, raises: [].} + +type Conv = proc(v: cstring): cint {.cdecl, raises: [].} + +var x: Conv = atoi + +# bug #17475 + +type + Callback = proc() + +proc f(callback: Callback) {.raises: [].} = + callback() + +proc main = + f(proc () = raise newException(IOError, "IO")) + +main() diff --git a/tests/effects/tstrict_effects3.nim b/tests/effects/tstrict_effects3.nim new file mode 100644 index 000000000..1ab15cc80 --- /dev/null +++ b/tests/effects/tstrict_effects3.nim @@ -0,0 +1,17 @@ +discard """ + action: compile +""" + +{.push warningAsError[Effect]: on.} + +{.experimental: "strictEffects".} + +proc fn(a: int, p1, p2: proc()) {.effectsOf: p1.} = + if a == 7: + p1() + if a<0: + raise newException(ValueError, $a) + +proc main() {.raises: [ValueError].} = + fn(1, proc()=discard, proc() = raise newException(IOError, "foo")) +main() diff --git a/tests/effects/tstrict_effects_sort.nim b/tests/effects/tstrict_effects_sort.nim new file mode 100644 index 000000000..8928ed0d3 --- /dev/null +++ b/tests/effects/tstrict_effects_sort.nim @@ -0,0 +1,27 @@ +discard """ + errormsg: "cmpE can raise an unlisted exception: Exception" + line: 27 +""" + +{.push warningAsError[Effect]: on.} + +{.experimental: "strictEffects".} + +import algorithm + +type + MyInt = distinct int + +var toSort = @[MyInt 1, MyInt 2, MyInt 3] + +proc cmpN(a, b: MyInt): int = + cmp(a.int, b.int) + +proc harmless {.raises: [].} = + toSort.sort cmpN + +proc cmpE(a, b: MyInt): int {.raises: [Exception].} = + cmp(a.int, b.int) + +proc harmfull {.raises: [].} = + toSort.sort cmpE diff --git a/tests/pragmas/tuserpragma2.nim b/tests/pragmas/tuserpragma2.nim index ce16c4649..c3f31cd5e 100644 --- a/tests/pragmas/tuserpragma2.nim +++ b/tests/pragmas/tuserpragma2.nim @@ -3,9 +3,10 @@ discard """ file: "tuserpragma2.nim" line: 11 """ - +{.push warningAsError[Effect]: on.} # bug #7216 {.pragma: my_pragma, raises: [].} proc test1 {.my_pragma.} = raise newException(Exception, "msg") +{.pop.} diff --git a/tests/stdlib/tstdlib_various.nim b/tests/stdlib/tstdlib_various.nim index b153fd2ba..bc90d6ef4 100644 --- a/tests/stdlib/tstdlib_various.nim +++ b/tests/stdlib/tstdlib_various.nim @@ -20,12 +20,6 @@ Hi Andreas! How do you feel, Rumpf? @[0, 2, 1] @[0, 1, 2] 055this should be the casehugh@["(", "+", " 1", " 2", ")"] -caught a crash! -caught a crash! -caught a crash! -caught a crash! -caught a crash! -caught a crash! [5] [4, 5] [3, 4, 5] @@ -161,18 +155,21 @@ block tropes: block tsegfaults: - proc main = - try: - var x: ptr int - echo x[] + when not defined(arm64): + var crashes = 0 + proc main = try: - raise newException(ValueError, "not a crash") - except ValueError: - discard - except NilAccessDefect: - echo "caught a crash!" - for i in 0..5: - main() + var x: ptr int + echo x[] + try: + raise newException(ValueError, "not a crash") + except ValueError: + discard + except NilAccessDefect: + inc crashes + for i in 0..5: + main() + assert crashes == 6 From 7c8ea490a2be36d171d362c43212195ea8ce489f Mon Sep 17 00:00:00 2001 From: Tomohiro Date: Thu, 2 Sep 2021 21:12:14 +0900 Subject: [PATCH 1183/2472] Fix initrand to avoid random number sequences overlapping (#18744) * Fix initrand to avoid random number sequences overlapping * Minor fix * Fix compile error on js backend * Disable new test for js backend * Minor fix * tempfiles module uses random.initRand() * Remove unused module import from lib/std/tempfiles.nim * Initialize baseState in initRand() * Run tests/stdlib/trandom.nim from tests/test_nimscript.nims * baseState is initialized only with sysrand.urandom and quit if failed * Add comments --- lib/pure/random.nim | 74 +++++++++++++++++++++++++++++++++------ lib/std/tempfiles.nim | 7 ++-- tests/stdlib/trandom.nim | 32 ++++++++++++++--- tests/test_nimscript.nims | 2 ++ 4 files changed, 95 insertions(+), 20 deletions(-) diff --git a/lib/pure/random.nim b/lib/pure/random.nim index a292386af..b8aeb86e0 100644 --- a/lib/pure/random.nim +++ b/lib/pure/random.nim @@ -108,10 +108,19 @@ when defined(js): a0: 0x69B4C98Cu32, a1: 0xFED1DD30u32) # global for backwards compatibility else: - # racy for multi-threading but good enough for now: - var state = Rand( + const DefaultRandSeed = Rand( a0: 0x69B4C98CB8530805u64, - a1: 0xFED1DD3004688D67CAu64) # global for backwards compatibility + a1: 0xFED1DD3004688D67CAu64) + + # racy for multi-threading but good enough for now: + var state = DefaultRandSeed # global for backwards compatibility + +func isValid(r: Rand): bool {.inline.} = + ## Check whether state of `r` is valid. + ## + ## In `xoroshiro128+`, if all bits of `a0` and `a1` are zero, + ## they are always zero after calling `next(r: var Rand)`. + not (r.a0 == 0 and r.a1 == 0) since (1, 5): template randState*(): untyped = @@ -617,15 +626,28 @@ proc shuffle*[T](x: var openArray[T]) = shuffle(state, x) -when not defined(nimscript) and not defined(standalone): - import times +when not defined(standalone): + when defined(js): + import std/times + else: + when defined(nimscript): + import std/hashes + else: + import std/[hashes, os, sysrand, monotimes] + + when compileOption("threads"): + import locks + var baseSeedLock: Lock + baseSeedLock.initLock + + var baseState: Rand proc initRand(): Rand = - ## Initializes a new Rand state with a seed based on the current time. + ## Initializes a new Rand state. ## ## The resulting state is independent of the default RNG's state. ## - ## **Note:** Does not work for NimScript or the compile-time VM. + ## **Note:** Does not work for the compile-time VM. ## ## See also: ## * `initRand proc<#initRand,int64>`_ that accepts a seed for a new Rand state @@ -635,20 +657,50 @@ when not defined(nimscript) and not defined(standalone): let time = int64(times.epochTime() * 1000) and 0x7fff_ffff result = initRand(time) else: - let now = times.getTime() - result = initRand(convert(Seconds, Nanoseconds, now.toUnix) + now.nanosecond) + proc getRandomState(): Rand = + when defined(nimscript): + result = Rand( + a0: CompileTime.hash.Ui, + a1: CompileDate.hash.Ui) + if not result.isValid: + result = DefaultRandSeed + else: + var urand: array[sizeof(Rand), byte] + + for i in 0 .. 7: + if sysrand.urandom(urand): + copyMem(result.addr, urand[0].addr, sizeof(Rand)) + if result.isValid: + break + + if not result.isValid: + # Don't try to get alternative random values from other source like time or process/thread id, + # because such code would be never tested and is a liability for security. + quit("Failed to initializes baseState in random module as sysrand.urandom doesn't work.") + + when compileOption("threads"): + baseSeedLock.withLock: + if not baseState.isValid: + baseState = getRandomState() + result = baseState + baseState.skipRandomNumbers + else: + if not baseState.isValid: + baseState = getRandomState() + result = baseState + baseState.skipRandomNumbers since (1, 5, 1): export initRand proc randomize*() {.benign.} = ## Initializes the default random number generator with a seed based on - ## the current time. + ## random number source. ## ## This proc only needs to be called once, and it should be called before ## the first usage of procs from this module that use the default RNG. ## - ## **Note:** Does not work for NimScript or the compile-time VM. + ## **Note:** Does not work for the compile-time VM. ## ## **See also:** ## * `randomize proc<#randomize,int64>`_ that accepts a seed diff --git a/lib/std/tempfiles.nim b/lib/std/tempfiles.nim index f1cfd5885..26786d463 100644 --- a/lib/std/tempfiles.nim +++ b/lib/std/tempfiles.nim @@ -17,7 +17,7 @@ See also: * `mkstemp` (posix), refs https://man7.org/linux/man-pages/man3/mkstemp.3.html ]# -import os, random, std/monotimes +import os, random const @@ -107,11 +107,8 @@ var nimTempPathState {.threadvar.}: NimTempPathState template randomPathName(length: Natural): string = var res = newString(length) if not nimTempPathState.isInit: - var time = getMonoTime().ticks - when compileOption("threads"): - time = time xor int64(getThreadId()) nimTempPathState.isInit = true - nimTempPathState.state = initRand(time) + nimTempPathState.state = initRand() for i in 0 ..< length: res[i] = nimTempPathState.state.sample(letters) diff --git a/tests/stdlib/trandom.nim b/tests/stdlib/trandom.nim index 39ccca85b..61e858f86 100644 --- a/tests/stdlib/trandom.nim +++ b/tests/stdlib/trandom.nim @@ -23,9 +23,10 @@ proc main() = doAssert a in [[0,1], [1,0]] doAssert rand(0) == 0 - doAssert sample("a") == 'a' + when not defined(nimscript): + doAssert sample("a") == 'a' - when compileOption("rangeChecks"): + when compileOption("rangeChecks") and not defined(nimscript): doAssertRaises(RangeDefect): discard rand(-1) @@ -92,7 +93,7 @@ block: # random int block: # again gives new numbers var rand1 = rand(1000000) - when not defined(js): + when not (defined(js) or defined(nimscript)): os.sleep(200) var rand2 = rand(1000000) @@ -122,7 +123,7 @@ block: # random float block: # again gives new numbers var rand1: float = rand(1000000.0) - when not defined(js): + when not (defined(js) or defined(nimscript)): os.sleep(200) var rand2: float = rand(1000000.0) @@ -248,3 +249,26 @@ block: # bug #17670 type UInt48 = range[0'u64..2'u64^48-1] let x = rand(UInt48) doAssert x is UInt48 + +block: # bug #17898 + # Checks whether `initRand()` generates unique states. + # size should be 2^64, but we don't have time and space. + + # Disable this test for js until js gets proper skipRandomNumbers. + when not defined(js): + const size = 1000 + var + rands: array[size, Rand] + randSet: HashSet[Rand] + for i in 0.. Date: Thu, 2 Sep 2021 14:35:25 +0200 Subject: [PATCH 1184/2472] sequtils now support strict effect tracking (#18782) --- lib/pure/collections/sequtils.nim | 23 ++++++++++++++--------- tests/stdlib/tsequtils.nim | 5 +++++ 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index 5b41ea00d..12b961371 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -84,6 +84,11 @@ import std/private/since import macros +when defined(nimHasEffectsOf): + {.experimental: "strictEffects".} +else: + {.pragma: effectsOf.} + macro evalOnceAs(expAlias, exp: untyped, letAssigneable: static[bool]): untyped = ## Injects `expAlias` in caller scope, to avoid bugs involving multiple @@ -356,7 +361,7 @@ func distribute*[T](s: seq[T], num: Positive, spread = true): seq[seq[T]] = first = last proc map*[T, S](s: openArray[T], op: proc (x: T): S {.closure.}): - seq[S]{.inline.} = + seq[S] {.inline, effectsOf: op.} = ## Returns a new sequence with the results of the `op` proc applied to every ## item in the container `s`. ## @@ -382,7 +387,7 @@ proc map*[T, S](s: openArray[T], op: proc (x: T): S {.closure.}): result[i] = op(s[i]) proc apply*[T](s: var openArray[T], op: proc (x: var T) {.closure.}) - {.inline.} = + {.inline, effectsOf: op.} = ## Applies `op` to every item in `s`, modifying it directly. ## ## Note that the container `s` must be declared as a `var`, @@ -401,7 +406,7 @@ proc apply*[T](s: var openArray[T], op: proc (x: var T) {.closure.}) for i in 0 ..< s.len: op(s[i]) proc apply*[T](s: var openArray[T], op: proc (x: T): T {.closure.}) - {.inline.} = + {.inline, effectsOf: op.} = ## Applies `op` to every item in `s` modifying it directly. ## ## Note that the container `s` must be declared as a `var` @@ -420,7 +425,7 @@ proc apply*[T](s: var openArray[T], op: proc (x: T): T {.closure.}) for i in 0 ..< s.len: s[i] = op(s[i]) -proc apply*[T](s: openArray[T], op: proc (x: T) {.closure.}) {.inline, since: (1, 3).} = +proc apply*[T](s: openArray[T], op: proc (x: T) {.closure.}) {.inline, since: (1, 3), effectsOf: op.} = ## Same as `apply` but for a proc that does not return anything ## and does not mutate `s` directly. runnableExamples: @@ -429,7 +434,7 @@ proc apply*[T](s: openArray[T], op: proc (x: T) {.closure.}) {.inline, since: (1 assert message == "01234" for i in 0 ..< s.len: op(s[i]) -iterator filter*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): T = +iterator filter*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): T {.effectsOf: pred.} = ## Iterates through a container `s` and yields every item that fulfills the ## predicate `pred` (a function that returns a `bool`). ## @@ -453,7 +458,7 @@ iterator filter*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): T = yield s[i] proc filter*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): seq[T] - {.inline.} = + {.inline, effectsOf: pred.} = ## Returns a new sequence with all the items of `s` that fulfill the ## predicate `pred` (a function that returns a `bool`). ## @@ -480,7 +485,7 @@ proc filter*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): seq[T] result.add(s[i]) proc keepIf*[T](s: var seq[T], pred: proc(x: T): bool {.closure.}) - {.inline.} = + {.inline, effectsOf: pred.} = ## Keeps the items in the passed sequence `s` if they fulfill the ## predicate `pred` (a function that returns a `bool`). ## @@ -685,7 +690,7 @@ since (1, 1): if pred: result += 1 result -proc all*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): bool = +proc all*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): bool {.effectsOf: pred.} = ## Iterates through a container and checks if every item fulfills the ## predicate. ## @@ -727,7 +732,7 @@ template allIt*(s, pred: untyped): bool = break result -proc any*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): bool = +proc any*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): bool {.effectsOf: pred.} = ## Iterates through a container and checks if at least one item ## fulfills the predicate. ## diff --git a/tests/stdlib/tsequtils.nim b/tests/stdlib/tsequtils.nim index 6c9c92873..338473407 100644 --- a/tests/stdlib/tsequtils.nim +++ b/tests/stdlib/tsequtils.nim @@ -8,6 +8,9 @@ import std/sequtils import strutils from algorithm import sorted +{.experimental: "strictEffects".} +{.push warningAsError[Effect]: on.} + # helper for testing double substitution side effects which are handled # by `evalOnceAs` var counter = 0 @@ -507,3 +510,5 @@ template main = static: main() main() + +{.pop.} From a7cae2bda24871c55348173816a43144b9e668a5 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 2 Sep 2021 17:15:11 +0200 Subject: [PATCH 1185/2472] fixes #16325 [backport:1.4] (#18784) --- compiler/sigmatch.nim | 3 +-- .../tproc_types_dont_like_subtypes.nim | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 tests/overload/tproc_types_dont_like_subtypes.nim diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 465a9a51d..9cf18277f 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -595,8 +595,7 @@ proc procParamTypeRel(c: var TCandidate, f, a: PType): TTypeRelation = # if f is metatype. result = typeRel(c, f, a) - # v--- is this correct? - if result <= isIntConv or inconsistentVarTypes(f, a): + if result <= isSubrange or inconsistentVarTypes(f, a): result = isNone #if result == isEqual: diff --git a/tests/overload/tproc_types_dont_like_subtypes.nim b/tests/overload/tproc_types_dont_like_subtypes.nim new file mode 100644 index 000000000..d322f41be --- /dev/null +++ b/tests/overload/tproc_types_dont_like_subtypes.nim @@ -0,0 +1,20 @@ +discard """ + errormsg: "got " + line: 20 +""" + +type + A = ref object of RootObj + B = ref object of A + + P = proc (a: A) + +# bug #16325 + +proc doThings(a: A, p: P) = + p(a) + +var x = proc (b: B) {.closure.} = + echo "B" + +doThings(B(), x) From f46569bafdfdd3290baf132f025d61a0e550dd58 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 2 Sep 2021 17:16:03 +0200 Subject: [PATCH 1186/2472] fixes #18494 (#18783) --- lib/system/alloc.nim | 46 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index 479458950..6d83a2c17 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -1053,22 +1053,48 @@ template instantiateForRegion(allocator: untyped) {.dirty.} = inc(result, it.size) it = it.next - proc getFreeMem(): int = - result = allocator.freeMem - #sysAssert(result == countFreeMem()) + when hasThreadSupport: + var sharedHeap: MemRegion + var heapLock: SysLock + initSysLock(heapLock) - proc getTotalMem(): int = return allocator.currMem - proc getOccupiedMem(): int = return allocator.occ #getTotalMem() - getFreeMem() - proc getMaxMem*(): int = return getMaxMem(allocator) + proc getFreeMem(): int = + #sysAssert(result == countFreeMem()) + when hasThreadSupport and defined(gcDestructors): + acquireSys(heapLock) + result = sharedHeap.freeMem + releaseSys(heapLock) + else: + result = allocator.freeMem + + proc getTotalMem(): int = + when hasThreadSupport and defined(gcDestructors): + acquireSys(heapLock) + result = sharedHeap.currMem + releaseSys(heapLock) + else: + result = allocator.currMem + + proc getOccupiedMem(): int = + when hasThreadSupport and defined(gcDestructors): + acquireSys(heapLock) + result = sharedHeap.occ + releaseSys(heapLock) + else: + result = allocator.occ #getTotalMem() - getFreeMem() + + proc getMaxMem*(): int = + when hasThreadSupport and defined(gcDestructors): + acquireSys(heapLock) + result = getMaxMem(sharedHeap) + releaseSys(heapLock) + else: + result = getMaxMem(allocator) when defined(nimTypeNames): proc getMemCounters*(): (int, int) = getMemCounters(allocator) # -------------------- shared heap region ---------------------------------- - when hasThreadSupport: - var sharedHeap: MemRegion - var heapLock: SysLock - initSysLock(heapLock) proc allocSharedImpl(size: Natural): pointer = when hasThreadSupport: From 64165f03fc96d6e8293f1b065a2e1e45f9fc1bf8 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 2 Sep 2021 22:15:08 -0700 Subject: [PATCH 1187/2472] testament --megatest:off now makes tests non-joinable so that they all run (#18787) --- testament/categories.nim | 2 +- testament/testament.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/testament/categories.nim b/testament/categories.nim index 6375105ee..6702d24a2 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -541,7 +541,7 @@ proc processSingleTest(r: var TResults, cat: Category, options, test: string, ta proc isJoinableSpec(spec: TSpec): bool = # xxx simplify implementation using a whitelist of fields that are allowed to be # set to non-default values (use `fieldPairs`), to avoid issues like bug #16576. - result = not spec.sortoutput and + result = useMegatest and not spec.sortoutput and spec.action == actionRun and not fileExists(spec.file.changeFileExt("cfg")) and not fileExists(spec.file.changeFileExt("nims")) and diff --git a/testament/testament.nim b/testament/testament.nim index ab6620c84..105f00156 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -28,6 +28,7 @@ var useColors = true var backendLogging = true var simulate = false var optVerbose = false +var useMegatest = true proc verboseCmd(cmd: string) = if optVerbose: @@ -645,7 +646,6 @@ proc main() = var targetsStr = "" var isMainProcess = true var skipFrom = "" - var useMegatest = true var p = initOptParser() p.next() From 0887dcc398a38c91e99a043acbb095afc7200460 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 3 Sep 2021 09:03:26 +0200 Subject: [PATCH 1188/2472] fixes #18786 (#18788) --- compiler/sempass2.nim | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index b584bb232..cae2f926c 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -296,12 +296,11 @@ proc useVarNoInitCheck(a: PEffects; n: PNode; s: PSym) = if {sfGlobal, sfThread} * s.flags != {} and s.kind in {skVar, skLet} and s.magic != mNimvm: if s.guard != nil: guardGlobal(a, n, s.guard) - if strictEffects notin a.c.features: - if {sfGlobal, sfThread} * s.flags == {sfGlobal} and - (tfHasGCedMem in s.typ.flags or s.typ.isGCedMem): - #if a.config.hasWarn(warnGcUnsafe): warnAboutGcUnsafe(n) - markGcUnsafe(a, s) - markSideEffect(a, s, n.info) + if {sfGlobal, sfThread} * s.flags == {sfGlobal} and + (tfHasGCedMem in s.typ.flags or s.typ.isGCedMem): + #if a.config.hasWarn(warnGcUnsafe): warnAboutGcUnsafe(n) + markGcUnsafe(a, s) + markSideEffect(a, s, n.info) if s.owner != a.owner and s.kind in {skVar, skLet, skForVar, skResult, skParam} and {sfGlobal, sfThread} * s.flags == {}: a.isInnerProc = true From 0635697a89b3aef2a188a0bddb46965bae7ee70f Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Fri, 3 Sep 2021 14:20:57 +0200 Subject: [PATCH 1189/2472] Document `-d:async_backend` flag (#18700) * Document `-d:asyncBackend` flag Since libraries may want to support multiple async backends, it's useful to standardize the way this is done, so as to simplify usage of any async backend within the ecosystem. A similar text is being added to chronos here: https://github.com/status-im/nim-chronos/pull/216 See also https://github.com/status-im/nim-chronos/issues/207 * typos, add none * Update lib/pure/asyncdispatch.nim * Update lib/pure/asyncdispatch.nim Co-authored-by: Andreas Rumpf --- lib/pure/asyncdispatch.nim | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 955e66f6b..d87e2360f 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -165,6 +165,34 @@ ## ================ ## ## * The effect system (`raises: []`) does not work with async procedures. +## +## +## Multiple async backend support +## ============================== +## +## Thanks to its powerful macro support, Nim allows ``async``/``await`` to be +## implemented in libraries with only minimal support from the language - as +## such, multiple ``async`` libraries exist, including ``asyncdispatch`` and +## ``chronos``, and more may come to be developed in the future. +## +## Libraries built on top of async/await may wish to support multiple async +## backends - the best way to do so is to create separate modules for each backend +## that may be imported side-by-side. +## +## An alternative way is to select backend using a global compile flag - this +## method makes it difficult to compose applications that use both backends as may +## happen with transitive dependencies, but may be appropriate in some cases - +## libraries choosing this path should call the flag `asyncBackend`, allowing +## applications to choose the backend with `-d:asyncBackend=`. +## +## Known `async` backends include: +## +## * `none` - ``-d:asyncBackend=none`` - disable ``async`` support completely +## * `asyncdispatch -``-d:asyncBackend=asyncdispatch`` +## * `chronos ` - ``-d:asyncBackend=chronos`` +## +## ``none`` can be used when a library supports both a synchronous and +## asynchronous API, to disable the latter. import os, tables, strutils, times, heapqueue, options, asyncstreams import options, math, std/monotimes From 06ff0e96249884f4ac6c49dadc96147be1c31097 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 3 Sep 2021 15:18:55 +0200 Subject: [PATCH 1190/2472] fixes #18769 (#18790) --- compiler/semgnrc.nim | 8 +++++++- tests/enum/toverloadable_enums.nim | 11 ++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/compiler/semgnrc.nim b/compiler/semgnrc.nim index 3df1e0c3f..e249d88e8 100644 --- a/compiler/semgnrc.nim +++ b/compiler/semgnrc.nim @@ -107,6 +107,12 @@ proc semGenericStmtSymbol(c: PContext, n: PNode, s: PSym, else: result = n onUse(n.info, s) + of skEnumField: + if overloadableEnums in c.features: + result = symChoice(c, n, s, scOpen) + else: + result = newSymNode(s, n.info) + onUse(n.info, s) else: result = newSymNode(s, n.info) onUse(n.info, s) @@ -407,7 +413,7 @@ proc semGenericStmt(c: PContext, n: PNode, a[^1] = semGenericStmt(c, a[^1], flags, ctx) for j in 0.. Date: Fri, 3 Sep 2021 22:41:00 +0800 Subject: [PATCH 1191/2472] remove unnecessary if statement (#18792) --- compiler/transf.nim | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/compiler/transf.nim b/compiler/transf.nim index 80794b581..3250f2ec4 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -434,12 +434,12 @@ proc transformYield(c: PTransf, n: PNode): PNode = else: # we need to introduce new local variables: result.add(introduceNewLocalVars(c, c.transCon.forLoopBody)) - if result.len > 0: - for idx in 0 ..< result.len: - var changeNode = result[idx] - changeNode.info = c.transCon.forStmt.info - for i, child in changeNode: - child.info = changeNode.info + + for idx in 0 ..< result.len: + var changeNode = result[idx] + changeNode.info = c.transCon.forStmt.info + for i, child in changeNode: + child.info = changeNode.info proc transformAddrDeref(c: PTransf, n: PNode, a, b: TNodeKind): PNode = result = transformSons(c, n) From c2b20516d33520b1d339b447ece32ade8625fefc Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 3 Sep 2021 17:31:16 +0200 Subject: [PATCH 1192/2472] implemented Unicode operators (#18789) * implemented Unicode operators; refs https://github.com/nim-lang/RFCs/issues/388 * bugfix * better test * arguably more elegant implementation * Update changelog.md Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- changelog.md | 7 +++ compiler/lexer.nim | 80 +++++++++++++++++++++++++++--- compiler/options.nim | 3 +- doc/manual.rst | 19 +++++++ tests/lexer/nim.cfg | 1 + tests/lexer/tunicode_operators.nim | 14 ++++++ 6 files changed, 117 insertions(+), 7 deletions(-) create mode 100644 tests/lexer/nim.cfg create mode 100644 tests/lexer/tunicode_operators.nim diff --git a/changelog.md b/changelog.md index 10824cfc6..bc4898c9d 100644 --- a/changelog.md +++ b/changelog.md @@ -445,6 +445,13 @@ proc mysort(s: seq; cmp: proc(a, b: T): int) {.effectsOf: cmp.} To enable the new effect system, use --experimental:strictEffects. +- Nim now supports a small subset of Unicode operators as operator symbols. + The supported symbols are: "∙ ∘ × ★ ⊗ ⊘ ⊙ ⊛ ⊠ ⊡ ∩ ∧ ⊓ ± ⊕ ⊖ ⊞ ⊟ ∪ ∨ ⊔". + To enable this feature, use `--experimental:unicodeOperators`. Note that due + to parser limitations you **cannot** enable this feature via a + pragma `{.experimental: "unicodeOperators".}` reliably. + + ## Compiler changes - Added `--declaredLocs` to show symbol declaration location in messages. diff --git a/compiler/lexer.nim b/compiler/lexer.nim index c7a9aa47f..ddf98661a 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -882,15 +882,66 @@ proc endOperator(L: var Lexer, tok: var Token, pos: int, else: tok.tokType = TokType(tok.ident.id - oprLow + ord(tkColon)) L.bufpos = pos +const + UnicodeOperatorStartChars = {'\226', '\194', '\195'} + # the allowed unicode characters ("∙ ∘ × ★ ⊗ ⊘ ⊙ ⊛ ⊠ ⊡ ∩ ∧ ⊓ ± ⊕ ⊖ ⊞ ⊟ ∪ ∨ ⊔") + # all start with one of these. + +type + UnicodeOprPred = enum + Mul, Add + +proc unicodeOprLen(buf: cstring; pos: int): (int8, UnicodeOprPred) = + template m(len): untyped = (int8(len), Mul) + template a(len): untyped = (int8(len), Add) + result = 0.m + case buf[pos] + of '\226': + if buf[pos+1] == '\136': + if buf[pos+2] == '\152': result = 3.m # ∘ + elif buf[pos+2] == '\153': result = 3.m # ∙ + elif buf[pos+2] == '\167': result = 3.m # ∧ + elif buf[pos+2] == '\168': result = 3.a # ∨ + elif buf[pos+2] == '\169': result = 3.m # ∩ + elif buf[pos+2] == '\170': result = 3.a # ∪ + elif buf[pos+1] == '\138': + if buf[pos+2] == '\147': result = 3.m # ⊓ + elif buf[pos+2] == '\148': result = 3.a # ⊔ + elif buf[pos+2] == '\149': result = 3.a # ⊕ + elif buf[pos+2] == '\150': result = 3.a # ⊖ + elif buf[pos+2] == '\151': result = 3.m # ⊗ + elif buf[pos+2] == '\152': result = 3.m # ⊘ + elif buf[pos+2] == '\153': result = 3.m # ⊙ + elif buf[pos+2] == '\155': result = 3.m # ⊛ + elif buf[pos+2] == '\158': result = 3.a # ⊞ + elif buf[pos+2] == '\159': result = 3.a # ⊟ + elif buf[pos+2] == '\160': result = 3.m # ⊠ + elif buf[pos+2] == '\161': result = 3.m # ⊡ + elif buf[pos+1] == '\152' and buf[pos+2] == '\133': result = 3.m # ★ + of '\194': + if buf[pos+1] == '\177': result = 2.a # ± + of '\195': + if buf[pos+1] == '\151': result = 2.m # × + else: + discard + proc getOperator(L: var Lexer, tok: var Token) = var pos = L.bufpos tokenBegin(tok, pos) var h: Hash = 0 while true: var c = L.buf[pos] - if c notin OpChars: break - h = h !& ord(c) - inc(pos) + if c in OpChars: + h = h !& ord(c) + inc(pos) + elif c in UnicodeOperatorStartChars and unicodeOperators in L.config.features: + let oprLen = unicodeOprLen(L.buf, pos)[0] + if oprLen == 0: break + for i in 0..', '!': result = 5 of '.': considerAsgn(6) of '?': result = 2 + of UnicodeOperatorStartChars: + if tok.ident.s[^1] == '=': + result = 1 + else: + let (len, pred) = unicodeOprLen(cstring(tok.ident.s), 0) + if len != 0: + result = if pred == Mul: MulPred else: PlusPred + else: + result = 2 else: considerAsgn(2) of tkDiv, tkMod, tkShl, tkShr: result = 9 of tkDotDot: result = 6 @@ -1167,10 +1230,15 @@ proc rawGetTok*(L: var Lexer, tok: var Token) = var c = L.buf[L.bufpos] tok.line = L.lineNumber tok.col = getColNumber(L, L.bufpos) - if c in SymStartChars - {'r', 'R'}: + if c in SymStartChars - {'r', 'R'} - UnicodeOperatorStartChars: getSymbol(L, tok) else: case c + of UnicodeOperatorStartChars: + if unicodeOperators in L.config.features and unicodeOprLen(L.buf, L.bufpos)[0] != 0: + getOperator(L, tok) + else: + getSymbol(L, tok) of '#': scanComment(L, tok) of '*': diff --git a/compiler/options.nim b/compiler/options.nim index 89a16a49c..ea302aed6 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -205,7 +205,8 @@ type views, strictNotNil, overloadableEnums, - strictEffects + strictEffects, + unicodeOperators LegacyFeature* = enum allowSemcheckedAstModification, diff --git a/doc/manual.rst b/doc/manual.rst index 02c688968..997253f16 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -700,6 +700,25 @@ contain a dot: `{..}` are the three tokens `{`:tok:, `..`:tok:, `}`:tok: and not the two tokens `{.`:tok:, `.}`:tok:. +Unicode Operators +----------------- + +Under the `--experimental:unicodeOperators` switch these Unicode operators are +also parsed as operators:: + + ∙ ∘ × ★ ⊗ ⊘ ⊙ ⊛ ⊠ ⊡ ∩ ∧ ⊓ # same priority as * (multiplication) + ± ⊕ ⊖ ⊞ ⊟ ∪ ∨ ⊔ # same priority as + (addition) + + +If enabled, Unicode operators can be combined with non-Unicode operator +symbols. The usual precedence extensions then apply, for example, `⊠=` is an +assignment like operator just like `*=` is. + +No Unicode normalization step is performed. + +**Note**: Due to parser limitations one **cannot** enable this feature via a +pragma `{.experimental: "unicodeOperators".}` reliably. + Syntax ====== diff --git a/tests/lexer/nim.cfg b/tests/lexer/nim.cfg new file mode 100644 index 000000000..f7a301a10 --- /dev/null +++ b/tests/lexer/nim.cfg @@ -0,0 +1 @@ +--experimental:unicodeOperators diff --git a/tests/lexer/tunicode_operators.nim b/tests/lexer/tunicode_operators.nim new file mode 100644 index 000000000..74fcbb763 --- /dev/null +++ b/tests/lexer/tunicode_operators.nim @@ -0,0 +1,14 @@ +#{.experimental: "unicodeOperators".} + +proc `⊙`(x, y: int): int = x * y +proc `⊙=`(x: var int, y: int) = x *= y + +proc `⊞++`(x, y: int): int = x + y + +var x = 45 +x ⊙= 9 ⊞++ 4 ⊙ 3 + +var y = 45 +y *= 9 + 4 * 3 + +assert x == y From cddf8ec6f684e5636a114e0e286bc6609e01f228 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 3 Sep 2021 21:52:24 +0200 Subject: [PATCH 1193/2472] implements https://github.com/nim-lang/RFCs/issues/407 (#18793) --- compiler/semdata.nim | 1 + compiler/semexprs.nim | 16 ++++++++++++---- compiler/semobjconstr.nim | 31 ++++++++++++++++-------------- compiler/sempass2.nim | 2 ++ compiler/semstmts.nim | 14 ++++++++++++++ compiler/sigmatch.nim | 14 ++++++++++++-- compiler/wordrecg.nim | 1 + doc/manual.rst | 35 ++++++++++++++++++++++++++++++++++ tests/objvariant/treassign.nim | 9 +++++++++ 9 files changed, 103 insertions(+), 20 deletions(-) diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 31affb24f..422d1223a 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -160,6 +160,7 @@ type importModuleMap*: Table[int, int] # (module.id, module.id) lastTLineInfo*: TLineInfo sideEffects*: Table[int, seq[(TLineInfo, PSym)]] # symbol.id index + inUncheckedAssignSection*: int template config*(c: PContext): ConfigRef = c.graph.config diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index f2fe54fad..3edeae324 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -691,8 +691,12 @@ proc newHiddenAddrTaken(c: PContext, n: PNode): PNode = else: result = newNodeIT(nkHiddenAddr, n.info, makeVarType(c, n.typ)) result.add n - if isAssignable(c, n) notin {arLValue, arLocalLValue}: - localError(c.config, n.info, errVarForOutParamNeededX % renderNotLValue(n)) + let aa = isAssignable(c, n) + if aa notin {arLValue, arLocalLValue}: + if aa == arDiscriminant and c.inUncheckedAssignSection > 0: + discard "allow access within a cast(unsafeAssign) section" + else: + localError(c.config, n.info, errVarForOutParamNeededX % renderNotLValue(n)) proc analyseIfAddressTaken(c: PContext, n: PNode): PNode = result = n @@ -738,9 +742,13 @@ proc analyseIfAddressTakenInCall(c: PContext, n: PNode) = if i < t.len and t[i] != nil and skipTypes(t[i], abstractInst-{tyTypeDesc}).kind in {tyVar}: let it = n[i] - if isAssignable(c, it) notin {arLValue, arLocalLValue}: + let aa = isAssignable(c, it) + if aa notin {arLValue, arLocalLValue}: if it.kind != nkHiddenAddr: - localError(c.config, it.info, errVarForOutParamNeededX % $it) + if aa == arDiscriminant and c.inUncheckedAssignSection > 0: + discard "allow access within a cast(unsafeAssign) section" + else: + localError(c.config, it.info, errVarForOutParamNeededX % $it) # bug #5113: disallow newSeq(result) where result is a 'var T': if n[0].sym.magic in {mNew, mNewFinalize, mNewSeq}: var arg = n[1] #.skipAddr diff --git a/compiler/semobjconstr.nim b/compiler/semobjconstr.nim index 1c97d32c8..1563e5c16 100644 --- a/compiler/semobjconstr.nim +++ b/compiler/semobjconstr.nim @@ -200,20 +200,22 @@ proc semConstructFields(c: PContext, n: PNode, if selectedBranch != -1: template badDiscriminatorError = - let fields = fieldsPresentInBranch(selectedBranch) - localError(c.config, constrCtx.initExpr.info, - ("cannot prove that it's safe to initialize $1 with " & - "the runtime value for the discriminator '$2' ") % - [fields, discriminator.sym.name.s]) + if c.inUncheckedAssignSection == 0: + let fields = fieldsPresentInBranch(selectedBranch) + localError(c.config, constrCtx.initExpr.info, + ("cannot prove that it's safe to initialize $1 with " & + "the runtime value for the discriminator '$2' ") % + [fields, discriminator.sym.name.s]) mergeInitStatus(result, initNone) template wrongBranchError(i) = - let fields = fieldsPresentInBranch(i) - localError(c.config, constrCtx.initExpr.info, - "a case selecting discriminator '$1' with value '$2' " & - "appears in the object construction, but the field(s) $3 " & - "are in conflict with this value." % - [discriminator.sym.name.s, discriminatorVal.renderTree, fields]) + if c.inUncheckedAssignSection == 0: + let fields = fieldsPresentInBranch(i) + localError(c.config, constrCtx.initExpr.info, + ("a case selecting discriminator '$1' with value '$2' " & + "appears in the object construction, but the field(s) $3 " & + "are in conflict with this value.") % + [discriminator.sym.name.s, discriminatorVal.renderTree, fields]) template valuesInConflictError(valsDiff) = localError(c.config, discriminatorVal.info, ("possible values " & @@ -251,9 +253,10 @@ proc semConstructFields(c: PContext, n: PNode, badDiscriminatorError() elif discriminatorVal.sym.kind notin {skLet, skParam} or discriminatorVal.sym.typ.kind in {tyVar}: - localError(c.config, discriminatorVal.info, - "runtime discriminator must be immutable if branch fields are " & - "initialized, a 'let' binding is required.") + if c.inUncheckedAssignSection == 0: + localError(c.config, discriminatorVal.info, + "runtime discriminator must be immutable if branch fields are " & + "initialized, a 'let' binding is required.") elif ctorCase[ctorIdx].kind == nkElifBranch: localError(c.config, discriminatorVal.info, "branch initialization " & "with a runtime discriminator is not supported inside of an " & diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index cae2f926c..2df6327b8 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -961,6 +961,8 @@ proc castBlock(tracked: PEffects, pragma: PNode, bc: var PragmaBlockContext) = else: bc.exc = newNodeI(nkArgList, pragma.info) bc.exc.add n + of wUncheckedAssign: + discard "handled in sempass1" else: localError(tracked.config, pragma.info, "invalid pragma block: " & $pragma) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index df0e2778b..2c5f3ba54 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -2223,7 +2223,21 @@ proc semPragmaBlock(c: PContext, n: PNode): PNode = checkSonsLen(n, 2, c.config) let pragmaList = n[0] pragma(c, nil, pragmaList, exprPragmas, isStatement = true) + + var inUncheckedAssignSection = 0 + for p in pragmaList: + if whichPragma(p) == wCast: + case whichPragma(p[1]) + of wGcSafe, wNoSideEffect, wTags, wRaises: + discard "handled in sempass2" + of wUncheckedAssign: + inUncheckedAssignSection = 1 + else: + localError(c.config, p.info, "invalid pragma block: " & $p) + + inc c.inUncheckedAssignSection, inUncheckedAssignSection n[1] = semExpr(c, n[1]) + dec c.inUncheckedAssignSection, inUncheckedAssignSection result = n result.typ = n[1].typ for i in 0.. 0 + else: + result = false + proc userConvMatch(c: PContext, m: var TCandidate, f, a: PType, arg: PNode): PNode = result = nil @@ -1895,7 +1905,7 @@ proc userConvMatch(c: PContext, m: var TCandidate, f, a: PType, let constraint = c.converters[i].typ.n[1].sym.constraint if not constraint.isNil and not matchNodeKinds(constraint, arg): continue - if src.kind in {tyVar, tyLent} and not arg.isLValue: + if src.kind in {tyVar, tyLent} and not isLValue(c, arg): continue let destIsGeneric = containsGenericType(dest) @@ -2338,7 +2348,7 @@ proc matchesAux(c: PContext, n, nOrig: PNode, m: var TCandidate, marker: var Int if argConverter.typ.kind notin {tyVar}: m.firstMismatch.kind = kVarNeeded noMatch() - elif not n.isLValue: + elif not isLValue(c, n): m.firstMismatch.kind = kVarNeeded noMatch() diff --git a/compiler/wordrecg.nim b/compiler/wordrecg.nim index 6ff68be75..4ba807518 100644 --- a/compiler/wordrecg.nim +++ b/compiler/wordrecg.nim @@ -36,6 +36,7 @@ type wMemTracker = "memtracker", wObjChecks = "objchecks", wIntDefine = "intdefine", wStrDefine = "strdefine", wBoolDefine = "booldefine", wCursor = "cursor", wNoalias = "noalias", wEffectsOf = "effectsOf", + wUncheckedAssign = "uncheckedAssign", wImmediate = "immediate", wConstructor = "constructor", wDestructor = "destructor", wDelegator = "delegator", wOverride = "override", wImportCpp = "importcpp", diff --git a/doc/manual.rst b/doc/manual.rst index 997253f16..714f98bdc 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -1854,6 +1854,41 @@ A small example: let unknownKindBounded = range[nkAdd..nkSub](unknownKind) z = Node(kind: unknownKindBounded, leftOp: Node(), rightOp: Node()) + +cast uncheckedAssign +-------------------- + +Some restrictions for case objects can be disabled via a `{.cast(unsafeAssign).}` section: + +.. code-block:: nim + :test: "nim c $1" + + type + TokenKind* = enum + strLit, intLit + Token = object + case kind*: TokenKind + of strLit: + s*: string + of intLit: + i*: int64 + + proc passToVar(x: var TokenKind) = discard + + var t = Token(kind: strLit, s: "abc") + + {.cast(uncheckedAssign).}: + # inside the 'cast' section it is allowed to pass 't.kind' to a 'var T' parameter: + passToVar(t.kind) + + # inside the 'cast' section it is allowed to set field 's' even though the + # constructed 'kind' field has an unknown value: + t = Token(kind: t.kind, s: "abc") + + # inside the 'cast' section it is allowed to assign to the 't.kind' field directly: + t.kind = intLit + + Set type -------- diff --git a/tests/objvariant/treassign.nim b/tests/objvariant/treassign.nim index 2938b30a3..9549cb29c 100644 --- a/tests/objvariant/treassign.nim +++ b/tests/objvariant/treassign.nim @@ -25,3 +25,12 @@ t.curr = TokenObject(kind: Token.bar, bar: BasicNumber(value: 12.34)) t.curr = TokenObject(kind: Token.foo, foo: "foo") echo "SUCCESS" + +proc passToVar(x: var Token) = discard + +{.cast(uncheckedAssign).}: + passToVar(t.curr.kind) + + t.curr = TokenObject(kind: t.curr.kind, foo: "abc") + + t.curr.kind = Token.foo From ac7acd827cd05ea916c53513e68d13be0d63a604 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 4 Sep 2021 08:18:00 +0200 Subject: [PATCH 1194/2472] =?UTF-8?q?we=20need=20something=20better=20than?= =?UTF-8?q?=20warningAsError=20for=20effect=20handling=20viol=E2=80=A6=20(?= =?UTF-8?q?#18796)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * we need something better than warningAsError for effect handling violations --- compiler/sempass2.nim | 4 ++-- lib/system.nim | 3 +-- nimdoc/test_out_index_dot_html/expected/index.html | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index 2df6327b8..6a0e8772a 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -1420,7 +1420,7 @@ proc trackProc*(c: PContext; s: PSym, body: PNode) = let p = s.ast[pragmasPos] let raisesSpec = effectSpec(p, wRaises) if not isNil(raisesSpec): - checkRaisesSpec(g, emitWarnings, raisesSpec, t.exc, "can raise an unlisted exception: ", + checkRaisesSpec(g, false, raisesSpec, t.exc, "can raise an unlisted exception: ", hints=on, subtypeRelation, hintsArg=s.ast[0]) # after the check, use the formal spec: effects[exceptionEffects] = raisesSpec @@ -1429,7 +1429,7 @@ proc trackProc*(c: PContext; s: PSym, body: PNode) = let tagsSpec = effectSpec(p, wTags) if not isNil(tagsSpec): - checkRaisesSpec(g, emitWarnings, tagsSpec, t.tags, "can have an unlisted effect: ", + checkRaisesSpec(g, false, tagsSpec, t.tags, "can have an unlisted effect: ", hints=off, subtypeRelation) # after the check, use the formal spec: effects[tagEffects] = tagsSpec diff --git a/lib/system.nim b/lib/system.nim index 740c4419f..c93af0854 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2004,8 +2004,7 @@ elif hasAlloc: inc(i) {.pop.} -proc echo*(x: varargs[typed, `$`]) {.magic: "Echo", tags: [WriteIOEffect], - benign, sideEffect.} +proc echo*(x: varargs[typed, `$`]) {.magic: "Echo", benign, sideEffect.} ## Writes and flushes the parameters to the standard output. ## ## Special built-in that takes a variable number of arguments. Each argument diff --git a/nimdoc/test_out_index_dot_html/expected/index.html b/nimdoc/test_out_index_dot_html/expected/index.html index 458a0394f..3499b5326 100644 --- a/nimdoc/test_out_index_dot_html/expected/index.html +++ b/nimdoc/test_out_index_dot_html/expected/index.html @@ -122,7 +122,7 @@ window.addEventListener('DOMContentLoaded', main);

    Procs

    -
    proc foo() {....raises: [], tags: [WriteIOEffect].}
    +
    proc foo() {....raises: [], tags: [].}
    I do foo From 686096a912107e620142e78ef98a380bb8a4764f Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 4 Sep 2021 12:52:24 +0200 Subject: [PATCH 1195/2472] documented overloadable enums and changelog improvements (#18797) --- changelog.md | 51 ++++++++++++++++++++++++++++---------------------- doc/manual.rst | 41 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 22 deletions(-) diff --git a/changelog.md b/changelog.md index bc4898c9d..d553864b6 100644 --- a/changelog.md +++ b/changelog.md @@ -114,16 +114,18 @@ ## Standard library additions and changes - `strformat`: - added support for parenthesized expressions. - added support for const string's instead of just string literals + - added support for parenthesized expressions. + - added support for const string's instead of just string literals +- `os.copyFile` is now 2.5x faster on OSX, by using `copyfile` from `copyfile.h`; + use `-d:nimLegacyCopyFile` for OSX < 10.5. - `system.addFloat` and `system.$` now can produce string representations of floating point numbers that are minimal in size and that "roundtrip" (via the "Dragonbox" algorithm). This currently has to be enabled via `-d:nimPreviewFloatRoundtrip`. It is expected that this behavior becomes the new default in upcoming versions, as with other `nimPreviewX` define flags. -- Fixed buffer overflow bugs in `net` +- Fixed buffer overflow bugs in `net`. - Exported `sslHandle` from `net` and `asyncnet`. @@ -187,10 +189,11 @@ - Added `std/sysrand` module to get random numbers from a secure source provided by the operating system. -- Added `std/enumutils` module. Added `genEnumCaseStmt` macro that generates case statement to parse string to enum. - Added `items` for enums with holes. - Added `symbolName` to return the enum symbol name ignoring the human readable name. - Added `symbolRank` to return the index in which an enum member is listed in an enum. +- Added `std/enumutils` module. Added `genEnumCaseStmt` macro that generates + case statement to parse string to enum. +- Added `items` for enums with holes. +- Added `symbolName` to return the enum symbol name ignoring the human readable name. +- Added `symbolRank` to return the index in which an enum member is listed in an enum. - Removed deprecated `iup` module from stdlib, it has already moved to [nimble](https://github.com/nim-lang/iup). @@ -381,15 +384,10 @@ ## Language changes -- `nimscript` now handles `except Exception as e`. - - The `cstring` doesn't support `[]=` operator in JS backend. - nil dereference is not allowed at compile time. `cast[ptr int](nil)[]` is rejected at compile time. -- `os.copyFile` is now 2.5x faster on OSX, by using `copyfile` from `copyfile.h`; - use `-d:nimLegacyCopyFile` for OSX < 10.5. - - The required name of case statement macros for the experimental `caseStmtMacros` feature has changed from `match` to `` `case` ``. @@ -403,18 +401,11 @@ - Tuple expressions are now parsed consistently as `nnkTupleConstr` node. Will affect macros expecting nodes to be of `nnkPar`. -- `nim e` now accepts arbitrary file extensions for the nimscript file, - although `.nims` is still the preferred extension in general. - - Added `iterable[T]` type class to match called iterators, which enables writing: `template fn(a: iterable)` instead of `template fn(a: untyped)` - A new import syntax `import foo {.all.}` now allows to import all symbols (public or private) - from `foo`. It works in combination with all pre-existing import features. - This reduces or eliminates the need for workarounds such as using `include` (which has known issues) - when you need a private symbol for testing or making some internal APIs public just because - another internal module needs those. - It also helps mitigate the lack of cyclic imports in some cases. + from `foo`. This can be useful for testing purposes. - Added a new module `std/importutils`, and an API `privateAccess`, which allows access to private fields for an object type in the current scope. @@ -422,7 +413,7 @@ - `typeof(voidStmt)` now works and returns `void`. - The `gc:orc` algorithm was refined so that custom container types can participate in the - cycle collection process. + cycle collection process. See the documentation of `=trace` for more details. - On embedded devices `malloc` can now be used instead of `mmap` via `-d:nimAllocPagesViaMalloc`. This is only supported for `--gc:orc` or `--gc:arc`. @@ -449,7 +440,18 @@ proc mysort(s: seq; cmp: proc(a, b: T): int) {.effectsOf: cmp.} The supported symbols are: "∙ ∘ × ★ ⊗ ⊘ ⊙ ⊛ ⊠ ⊡ ∩ ∧ ⊓ ± ⊕ ⊖ ⊞ ⊟ ∪ ∨ ⊔". To enable this feature, use `--experimental:unicodeOperators`. Note that due to parser limitations you **cannot** enable this feature via a - pragma `{.experimental: "unicodeOperators".}` reliably. + pragma `{.experimental: "unicodeOperators".}` reliably, you need to enable + it via the command line or in a configuration file. + +- There is a new `cast` section `{.cast(uncheckedAssign).}: body` that disables some + compiler checks regarding `case objects`. This allows serialization libraries + to avoid ugly, non-portable solutions. See https://github.com/nim-lang/RFCs/issues/407 + for more details. + +- Enum values can now be overloaded. This needs to be enabled + via `{.experimental: "overloadableEnums".}`. We hope that this feature allows for the + development of more fluent (less ugly) APIs. See https://github.com/nim-lang/RFCs/issues/373 + for more details. ## Compiler changes @@ -529,6 +531,11 @@ proc mysort(s: seq; cmp: proc(a, b: T): int) {.effectsOf: cmp.} ## Tool changes +- `nimscript` now handles `except Exception as e`. + +- `nim e` now accepts arbitrary file extensions for the nimscript file, + although `.nims` is still the preferred extension in general. + - Latex doc generation is revised: output `.tex` files should be compiled by `xelatex` (not by `pdflatex` as before). Now default Latex settings provide support for Unicode and do better job for avoiding margin overflows. diff --git a/doc/manual.rst b/doc/manual.rst index 714f98bdc..e9c730c7b 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -1323,6 +1323,47 @@ as `MyEnum.value`: To implement bit fields with enums see `Bit fields <#set-type-bit-fields>`_ +Overloadable enum field names +----------------------------- + +To be enabled via `{.experimental: "overloadableEnums".}`. + +Enum field names are overloadable much like routines. When an overloaded +enum field is used, it produces a closed sym choice construct, here +written as `(E|E)`. +During overload resolution the right `E` is picked, if possible. +For (array/object...) constructors the right `E` is picked, comparable to +how `[byte(1), 2, 3]` works, one needs to use `[T.E, E2, E3]`. Ambiguous +enum fields produce a static error: + +.. code-block:: nim + :test: "nim c $1" + + {.experimental: "overloadableEnums".} + + type + E1 = enum + value1, + value2 + E2 = enum + value1, + value2 = 4 + + const + Lookuptable = [ + E1.value1: "1", + value2: "2" + ] + + proc p(e: E1) = + # disambiguation in 'case' statements: + case e + of value1: echo "A" + of value2: echo "B" + + p value2 + + String type ----------- All string literals are of the type `string`. A string in Nim is very From 3c2edd142b527c34163cbd7ea9ad94518f46e9ae Mon Sep 17 00:00:00 2001 From: Sven Keller Date: Sat, 4 Sep 2021 12:53:02 +0200 Subject: [PATCH 1196/2472] fix for js strict mode (#18799) * Assignments, which would accidentally create global variables, instead throw an error in strict mode * Assignment to a getter-only property Co-authored-by: Sven Keller --- compiler/jsgen.nim | 2 +- lib/system/jssys.nim | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 931e56475..f009c83f2 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -1769,7 +1769,7 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) = inc p.extraIndent elif useGlobalPragmas: lineF(p, "if (globalThis.$1 === undefined) {$n", varName) - varCode = $varName + varCode = "globalThis." & $varName inc p.extraIndent else: varCode = "var $2" diff --git a/lib/system/jssys.nim b/lib/system/jssys.nim index 63e7b3d9d..cd4f378be 100644 --- a/lib/system/jssys.nim +++ b/lib/system/jssys.nim @@ -593,12 +593,9 @@ proc nimCopy(dest, src: JSRef, ti: PNimType): JSRef = `result` = null; } else { - if (`dest` === null || `dest` === undefined) { + if (`dest` === null || `dest` === undefined || `dest`.length != `src`.length) { `dest` = new Array(`src`.length); } - else { - `dest`.length = `src`.length; - } `result` = `dest`; for (var i = 0; i < `src`.length; ++i) { `result`[i] = nimCopy(`result`[i], `src`[i], `ti`.base); From b3ad68edea72f04275c9b1e330cafb77254d945c Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 4 Sep 2021 21:57:02 +0800 Subject: [PATCH 1197/2472] remove channels (#18801) * remove channels * test --- changelog.md | 2 - doc/manual.rst | 2 +- lib/std/channels.nim | 498 ----------------------------- tests/stdlib/tchannels.nim | 33 -- tests/stdlib/tchannels_pthread.nim | 322 ------------------- tests/stdlib/tchannels_simple.nim | 67 ---- 6 files changed, 1 insertion(+), 923 deletions(-) delete mode 100644 lib/std/channels.nim delete mode 100644 tests/stdlib/tchannels.nim delete mode 100644 tests/stdlib/tchannels_pthread.nim delete mode 100644 tests/stdlib/tchannels_simple.nim diff --git a/changelog.md b/changelog.md index d553864b6..3658fb105 100644 --- a/changelog.md +++ b/changelog.md @@ -341,8 +341,6 @@ - Added `jscore.debugger` to [call any available debugging functionality, such as breakpoints.](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/debugger). -- Added `std/channels`. - - Added `htmlgen.portal` for [making "SPA style" pages using HTML only](https://web.dev/hands-on-portals). - `std/times`: diff --git a/doc/manual.rst b/doc/manual.rst index e9c730c7b..7976ffe0f 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -8016,7 +8016,7 @@ Threads To enable thread support the `--threads:on`:option: command-line switch needs to be used. The system_ module then contains several threading primitives. -See the `threads `_ and `channels `_ modules +See the `channels `_ modules for the low-level thread API. There are also high-level parallelism constructs available. See `spawn `_ for further details. diff --git a/lib/std/channels.nim b/lib/std/channels.nim deleted file mode 100644 index 5ff141384..000000000 --- a/lib/std/channels.nim +++ /dev/null @@ -1,498 +0,0 @@ -# -# -# The Nim Compiler -# (c) Copyright 2021 Andreas Prell, Mamy André-Ratsimbazafy & Nim Contributors -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - - -# Based on https://github.com/mratsim/weave/blob/5696d94e6358711e840f8c0b7c684fcc5cbd4472/unused/channels/channels_legacy.nim -# Those are translations of @aprell (Andreas Prell) original channels from C to Nim -# (https://github.com/aprell/tasking-2.0/blob/master/src/channel_shm/channel.c) -# And in turn they are an implementation of Michael & Scott lock-based queues -# (note the paper has 2 channels: lock-free and lock-based) with additional caching: -# Simple, Fast, and Practical Non-Blocking and Blocking Concurrent Queue Algorithms -# Maged M. Michael, Michael L. Scott, 1996 -# https://www.cs.rochester.edu/~scott/papers/1996_PODC_queues.pdf - -## This module only works with `--gc:arc` or `--gc:orc`. -## -## .. warning:: This module is experimental and its interface may change. -## -## The following is a simple example of two different ways to use channels: -## blocking and non-blocking. -## - -runnableExamples("--threads:on --gc:orc"): - import std/os - - # In this example a channel is declared at module scope. - # Channels are generic, and they include support for passing objects between - # threads. - # Note that isolated data passed through channels is moved around. - var chan = newChannel[string]() - - # This proc will be run in another thread using the threads module. - proc firstWorker() = - chan.send("Hello World!") - - # This is another proc to run in a background thread. This proc takes a while - # to send the message since it sleeps for 2 seconds (or 2000 milliseconds). - proc secondWorker() = - sleep(2000) - chan.send("Another message") - - # Launch the worker. - var worker1: Thread[void] - createThread(worker1, firstWorker) - - # Block until the message arrives, then print it out. - let dest = chan.recv() - assert dest == "Hello World!" - - # Wait for the thread to exit before moving on to the next example. - worker1.joinThread() - - # Launch the other worker. - var worker2: Thread[void] - createThread(worker2, secondWorker) - # This time, use a non-blocking approach with tryRecv. - # Since the main thread is not blocked, it could be used to perform other - # useful work while it waits for data to arrive on the channel. - var messages: seq[string] - while true: - var msg = "" - if chan.tryRecv(msg): - messages.add msg # "Another message" - break - - messages.add "Pretend I'm doing useful work..." - # For this example, sleep in order not to flood stdout with the above - # message. - sleep(400) - - # Wait for the second thread to exit before cleaning up the channel. - worker2.joinThread() - - # Clean up the channel. - assert chan.close() - - assert messages[^1] == "Another message" - assert messages.len >= 2 - - -when not defined(gcArc) and not defined(gcOrc) and not defined(nimdoc): - {.error: "This channel implementation requires --gc:arc or --gc:orc".} - -import std/[locks, atomics, isolation] -import system/ansi_c - -# Channel (Shared memory channels) -# ---------------------------------------------------------------------------------- - -const - cacheLineSize {.intdefine.} = 64 # TODO: some Samsung phone have 128 cache-line - nimChannelCacheSize* {.intdefine.} = 100 - -type - ChannelRaw = ptr ChannelObj - ChannelObj = object - headLock, tailLock: Lock - notFullCond, notEmptyCond: Cond - closed: Atomic[bool] - size: int - itemsize: int # up to itemsize bytes can be exchanged over this channel - head {.align: cacheLineSize.} : int # Items are taken from head and new items are inserted at tail - tail: int - buffer: ptr UncheckedArray[byte] - atomicCounter: Atomic[int] - - ChannelCache = ptr ChannelCacheObj - ChannelCacheObj = object - next: ChannelCache - chanSize: int - chanN: int - numCached: int - cache: array[nimChannelCacheSize, ChannelRaw] - -# ---------------------------------------------------------------------------------- - -proc numItems(chan: ChannelRaw): int {.inline.} = - result = chan.tail - chan.head - if result < 0: - inc(result, 2 * chan.size) - - assert result <= chan.size - -template isFull(chan: ChannelRaw): bool = - abs(chan.tail - chan.head) == chan.size - -template isEmpty(chan: ChannelRaw): bool = - chan.head == chan.tail - -# Unbuffered / synchronous channels -# ---------------------------------------------------------------------------------- - -template numItemsUnbuf(chan: ChannelRaw): int = - chan.head - -template isFullUnbuf(chan: ChannelRaw): bool = - chan.head == 1 - -template isEmptyUnbuf(chan: ChannelRaw): bool = - chan.head == 0 - -# ChannelRaw kinds -# ---------------------------------------------------------------------------------- - -func isUnbuffered(chan: ChannelRaw): bool = - chan.size - 1 == 0 - -# ChannelRaw status and properties -# ---------------------------------------------------------------------------------- - -proc isClosed(chan: ChannelRaw): bool {.inline.} = load(chan.closed, moRelaxed) - -proc peek(chan: ChannelRaw): int {.inline.} = - (if chan.isUnbuffered: numItemsUnbuf(chan) else: numItems(chan)) - -# Per-thread channel cache -# ---------------------------------------------------------------------------------- - -var channelCache {.threadvar.}: ChannelCache -var channelCacheLen {.threadvar.}: int - -proc allocChannelCache(size, n: int): bool = - ## Allocate a free list for storing channels of a given type - var p = channelCache - - # Avoid multiple free lists for the exact same type of channel - while not p.isNil: - if size == p.chanSize and n == p.chanN: - return false - p = p.next - - p = cast[ptr ChannelCacheObj](c_malloc(csize_t sizeof(ChannelCacheObj))) - if p.isNil: - raise newException(OutOfMemDefect, "Could not allocate memory") - - p.chanSize = size - p.chanN = n - p.numCached = 0 - - p.next = channelCache - channelCache = p - inc channelCacheLen - result = true - -proc freeChannelCache*() = - ## Frees the entire channel cache, including all channels - var p = channelCache - var q: ChannelCache - - while not p.isNil: - q = p.next - for i in 0 ..< p.numCached: - let chan = p.cache[i] - if not chan.buffer.isNil: - c_free(chan.buffer) - deinitLock(chan.headLock) - deinitLock(chan.tailLock) - deinitCond(chan.notFullCond) - deinitCond(chan.notEmptyCond) - c_free(chan) - c_free(p) - dec channelCacheLen - p = q - - assert(channelCacheLen == 0) - channelCache = nil - -# Channels memory ops -# ---------------------------------------------------------------------------------- - -proc allocChannel(size, n: int): ChannelRaw = - when nimChannelCacheSize > 0: - var p = channelCache - - while not p.isNil: - if size == p.chanSize and n == p.chanN: - # Check if free list contains channel - if p.numCached > 0: - dec p.numCached - result = p.cache[p.numCached] - assert(result.isEmpty) - return - else: - # All the other lists in cache won't match - break - p = p.next - - result = cast[ChannelRaw](c_malloc(csize_t sizeof(ChannelObj))) - if result.isNil: - raise newException(OutOfMemDefect, "Could not allocate memory") - - # To buffer n items, we allocate for n - result.buffer = cast[ptr UncheckedArray[byte]](c_malloc(csize_t n*size)) - if result.buffer.isNil: - raise newException(OutOfMemDefect, "Could not allocate memory") - - initLock(result.headLock) - initLock(result.tailLock) - initCond(result.notFullCond) - initCond(result.notEmptyCond) - - result.closed.store(false, moRelaxed) # We don't need atomic here, how to? - result.size = n - result.itemsize = size - result.head = 0 - result.tail = 0 - result.atomicCounter.store(0, moRelaxed) - - when nimChannelCacheSize > 0: - # Allocate a cache as well if one of the proper size doesn't exist - discard allocChannelCache(size, n) - -proc freeChannel(chan: ChannelRaw) = - if chan.isNil: - return - - when nimChannelCacheSize > 0: - var p = channelCache - while not p.isNil: - if chan.itemsize == p.chanSize and - chan.size == p.chanN: - if p.numCached < nimChannelCacheSize: - # If space left in cache, cache it - p.cache[p.numCached] = chan - inc p.numCached - return - else: - # All the other lists in cache won't match - break - p = p.next - - if not chan.buffer.isNil: - c_free(chan.buffer) - - deinitLock(chan.headLock) - deinitLock(chan.tailLock) - deinitCond(chan.notFullCond) - deinitCond(chan.notEmptyCond) - - c_free(chan) - -# MPMC Channels (Multi-Producer Multi-Consumer) -# ---------------------------------------------------------------------------------- - -proc sendUnbufferedMpmc(chan: ChannelRaw, data: sink pointer, size: int, nonBlocking: bool): bool = - if nonBlocking and chan.isFullUnbuf: - return false - - acquire(chan.headLock) - - if nonBlocking and chan.isFullUnbuf: - # Another thread was faster - release(chan.headLock) - return false - - while chan.isFullUnbuf: - wait(chan.notFullcond, chan.headLock) - - assert chan.isEmptyUnbuf - assert size <= chan.itemsize - copyMem(chan.buffer, data, size) - - chan.head = 1 - - signal(chan.notEmptyCond) - release(chan.headLock) - result = true - -proc sendMpmc(chan: ChannelRaw, data: sink pointer, size: int, nonBlocking: bool): bool = - assert not chan.isNil - assert not data.isNil - - if isUnbuffered(chan): - return sendUnbufferedMpmc(chan, data, size, nonBlocking) - - if nonBlocking and chan.isFull: - return false - - acquire(chan.tailLock) - - if nonBlocking and chan.isFull: - # Another thread was faster - release(chan.tailLock) - return false - - while chan.isFull: - wait(chan.notFullcond, chan.tailLock) - - assert not chan.isFull - assert size <= chan.itemsize - - let writeIdx = if chan.tail < chan.size: chan.tail - else: chan.tail - chan.size - - copyMem(chan.buffer[writeIdx * chan.itemsize].addr, data, size) - - inc chan.tail - if chan.tail == 2 * chan.size: - chan.tail = 0 - - signal(chan.notEmptyCond) - release(chan.tailLock) - result = true - -proc recvUnbufferedMpmc(chan: ChannelRaw, data: pointer, size: int, nonBlocking: bool): bool = - if nonBlocking and chan.isEmptyUnbuf: - return false - - acquire(chan.headLock) - - if nonBlocking and chan.isEmptyUnbuf: - # Another thread was faster - release(chan.headLock) - return false - - while chan.isEmptyUnbuf: - wait(chan.notEmptyCond, chan.headLock) - - assert chan.isFullUnbuf - assert size <= chan.itemsize - - copyMem(data, chan.buffer, size) - - chan.head = 0 - - signal(chan.notFullCond) - release(chan.headLock) - result = true - -proc recvMpmc(chan: ChannelRaw, data: pointer, size: int, nonBlocking: bool): bool = - assert not chan.isNil - assert not data.isNil - - if isUnbuffered(chan): - return recvUnbufferedMpmc(chan, data, size, nonBlocking) - - if nonBlocking and chan.isEmpty: - return false - - acquire(chan.headLock) - - if nonBlocking and chan.isEmpty: - # Another thread took the last data - release(chan.headLock) - return false - - while chan.isEmpty: - wait(chan.notEmptyCond, chan.headLock) - - assert not chan.isEmpty - assert size <= chan.itemsize - - let readIdx = if chan.head < chan.size: chan.head - else: chan.head - chan.size - - copyMem(data, chan.buffer[readIdx * chan.itemsize].addr, size) - - inc chan.head - if chan.head == 2 * chan.size: - chan.head = 0 - - signal(chan.notFullCond) - release(chan.headLock) - result = true - -proc channelCloseMpmc(chan: ChannelRaw): bool = - # Unsynchronized - - if chan.isClosed: - # ChannelRaw already closed - return false - - store(chan.closed, true, moRelaxed) - result = true - -proc channelOpenMpmc(chan: ChannelRaw): bool = - # Unsynchronized - - if not chan.isClosed: - # ChannelRaw already open - return false - - store(chan.closed, false, moRelaxed) - result = true - -# Public API -# ---------------------------------------------------------------------------------- - -type - Channel*[T] = object ## Typed channels - d: ChannelRaw - -proc `=destroy`*[T](c: var Channel[T]) = - if c.d != nil: - if load(c.d.atomicCounter, moAcquire) == 0: - if c.d.buffer != nil: - freeChannel(c.d) - else: - atomicDec(c.d.atomicCounter) - -proc `=copy`*[T](dest: var Channel[T], src: Channel[T]) = - ## Shares `Channel` by reference counting. - if src.d != nil: - atomicInc(src.d.atomicCounter) - - if dest.d != nil: - `=destroy`(dest) - dest.d = src.d - -func trySend*[T](c: Channel[T], src: var Isolated[T]): bool {.inline.} = - ## Sends item to the channel(non blocking). - var data = src.extract - result = sendMpmc(c.d, data.addr, sizeof(T), true) - if result: - wasMoved(data) - -template trySend*[T](c: Channel[T], src: T): bool = - ## Helper templates for `trySend`. - trySend(c, isolate(src)) - -func tryRecv*[T](c: Channel[T], dst: var T): bool {.inline.} = - ## Receives item from the channel(non blocking). - recvMpmc(c.d, dst.addr, sizeof(T), true) - -func send*[T](c: Channel[T], src: sink Isolated[T]) {.inline.} = - ## Sends item to the channel(blocking). - var data = src.extract - discard sendMpmc(c.d, data.addr, sizeof(T), false) - wasMoved(data) - -template send*[T](c: Channel[T]; src: T) = - ## Helper templates for `send`. - send(c, isolate(src)) - -func recv*[T](c: Channel[T]): T {.inline.} = - ## Receives item from the channel(blocking). - discard recvMpmc(c.d, result.addr, sizeof(result), false) - -func open*[T](c: Channel[T]): bool {.inline.} = - result = c.d.channelOpenMpmc() - -func close*[T](c: Channel[T]): bool {.inline.} = - result = c.d.channelCloseMpmc() - -func peek*[T](c: Channel[T]): int {.inline.} = peek(c.d) - -proc newChannel*[T](elements = 30): Channel[T] = - ## Returns a new `Channel`. `elements` should be positive. - ## `elements` is used to specify whether a channel is buffered or not. - ## If `elements` = 1, the channel is unbuffered. If `elements` > 1, the - ## channel is buffered. - assert elements >= 1, "Elements must be positive!" - result = Channel[T](d: allocChannel(sizeof(T), elements)) diff --git a/tests/stdlib/tchannels.nim b/tests/stdlib/tchannels.nim deleted file mode 100644 index 492faf500..000000000 --- a/tests/stdlib/tchannels.nim +++ /dev/null @@ -1,33 +0,0 @@ -discard """ - timeout: 60.0 # but typically < 1s (in isolation but other tests running in parallel can affect this since based on epochTime) - disabled: "freebsd" - matrix: "--gc:arc --threads:on; --gc:arc --threads:on -d:danger" -""" - -when true: - # bug #17380: this was either blocking (without -d:danger) or crashing with SIGSEGV (with -d:danger) - import std/[channels, isolation] - const - N1 = 10 - N2 = 100 - var - sender: array[N1, Thread[void]] - receiver: array[5, Thread[void]] - - var chan = newChannel[seq[string]](N1 * N2) # large enough to not block - proc sendHandler() = - chan.send(isolate(@["Hello, Nim"])) - proc recvHandler() = - template fn = - let x = chan.recv() - fn() - - template benchmark() = - for t in mitems(sender): - t.createThread(sendHandler) - joinThreads(sender) - for t in mitems(receiver): - t.createThread(recvHandler) - joinThreads(receiver) - for i in 0.. 0 - -when defined(blockingTest): - const nonBlocking = false -else: - const nonBlocking = true - -type - Pthread {.importc: "pthread_t", header: "".} = distinct culong - PthreadAttr* {.byref, importc: "pthread_attr_t", header: "".} = object - Errno* = distinct cint - -proc pthread_create[T]( - thread: var Pthread, - attr: ptr PthreadAttr, # In Nim this is a var and how Nim sets a custom stack - fn: proc (x: ptr T): pointer {.thread, noconv.}, - arg: ptr T - ): Errno {.header: "".} - -proc pthread_join( - thread: Pthread, - thread_exit_status: ptr pointer - ): Errno {.header: "".} - -template channel_send_loop(chan: ChannelRaw, - data: sink pointer, - size: int, - body: untyped): untyped = - while not sendMpmc(chan, data, size, nonBlocking): - body - -template channel_receive_loop(chan: ChannelRaw, - data: pointer, - size: int, - body: untyped): untyped = - while not recvMpmc(chan, data, size, nonBlocking): - body - - -# Without threads:on or release, -# worker threads will crash on popFrame - -import std/unittest - -type ThreadArgs = object - ID: int - chan: ChannelRaw - -template Worker(id: int, body: untyped): untyped {.dirty.} = - if args.ID == id: - body - - -const Sender = 1 -const Receiver = 0 - -proc runSuite( - name: string, - fn: proc(args: ptr ThreadArgs): pointer {.noconv, gcsafe.} - ) = - var chan: ChannelRaw - - for i in Unbuffered .. Buffered: - if i == Unbuffered: - chan = allocChannel(size = 32, n = 1) - check: - peek(chan) == 0 - capacity(chan) == 1 - isBuffered(chan) == false - isUnbuffered(chan) == true - - else: - chan = allocChannel(size = int.sizeof.int, n = 7) - check: - peek(chan) == 0 - capacity(chan) == 7 - isBuffered(chan) == true - isUnbuffered(chan) == false - - var threads: array[2, Pthread] - var args = [ - ThreadArgs(ID: 0, chan: chan), - ThreadArgs(ID: 1, chan: chan) - ] - - discard pthread_create(threads[0], nil, fn, args[0].addr) - discard pthread_create(threads[1], nil, fn, args[1].addr) - - discard pthread_join(threads[0], nil) - discard pthread_join(threads[1], nil) - - freeChannel(chan) - -# ---------------------------------------------------------------------------------- - -proc thread_func(args: ptr ThreadArgs): pointer {.noconv.} = - - # Worker RECEIVER: - # --------- - # <- chan - # <- chan - # <- chan - # - # Worker SENDER: - # --------- - # chan <- 42 - # chan <- 53 - # chan <- 64 - # - - Worker(Receiver): - var val: int - for j in 0 ..< 3: - channel_receive_loop(args.chan, val.addr, val.sizeof.int): - # Busy loop, normally it should yield - discard - check: val == 42 + j*11 - - Worker(Sender): - var val: int - check: peek(args.chan) == 0 - for j in 0 ..< 3: - val = 42 + j*11 - channel_send_loop(args.chan, val.addr, val.sizeof.int): - # Busy loop, normally it should yield - discard - - return nil - -runSuite("[ChannelRaw] 2 threads can send data", thread_func) - -# ---------------------------------------------------------------------------------- - -iterator pairs(chan: ChannelRaw, T: typedesc): (int, T) = - var i = 0 - var x: T - while not isClosed(chan) or peek(chan) > 0: - let r = recvMpmc(chan, x.addr, x.sizeof.int, true) - # printf("x: %d, r: %d\n", x, r) - if r: - yield (i, x) - inc i - -proc thread_func_2(args: ptr ThreadArgs): pointer {.noconv.} = - # Worker RECEIVER: - # --------- - # <- chan until closed and empty - # - # Worker SENDER: - # --------- - # chan <- 42, 53, 64, ... - - const N = 100 - - Worker(Receiver): - for j, val in pairs(args.chan, int): - # TODO: Need special handling that doesn't allocate - # in thread with no GC - # when check fails - # - check: val == 42 + j*11 - - Worker(Sender): - var val: int - check: peek(args.chan) == 0 - for j in 0 ..< N: - val = 42 + j*11 - channel_send_loop(args.chan, val.addr, int.sizeof.int): - discard - discard channelCloseMpmc(args.chan) - - return nil - -runSuite("[ChannelRaw] channel_close, freeChannel, channelCache", thread_func_2) - -# ---------------------------------------------------------------------------------- - -proc isCached(chan: ChannelRaw): bool = - assert not chan.isNil - - var p = channelCache - while not p.isNil: - if chan.itemsize == p.chanSize and - chan.size == p.chanN: - for i in 0 ..< p.numCached: - if chan == p.cache[i]: - return true - # No more channel in cache can match - return false - p = p.next - return false - -block: # [ChannelRaw] ChannelRaw caching implementation - - # Start from clean cache slate - freeChannelCache() - - block: # Explicit caches allocation - check: - allocChannelCache(int sizeof(char), 4) - allocChannelCache(int sizeof(int), 8) - allocChannelCache(int sizeof(ptr float64), 16) - - # Don't create existing channel cache - not allocChannelCache(int sizeof(char), 4) - not allocChannelCache(int sizeof(int), 8) - not allocChannelCache(int sizeof(ptr float64), 16) - - check: - channelCacheLen == 3 - - # --------------------------------- - var chan, stash: array[10, ChannelRaw] - - block: # Implicit caches allocation - - chan[0] = allocChannel(sizeof(char), 4) - chan[1] = allocChannel(sizeof(int32), 8) - chan[2] = allocChannel(sizeof(ptr float64), 16) - - chan[3] = allocChannel(sizeof(char), 5) - chan[4] = allocChannel(sizeof(int64), 8) - chan[5] = allocChannel(sizeof(ptr float32), 24) - - # We have caches ready to store specific channel kinds - check: channelCacheLen == 6 # Cumulated with previous test - # But they are not in cache while in use - check: - not chan[0].isCached - not chan[1].isCached - not chan[2].isCached - not chan[3].isCached - not chan[4].isCached - not chan[5].isCached - - block: # Freed channels are returned to cache - stash[0..5] = chan.toOpenArray(0, 5) - for i in 0 .. 5: - # Free the channels - freeChannel(chan[i]) - - check: - stash[0].isCached - stash[1].isCached - stash[2].isCached - stash[3].isCached - stash[4].isCached - stash[5].isCached - - block: # Cached channels are being reused - - chan[6] = allocChannel(sizeof(char), 4) - chan[7] = allocChannel(sizeof(int32), 8) - chan[8] = allocChannel(sizeof(ptr float32), 16) - chan[9] = allocChannel(sizeof(ptr float64), 16) - - # All (itemsize, queue size, implementation) were already allocated - check: channelCacheLen == 6 - - # We reused old channels from cache - check: - chan[6] == stash[0] - chan[7] == stash[1] - chan[8] == stash[2] - # chan[9] - required a fresh alloc - - block: # Clearing the cache - - stash[6..9] = chan.toOpenArray(6, 9) - - for i in 6 .. 9: - freeChannel(chan[i]) - - check: - stash[6].isCached - stash[7].isCached - stash[8].isCached - stash[9].isCached - - freeChannelCache() - - # Check that nothing is cached anymore - for i in 0 .. 9: - check: not stash[i].isCached - # And length is reset to 0 - check: channelCacheLen == 0 - - # Cache can grow again - chan[0] = allocChannel(sizeof((int, float, int32, uint)), 1) - chan[1] = allocChannel(sizeof(int32), 0) - chan[2] = allocChannel(sizeof(int32), 0) - - check: channelCacheLen == 2 - - # Interleave cache clear and channel free - freeChannelCache() - check: channelCacheLen == 0 - - freeChannel(chan[0]) - freeChannel(chan[1]) - freeChannel(chan[2]) diff --git a/tests/stdlib/tchannels_simple.nim b/tests/stdlib/tchannels_simple.nim deleted file mode 100644 index 56e5fb8f1..000000000 --- a/tests/stdlib/tchannels_simple.nim +++ /dev/null @@ -1,67 +0,0 @@ -discard """ - matrix: "--threads:on --gc:orc; --threads:on --gc:arc" - disabled: "freebsd" -""" - -import std/channels -import std/os - -var chan = newChannel[string]() - -# This proc will be run in another thread using the threads module. -proc firstWorker() = - chan.send("Hello World!") - -# This is another proc to run in a background thread. This proc takes a while -# to send the message since it sleeps for 2 seconds (or 2000 milliseconds). -proc secondWorker() = - sleep(2000) - chan.send("Another message") - - -# Launch the worker. -var worker1: Thread[void] -createThread(worker1, firstWorker) - -# Block until the message arrives, then print it out. -let dest = chan.recv() -doAssert dest == "Hello World!" - -# Wait for the thread to exit before moving on to the next example. -worker1.joinThread() - -# Launch the other worker. -var worker2: Thread[void] -createThread(worker2, secondWorker) -# This time, use a non-blocking approach with tryRecv. -# Since the main thread is not blocked, it could be used to perform other -# useful work while it waits for data to arrive on the channel. - -var messages: seq[string] -var msg = "" -while true: - let tried = chan.tryRecv(msg) - if tried: - messages.add move(msg) - break - - messages.add "Pretend I'm doing useful work..." - # For this example, sleep in order not to flood stdout with the above - # message. - sleep(400) - -# Wait for the second thread to exit before cleaning up the channel. -worker2.joinThread() - -# Clean up the channel. -doAssert chan.close() -doAssert messages[^1] == "Another message" -doAssert messages.len >= 2 - - -block: - let chan0 = newChannel[int]() - let chan1 = chan0 - block: - let chan3 = chan0 - let chan4 = chan0 From e8dad482a309b1c33056aba22550d691845414d7 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 4 Sep 2021 15:57:16 +0200 Subject: [PATCH 1198/2472] fixes #16246 (#18800) --- compiler/seminst.nim | 3 ++- compiler/semtypinst.nim | 2 +- compiler/types.nim | 8 ++++++++ tests/arc/tarcmisc.nim | 11 +++++++++++ 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/compiler/seminst.nim b/compiler/seminst.nim index e1fad236c..456e40a94 100644 --- a/compiler/seminst.nim +++ b/compiler/seminst.nim @@ -91,7 +91,8 @@ proc sameInstantiation(a, b: TInstantiation): bool = for i in 0..a.concreteTypes.high: if not compareTypes(a.concreteTypes[i], b.concreteTypes[i], flags = {ExactTypeDescValues, - ExactGcSafety}): return + ExactGcSafety, + PickyCAliases}): return result = true proc genericCacheGet(g: ModuleGraph; genericSym: PSym, entry: TInstantiation; diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim index 18febb52e..9f9eb9489 100644 --- a/compiler/semtypinst.nim +++ b/compiler/semtypinst.nim @@ -51,7 +51,7 @@ proc searchInstTypes*(g: ModuleGraph; key: PType): PType = for j in 1..high(key.sons): # XXX sameType is not really correct for nested generics? if not compareTypes(inst[j], key[j], - flags = {ExactGenericParams}): + flags = {ExactGenericParams, PickyCAliases}): break matchType return inst diff --git a/compiler/types.nim b/compiler/types.nim index 61f563514..cf6579317 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -896,6 +896,7 @@ type ExactConstraints ExactGcSafety AllowCommonBase + PickyCAliases # be picky about the distinction between 'cint' and 'int32' TTypeCmpFlags* = set[TTypeCmpFlag] @@ -1144,6 +1145,13 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool = of tyEmpty, tyChar, tyBool, tyNil, tyPointer, tyString, tyCstring, tyInt..tyUInt64, tyTyped, tyUntyped, tyVoid: result = sameFlags(a, b) + if result and PickyCAliases in c.flags: + # additional requirement for the caching of generics for importc'ed types: + # the symbols must be identical too: + let symFlagsA = if a.sym != nil: a.sym.flags else: {} + let symFlagsB = if b.sym != nil: b.sym.flags else: {} + if (symFlagsA+symFlagsB) * {sfImportc, sfExportc} != {}: + result = symFlagsA == symFlagsB of tyStatic, tyFromExpr: result = exprStructuralEquivalent(a.n, b.n) and sameFlags(a, b) if result and a.len == b.len and a.len == 1: diff --git a/tests/arc/tarcmisc.nim b/tests/arc/tarcmisc.nim index 7daea62c8..951ae25f3 100644 --- a/tests/arc/tarcmisc.nim +++ b/tests/arc/tarcmisc.nim @@ -463,3 +463,14 @@ proc putValue[T](n: T) = echo b.n useForward() + + +# bug #16246 + +proc testWeirdTypeAliases() = + var values = newSeq[cuint](8) + # var values: seq[cuint] does not produce codegen error + var drawCb = proc(): seq[uint32] = + result = newSeq[uint32](10) + +testWeirdTypeAliases() From 5c85e480a6a13ac12c69f7e5500322eb5474be11 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 4 Sep 2021 17:49:27 +0200 Subject: [PATCH 1199/2472] unicode operator bugfixes (#18802) --- compiler/lexer.nim | 106 ++++++++++++++++------------- tests/lexer/tunicode_operators.nim | 4 +- 2 files changed, 63 insertions(+), 47 deletions(-) diff --git a/compiler/lexer.nim b/compiler/lexer.nim index ddf98661a..ede16fdf1 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -838,50 +838,6 @@ proc getCharacter(L: var Lexer; tok: var Token) = lexMessage(L, errGenerated, "missing closing ' for character literal") tokenEndIgnore(tok, L.bufpos) -proc getSymbol(L: var Lexer, tok: var Token) = - var h: Hash = 0 - var pos = L.bufpos - tokenBegin(tok, pos) - var suspicious = false - while true: - var c = L.buf[pos] - case c - of 'a'..'z', '0'..'9', '\x80'..'\xFF': - h = h !& ord(c) - inc(pos) - of 'A'..'Z': - c = chr(ord(c) + (ord('a') - ord('A'))) # toLower() - h = h !& ord(c) - inc(pos) - suspicious = true - of '_': - if L.buf[pos+1] notin SymChars: - lexMessage(L, errGenerated, "invalid token: trailing underscore") - break - inc(pos) - suspicious = true - else: break - tokenEnd(tok, pos-1) - h = !$h - tok.ident = L.cache.getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h) - if (tok.ident.id < ord(tokKeywordLow) - ord(tkSymbol)) or - (tok.ident.id > ord(tokKeywordHigh) - ord(tkSymbol)): - tok.tokType = tkSymbol - else: - tok.tokType = TokType(tok.ident.id + ord(tkSymbol)) - if suspicious and {optStyleHint, optStyleError} * L.config.globalOptions != {}: - lintReport(L.config, getLineInfo(L), tok.ident.s.normalize, tok.ident.s) - L.bufpos = pos - - -proc endOperator(L: var Lexer, tok: var Token, pos: int, - hash: Hash) {.inline.} = - var h = !$hash - tok.ident = L.cache.getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h) - if (tok.ident.id < oprLow) or (tok.ident.id > oprHigh): tok.tokType = tkOpr - else: tok.tokType = TokType(tok.ident.id - oprLow + ord(tkColon)) - L.bufpos = pos - const UnicodeOperatorStartChars = {'\226', '\194', '\195'} # the allowed unicode characters ("∙ ∘ × ★ ⊗ ⊘ ⊙ ⊛ ⊠ ⊡ ∩ ∧ ⊓ ± ⊕ ⊖ ⊞ ⊟ ∪ ∨ ⊔") @@ -925,6 +881,56 @@ proc unicodeOprLen(buf: cstring; pos: int): (int8, UnicodeOprPred) = else: discard +proc getSymbol(L: var Lexer, tok: var Token) = + var h: Hash = 0 + var pos = L.bufpos + tokenBegin(tok, pos) + var suspicious = false + while true: + var c = L.buf[pos] + case c + of 'a'..'z', '0'..'9': + h = h !& ord(c) + inc(pos) + of 'A'..'Z': + c = chr(ord(c) + (ord('a') - ord('A'))) # toLower() + h = h !& ord(c) + inc(pos) + suspicious = true + of '_': + if L.buf[pos+1] notin SymChars: + lexMessage(L, errGenerated, "invalid token: trailing underscore") + break + inc(pos) + suspicious = true + of '\x80'..'\xFF': + if c in UnicodeOperatorStartChars and unicodeOperators in L.config.features and unicodeOprLen(L.buf, pos)[0] != 0: + break + else: + h = h !& ord(c) + inc(pos) + else: break + tokenEnd(tok, pos-1) + h = !$h + tok.ident = L.cache.getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h) + if (tok.ident.id < ord(tokKeywordLow) - ord(tkSymbol)) or + (tok.ident.id > ord(tokKeywordHigh) - ord(tkSymbol)): + tok.tokType = tkSymbol + else: + tok.tokType = TokType(tok.ident.id + ord(tkSymbol)) + if suspicious and {optStyleHint, optStyleError} * L.config.globalOptions != {}: + lintReport(L.config, getLineInfo(L), tok.ident.s.normalize, tok.ident.s) + L.bufpos = pos + + +proc endOperator(L: var Lexer, tok: var Token, pos: int, + hash: Hash) {.inline.} = + var h = !$hash + tok.ident = L.cache.getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h) + if (tok.ident.id < oprLow) or (tok.ident.id > oprHigh): tok.tokType = tkOpr + else: tok.tokType = TokType(tok.ident.id - oprLow + ord(tkColon)) + L.bufpos = pos + proc getOperator(L: var Lexer, tok: var Token) = var pos = L.bufpos tokenBegin(tok, pos) @@ -1346,7 +1352,11 @@ proc rawGetTok*(L: var Lexer, tok: var Token) = getNumber(L, tok) let c = L.buf[L.bufpos] if c in SymChars+{'_'}: - lexMessage(L, errGenerated, "invalid token: no whitespace between number and identifier") + if c in UnicodeOperatorStartChars and unicodeOperators in L.config.features and + unicodeOprLen(L.buf, L.bufpos)[0] != 0: + discard + else: + lexMessage(L, errGenerated, "invalid token: no whitespace between number and identifier") of '-': if L.buf[L.bufpos+1] in {'0'..'9'} and (L.bufpos-1 == 0 or L.buf[L.bufpos-1] in UnaryMinusWhitelist): @@ -1357,7 +1367,11 @@ proc rawGetTok*(L: var Lexer, tok: var Token) = getNumber(L, tok) let c = L.buf[L.bufpos] if c in SymChars+{'_'}: - lexMessage(L, errGenerated, "invalid token: no whitespace between number and identifier") + if c in UnicodeOperatorStartChars and unicodeOperators in L.config.features and + unicodeOprLen(L.buf, L.bufpos)[0] != 0: + discard + else: + lexMessage(L, errGenerated, "invalid token: no whitespace between number and identifier") else: getOperator(L, tok) else: diff --git a/tests/lexer/tunicode_operators.nim b/tests/lexer/tunicode_operators.nim index 74fcbb763..6ad40beab 100644 --- a/tests/lexer/tunicode_operators.nim +++ b/tests/lexer/tunicode_operators.nim @@ -5,8 +5,10 @@ proc `⊙=`(x: var int, y: int) = x *= y proc `⊞++`(x, y: int): int = x + y +const a = 9 + var x = 45 -x ⊙= 9 ⊞++ 4 ⊙ 3 +x ⊙= a⊞++4⊙3 var y = 45 y *= 9 + 4 * 3 From 2e2d9288465709d84a5d91abbb035583c23efe45 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 4 Sep 2021 19:52:57 +0200 Subject: [PATCH 1200/2472] formal grammar updates [backport:1.2] (#18803) --- compiler/parser.nim | 50 +++++++++++++++++++---------------------- doc/grammar.txt | 37 +++++++++++++----------------- tools/grammar_nanny.nim | 1 + 3 files changed, 40 insertions(+), 48 deletions(-) diff --git a/compiler/parser.nim b/compiler/parser.nim index 225493902..7d3c913da 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -275,12 +275,6 @@ proc isRightAssociative(tok: Token): bool {.inline.} = result = tok.tokType == tkOpr and tok.ident.s[0] == '^' # or (tok.ident.s.len > 1 and tok.ident.s[^1] == '>') -proc isOperator(tok: Token): bool = - ## Determines if the given token is an operator type token. - tok.tokType in {tkOpr, tkDiv, tkMod, tkShl, tkShr, tkIn, tkNotin, tkIs, - tkIsnot, tkNot, tkOf, tkAs, tkFrom, tkDotDot, tkAnd, - tkOr, tkXor} - proc isUnary(tok: Token): bool = ## Check if the given token is a unary operator tok.tokType in {tkOpr, tkDotDot} and @@ -323,6 +317,14 @@ proc checkBinary(p: Parser) {.inline.} = #| mulExpr = dollarExpr (OP9 optInd dollarExpr)* #| dollarExpr = primary (OP10 optInd primary)* +proc isOperator(tok: Token): bool = + #| operatorB = OP0 | OP1 | OP2 | OP3 | OP4 | OP5 | OP6 | OP7 | OP8 | OP9 | + #| 'div' | 'mod' | 'shl' | 'shr' | 'in' | 'notin' | + #| 'is' | 'isnot' | 'not' | 'of' | 'as' | 'from' | '..' | 'and' | 'or' | 'xor' + tok.tokType in {tkOpr, tkDiv, tkMod, tkShl, tkShr, tkIn, tkNotin, tkIs, + tkIsnot, tkNot, tkOf, tkAs, tkFrom, tkDotDot, tkAnd, + tkOr, tkXor} + proc colcom(p: var Parser, n: PNode) = eat(p, tkColon) skipComment(p, n) @@ -447,8 +449,6 @@ proc exprColonEqExprList(p: var Parser, kind: TNodeKind, exprColonEqExprListAux(p, endTok, result) proc dotExpr(p: var Parser, a: PNode): PNode = - #| dotExpr = expr '.' optInd (symbol | '[:' exprList ']') - #| explicitGenericInstantiation = '[:' exprList ']' ( '(' exprColonEqExpr ')' )? var info = p.parLineInfo getTok(p) result = newNodeI(nkDotExpr, info) @@ -469,7 +469,6 @@ proc dotExpr(p: var Parser, a: PNode): PNode = result = y proc dotLikeExpr(p: var Parser, a: PNode): PNode = - #| dotLikeExpr = expr DOTLIKEOP optInd symbol var info = p.parLineInfo result = newNodeI(nkInfix, info) optInd(p, result) @@ -661,7 +660,7 @@ proc identOrLiteral(p: var Parser, mode: PrimaryMode): PNode = #| | NIL #| generalizedLit = GENERALIZED_STR_LIT | GENERALIZED_TRIPLESTR_LIT #| identOrLiteral = generalizedLit | symbol | literal - #| | par | arrayConstr | setOrTableConstr + #| | par | arrayConstr | setOrTableConstr | tupleConstr #| | castExpr #| tupleConstr = '(' optInd (exprColonEqExpr comma?)* optPar ')' #| arrayConstr = '[' optInd (exprColonEqExpr comma?)* optPar ']' @@ -808,7 +807,7 @@ proc isDotLike(tok: Token): bool = proc primarySuffix(p: var Parser, r: PNode, baseIndent: int, mode: PrimaryMode): PNode = #| primarySuffix = '(' (exprColonEqExpr comma?)* ')' - #| | '.' optInd symbol generalizedLit? + #| | '.' optInd symbol ('[:' exprList ']' ( '(' exprColonEqExpr ')' )?)? generalizedLit? #| | DOTLIKEOP optInd symbol generalizedLit? #| | '[' optInd exprColonEqExprList optPar ']' #| | '{' optInd exprColonEqExprList optPar '}' @@ -1013,11 +1012,9 @@ proc parseIdentColonEquals(p: var Parser, flags: DeclaredIdentFlags): PNode = result.add(newNodeP(nkEmpty, p)) proc parseTuple(p: var Parser, indentAllowed = false): PNode = - #| inlTupleDecl = 'tuple' - #| '[' optInd (identColonEquals (comma/semicolon)?)* optPar ']' - #| extTupleDecl = 'tuple' + #| tupleDecl = 'tuple' + #| '[' optInd (identColonEquals (comma/semicolon)?)* optPar ']' | #| COMMENT? (IND{>} identColonEquals (IND{=} identColonEquals)*)? - #| tupleClass = 'tuple' result = newNodeP(nkTupleTy, p) getTok(p) if p.tok.tokType == tkBracketLe: @@ -1123,7 +1120,7 @@ proc parseDoBlock(p: var Parser; info: TLineInfo): PNode = genericParams = p.emptyNode, pragmas = pragmas, exceptions = p.emptyNode) proc parseProcExpr(p: var Parser; isExpr: bool; kind: TNodeKind): PNode = - #| procExpr = 'proc' paramListColon pragma? ('=' COMMENT? stmt)? + #| routineExpr = ('proc' | 'func' | 'iterator') paramListColon pragma? ('=' COMMENT? stmt)? # either a proc type or a anonymous proc let info = parLineInfo(p) getTok(p) @@ -1165,7 +1162,6 @@ proc parseSymbolList(p: var Parser, result: PNode) = proc parseTypeDescKAux(p: var Parser, kind: TNodeKind, mode: PrimaryMode): PNode = - #| distinct = 'distinct' optInd typeDesc result = newNodeP(kind, p) getTok(p) if p.tok.indent != -1 and p.tok.indent <= p.currInd: return @@ -1254,12 +1250,14 @@ proc parseObject(p: var Parser): PNode proc parseTypeClass(p: var Parser): PNode proc primary(p: var Parser, mode: PrimaryMode): PNode = - #| typeKeyw = 'var' | 'out' | 'ref' | 'ptr' | 'shared' | 'tuple' - #| | 'proc' | 'iterator' | 'distinct' | 'object' | 'enum' - #| primary = typeKeyw optInd typeDesc + #| primary = operatorB primary primarySuffix* | + #| tupleDecl | routineExpr | enumDecl + #| objectDecl | conceptDecl | ('bind' primary) + #| ('var' | 'out' | 'ref' | 'ptr' | 'distinct') primary #| / prefixOperator* identOrLiteral primarySuffix* - #| / 'bind' primary if isOperator(p.tok): + # Note 'sigil like' operators are currently not reflected in the grammar + # and should be removed for Nim 2.0, I don't think anybody uses them. let isSigil = isSigilLike(p.tok) result = newNodeP(nkPrefix, p) var a = newIdentNodeP(p.tok.ident, p) @@ -1342,7 +1340,6 @@ proc parseTypeDesc(p: var Parser): PNode = proc parseTypeDefAux(p: var Parser): PNode = #| typeDefAux = simpleExpr ('not' expr)? - #| | 'concept' typeClass result = simpleExpr(p, pmTypeDef) result = binaryNot(p, result) @@ -1695,7 +1692,6 @@ proc parseTry(p: var Parser; isExpr: bool): PNode = if b == nil: parMessage(p, "expected 'except'") proc parseExceptBlock(p: var Parser, kind: TNodeKind): PNode = - #| exceptBlock = 'except' colcom stmt result = newNodeP(kind, p) getTok(p) colcom(p, result) @@ -1867,7 +1863,7 @@ proc parseSection(p: var Parser, kind: TNodeKind, parMessage(p, errIdentifierExpected, p.tok) proc parseEnum(p: var Parser): PNode = - #| enum = 'enum' optInd (symbol pragma? optInd ('=' optInd expr COMMENT?)? comma?)+ + #| enumDecl = 'enum' optInd (symbol pragma? optInd ('=' optInd expr COMMENT?)? comma?)+ result = newNodeP(nkEnumTy, p) getTok(p) result.add(p.emptyNode) @@ -2015,7 +2011,7 @@ proc parseObjectPart(p: var Parser): PNode = result = p.emptyNode proc parseObject(p: var Parser): PNode = - #| object = 'object' pragma? ('of' typeDesc)? COMMENT? objectPart + #| objectDecl = 'object' pragma? ('of' typeDesc)? COMMENT? objectPart result = newNodeP(nkObjectTy, p) getTok(p) if p.tok.tokType == tkCurlyDotLe and p.validInd: @@ -2057,8 +2053,8 @@ proc parseTypeClassParam(p: var Parser): PNode = result = p.parseSymbol proc parseTypeClass(p: var Parser): PNode = - #| typeClassParam = ('var' | 'out')? symbol - #| typeClass = typeClassParam ^* ',' (pragma)? ('of' typeDesc ^* ',')? + #| conceptParam = ('var' | 'out')? symbol + #| conceptDecl = 'concept' conceptParam ^* ',' (pragma)? ('of' typeDesc ^* ',')? #| &IND{>} stmt result = newNodeP(nkTypeClassTy, p) getTok(p) diff --git a/doc/grammar.txt b/doc/grammar.txt index f58621b97..8f86dd98c 100644 --- a/doc/grammar.txt +++ b/doc/grammar.txt @@ -22,14 +22,14 @@ ampExpr = plusExpr (OP7 optInd plusExpr)* plusExpr = mulExpr (OP8 optInd mulExpr)* mulExpr = dollarExpr (OP9 optInd dollarExpr)* dollarExpr = primary (OP10 optInd primary)* +operatorB = OP0 | OP1 | OP2 | OP3 | OP4 | OP5 | OP6 | OP7 | OP8 | OP9 | + 'div' | 'mod' | 'shl' | 'shr' | 'in' | 'notin' | + 'is' | 'isnot' | 'not' | 'of' | 'as' | 'from' | '..' | 'and' | 'or' | 'xor' symbol = '`' (KEYW|IDENT|literal|(operator|'('|')'|'['|']'|'{'|'}'|'=')+)+ '`' | IDENT | KEYW exprColonEqExpr = expr (':'|'=' expr)? exprList = expr ^+ comma exprColonEqExprList = exprColonEqExpr (comma exprColonEqExpr)* (comma)? -dotExpr = expr '.' optInd (symbol | '[:' exprList ']') -explicitGenericInstantiation = '[:' exprList ']' ( '(' exprColonEqExpr ')' )? -dotLikeExpr = expr DOTLIKEOP optInd symbol qualifiedIdent = symbol ('.' optInd symbol)? setOrTableConstr = '{' ((exprColonEqExpr comma)* | ':' ) '}' castExpr = 'cast' ('[' optInd typeDesc optPar ']' '(' optInd expr optPar ')') / @@ -51,12 +51,12 @@ literal = | INT_LIT | INT8_LIT | INT16_LIT | INT32_LIT | INT64_LIT | NIL generalizedLit = GENERALIZED_STR_LIT | GENERALIZED_TRIPLESTR_LIT identOrLiteral = generalizedLit | symbol | literal - | par | arrayConstr | setOrTableConstr + | par | arrayConstr | setOrTableConstr | tupleConstr | castExpr tupleConstr = '(' optInd (exprColonEqExpr comma?)* optPar ')' arrayConstr = '[' optInd (exprColonEqExpr comma?)* optPar ']' primarySuffix = '(' (exprColonEqExpr comma?)* ')' - | '.' optInd symbol generalizedLit? + | '.' optInd symbol ('[:' exprList ']' ( '(' exprColonEqExpr ')' )?)? generalizedLit? | DOTLIKEOP optInd symbol generalizedLit? | '[' optInd exprColonEqExprList optPar ']' | '{' optInd exprColonEqExprList optPar '}' @@ -70,17 +70,14 @@ declColonEquals = identWithPragma (comma identWithPragma)* comma? (':' optInd typeDesc)? ('=' optInd expr)? identColonEquals = IDENT (comma IDENT)* comma? (':' optInd typeDesc)? ('=' optInd expr)?) -inlTupleDecl = 'tuple' - '[' optInd (identColonEquals (comma/semicolon)?)* optPar ']' -extTupleDecl = 'tuple' +tupleDecl = 'tuple' + '[' optInd (identColonEquals (comma/semicolon)?)* optPar ']' | COMMENT? (IND{>} identColonEquals (IND{=} identColonEquals)*)? -tupleClass = 'tuple' paramList = '(' declColonEquals ^* (comma/semicolon) ')' paramListArrow = paramList? ('->' optInd typeDesc)? paramListColon = paramList? (':' optInd typeDesc)? doBlock = 'do' paramListArrow pragma? colcom stmt -procExpr = 'proc' paramListColon pragma? ('=' COMMENT? stmt)? -distinct = 'distinct' optInd typeDesc +routineExpr = ('proc' | 'func' | 'iterator') paramListColon pragma? ('=' COMMENT? stmt)? forStmt = 'for' (identWithPragma ^+ comma) 'in' expr colcom stmt forExpr = forStmt expr = (blockExpr @@ -90,14 +87,13 @@ expr = (blockExpr | forExpr | tryExpr) / simpleExpr -typeKeyw = 'var' | 'out' | 'ref' | 'ptr' | 'shared' | 'tuple' - | 'proc' | 'iterator' | 'distinct' | 'object' | 'enum' -primary = typeKeyw optInd typeDesc +primary = operatorB primary primarySuffix* | + tupleDecl | routineExpr | enumDecl + objectDecl | conceptDecl | ('bind' primary) + ('var' | 'out' | 'ref' | 'ptr' | 'distinct') primary / prefixOperator* identOrLiteral primarySuffix* - / 'bind' primary typeDesc = simpleExpr ('not' expr)? typeDefAux = simpleExpr ('not' expr)? - | 'concept' typeClass postExprBlocks = ':' stmt? ( IND{=} doBlock | IND{=} 'of' exprList ':' stmt | IND{=} 'elif' expr ':' stmt @@ -147,7 +143,6 @@ tryStmt = 'try' colcom stmt &(IND{=}? 'except'|'finally') tryExpr = 'try' colcom stmt &(optInd 'except'|'finally') (optInd 'except' exprList colcom stmt)* (optInd 'finally' colcom stmt)? -exceptBlock = 'except' colcom stmt blockStmt = 'block' symbol? colcom stmt blockExpr = 'block' symbol? colcom stmt staticStmt = 'static' colcom stmt @@ -162,7 +157,7 @@ routine = optInd identVis pattern? genericParamList? paramListColon pragma? ('=' COMMENT? stmt)? indAndComment commentStmt = COMMENT section(RULE) = COMMENT? RULE / (IND{>} (RULE / COMMENT)^+IND{=} DED) -enum = 'enum' optInd (symbol pragma? optInd ('=' optInd expr COMMENT?)? comma?)+ +enumDecl = 'enum' optInd (symbol pragma? optInd ('=' optInd expr COMMENT?)? comma?)+ objectWhen = 'when' expr colcom objectPart COMMENT? ('elif' expr colcom objectPart COMMENT?)* ('else' colcom objectPart COMMENT?)? @@ -175,9 +170,9 @@ objectCase = 'case' identWithPragma ':' typeDesc ':'? COMMENT? | IND{=} objectBranches) objectPart = IND{>} objectPart^+IND{=} DED / objectWhen / objectCase / 'nil' / 'discard' / declColonEquals -object = 'object' pragma? ('of' typeDesc)? COMMENT? objectPart -typeClassParam = ('var' | 'out')? symbol -typeClass = typeClassParam ^* ',' (pragma)? ('of' typeDesc ^* ',')? +objectDecl = 'object' pragma? ('of' typeDesc)? COMMENT? objectPart +conceptParam = ('var' | 'out')? symbol +conceptDecl = 'concept' conceptParam ^* ',' (pragma)? ('of' typeDesc ^* ',')? &IND{>} stmt typeDef = identWithPragmaDot genericParamList? '=' optInd typeDefAux indAndComment? / identVisDot genericParamList? pragma '=' optInd typeDefAux diff --git a/tools/grammar_nanny.nim b/tools/grammar_nanny.nim index 5120ef976..502412c3c 100644 --- a/tools/grammar_nanny.nim +++ b/tools/grammar_nanny.nim @@ -13,6 +13,7 @@ proc checkGrammarFileImpl(cache: IdentCache, config: ConfigRef) = var stream = llStreamOpen(data) var declaredSyms = initHashSet[string]() var usedSyms = initHashSet[string]() + usedSyms.incl "module" # 'module' is the start rule. if stream != nil: declaredSyms.incl "section" # special case for 'section(RULE)' in the grammar var From 4320b158dd2e5d7420cc813a1e9e2aecc946de7d Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 5 Sep 2021 16:50:54 +0800 Subject: [PATCH 1201/2472] add weave to important packages (#17665) * add weave to important packages * Update testament/important_packages.nim --- testament/important_packages.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index ab29dcd0d..a91246562 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -158,6 +158,7 @@ pkg "tiny_sqlite" pkg "unicodedb", "nim c -d:release -r tests/tests.nim" pkg "unicodeplus", "nim c -d:release -r tests/tests.nim" pkg "unpack" +pkg "weave", "nimble test_gc_arc", allowFailure = true pkg "websocket", "nim c websocket.nim" pkg "winim", allowFailure = true pkg "with" From f373c17ad926b669bb3b5819ae1dff4bde1da88a Mon Sep 17 00:00:00 2001 From: Miran Date: Mon, 6 Sep 2021 09:40:31 +0200 Subject: [PATCH 1202/2472] use new Nimble, with lockfiles (#18810) --- koch.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/koch.nim b/koch.nim index 8dd897028..e42ea53de 100644 --- a/koch.nim +++ b/koch.nim @@ -10,7 +10,7 @@ # const - NimbleStableCommit = "d13f3b8ce288b4dc8c34c219a4e050aaeaf43fc9" # master + NimbleStableCommit = "795704833ddfd0cdaefb45c60551d3ea205279ef" # master # examples of possible values: #head, #ea82b54, 1.2.3 FusionStableHash = "#372ee4313827ef9f2ea388840f7d6b46c2b1b014" HeadHash = "#head" From 7ae52d779184cd8a16b760e384777a75ed54464b Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Mon, 6 Sep 2021 01:43:26 -0600 Subject: [PATCH 1203/2472] Fix recursive generic typed defs (#18809) --- compiler/semtypinst.nim | 2 +- tests/generics/trecursivegenerics.nim | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 tests/generics/trecursivegenerics.nim diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim index 9f9eb9489..01b21498b 100644 --- a/compiler/semtypinst.nim +++ b/compiler/semtypinst.nim @@ -378,7 +378,7 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType = cl.typeMap = newTypeMapLayer(cl) for i in 1.. Date: Mon, 6 Sep 2021 13:12:14 +0200 Subject: [PATCH 1204/2472] fixes #14165, fixes #18739, fix the second example of #6269 (#18812) --- compiler/transf.nim | 11 ++++++---- tests/iter/titer.nim | 50 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/compiler/transf.nim b/compiler/transf.nim index 3250f2ec4..5846e6e3b 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -738,7 +738,7 @@ proc transformFor(c: PTransf, n: PNode): PNode = pushInfoContext(c.graph.config, n.info) inc(c.inlining) stmtList.add(transform(c, body)) - #findWrongOwners(c, stmtList.pnode) + #findWrongOwners(c, stmtList.PNode) dec(c.inlining) popInfoContext(c.graph.config) popTransCon(c) @@ -1019,10 +1019,11 @@ proc transform(c: PTransf, n: PNode): PNode = of nkAsgn: result = transformAsgn(c, n) of nkIdentDefs, nkConstDef: - result = n + result = newTransNode(n) result[0] = transform(c, n[0]) # Skip the second son since it only contains an unsemanticized copy of the # variable type used by docgen + result[1] = n[1] result[2] = transform(c, n[2]) # XXX comment handling really sucks: if importantComments(c.graph.config): @@ -1033,8 +1034,10 @@ proc transform(c: PTransf, n: PNode): PNode = # (bug #2604). We need to patch this environment here too: let a = n[1] if a.kind == nkSym: - n[1] = transformSymAux(c, a) - return n + result = copyTree(n) + result[1] = transformSymAux(c, a) + else: + result = n of nkExceptBranch: result = transformExceptBranch(c, n) of nkCheckedFieldExpr: diff --git a/tests/iter/titer.nim b/tests/iter/titer.nim index 8e1c13e82..3a2450ae3 100644 --- a/tests/iter/titer.nim +++ b/tests/iter/titer.nim @@ -62,3 +62,53 @@ type Rule[T] = (int, T) var t: seq[Rule[int]] for (c, t) in t: discard + + + +import std/sugar + +# bug #14165 +iterator log_nodups_hamming(): int {.inline.} = + let lb3 = 1 + let lb4 = 123 + proc mul3(): int = lb3 + lb4 + yield mul3() + +for h in log_nodups_hamming(): + break +for h in log_nodups_hamming(): + break +for h in log_nodups_hamming(): + break + +# bug #18536 +iterator envPairs*(): int = + var foo: seq[int] + proc fun() = + foo = @[] + fun() + yield 3 + +proc main() = + for a in envPairs(): + discard + for a in envPairs(): + discard +static: main() +main() + +# bug #6269 +iterator makeFn(outer_val: int): proc(a: int): int = + for i in 0..1: + yield proc(a:int): int = + return a + i.int + +let v1 = 42 + +let res = collect: + for fn1 in makeFn(v1): + let v2 = fn1(v1) + for fn2 in makeFn(v2): + fn2(v2) + +doAssert res == @[42, 43, 43, 44] From cc5422ae5070a2cdd69dbd5b78663305623a0fbc Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 6 Sep 2021 13:19:11 +0200 Subject: [PATCH 1205/2472] Atlas: added 'extract' command, as promised (#18813) --- tools/atlas/atlas.nim | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tools/atlas/atlas.nim b/tools/atlas/atlas.nim index 761ca5db8..370138029 100644 --- a/tools/atlas/atlas.nim +++ b/tools/atlas/atlas.nim @@ -9,7 +9,7 @@ ## Simple tool to automate frequent workflows: Can "clone" ## a Nimble dependency and its dependencies recursively. -import std/[parseopt, strutils, os, osproc, unicode, tables, sets] +import std/[parseopt, strutils, os, osproc, unicode, tables, sets, json, jsonutils] import parse_requires, osutils, packagesjson const @@ -22,6 +22,8 @@ Usage: Command: clone url|pkgname clone a package and all of its dependencies search keyw keywB... search for package that contains the given keywords + extract file.nimble extract the requirements and custom commands from + the given Nimble file Options: --keepCommits do not perform any `git checkouts` @@ -460,6 +462,12 @@ proc main = of "search", "list": updatePackages(c) search getPackages(c.workspace), args + of "extract": + singleArg() + if fileExists(args[0]): + echo toJson(extractRequiresInfo(args[0])) + else: + error "File does not exist: " & args[0] else: error "Invalid action: " & action From 90bfd342504fd1a6a9e4c8f232bf7c35eab92f82 Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Mon, 6 Sep 2021 07:30:49 -0600 Subject: [PATCH 1206/2472] '[]' can now be used for iterators (#18814) --- compiler/semexprs.nim | 2 +- tests/iter/tarrayiter.nim | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 tests/iter/tarrayiter.nim diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 3edeae324..c0a2a346a 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1596,7 +1596,7 @@ proc semArrayAccess(c: PContext, n: PNode, flags: TExprFlags): PNode = result = semSubscript(c, n, flags) if result == nil: # overloaded [] operator: - result = semExpr(c, buildOverloadedSubscripts(n, getIdent(c.cache, "[]"))) + result = semExpr(c, buildOverloadedSubscripts(n, getIdent(c.cache, "[]")), flags) proc propertyWriteAccess(c: PContext, n, nOrig, a: PNode): PNode = var id = considerQuotedIdent(c, a[1], a) diff --git a/tests/iter/tarrayiter.nim b/tests/iter/tarrayiter.nim new file mode 100644 index 000000000..eb7ba591a --- /dev/null +++ b/tests/iter/tarrayiter.nim @@ -0,0 +1,14 @@ +block: + iterator `[]`(a: int, r: int): int = + for q in 0 .. r: + yield a + + for val in 10[2]: discard + + type Custom = distinct string + + iterator `[]`(a: Custom, r: int): char = + for q in 0 .. r: + yield a.string[q] + + for val in Custom("test")[2]: discard \ No newline at end of file From 34a53e804943ceaf3900feb6e89194aa03236c0a Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 6 Sep 2021 17:43:03 +0200 Subject: [PATCH 1207/2472] fixes #12642 (#18811) * fixes #12642 * update important packages; refs #18804 * fixes #18805; refs #18806 * fixes a regression * Update testament/categories.nim Co-authored-by: flywind * progress * progress Co-authored-by: flywind --- compiler/ccgcalls.nim | 1 + compiler/sem.nim | 6 ++++++ compiler/transf.nim | 5 +++-- testament/categories.nim | 6 ++++-- testament/important_packages.nim | 5 ++--- tests/effects/teffects6.nim | 20 ++++++++++++++++++++ 6 files changed, 36 insertions(+), 7 deletions(-) diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index 828e666a8..f6cc6da74 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -369,6 +369,7 @@ proc genParams(p: BProc, ri: PNode, typ: PType): Rope = if ri[i].skipTrivialIndirections.kind == nkSym: needTmp[i - 1] = potentialAlias(ri[i], potentialWrites) else: + #if not ri[i].typ.isCompileTimeOnly: var potentialReads: seq[PNode] getPotentialReads(ri[i], potentialReads) for n in potentialReads: diff --git a/compiler/sem.nim b/compiler/sem.nim index 24709cf21..ad0ba1f7c 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -174,6 +174,12 @@ proc commonType*(c: PContext; x, y: PType): PType = result = b #.skipIntLit elif a.kind in IntegralTypes and a.n != nil: result = a #.skipIntLit + elif a.kind == tyProc and b.kind == tyProc: + if a.callConv == ccClosure and b.callConv != ccClosure: + result = x + elif compatibleEffects(a, b) != efCompat or + (b.flags * {tfNoSideEffect, tfGcSafe}) < (a.flags * {tfNoSideEffect, tfGcSafe}): + result = y else: var k = tyNone if a.kind in {tyRef, tyPtr}: diff --git a/compiler/transf.nim b/compiler/transf.nim index 5846e6e3b..89fa89701 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -1023,8 +1023,9 @@ proc transform(c: PTransf, n: PNode): PNode = result[0] = transform(c, n[0]) # Skip the second son since it only contains an unsemanticized copy of the # variable type used by docgen - result[1] = n[1] - result[2] = transform(c, n[2]) + let last = n.len-1 + for i in 1.. Date: Mon, 6 Sep 2021 12:11:21 -0600 Subject: [PATCH 1208/2472] Fixes implicit and explicit generics in procedures (#18808) * Fixes implicit and explicit generics * moved block logic into 'maybeInstantiateGeneric' * Added more tests * Update compiler/semexprs.nim Co-authored-by: Andreas Rumpf --- compiler/semexprs.nim | 28 +++++++++++--- tests/generics/timplicit_and_explicit.nim | 45 +++++++++++++++++++++++ 2 files changed, 68 insertions(+), 5 deletions(-) create mode 100644 tests/generics/timplicit_and_explicit.nim diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index c0a2a346a..0f6f71cc2 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1493,6 +1493,28 @@ proc semDeref(c: PContext, n: PNode): PNode = else: result = nil #GlobalError(n[0].info, errCircumNeedsPointer) +proc maybeInstantiateGeneric(c: PContext, n: PNode, s: PSym): PNode = + ## Instantiates generic if not lacking implicit generics, + ## otherwise returns n. + let + neededGenParams = s.ast[genericParamsPos].len + heldGenParams = n.len - 1 + var implicitParams = 0 + for x in s.ast[genericParamsPos]: + if tfImplicitTypeParam in x.typ.flags: + inc implicitParams + if heldGenParams != neededGenParams and implicitParams + heldGenParams == neededGenParams: + # This is an implicit + explicit generic procedure without all args passed, + # kicking back the sem'd symbol fixes #17212 + # Uncertain the hackiness of this solution. + result = n + else: + result = explicitGenericInstantiation(c, n, s) + if result == n: + n[0] = copyTree(result) + else: + n[0] = result + proc semSubscript(c: PContext, n: PNode, flags: TExprFlags): PNode = ## returns nil if not a built-in subscript operator; also called for the ## checking of assignments @@ -1568,11 +1590,7 @@ proc semSubscript(c: PContext, n: PNode, flags: TExprFlags): PNode = of skProc, skFunc, skMethod, skConverter, skIterator: # type parameters: partial generic specialization n[0] = semSymGenericInstantiation(c, n[0], s) - result = explicitGenericInstantiation(c, n, s) - if result == n: - n[0] = copyTree(result) - else: - n[0] = result + result = maybeInstantiateGeneric(c, n, s) of skMacro, skTemplate: if efInCall in flags: # We are processing macroOrTmpl[] in macroOrTmpl[](...) call. diff --git a/tests/generics/timplicit_and_explicit.nim b/tests/generics/timplicit_and_explicit.nim new file mode 100644 index 000000000..ba24b79e9 --- /dev/null +++ b/tests/generics/timplicit_and_explicit.nim @@ -0,0 +1,45 @@ + +block: # basic test + proc doStuff[T](a: SomeInteger): T = discard + proc doStuff[T;Y](a: SomeInteger, b: Y): Y = discard + assert typeof(doStuff[int](100)) is int + assert typeof(doStuff[int](100, 1.0)) is float + assert typeof(doStuff[int](100, "Hello")) is string + + proc t[T](x: T; z: int | float): seq[T] = result.add(x & $z) + + assert t[string]("Hallo", 2.0) == @["Hallo" & $2.0] + + proc t2[T](z: int | float): seq[T] = result.add($z) + + assert t2[string](2.0) == @[$2.0] + +block: # template test + template someThing[T;Y](a: SomeFloat, b: SomeOrdinal): (T, Y) = (a, b) + assert typeof(someThing[float64, int](1.0, 100)) is (float64, int) + +block: # static test + proc t[T](s: static bool) = discard + proc t2[T](s: static string) = discard + t[string](true) + t2[int]("hello") + t2[string]("world") + t2[float]("test222222") + +block: #11152 + proc f[T](X: typedesc) = discard + f[int](string) + +block: #15622 + proc test1[T](a: T, b: static[string] = "") = discard + test1[int64](123) + proc test2[T](a: T, b: static[string] = "") = discard + test2[int64, static[string]](123) + +block: #4688 + proc convertTo[T](v: int or float): T = (T)(v) + discard convertTo[float](1) + +block: #4164 + proc printStr[T](s: static[string]): T = discard + discard printStr[int]("hello static") \ No newline at end of file From ee2eb5cae2585c5cd0a476814bd5879c130e5d30 Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Tue, 7 Sep 2021 09:11:08 -0600 Subject: [PATCH 1209/2472] Fix subranges of distinct types (#18816) [backport] --- compiler/typeallowed.nim | 2 +- tests/distinct/tdistinct.nim | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/compiler/typeallowed.nim b/compiler/typeallowed.nim index 2f0c039a4..d00aa538f 100644 --- a/compiler/typeallowed.nim +++ b/compiler/typeallowed.nim @@ -124,7 +124,7 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, result = typeAllowedAux(marker, lastSon(t), kind, c, flags) of tyRange: if skipTypes(t[0], abstractInst-{tyTypeDesc}).kind notin - {tyChar, tyEnum, tyInt..tyFloat128, tyInt..tyUInt64}: result = t + {tyChar, tyEnum, tyInt..tyFloat128, tyInt..tyUInt64, tyRange}: result = t of tyOpenArray: # you cannot nest openArrays/sinks/etc. if (kind != skParam or taIsOpenArray in flags) and views notin c.features: diff --git a/tests/distinct/tdistinct.nim b/tests/distinct/tdistinct.nim index 876975a7c..fd60b4ac0 100644 --- a/tests/distinct/tdistinct.nim +++ b/tests/distinct/tdistinct.nim @@ -167,6 +167,20 @@ template main() = s.Foo.add('c') doAssert s.string == "c" # was failing test() + block: #18061 + type + A = distinct (0..100) + B = A(0) .. A(10) + proc test(b: B) = discard + let + a = A(10) + b = B(a) + test(b) + + proc test(a: A) = discard + discard cast[B](A(1)) + var c: B + static: main() main() From c56008450a86daf454fa0e2e0722711831cd78ae Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 8 Sep 2021 17:47:14 +0800 Subject: [PATCH 1210/2472] fix CI failure(upstream changes) (#18820) * fix upstream changes * Update tests/js/tunittest_error2.nim Co-authored-by: flywind Co-authored-by: Andreas Rumpf --- tests/js/tunittest_error2.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/js/tunittest_error2.nim b/tests/js/tunittest_error2.nim index 273e39d9d..9c5af7529 100644 --- a/tests/js/tunittest_error2.nim +++ b/tests/js/tunittest_error2.nim @@ -1,7 +1,7 @@ discard """ exitcode: 1 outputsub: ''' -Unhandled exception: Cannot read property 'charCodeAt' of null [] +[] [FAILED] Bad test ''' matrix: "-d:nodejs" From 23e10ea85ea35c1a5963bfd12fb69efe6358076a Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 9 Sep 2021 14:46:55 +0800 Subject: [PATCH 1211/2472] active some packages (#18825) --- testament/important_packages.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 025121990..657ddb2d1 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -63,7 +63,7 @@ pkg "delaunay" pkg "docopt" pkg "easygl", "nim c -o:egl -r src/easygl.nim", "https://github.com/jackmott/easygl" pkg "elvis" -pkg "fidget", allowFailure = true # pending https://github.com/treeform/fidget/issues/155 +pkg "fidget" pkg "fragments", "nim c -r fragments/dsl.nim" pkg "fusion" pkg "gara" @@ -76,7 +76,7 @@ pkg "gnuplot", "nim c gnuplot.nim" pkg "hts", "nim c -o:htss src/hts.nim" pkg "httpauth" pkg "illwill", "nimble examples" -pkg "inim", allowFailure=true +pkg "inim" pkg "itertools", "nim doc src/itertools.nim" pkg "iterutils" pkg "jstin" @@ -100,7 +100,7 @@ pkg "nimcrypto", "nim r --path:. tests/testall.nim" # `--path:.` workaround need pkg "NimData", "nim c -o:nimdataa src/nimdata.nim" pkg "nimes", "nim c src/nimes.nim" pkg "nimfp", "nim c -o:nfp -r src/fp.nim" -pkg "nimgame2", "nim c -d:nimLegacyConvEnumEnum nimgame2/nimgame.nim", allowFailure = true +pkg "nimgame2", "nim c -d:nimLegacyConvEnumEnum nimgame2/nimgame.nim" # XXX Doesn't work with deprecated 'randomize', will create a PR. pkg "nimgen", "nim c -o:nimgenn -r src/nimgen/runcfg.nim" pkg "nimlsp" @@ -145,7 +145,7 @@ pkg "strslice" pkg "strunicode", "nim c -r src/strunicode.nim" pkg "supersnappy" pkg "synthesis" -pkg "telebot", "nim c -o:tbot -r src/telebot.nim", allowFailure = true # pending https://github.com/ba0f3/telebot.nim/issues/59 +pkg "telebot", "nim c -o:tbot -r src/telebot.nim" pkg "tempdir" pkg "templates" pkg "tensordsl", "nim c -r tests/tests.nim", "https://krux02@bitbucket.org/krux02/tensordslnim.git" From a896f9f19eec221b8651849297bdbc8be72b00af Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Thu, 9 Sep 2021 12:57:21 +0100 Subject: [PATCH 1212/2472] Clarify unknown processor error (#18829) --- tools/niminst/makefile.nimf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/niminst/makefile.nimf b/tools/niminst/makefile.nimf index d14960ac5..a0e9a40f7 100644 --- a/tools/niminst/makefile.nimf +++ b/tools/niminst/makefile.nimf @@ -164,7 +164,7 @@ ifeq ($(ucpu),riscv64) mycpu = riscv64 endif ifndef mycpu - $(error unknown processor: $(ucpu)) + $(error unknown CPU architecture: $(ucpu) See makefile.nimf) endif # for osA in 1..c.oses.len: From 0ef830577b6f6ee2314ecdfe56a631ffd4f8e4c7 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Fri, 10 Sep 2021 05:20:32 -0300 Subject: [PATCH 1213/2472] distros.foreignDeps made public (#18830) * Deprecate distros.echoForeignDeps * Deprecate distros.echoForeignDeps * https://github.com/nim-lang/Nim/pull/18830#discussion_r705364162 * https://github.com/nim-lang/Nim/pull/18830#discussion_r705364162 --- changelog.md | 1 + lib/pure/distros.nim | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/changelog.md b/changelog.md index 3658fb105..9e449dcaa 100644 --- a/changelog.md +++ b/changelog.md @@ -380,6 +380,7 @@ - Deprecated `sequtils.delete` and added an overload taking a `Slice` that raises a defect if the slice is out of bounds, likewise with `strutils.delete`. + ## Language changes - The `cstring` doesn't support `[]=` operator in JS backend. diff --git a/lib/pure/distros.nim b/lib/pure/distros.nim index ddf0fd30c..797698d61 100644 --- a/lib/pure/distros.nim +++ b/lib/pure/distros.nim @@ -27,11 +27,11 @@ ## ## See `packaging `_ for hints on distributing Nim using OS packages. -from strutils import contains, toLowerAscii +from std/strutils import contains, toLowerAscii when not defined(nimscript): - from osproc import execProcess - from os import existsEnv + from std/osproc import execProcess + from std/os import existsEnv type Distribution* {.pure.} = enum ## the list of known distributions @@ -211,7 +211,7 @@ template detectOs*(d: untyped): bool = detectOsImpl(Distribution.d) when not defined(nimble): - var foreignDeps: seq[string] = @[] + var foreignDeps*: seq[string] = @[] ## Registered foreign deps. proc foreignCmd*(cmd: string; requiresSudo = false) = ## Registers a foreign command to the internal list of commands From 6c2f041368e36bb7a6b217c7ac0c5de94c7b4a7a Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 11 Sep 2021 09:25:48 +0800 Subject: [PATCH 1214/2472] fix code-block which causes missing docs --- lib/system/channels_builtin.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/system/channels_builtin.nim b/lib/system/channels_builtin.nim index c7a445766..30267e375 100644 --- a/lib/system/channels_builtin.nim +++ b/lib/system/channels_builtin.nim @@ -26,7 +26,7 @@ ## The following is a simple example of two different ways to use channels: ## blocking and non-blocking. ## -## .. code-block :: Nim +## .. code-block:: Nim ## # Be sure to compile with --threads:on. ## # The channels and threads modules are part of system and should not be ## # imported. @@ -112,7 +112,7 @@ ## using e.g. `system.allocShared0` and pass these pointers through thread ## arguments: ## -## .. code-block :: Nim +## .. code-block:: Nim ## proc worker(channel: ptr Channel[string]) = ## let greeting = channel[].recv() ## echo greeting From 1f68f71ec24d3ec6b8e83411a6f1604277f9d493 Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 11 Sep 2021 17:33:52 +0800 Subject: [PATCH 1215/2472] [minor] fix docs (#18834) --- lib/pure/collections/sets.nim | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/pure/collections/sets.nim b/lib/pure/collections/sets.nim index 3f91d9030..7b1c58ac8 100644 --- a/lib/pure/collections/sets.nim +++ b/lib/pure/collections/sets.nim @@ -603,12 +603,10 @@ proc isValid*[A](s: HashSet[A]): bool {.deprecated: ## Returns `true` if the set has been initialized (with `initHashSet proc ## <#initHashSet>`_ or `init proc <#init,HashSet[A]>`_). ## - ## **Examples:** - ## - ## .. code-block :: - ## proc savePreferences(options: HashSet[string]) = - ## assert options.isValid, "Pass an initialized set!" - ## # Do stuff here, may crash in release builds! + runnableExamples: + proc savePreferences(options: HashSet[string]) = + assert options.isValid, "Pass an initialized set!" + # Do stuff here, may crash in release builds! result = s.data.len > 0 From 66e53bdd7b465edd9045314d4d6a60ef6e0b5e32 Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Sat, 11 Sep 2021 05:05:53 -0600 Subject: [PATCH 1216/2472] Fixed type inference for 'set` and 'tuple' (#18827) * improved built in typeclass inference * Smarter logic to fit node * Forgot the untyped check --- compiler/sem.nim | 3 ++- tests/metatype/typeclassinference.nim | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/compiler/sem.nim b/compiler/sem.nim index ad0ba1f7c..bdecbe602 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -77,7 +77,8 @@ template semIdeForTemplateOrGeneric(c: PContext; n: PNode; proc fitNodePostMatch(c: PContext, formal: PType, arg: PNode): PNode = let x = arg.skipConv - if x.kind in {nkPar, nkTupleConstr, nkCurly} and formal.kind != tyUntyped: + if (x.kind == nkCurly and formal.kind == tySet and formal.base.kind != tyGenericParam) or + (x.kind in {nkPar, nkTupleConstr}) and formal.kind notin {tyUntyped, tyBuiltInTypeClass}: changeType(c, x, formal, check=true) result = arg result = skipHiddenSubConv(result, c.graph, c.idgen) diff --git a/tests/metatype/typeclassinference.nim b/tests/metatype/typeclassinference.nim index c845e04f7..b3f197718 100644 --- a/tests/metatype/typeclassinference.nim +++ b/tests/metatype/typeclassinference.nim @@ -19,3 +19,25 @@ var ptr1: ptr = addr(str1) var str2: string = "hello, world!" var ptr2: ptr = str2 +block: # built in typeclass inference + proc tupleA(): tuple = return (1, 2) + proc tupleB(): tuple = (1f, 2f) + assert typeof(tupleA()) is (int, int) + assert typeof(tupleB()) is (float32, float32) + + proc a(val: int or float): tuple = + when typeof(val) is int: + (10, 10) + else: + (30f, 30f) + + assert typeof(a(10)) is (int, int) + assert typeof(a(10.0)) is (float32, float32) + + proc b(val: int or float): set = + when typeof(val) is int: + {10u8, 3} + else: + {'a', 'b'} + assert typeof(b(10)) is set[uint8] + assert typeof(b(10.0)) is set[char] \ No newline at end of file From 5d1608c9764416e74b36f6a772f16a66d58ddc28 Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Sat, 11 Sep 2021 14:20:22 -0600 Subject: [PATCH 1217/2472] Generic pointer procs now error if no types supplied (#18832) * more precise logic for pointer procs * added test for generic pointer procs * Fixed generic getting bracket expr if erroring --- compiler/semexprs.nim | 2 +- compiler/semstmts.nim | 2 +- tests/generics/tpointerprocs.nim | 28 ++++++++++++++++++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 tests/generics/tpointerprocs.nim diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 0f6f71cc2..41381b057 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1511,7 +1511,7 @@ proc maybeInstantiateGeneric(c: PContext, n: PNode, s: PSym): PNode = else: result = explicitGenericInstantiation(c, n, s) if result == n: - n[0] = copyTree(result) + n[0] = copyTree(result[0]) else: n[0] = result diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 2c5f3ba54..364444c98 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -553,7 +553,7 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = typ = typ.lastSon if hasEmpty(typ): localError(c.config, def.info, errCannotInferTypeOfTheLiteral % typ.kind.toHumanStr) - elif typ.kind == tyProc and hasUnresolvedParams(def, {}): + elif typ.kind == tyProc and def.kind == nkSym and isGenericRoutine(def.sym.ast): # tfUnresolved in typ.flags: localError(c.config, def.info, errProcHasNoConcreteType % def.renderTree) when false: diff --git a/tests/generics/tpointerprocs.nim b/tests/generics/tpointerprocs.nim new file mode 100644 index 000000000..2bcaf15b3 --- /dev/null +++ b/tests/generics/tpointerprocs.nim @@ -0,0 +1,28 @@ +discard """ +cmd: "nim check $options --hints:off $file" +action: "reject" +nimout:''' +tpointerprocs.nim(15, 11) Error: 'foo' doesn't have a concrete type, due to unspecified generic parameters. +tpointerprocs.nim(27, 11) Error: cannot instantiate: 'foo[int]'; got 1 typeof(s) but expected 2 +tpointerprocs.nim(27, 14) Error: expression 'foo[int]' has no type (or is ambiguous) +tpointerprocs.nim(28, 11) Error: expression 'bar' has no type (or is ambiguous) +''' +""" + +block: + proc foo(x: int | float): float = result = 1.0 + let + bar = foo + baz = bar + +block: + proc foo(x: int | float): float = result = 1.0 + let + bar = foo[int] + baz = bar + +block: + proc foo(x: int | float, y: int or string): float = result = 1.0 + let + bar = foo[int] + baz = bar \ No newline at end of file From 3f3e0fa303a2729d0d6a9ffe54936c1c830e2d7e Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Mon, 13 Sep 2021 01:35:19 -0600 Subject: [PATCH 1218/2472] Fixed #18838 (#18841) [backport] --- compiler/semtypinst.nim | 6 ++++- tests/generics/trecursivegenerics.nim | 36 ++++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim index 01b21498b..cb1ead027 100644 --- a/compiler/semtypinst.nim +++ b/compiler/semtypinst.nim @@ -344,7 +344,11 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType = x = lookupTypeVar(cl, x) if x != nil: if header == t: header = instCopyType(cl, t) - header[i] = x + header[i] = + if x.kind == tyGenericInst: + t[i] + else: + x propagateToOwner(header, x) else: propagateToOwner(header, x) diff --git a/tests/generics/trecursivegenerics.nim b/tests/generics/trecursivegenerics.nim index 56c074e3f..e4c0e720f 100644 --- a/tests/generics/trecursivegenerics.nim +++ b/tests/generics/trecursivegenerics.nim @@ -1,6 +1,7 @@ block: # Replicates #18728 type FlipFlop[A, B] = ref object + val: A next: FlipFlop[B, A] Trinary[A, B, C] = ref object @@ -9,4 +10,37 @@ block: # Replicates #18728 assert typeof(FlipFlop[int, string]().next) is FlipFlop[string, int] assert typeof(FlipFlop[string, int]().next) is FlipFlop[int, string] assert typeof(Trinary[int, float, string]().next) is Trinary[float, string, int] - assert typeof(Trinary[int, float, string]().next.next) is Trinary[string, int, float] \ No newline at end of file + assert typeof(Trinary[int, float, string]().next.next) is Trinary[string, int, float] + var a = FlipFlop[int, string](val: 100, next: FlipFlop[string, int](val: "Hello")) + assert a.val == 100 + assert a.next.val == "Hello" + +block: # 18838 + type + DoublyLinkedNodeObj[T] = object + value: T + + DoublyLinkedNode[T] = ref DoublyLinkedNodeObj[T] + + Item[T] = ref object + link: DoublyLinkedNode[Item[T]] + + Box = object + + proc newDoublyLinkedNode[T](value: T): DoublyLinkedNode[T] = + new(result) + result.value = value + + let link = newDoublyLinkedNode(Item[Box]()) + +import lists +block: + type + Box = object + Item[T] = ref object + link:DoublyLinkedNode[ Item[T] ] + + ItemSimple = ref object + link:DoublyLinkedNode[ ItemSimple ] + + let link = newDoublyLinkedNode( Item[Box]() ) \ No newline at end of file From 179fad934d52641c5e064efce4bdd98578cc5e9e Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 13 Sep 2021 11:05:09 +0200 Subject: [PATCH 1219/2472] doc improvements (#18843) * cleaned up destructors documentation [backport] * Spec updates [backport:1.0] --- doc/destructors.rst | 53 +++++--------------- doc/manual.rst | 118 ++++++++------------------------------------ 2 files changed, 33 insertions(+), 138 deletions(-) diff --git a/doc/destructors.rst b/doc/destructors.rst index a3fd7514e..28d8fa5f9 100644 --- a/doc/destructors.rst +++ b/doc/destructors.rst @@ -222,7 +222,14 @@ The prototype of this hook for a type `T` needs to be: `env` is used by ORC to keep track of its internal state, it should be passed around to calls of the built-in `=trace` operation. -Usually there will only be a need for a custom `=trace` when a custom `=destroy` that deallocates manually allocated resources is also used, and then only when there is a chance of cyclic references from items within the manually allocated resources when it is desired that `--gc:orc` be able to break and collect these cyclic referenced resources. Currently however, there is a mutual use problem in that whichever of `=destroy`/`=trace` is used first will automatically create a version of the other which will then conflict with the creation of the second of the pair. The work around for this problem is to forward declare the second of the "hooks" to prevent the automatic creation. +Usually there will only be a need for a custom `=trace` when a custom `=destroy` that deallocates +manually allocated resources is also used, and then only when there is a chance of cyclic +references from items within the manually allocated resources when it is desired that `--gc:orc` +be able to break and collect these cyclic referenced resources. Currently however, there is a +mutual use problem in that whichever of `=destroy`/`=trace` is used first will automatically +create a version of the other which will then conflict with the creation of the second of the +pair. The work around for this problem is to forward declare the second of the "hooks" to +prevent the automatic creation. The general pattern in using `=destroy` with `=trace` looks like: @@ -696,43 +703,22 @@ should eventually be replaced by a better solution. Copy on write ============= -String literals are implemented as [copy-on-write](https://en.wikipedia.org/wiki/Copy-on-write). +String literals are implemented as "copy on write". When assigning a string literal to a variable, a copy of the literal won't be created. Instead the variable simply points to the literal. The literal is shared between different variables which are pointing to it. The copy operation is deferred until the first write. -.. code-block:: nim - var x = "abc" # no copy - var y = x # no copy - -The string literal "abc" is stored in static memory and not allocated on the heap. -The variable `x` points to the literal and the variable `y` points to the literal too. -There is no copy during assigning operations. +For example: .. code-block:: nim var x = "abc" # no copy var y = x # no copy y[0] = 'h' # copy -The program above shows when the copy operations happen. -When mutating the variable `y`, the Nim compiler creates a fresh copy of `x`, -the variable `y` won't point to the string literal anymore. -Instead it points to the copy of `x` of which the memory can be mutated -and the variable `y` becomes a mutable string. -.. Note:: The abstraction fails for `addr x` because whether the address is going to be used for mutations is unknown. - -Let's look at a silly example demonstrating this behaviour: - -.. code-block:: nim - var x = "abc" - var y = x - - moveMem(addr y[0], addr x[0], 3) - -The program fails because we need to prepare a fresh copy for the variable `y`. -`prepareMutation` should be called before the address operation. +The abstraction fails for `addr x` because whether the address is going to be used for mutations is unknown. +`prepareMutation` needs to be called before the "address of" operation. For example: .. code-block:: nim var x = "abc" @@ -741,18 +727,3 @@ The program fails because we need to prepare a fresh copy for the variable `y`. prepareMutation(y) moveMem(addr y[0], addr x[0], 3) assert y == "abc" - -Now `prepareMutation` solves the problem. -It manually creates a fresh copy and makes the variable `y` mutable. - -.. code-block:: nim - var x = "abc" - var y = x - - prepareMutation(y) - moveMem(addr y[0], addr x[0], 3) - moveMem(addr y[0], addr x[0], 3) - moveMem(addr y[0], addr x[0], 3) - assert y == "abc" - -No matter how many times `moveMem` is called, the program compiles and runs. diff --git a/doc/manual.rst b/doc/manual.rst index 7976ffe0f..af163bceb 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -2400,121 +2400,38 @@ describe the type checking done by the compiler. Type equality ------------- + Nim uses structural type equivalence for most types. Only for objects, -enumerations and distinct types name equivalence is used. The following -algorithm, *in pseudo-code*, determines type equality: - -.. code-block:: nim - proc typeEqualsAux(a, b: PType, - s: var HashSet[(PType, PType)]): bool = - if (a,b) in s: return true - incl(s, (a,b)) - if a.kind == b.kind: - case a.kind - of int, intXX, float, floatXX, char, string, cstring, pointer, - bool, nil, void: - # leaf type: kinds identical; nothing more to check - result = true - of ref, ptr, var, set, seq, openarray: - result = typeEqualsAux(a.baseType, b.baseType, s) - of range: - result = typeEqualsAux(a.baseType, b.baseType, s) and - (a.rangeA == b.rangeA) and (a.rangeB == b.rangeB) - of array: - result = typeEqualsAux(a.baseType, b.baseType, s) and - typeEqualsAux(a.indexType, b.indexType, s) - of tuple: - if a.tupleLen == b.tupleLen: - for i in 0..a.tupleLen-1: - if not typeEqualsAux(a[i], b[i], s): return false - result = true - of object, enum, distinct: - result = a == b - of proc: - result = typeEqualsAux(a.parameterTuple, b.parameterTuple, s) and - typeEqualsAux(a.resultType, b.resultType, s) and - a.callingConvention == b.callingConvention - - proc typeEquals(a, b: PType): bool = - var s: HashSet[(PType, PType)] = {} - result = typeEqualsAux(a, b, s) - -Since types are graphs which can have cycles, the above algorithm needs an -auxiliary set `s` to detect this case. - - -Type equality modulo type distinction -------------------------------------- - -The following algorithm (in pseudo-code) determines whether two types -are equal with no respect to `distinct` types. For brevity the cycle check -with an auxiliary set `s` is omitted: - -.. code-block:: nim - proc typeEqualsOrDistinct(a, b: PType): bool = - if a.kind == b.kind: - case a.kind - of int, intXX, float, floatXX, char, string, cstring, pointer, - bool, nil, void: - # leaf type: kinds identical; nothing more to check - result = true - of ref, ptr, var, set, seq, openarray: - result = typeEqualsOrDistinct(a.baseType, b.baseType) - of range: - result = typeEqualsOrDistinct(a.baseType, b.baseType) and - (a.rangeA == b.rangeA) and (a.rangeB == b.rangeB) - of array: - result = typeEqualsOrDistinct(a.baseType, b.baseType) and - typeEqualsOrDistinct(a.indexType, b.indexType) - of tuple: - if a.tupleLen == b.tupleLen: - for i in 0..a.tupleLen-1: - if not typeEqualsOrDistinct(a[i], b[i]): return false - result = true - of distinct: - result = typeEqualsOrDistinct(a.baseType, b.baseType) - of object, enum: - result = a == b - of proc: - result = typeEqualsOrDistinct(a.parameterTuple, b.parameterTuple) and - typeEqualsOrDistinct(a.resultType, b.resultType) and - a.callingConvention == b.callingConvention - elif a.kind == distinct: - result = typeEqualsOrDistinct(a.baseType, b) - elif b.kind == distinct: - result = typeEqualsOrDistinct(a, b.baseType) +enumerations and distinct types and for generic types name equivalence is used. Subtype relation ---------------- -If object `a` inherits from `b`, `a` is a subtype of `b`. This subtype -relation is extended to the types `var`, `ref`, `ptr`: -.. code-block:: nim - proc isSubtype(a, b: PType): bool = - if a.kind == b.kind: - case a.kind - of object: - var aa = a.baseType - while aa != nil and aa != b: aa = aa.baseType - result = aa == b - of var, ref, ptr: - result = isSubtype(a.baseType, b.baseType) +If object `a` inherits from `b`, `a` is a subtype of `b`. -.. XXX nil is a special value! +This subtype relation is extended to the types `var`, `ref`, `ptr`. +If `A` is a subtype of `B` and `A` and `B` are `object` types then: +- `var A` is a subtype of `var B` +- `ref A` is a subtype of `ref B` +- `ptr A` is a subtype of `ptr B`. +**Note**: In later versions of the language the subtype relation might +be changed to *require* the pointer indirection in order to prevent +"object slicing". Convertible relation -------------------- + A type `a` is **implicitly** convertible to type `b` iff the following algorithm returns true: .. code-block:: nim proc isImplicitlyConvertible(a, b: PType): bool = - if isSubtype(a, b) or isCovariant(a, b): + if isSubtype(a, b): return true if isIntLiteral(a): return b in {int8, int16, int32, int64, int, uint, uint8, uint16, @@ -2540,7 +2457,12 @@ algorithm returns true: result = b == pointer of string: result = b == cstring + of proc: + result = typeEquals(a, b) or compatibleParametersAndEffects(a, b) +We used the predicate `typeEquals(a, b)` for the "type equality" property +and the predicate `isSubtype(a, b)` for the "subtype relation". +`compatibleParametersAndEffects(a, b)` is currently not specified. Implicit conversions are also performed for Nim's `range` type constructor. @@ -2563,7 +2485,9 @@ algorithm returns true: proc isExplicitlyConvertible(a, b: PType): bool = result = false if isImplicitlyConvertible(a, b): return true - if typeEqualsOrDistinct(a, b): return true + if typeEquals(a, b): return true + if a == distinct and typeEquals(a.baseType, b): return true + if b == distinct and typeEquals(b.baseType, a): return true if isIntegralType(a) and isIntegralType(b): return true if isSubtype(a, b) or isSubtype(b, a): return true From ef390e6a68db74a61137d7690dd2b10ec3dee050 Mon Sep 17 00:00:00 2001 From: Christian Ulrich Date: Mon, 13 Sep 2021 18:47:33 +0200 Subject: [PATCH 1220/2472] add changelog entry for #18656 (#18839) [backport] --- changelog.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/changelog.md b/changelog.md index 9e449dcaa..ab037c328 100644 --- a/changelog.md +++ b/changelog.md @@ -111,6 +111,11 @@ - The allocator for Nintendo Switch, which was nonfunctional because of breaking changes in libnx, was removed, in favour of the new `-d:nimAllocPagesViaMalloc` option. +- `net.parseIpAddress` now only allows IPv4 addresses in strict form as defined + in [RFC 6943](https://www.rfc-editor.org/rfc/rfc6943#section-3.1.1). + Specifically, octal numbers in IPv4 addresses are no longer accepted (before + they were parsed as decimal numbers). + ## Standard library additions and changes - `strformat`: From 172253cb551a475d7d32093e562521990e71a1ed Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Tue, 14 Sep 2021 11:34:52 -0600 Subject: [PATCH 1221/2472] Dotborrow now works with generic distincts (#18848) --- compiler/semexprs.nim | 4 ++-- compiler/semstmts.nim | 3 ++- tests/borrow/tborrow.nim | 20 ++++++++++++++++++++ tests/borrow/tinvalidborrow.nim | 17 +++++++++++++---- 4 files changed, 37 insertions(+), 7 deletions(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 41381b057..ebdfe02f0 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1280,7 +1280,7 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode = if p != nil and p.selfSym != nil: var ty = skipTypes(p.selfSym.typ, {tyGenericInst, tyVar, tyLent, tyPtr, tyRef, tyAlias, tySink, tyOwned}) - while tfBorrowDot in ty.flags: ty = ty.skipTypes({tyDistinct}) + while tfBorrowDot in ty.flags: ty = ty.skipTypes({tyDistinct, tyGenericInst}) var check: PNode = nil if ty.kind == tyObject: while true: @@ -1412,7 +1412,7 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode = if ty.kind in tyUserTypeClasses and ty.isResolvedUserTypeClass: ty = ty.lastSon ty = skipTypes(ty, {tyGenericInst, tyVar, tyLent, tyPtr, tyRef, tyOwned, tyAlias, tySink}) - while tfBorrowDot in ty.flags: ty = ty.skipTypes({tyDistinct}) + while tfBorrowDot in ty.flags: ty = ty.skipTypes({tyDistinct, tyGenericInst}) var check: PNode = nil if ty.kind == tyObject: while true: diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 364444c98..c2f05ccce 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -1296,7 +1296,8 @@ proc typeSectionRightSidePass(c: PContext, n: PNode) = incl a[2].flags, nfSem # bug #10548 if sfExportc in s.flags and s.typ.kind == tyAlias: localError(c.config, name.info, "{.exportc.} not allowed for type aliases") - if tfBorrowDot in s.typ.flags and s.typ.kind != tyDistinct: + + if tfBorrowDot in s.typ.flags and s.typ.skipTypes({tyGenericBody}).kind != tyDistinct: excl s.typ.flags, tfBorrowDot localError(c.config, name.info, "only a 'distinct' type can borrow `.`") let aa = a[2] diff --git a/tests/borrow/tborrow.nim b/tests/borrow/tborrow.nim index 9c403021e..2fc788fa0 100644 --- a/tests/borrow/tborrow.nim +++ b/tests/borrow/tborrow.nim @@ -33,3 +33,23 @@ let b = Radians(1.0) a -= b echo a.float64 + +block: #14449 + type + Foo[T] = object + foo: T + + Bar[T] {.borrow:`.`.} = distinct Foo[T] + SomeThing {.borrow:`.`.} = distinct Foo[float] + OtherThing {.borrow:`.`.} = distinct SomeThing + + var + a: Bar[int] + b: SomeThing + c: OtherThing + a.foo = 300 + b.foo = 400 + c.foo = 42 + assert a.foo == 300 + assert b.foo == 400d + assert c.foo == 42d \ No newline at end of file diff --git a/tests/borrow/tinvalidborrow.nim b/tests/borrow/tinvalidborrow.nim index 89aa4e2e8..08148608d 100644 --- a/tests/borrow/tinvalidborrow.nim +++ b/tests/borrow/tinvalidborrow.nim @@ -1,16 +1,25 @@ discard """ - errormsg: "no symbol to borrow from found" - line: 11 + cmd: "nim check --hints:off --warnings:off $file" + action: "reject" + nimout:''' +tinvalidborrow.nim(18, 3) Error: only a 'distinct' type can borrow `.` +tinvalidborrow.nim(19, 3) Error: only a 'distinct' type can borrow `.` +tinvalidborrow.nim(20, 1) Error: no symbol to borrow from found +''' """ # bug #516 type TAtom = culong - + Test {.borrow:`.`.} = distinct int + Foo[T] = object + a: int + Bar[T] {.borrow:`.`.} = Foo[T] + OtherFoo {.borrow:`.`.} = Foo[int] proc `==`*(a, b: TAtom): bool {.borrow.} var d, e: TAtom -echo( $(d == e) ) +discard( $(d == e) ) From bf1700bab17da3c8776914ac5bdfff016eb70442 Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 15 Sep 2021 01:39:55 +0800 Subject: [PATCH 1222/2472] add testcase for #7308 (#18849) --- tests/arc/tarcmisc.nim | 11 ---------- tests/ccgbugs/tctypes.nim | 43 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 11 deletions(-) create mode 100644 tests/ccgbugs/tctypes.nim diff --git a/tests/arc/tarcmisc.nim b/tests/arc/tarcmisc.nim index 951ae25f3..7daea62c8 100644 --- a/tests/arc/tarcmisc.nim +++ b/tests/arc/tarcmisc.nim @@ -463,14 +463,3 @@ proc putValue[T](n: T) = echo b.n useForward() - - -# bug #16246 - -proc testWeirdTypeAliases() = - var values = newSeq[cuint](8) - # var values: seq[cuint] does not produce codegen error - var drawCb = proc(): seq[uint32] = - result = newSeq[uint32](10) - -testWeirdTypeAliases() diff --git a/tests/ccgbugs/tctypes.nim b/tests/ccgbugs/tctypes.nim new file mode 100644 index 000000000..be6009115 --- /dev/null +++ b/tests/ccgbugs/tctypes.nim @@ -0,0 +1,43 @@ +discard """ + targets: "c cpp" + matrix: "--gc:refc; --gc:arc" +""" + +# bug #7308 +proc foo(x: seq[int32]) = + var y = newSeq[cint](1) + +proc bar = + var t = newSeq[int32](1) + foo(t) + +bar() + + +# bug #16246 + +proc testWeirdTypeAliases() = + var values = newSeq[cuint](8) + # var values: seq[cuint] does not produce codegen error + var drawCb = proc(): seq[uint32] = + result = newSeq[uint32](10) + +testWeirdTypeAliases() + +block: # bug #11797 + block: + type cdouble2 = cdouble + type Foo1 = seq[cdouble] + type Foo2 = seq[cdouble2] + static: doAssert Foo1 is Foo2 + var a1: Foo1 + var a2: Foo2 + doAssert a1 == @[] + doAssert a2 == @[] + + block: + proc foo[T: int|cint](fun: proc(): T) = discard + proc foo1(): cint = 1 + proc foo3(): int32 = 2 + foo(proc(): cint = foo1()) + foo(proc(): int32 = foo3()) From cebf7cdc1ede7385e8d774b83c52d17e7021899b Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 15 Sep 2021 01:40:42 +0800 Subject: [PATCH 1223/2472] fix #10128 (#18852) --- compiler/ccgexprs.nim | 6 +++--- tests/ccgbugs/t10128.nim | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 tests/ccgbugs/t10128.nim diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 374876367..7a3d769aa 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1006,14 +1006,14 @@ proc genOpenArrayElem(p: BProc, n, x, y: PNode, d: var TLoc) = # emit range check: if optBoundsCheck in p.options: linefmt(p, cpsStmts, "if ((NU)($1) >= (NU)($2Len_0)){ #raiseIndexError2($1,$2Len_0-1); $3}$n", - [rdLoc(b), rdLoc(a), raiseInstr(p)]) # BUGFIX: ``>=`` and not ``>``! + [rdCharLoc(b), rdLoc(a), raiseInstr(p)]) # BUGFIX: ``>=`` and not ``>``! inheritLocation(d, a) putIntoDest(p, d, n, ropecg(p.module, "$1[$2]", [rdLoc(a), rdCharLoc(b)]), a.storage) else: if optBoundsCheck in p.options: linefmt(p, cpsStmts, "if ((NU)($1) >= (NU)($2.Field1)){ #raiseIndexError2($1,$2.Field1-1); $3}$n", - [rdLoc(b), rdLoc(a), raiseInstr(p)]) # BUGFIX: ``>=`` and not ``>``! + [rdCharLoc(b), rdLoc(a), raiseInstr(p)]) # BUGFIX: ``>=`` and not ``>``! inheritLocation(d, a) putIntoDest(p, d, n, ropecg(p.module, "$1.Field0[$2]", [rdLoc(a), rdCharLoc(b)]), a.storage) @@ -1028,7 +1028,7 @@ proc genSeqElem(p: BProc, n, x, y: PNode, d: var TLoc) = if optBoundsCheck in p.options: linefmt(p, cpsStmts, "if ((NU)($1) >= (NU)$2){ #raiseIndexError2($1,$2-1); $3}$n", - [rdLoc(b), lenExpr(p, a), raiseInstr(p)]) + [rdCharLoc(b), lenExpr(p, a), raiseInstr(p)]) if d.k == locNone: d.storage = OnHeap if skipTypes(a.t, abstractVar).kind in {tyRef, tyPtr}: a.r = ropecg(p.module, "(*$1)", [a.r]) diff --git a/tests/ccgbugs/t10128.nim b/tests/ccgbugs/t10128.nim new file mode 100644 index 000000000..48970916f --- /dev/null +++ b/tests/ccgbugs/t10128.nim @@ -0,0 +1,18 @@ +# bug #10128 +let data = "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz" +var seq2 = newSeq[char](data.len) +for i in 0.. Date: Thu, 16 Sep 2021 00:48:58 -0600 Subject: [PATCH 1224/2472] Fixed borrowing dot from aliases (#18854) --- compiler/semexprs.nim | 4 ++-- tests/borrow/tborrow.nim | 38 +++++++++++++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index ebdfe02f0..5a1426f2b 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1280,7 +1280,7 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode = if p != nil and p.selfSym != nil: var ty = skipTypes(p.selfSym.typ, {tyGenericInst, tyVar, tyLent, tyPtr, tyRef, tyAlias, tySink, tyOwned}) - while tfBorrowDot in ty.flags: ty = ty.skipTypes({tyDistinct, tyGenericInst}) + while tfBorrowDot in ty.flags: ty = ty.skipTypes({tyDistinct, tyGenericInst, tyAlias}) var check: PNode = nil if ty.kind == tyObject: while true: @@ -1412,7 +1412,7 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode = if ty.kind in tyUserTypeClasses and ty.isResolvedUserTypeClass: ty = ty.lastSon ty = skipTypes(ty, {tyGenericInst, tyVar, tyLent, tyPtr, tyRef, tyOwned, tyAlias, tySink}) - while tfBorrowDot in ty.flags: ty = ty.skipTypes({tyDistinct, tyGenericInst}) + while tfBorrowDot in ty.flags: ty = ty.skipTypes({tyDistinct, tyGenericInst, tyAlias}) var check: PNode = nil if ty.kind == tyObject: while true: diff --git a/tests/borrow/tborrow.nim b/tests/borrow/tborrow.nim index 2fc788fa0..35652e2e0 100644 --- a/tests/borrow/tborrow.nim +++ b/tests/borrow/tborrow.nim @@ -52,4 +52,40 @@ block: #14449 c.foo = 42 assert a.foo == 300 assert b.foo == 400d - assert c.foo == 42d \ No newline at end of file + assert c.foo == 42d + +block: # Borrow from muliple aliasses #16666 + type + AImpl = object + i: int + + A = AImpl + + B {.borrow: `.`.} = distinct A + C = B + D {.borrow: `.`.} = distinct C + E {.borrow: `.`.} = distinct D + + let + b = default(B) + d = default(D) + e = default(E) + + assert b.i == 0 + assert d.i == 0 + assert e.i == 0 + +block: # Borrow from generic alias + type + AImpl[T] = object + i: T + B[T] = AImpl[T] + C {.borrow: `.`.} = distinct B[int] + D = B[float] + E {.borrow: `.`.} = distinct D + + let + c = default(C) + e = default(E) + assert c.i == int(0) + assert e.i == 0d \ No newline at end of file From 6cd219c3a38c5f1a0712b0b5d07a3ea4ab02ff74 Mon Sep 17 00:00:00 2001 From: alaviss Date: Fri, 17 Sep 2021 04:51:26 +0000 Subject: [PATCH 1225/2472] semtypinst: don't wrap type nodes from expressions in static[T] (#18860) --- compiler/semtypinst.nim | 2 +- tests/generics/t18859.nim | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 tests/generics/t18859.nim diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim index cb1ead027..2ca355e7d 100644 --- a/compiler/semtypinst.nim +++ b/compiler/semtypinst.nim @@ -548,7 +548,7 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType = result = n.typ.skipTypes({tyTypeDesc}) # result = n.typ.base else: - if n.typ.kind != tyStatic: + if n.typ.kind != tyStatic and n.kind != nkType: # XXX: In the future, semConstExpr should # return tyStatic values to let anyone make # use of this knowledge. The patching here diff --git a/tests/generics/t18859.nim b/tests/generics/t18859.nim new file mode 100644 index 000000000..ca6c3d10b --- /dev/null +++ b/tests/generics/t18859.nim @@ -0,0 +1,17 @@ +import macros + +macro symFromDesc(T: typedesc): untyped = + let typ = getType(T) + typ[1] + +template produceType(T: typedesc): untyped = + type + XT = object + x: symFromDesc(T) + + XT + +type + X[T] = produceType(T) + +var x: X[int] From ca51bb8fd97e36e001afc7e9ad150dc77e9ca83b Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 19 Sep 2021 18:31:55 +0200 Subject: [PATCH 1226/2472] documented overload disambiguation [backport] (#18865) * documented overload disambiguation [backport] * Update doc/manual.rst Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * documented overload disambiguation [backport] * documented overload disambiguation [backport] Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- doc/manual.rst | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/doc/manual.rst b/doc/manual.rst index af163bceb..09826441c 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -2708,6 +2708,24 @@ a parameter typed as `untyped` (for unresolved expressions) or the type class assert toSeq2(items(@[1,2])) == @[1, 2] # but items(@[1,2]) is +Overload disambiguation +======================= + +For routine calls "overload resolution" is performed. There is a weaker form of +overload resolution called *overload disambiguation* that is performed when an +overloaded symbol is used in a context where there is additional type information +available. Let `p` be an overloaded symbol. These contexts are: + +- In a function call `q(..., p, ...)` when the corresponding formal parameter + of `q` is a `proc` type. If `q` itself is overloaded then the cartesian product + of every interpretation of `q` and `p` must be considered. +- In an object constructor `Obj(..., field: p, ...)` when `field` is a `proc` + type. Analogous rules exist for array/set/tuple constructors. +- In a declaration like `x: T = p` when `T` is a `proc` type. + +As usual, ambiguous matches produce a compile-time error. + + Statements and expressions ========================== @@ -4414,7 +4432,7 @@ would. For example: if n > 0: yield n for e in toItr(recCountDown(n - 1)): - yield e + yield e for i in toItr(recCountDown(6)): # Emits: 6 5 4 3 2 1 echo i From 3241df2a13662e0158ec9d0b0c07996b22fb0939 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 19 Sep 2021 23:35:50 +0200 Subject: [PATCH 1227/2472] fixes #18858 [backport] (#18868) * fixes #18858 [backport] * ensure async tests work with --experimental:strictEffects [backport] * ensure async tests work with --experimental:strictEffects [backport] --- lib/pure/asyncfutures.nim | 2 +- lib/pure/osproc.nim | 6 +++++- lib/pure/unicode.nim | 5 ++++- tests/async/nim.cfg | 1 + 4 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 tests/async/nim.cfg diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index af3fbb3ec..6906d03fd 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -93,7 +93,7 @@ proc setCallSoonProc*(p: (proc(cbproc: proc ()) {.gcsafe.})) = ## Change current implementation of `callSoon`. This is normally called when dispatcher from `asyncdispatcher` is initialized. callSoonProc = p -proc callSoon*(cbproc: proc ()) = +proc callSoon*(cbproc: proc () {.gcsafe.}) = ## Call `cbproc` "soon". ## ## If async dispatcher is running, `cbproc` will be executed during next dispatcher tick. diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 1ac37661b..6a0ac9a8b 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -330,12 +330,16 @@ proc countProcessors*(): int {.rtl, extern: "nosp$1".} = ## It is implemented just calling `cpuinfo.countProcessors`. result = cpuinfo.countProcessors() +when not defined(nimHasEffectsOf): + {.pragma: effectsOf.} + proc execProcesses*(cmds: openArray[string], options = {poStdErrToStdOut, poParentStreams}, n = countProcessors(), beforeRunEvent: proc(idx: int) = nil, afterRunEvent: proc(idx: int, p: Process) = nil): int {.rtl, extern: "nosp$1", - tags: [ExecIOEffect, TimeEffect, ReadEnvEffect, RootEffect].} = + tags: [ExecIOEffect, TimeEffect, ReadEnvEffect, RootEffect], + effectsOf: [beforeRunEvent, afterRunEvent].} = ## Executes the commands `cmds` in parallel. ## Creates `n` processes that execute in parallel. ## diff --git a/lib/pure/unicode.nim b/lib/pure/unicode.nim index dda631633..660be7814 100644 --- a/lib/pure/unicode.nim +++ b/lib/pure/unicode.nim @@ -705,8 +705,11 @@ proc capitalize*(s: string): string {.noSideEffect, fastRuneAt(s, i, rune, doInc = true) result = $toUpper(rune) & substr(s, i) +when not defined(nimHasEffectsOf): + {.pragma: effectsOf.} + proc translate*(s: string, replacements: proc(key: string): string): string {. - rtl, extern: "nuc$1".} = + rtl, extern: "nuc$1", effectsOf: replacements.} = ## Translates words in a string using the ``replacements`` proc to substitute ## words inside ``s`` with their replacements. ## diff --git a/tests/async/nim.cfg b/tests/async/nim.cfg new file mode 100644 index 000000000..be50f572c --- /dev/null +++ b/tests/async/nim.cfg @@ -0,0 +1 @@ +--experimental:strictEffects From c56ba3f06a3d77ff5eabb3b48ecd6a752e5a3ea8 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 20 Sep 2021 09:09:53 +0200 Subject: [PATCH 1228/2472] fixes #18847 [backport] (#18870) --- compiler/parser.nim | 5 +++-- tests/parser/tdoc_comments.nim | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/compiler/parser.nim b/compiler/parser.nim index 7d3c913da..5e9a7424a 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -1824,8 +1824,9 @@ proc parseRoutine(p: var Parser, kind: TNodeKind): PNode = # => moves comment `foo` to `fn` result.comment = body[0].comment body[0].comment = "" - else: - assert false, p.lex.config$body.info # avoids hard to track bugs, fail early. + #else: + # assert false, p.lex.config$body.info # avoids hard to track bugs, fail early. + # Yeah, that worked so well. There IS a bug in this logic, now what? proc newCommentStmt(p: var Parser): PNode = #| commentStmt = COMMENT diff --git a/tests/parser/tdoc_comments.nim b/tests/parser/tdoc_comments.nim index fa1374b45..3c86e69ea 100644 --- a/tests/parser/tdoc_comments.nim +++ b/tests/parser/tdoc_comments.nim @@ -69,3 +69,7 @@ type MyEnum3* = enum value5 ## only document the enum value + +# bug #18847 +proc close*() = ## asdfasdfsdfa + discard ## adsfasdfads From ee9795f76f1125e0addbf8cf5471cf188ad7f57a Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 20 Sep 2021 15:50:20 +0800 Subject: [PATCH 1229/2472] fix #18844 (#18851) --- lib/pure/concurrency/atomics.nim | 8 ++++---- tests/stdlib/concurrency/tatomics.nim | 22 +++++++++++++++++++++- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/lib/pure/concurrency/atomics.nim b/lib/pure/concurrency/atomics.nim index bdf1e8cc2..b252789fb 100644 --- a/lib/pure/concurrency/atomics.nim +++ b/lib/pure/concurrency/atomics.nim @@ -288,10 +288,10 @@ else: type # Atomic* {.importcpp: "_Atomic('0)".} [T] = object - AtomicInt8 {.importc: "_Atomic NI8", size: 1.} = object - AtomicInt16 {.importc: "_Atomic NI16", size: 2.} = object - AtomicInt32 {.importc: "_Atomic NI32", size: 4.} = object - AtomicInt64 {.importc: "_Atomic NI64", size: 8.} = object + AtomicInt8 {.importc: "_Atomic NI8".} = int8 + AtomicInt16 {.importc: "_Atomic NI16".} = int16 + AtomicInt32 {.importc: "_Atomic NI32".} = int32 + AtomicInt64 {.importc: "_Atomic NI64".} = int64 template atomicType*(T: typedesc[Trivial]): untyped = # Maps the size of a trivial type to it's internal atomic type diff --git a/tests/stdlib/concurrency/tatomics.nim b/tests/stdlib/concurrency/tatomics.nim index 260d00990..beae0ed6d 100644 --- a/tests/stdlib/concurrency/tatomics.nim +++ b/tests/stdlib/concurrency/tatomics.nim @@ -1,6 +1,6 @@ # test atomic operations -import atomics, bitops +import std/[atomics, bitops] type Object = object @@ -607,3 +607,23 @@ block clear: doAssert not location.testAndSet location.clear(moRelease) doAssert not location.testAndSet + +block: # bug #18844 + when not defined(cpp): # cpp pending pr #18836 + type + Deprivation = object of RootObj + memes: Atomic[int] + Zoomer = object + dopamine: Deprivation + + block: + var x = Deprivation() + var y = Zoomer() + doAssert x.memes.load == 0 + doAssert y.dopamine.memes.load == 0 + + block: + var x: Deprivation + var y: Zoomer + doAssert x.memes.load == 0 + doAssert y.dopamine.memes.load == 0 From 14ced06bb1ef7f23fc6391d16763f1421a3edc80 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 20 Sep 2021 11:37:54 +0200 Subject: [PATCH 1230/2472] fixes #18863 [backport] (#18871) --- compiler/semstmts.nim | 2 +- compiler/semtempl.nim | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index c2f05ccce..81d6eeda2 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -1905,7 +1905,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, if tfTriggersCompileTime in s.typ.flags: incl(s.flags, sfCompileTime) if n[patternPos].kind != nkEmpty: - n[patternPos] = semPattern(c, n[patternPos]) + n[patternPos] = semPattern(c, n[patternPos], s) if s.kind == skIterator: s.typ.flags.incl(tfIterator) elif s.kind == skFunc: diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim index 2636784af..eda4ca6cc 100644 --- a/compiler/semtempl.nim +++ b/compiler/semtempl.nim @@ -334,7 +334,7 @@ proc semTemplSomeDecl(c: var TemplCtx, n: PNode, symKind: TSymKind; start = 0) = illFormedAst(a, c.c.config) -proc semPattern(c: PContext, n: PNode): PNode +proc semPattern(c: PContext, n: PNode; s: PSym): PNode proc semTemplBodySons(c: var TemplCtx, n: PNode): PNode = result = n @@ -645,7 +645,7 @@ proc semTemplateDef(c: PContext, n: PNode): PNode = if allUntyped: incl(s.flags, sfAllUntyped) if n[patternPos].kind != nkEmpty: - n[patternPos] = semPattern(c, n[patternPos]) + n[patternPos] = semPattern(c, n[patternPos], s) var ctx: TemplCtx ctx.toBind = initIntSet() @@ -798,7 +798,7 @@ proc semPatternBody(c: var TemplCtx, n: PNode): PNode = for i in 0.. Date: Mon, 20 Sep 2021 13:24:44 +0200 Subject: [PATCH 1231/2472] [backport] Revert "use new Nimble, with lockfiles (#18810)" (#18872) This reverts commit f373c17ad926b669bb3b5819ae1dff4bde1da88a. For more details, see: - https://github.com/nim-lang/nimble/issues/940 - https://github.com/nim-lang/Nim/issues/18840 - https://forum.nim-lang.org/t/8404 --- koch.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/koch.nim b/koch.nim index e42ea53de..8dd897028 100644 --- a/koch.nim +++ b/koch.nim @@ -10,7 +10,7 @@ # const - NimbleStableCommit = "795704833ddfd0cdaefb45c60551d3ea205279ef" # master + NimbleStableCommit = "d13f3b8ce288b4dc8c34c219a4e050aaeaf43fc9" # master # examples of possible values: #head, #ea82b54, 1.2.3 FusionStableHash = "#372ee4313827ef9f2ea388840f7d6b46c2b1b014" HeadHash = "#head" From 928ea6bb4c0cc791f1b81e55dbebd14d5c6a7315 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 20 Sep 2021 13:32:33 +0200 Subject: [PATCH 1232/2472] atlas: added --cfgHere switch [backport] (#18873) --- tools/atlas/atlas.nim | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/tools/atlas/atlas.nim b/tools/atlas/atlas.nim index 370138029..e604c3167 100644 --- a/tools/atlas/atlas.nim +++ b/tools/atlas/atlas.nim @@ -27,6 +27,8 @@ Command: Options: --keepCommits do not perform any `git checkouts` + --cfgHere also create/maintain a nim.cfg in the current + working directory --version show the version --help show this help """ @@ -58,6 +60,7 @@ type projectDir, workspace: string hasPackageList: bool keepCommits: bool + cfgHere: bool p: Table[string, string] # name -> url mapping processed: HashSet[string] # the key is (url / commit) errors: int @@ -353,7 +356,7 @@ proc clone(c: var AtlasContext; start: string): seq[string] = error c, toName(start), "cannot resolve package name" return - c.projectDir = work[0].name.string + c.projectDir = c.workspace / work[0].name.string result = @[] var i = 0 while i < work.len: @@ -376,10 +379,16 @@ const configPatternBegin = "############# begin Atlas config section ##########\n" configPatternEnd = "############# end Atlas config section ##########\n" -proc patchNimCfg(c: var AtlasContext; deps: seq[string]) = +proc patchNimCfg(c: var AtlasContext; deps: seq[string]; cfgHere: bool) = var paths = "--noNimblePath\n" - for d in deps: - paths.add "--path:\"../" & d.replace("\\", "/") & "\"\n" + if cfgHere: + let cwd = getCurrentDir() + for d in deps: + let x = relativePath(c.workspace / d, cwd, '/') + paths.add "--path:\"" & x & "\"\n" + else: + for d in deps: + paths.add "--path:\"../" & d.replace("\\", "/") & "\"\n" var cfgContent = configPatternBegin & paths & configPatternEnd @@ -387,7 +396,8 @@ proc patchNimCfg(c: var AtlasContext; deps: seq[string]) = assert readFile(TestsDir / "nim.cfg") == cfgContent c.mockupSuccess = true else: - let cfg = c.projectDir / "nim.cfg" + let cfg = if cfgHere: getCurrentDir() / "nim.cfg" + else: c.projectDir / "nim.cfg" if not fileExists(cfg): writeFile(cfg, cfgContent) else: @@ -437,6 +447,7 @@ proc main = of "help", "h": writeHelp() of "version", "v": writeVersion() of "keepcommits": c.keepCommits = true + of "cfghere": c.cfgHere = true else: writeHelp() of cmdEnd: assert false, "cannot happen" @@ -449,7 +460,9 @@ proc main = of "clone": singleArg() let deps = clone(c, args[0]) - patchNimCfg c, deps + patchNimCfg c, deps, false + if c.cfgHere: + patchNimCfg c, deps, true when MockupRun: if not c.mockupSuccess: error "There were problems." From 0ad601d3c1657216d27be839173861594e3e2421 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 22 Sep 2021 09:43:06 +0200 Subject: [PATCH 1233/2472] fixes #18856 [backport] (#18879) --- compiler/semexprs.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 5a1426f2b..ee2626922 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1661,7 +1661,7 @@ proc takeImplicitAddr(c: PContext, n: PNode; isLent: bool): PNode = localError(c.config, n.info, errXStackEscape % renderTree(n, {renderNoComments})) else: localError(c.config, n.info, errExprHasNoAddress) - result = newNodeIT(nkHiddenAddr, n.info, makePtrType(c, n.typ)) + result = newNodeIT(nkHiddenAddr, n.info, if n.typ.kind in {tyVar, tyLent}: n.typ else: makePtrType(c, n.typ)) result.add(n) proc asgnToResultVar(c: PContext, n, le, ri: PNode) {.inline.} = From 6163bdd279a2c33d4290223cc7f466c8070b8ce4 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 22 Sep 2021 15:07:36 +0200 Subject: [PATCH 1234/2472] closes #16132 [backport] (#18880) * closes #16132 [backport] * fixes #16132 [backport] --- compiler/semexprs.nim | 13 ++++++++----- tests/views/tviews1.nim | 14 +++++++++++++- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index ee2626922..9134c1301 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1667,11 +1667,14 @@ proc takeImplicitAddr(c: PContext, n: PNode; isLent: bool): PNode = proc asgnToResultVar(c: PContext, n, le, ri: PNode) {.inline.} = if le.kind == nkHiddenDeref: var x = le[0] - if x.kind == nkSym and x.sym.kind == skResult and (x.typ.kind in {tyVar, tyLent} or classifyViewType(x.typ) != noView): - n[0] = x # 'result[]' --> 'result' - n[1] = takeImplicitAddr(c, ri, x.typ.kind == tyLent) - x.typ.flags.incl tfVarIsPtr - #echo x.info, " setting it for this type ", typeToString(x.typ), " ", n.info + if x.kind == nkSym: + if x.sym.kind == skResult and (x.typ.kind in {tyVar, tyLent} or classifyViewType(x.typ) != noView): + n[0] = x # 'result[]' --> 'result' + n[1] = takeImplicitAddr(c, ri, x.typ.kind == tyLent) + x.typ.flags.incl tfVarIsPtr + #echo x.info, " setting it for this type ", typeToString(x.typ), " ", n.info + elif sfGlobal in x.sym.flags: + x.typ.flags.incl tfVarIsPtr proc borrowCheck(c: PContext, n, le, ri: PNode) = const diff --git a/tests/views/tviews1.nim b/tests/views/tviews1.nim index 51f17b9d6..49d79c5b5 100644 --- a/tests/views/tviews1.nim +++ b/tests/views/tviews1.nim @@ -5,7 +5,8 @@ discard """ 3 2 3 -3''' +3 +15''' targets: "c cpp" """ @@ -26,3 +27,14 @@ proc main(s: seq[int]) = take x main(@[11, 22, 33]) + +var x: int + +proc foo(x: var int): var int = + once: x = 42 + return x + +var y: var int = foo(x) +y = 15 +echo foo(x) +# bug #16132 From 7d2a4c0880223c2c80a5c30227c26fa497898f39 Mon Sep 17 00:00:00 2001 From: Miran Date: Thu, 23 Sep 2021 11:00:36 +0200 Subject: [PATCH 1235/2472] update csources hash (#18885) --- config/build_config.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/build_config.txt b/config/build_config.txt index 75dbb1da3..aee15e695 100644 --- a/config/build_config.txt +++ b/config/build_config.txt @@ -2,4 +2,4 @@ nim_comment="key-value pairs for windows/posix bootstrapping build scripts" nim_csourcesDir=csources_v1 nim_csourcesUrl=https://github.com/nim-lang/csources_v1.git nim_csourcesBranch=master -nim_csourcesHash=9a7f751d23c49c75a0b6f63a234c575dc0df7231 +nim_csourcesHash=5e9be0383f842c9c22ae86e41946a896c43e95d0 From 90a2b5afd8368777e5da9ab97c28130ba683e1d3 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 23 Sep 2021 16:47:24 +0200 Subject: [PATCH 1236/2472] correct effect tracking for .borrowed procs [backport] (#18882) * correct effect tracking for .borrowed procs [backport] * progress * fix error message in a test * correctly fix it Co-authored-by: narimiran --- compiler/ast.nim | 2 +- compiler/ccgexprs.nim | 4 ++-- compiler/jsgen.nim | 2 +- compiler/sempass2.nim | 13 +++++++++---- compiler/semstmts.nim | 5 ++++- compiler/vmgen.nim | 2 +- lib/system.nim | 2 +- tests/effects/tgcsafe3.nim | 2 +- 8 files changed, 20 insertions(+), 12 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 50d048edd..0a3b1b72d 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -673,7 +673,7 @@ type mSwap, mIsNil, mArrToSeq, mNewString, mNewStringOfCap, mParseBiggestFloat, mMove, mWasMoved, mDestroy, mTrace, - mDefault, mUnown, mIsolate, mAccessEnv, mReset, + mDefault, mUnown, mFinished, mIsolate, mAccessEnv, mReset, mArray, mOpenArray, mRange, mSet, mSeq, mVarargs, mRef, mPtr, mVar, mDistinct, mVoid, mTuple, mOrdinal, mIterableType, diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 7a3d769aa..ac4a26bd6 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -2335,7 +2335,7 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = genDollar(p, e, d, "#nimFloatToStr($1)") of mCStrToStr: genDollar(p, e, d, "#cstrToNimstr($1)") of mStrToStr, mUnown: expr(p, e[1], d) - of mIsolate: genCall(p, e, d) + of mIsolate, mFinished: genCall(p, e, d) of mEnumToStr: if optTinyRtti in p.config.globalOptions: genEnumToStr(p, e, d) @@ -2929,7 +2929,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = if n[genericParamsPos].kind == nkEmpty: var prc = n[namePos].sym if useAliveDataFromDce in p.module.flags: - if p.module.alive.contains(prc.itemId.item) and prc.magic in {mNone, mIsolate}: + if p.module.alive.contains(prc.itemId.item) and prc.magic in {mNone, mIsolate, mFinished}: genProc(p.module, prc) elif prc.skipGenericOwner.kind == skModule and sfCompileTime notin prc.flags: if ({sfExportc, sfCompilerProc} * prc.flags == {sfExportc}) or diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index f009c83f2..ff4d2839e 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -657,7 +657,7 @@ proc arithAux(p: PProc, n: PNode, r: var TCompRes, op: TMagic) = of mIntToStr: applyFormat("cstrToNimstr(($1) + \"\")", "cstrToNimstr(($1) + \"\")") of mInt64ToStr: applyFormat("cstrToNimstr(($1) + \"\")", "cstrToNimstr(($1) + \"\")") of mCStrToStr: applyFormat("cstrToNimstr($1)", "cstrToNimstr($1)") - of mStrToStr, mUnown, mIsolate: applyFormat("$1", "$1") + of mStrToStr, mUnown, mIsolate, mFinished: applyFormat("$1", "$1") else: assert false, $op diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index 6a0e8772a..9a27d14fa 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -245,9 +245,14 @@ proc listGcUnsafety(s: PSym; onlyWarning: bool; cycleCheck: var IntSet; conf: Co let msgKind = if onlyWarning: warnGcUnsafe2 else: errGenerated case u.kind of skLet, skVar: - message(conf, s.info, msgKind, - ("'$#' is not GC-safe as it accesses '$#'" & - " which is a global using GC'ed memory") % [s.name.s, u.name.s]) + if u.typ.skipTypes(abstractInst).kind == tyProc: + message(conf, s.info, msgKind, + "'$#' is not GC-safe as it calls '$#'" % + [s.name.s, u.name.s]) + else: + message(conf, s.info, msgKind, + ("'$#' is not GC-safe as it accesses '$#'" & + " which is a global using GC'ed memory") % [s.name.s, u.name.s]) of routineKinds: # recursive call *always* produces only a warning so the full error # message is printed: @@ -850,7 +855,7 @@ proc trackCall(tracked: PEffects; n: PNode) = mergeRaises(tracked, effectList[exceptionEffects], n) mergeTags(tracked, effectList[tagEffects], n) gcsafeAndSideeffectCheck() - if a.kind != nkSym or a.sym.magic != mNBindSym: + if a.kind != nkSym or a.sym.magic notin {mNBindSym, mFinished}: for i in 1.. 0: + s.typ.n[0] = b.typ.n[0] + s.typ.flags = b.typ.flags else: localError(c.config, n.info, errNoSymbolToBorrowFromFound) @@ -1954,7 +1957,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, if not hasProto: implicitPragmas(c, s, n.info, validPragmas) - if n[pragmasPos].kind != nkEmpty: + if n[pragmasPos].kind != nkEmpty and sfBorrow notin s.flags: setEffectsForProcType(c.graph, s.typ, n[pragmasPos], s) s.typ.flags.incl tfEffectSystemWorkaround diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index f2758ff37..bb095d3ee 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1029,7 +1029,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = c.genAsgnPatch(n[1], d) c.freeTemp(d) of mOrd, mChr, mArrToSeq, mUnown: c.gen(n[1], dest) - of mIsolate: + of mIsolate, mFinished: genCall(c, n, dest) of mNew, mNewFinalize: unused(c, n, dest) diff --git a/lib/system.nim b/lib/system.nim index c93af0854..a5356cb54 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2453,7 +2453,7 @@ when notJSnotNims: else: {.error: "Only closure function and iterator are allowed!".} - proc finished*[T: proc](x: T): bool {.noSideEffect, inline.} = + proc finished*[T: proc](x: T): bool {.noSideEffect, inline, magic: "Finished".} = ## It can be used to determine if a first class iterator has finished. when T is "iterator": {.emit: """ diff --git a/tests/effects/tgcsafe3.nim b/tests/effects/tgcsafe3.nim index 5137efe4c..36ea5112c 100644 --- a/tests/effects/tgcsafe3.nim +++ b/tests/effects/tgcsafe3.nim @@ -1,5 +1,5 @@ discard """ - errormsg: "'myproc' is not GC-safe as it accesses 'global_proc' which is a global using GC'ed memory" + errormsg: "'myproc' is not GC-safe as it calls 'global_proc'" line: 12 cmd: "nim $target --hints:on --threads:on $options $file" """ From 561b01004cfeb572f82327cd8a3de55d076cb5ff Mon Sep 17 00:00:00 2001 From: Miran Date: Fri, 24 Sep 2021 11:24:55 +0200 Subject: [PATCH 1237/2472] [backport] change some examples in tut1.rst (#18884) * [backport] change some examples in tut1.rst * Update doc/tut1.rst * Update doc/tut1.rst Co-authored-by: Andreas Rumpf --- doc/tut1.rst | 47 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/doc/tut1.rst b/doc/tut1.rst index 210ea324e..f492f12b0 100644 --- a/doc/tut1.rst +++ b/doc/tut1.rst @@ -1643,9 +1643,25 @@ variables! For example: echo badname echo badext +Tuple unpacking is also supported in for-loops: + +.. code-block:: nim + :test: "nim c $1" + let a = [(10, 'a'), (20, 'b'), (30, 'c')] + + for (x, c) in a: + echo x + # This will output: 10; 20; 30 + + # Accessing the index is also possible: + for i, (x, c) in a: + echo i, c + # This will output: 0a; 1b; 2c + Fields of tuples are always public, they don't need to be explicity marked to be exported, unlike for example fields in an object type. + Reference and pointer types --------------------------- @@ -1675,13 +1691,18 @@ operators perform implicit dereferencing operations for reference types: Node = ref object le, ri: Node data: int - var - n: Node - new(n) - n.data = 9 + + var n = Node(data: 9) + echo n.data # no need to write n[].data; in fact n[].data is highly discouraged! -To allocate a new traced object, the built-in procedure `new` must be used. +To allocate a new traced object, the built-in procedure `new` can be used: + +.. code-block:: nim + + var n: Node + new(n) + To deal with untraced memory, the procedures `alloc`, `dealloc` and `realloc` can be used. The `system `_ module's documentation contains further details. @@ -1701,15 +1722,17 @@ Example: .. code-block:: nim :test: "nim c $1" - proc echoItem(x: int) = echo x + proc greet(name: string): string = + "Hello, " & name & "!" - proc forEach(action: proc (x: int)) = - const - data = [2, 3, 5, 7, 11] - for d in items(data): - action(d) + proc bye(name: string): string = + "Goodbye, " & name & "." - forEach(echoItem) + proc communicate(greeting: proc (x: string): string, name: string) = + echo greeting(name) + + communicate(greet, "John") + communicate(bye, "Mary") A subtle issue with procedural types is that the calling convention of the procedure influences the type compatibility: procedural types are only compatible From 5d315ebcc2d4f46b4a74c6ab10146466c894b9de Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 24 Sep 2021 13:24:41 +0200 Subject: [PATCH 1238/2472] ported Nim to proprietary CrossOS [backport] (#18889) --- ci/funs.sh | 2 ++ compiler/installer.ini | 1 + compiler/options.nim | 6 ++++-- compiler/platform.nim | 6 +++++- lib/pure/os.nim | 2 +- tools/niminst/buildsh.nimf | 5 +++++ 6 files changed, 18 insertions(+), 4 deletions(-) diff --git a/ci/funs.sh b/ci/funs.sh index e25922c8c..1638931c6 100644 --- a/ci/funs.sh +++ b/ci/funs.sh @@ -74,6 +74,8 @@ _nimBuildCsourcesIfNeeded(){ makeX=gmake elif [ "$unamestr" = 'NetBSD' ]; then makeX=gmake + elif [ "$unamestr" = 'CROSSOS' ]; then + makeX=gmake else makeX=make fi diff --git a/compiler/installer.ini b/compiler/installer.ini index 3f1630a92..9e31b4e3f 100644 --- a/compiler/installer.ini +++ b/compiler/installer.ini @@ -13,6 +13,7 @@ Platforms: """ netbsd: i386;amd64 openbsd: i386;amd64;arm;arm64 dragonfly: i386;amd64 + crossos: amd64 haiku: i386;amd64 android: i386;arm;arm64 nintendoswitch: arm64 diff --git a/compiler/options.nim b/compiler/options.nim index ea302aed6..eafcd816d 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -577,11 +577,13 @@ proc isDefined*(conf: ConfigRef; symbol: string): bool = osQnx, osAtari, osAix, osHaiku, osVxWorks, osSolaris, osNetbsd, osFreebsd, osOpenbsd, osDragonfly, osMacosx, osIos, - osAndroid, osNintendoSwitch, osFreeRTOS} + osAndroid, osNintendoSwitch, osFreeRTOS, osCrossos} of "linux": result = conf.target.targetOS in {osLinux, osAndroid} of "bsd": - result = conf.target.targetOS in {osNetbsd, osFreebsd, osOpenbsd, osDragonfly} + result = conf.target.targetOS in {osNetbsd, osFreebsd, osOpenbsd, osDragonfly, osCrossos} + of "freebsd": + result = conf.target.targetOS in {osFreebsd, osCrossos} of "emulatedthreadvars": result = platform.OS[conf.target.targetOS].props.contains(ospLacksThreadVars) of "msdos": result = conf.target.targetOS == osDos diff --git a/compiler/platform.nim b/compiler/platform.nim index 7bc77d809..1bc00b629 100644 --- a/compiler/platform.nim +++ b/compiler/platform.nim @@ -20,7 +20,7 @@ type TSystemOS* = enum # Also add OS in initialization section and alias # conditionals to condsyms (end of module). osNone, osDos, osWindows, osOs2, osLinux, osMorphos, osSkyos, osSolaris, - osIrix, osNetbsd, osFreebsd, osOpenbsd, osDragonfly, osAix, osPalmos, osQnx, + osIrix, osNetbsd, osFreebsd, osOpenbsd, osDragonfly, osCrossos, osAix, osPalmos, osQnx, osAmiga, osAtari, osNetware, osMacos, osMacosx, osIos, osHaiku, osAndroid, osVxWorks osGenode, osJS, osNimVM, osStandalone, osNintendoSwitch, osFreeRTOS, osAny @@ -105,6 +105,10 @@ const scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".", props: {ospNeedsPIC, ospPosix}), + (name: "CROSSOS", parDir: "..", dllFrmt: "lib$1.so", altDirSep: "/", + objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/", + scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".", + props: {ospNeedsPIC, ospPosix}), (name: "AIX", parDir: "..", dllFrmt: "lib$1.so", altDirSep: "/", objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/", scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".", diff --git a/lib/pure/os.nim b/lib/pure/os.nim index cfdacdb3c..dd1f9f0b6 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -3007,7 +3007,7 @@ when not weirdTarget and (defined(freebsd) or defined(dragonfly) or defined(netb proc sysctl(name: ptr cint, namelen: cuint, oldp: pointer, oldplen: ptr csize_t, newp: pointer, newplen: csize_t): cint {.importc: "sysctl",header: """#include - #include """} + #include """.} const CTL_KERN = 1 KERN_PROC = 14 diff --git a/tools/niminst/buildsh.nimf b/tools/niminst/buildsh.nimf index f74413c09..7b30cd86c 100644 --- a/tools/niminst/buildsh.nimf +++ b/tools/niminst/buildsh.nimf @@ -109,6 +109,11 @@ case $uos in CC="clang" LINK_FLAGS="$LINK_FLAGS -lm" ;; + *crossos* ) + myos="crossos" + CC="clang" + LINK_FLAGS="$LINK_FLAGS -lm" + ;; *openbsd* ) myos="openbsd" CC="clang" From f7d642f2f3925feb5addc315eb59caf326cad470 Mon Sep 17 00:00:00 2001 From: Clyybber Date: Fri, 24 Sep 2021 14:59:48 +0200 Subject: [PATCH 1239/2472] [backport] arc: improve compile time of (nested) loops (#18890) --- compiler/dfa.nim | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/compiler/dfa.nim b/compiler/dfa.nim index 65c483e9c..0539f6699 100644 --- a/compiler/dfa.nim +++ b/compiler/dfa.nim @@ -308,6 +308,11 @@ when true: # We unroll every loop 3 times. We emulate 0, 1, 2 iterations # through the loop. We need to prove this is correct for our # purposes. But Herb Sutter claims it is. (Proof by authority.) + # + # EDIT: Actually, we only need to unroll 2 times + # because Nim doesn't have a way of breaking/goto-ing into + # a loop iteration. Unrolling 2 times is much better for compile + # times of nested loops than 3 times, so we do that here. #[ while cond: body @@ -347,12 +352,12 @@ when true: # 'while true' is an idiom in Nim and so we produce # better code for it: withBlock(nil): - for i in 0..2: + for i in 0..1: c.gen(n[1]) else: withBlock(nil): - var endings: array[3, TPosition] - for i in 0..2: + var endings: array[2, TPosition] + for i in 0..1: c.gen(n[0]) endings[i] = c.forkI(n) c.gen(n[1]) From 7e5eab571e6e8e57928b40f535802aec04e04633 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 24 Sep 2021 16:27:34 +0200 Subject: [PATCH 1240/2472] closes #18690; make view types stricter [backport] (#18891) * closes #18690 * don't allow capturing of view types [backport] --- compiler/lambdalifting.nim | 6 ++---- tests/views/tviews1.nim | 13 ++++++++++++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index c555cedfe..a622f6de6 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -12,7 +12,7 @@ import intsets, strutils, options, ast, astalgo, msgs, idents, renderer, types, magicsys, lowerings, tables, modulegraphs, lineinfos, - transf, liftdestructors + transf, liftdestructors, typeallowed discard """ The basic approach is that captured vars need to be put on the heap and @@ -191,9 +191,7 @@ proc interestingVar(s: PSym): bool {.inline.} = s.typ.kind notin {tyStatic, tyTypeDesc} proc illegalCapture(s: PSym): bool {.inline.} = - result = skipTypes(s.typ, abstractInst).kind in - {tyVar, tyOpenArray, tyVarargs, tyLent} or - s.kind == skResult + result = classifyViewType(s.typ) != noView or s.kind == skResult proc isInnerProc(s: PSym): bool = if s.kind in {skProc, skFunc, skMethod, skConverter, skIterator} and s.magic == mNone: diff --git a/tests/views/tviews1.nim b/tests/views/tviews1.nim index 49d79c5b5..3dbf664fc 100644 --- a/tests/views/tviews1.nim +++ b/tests/views/tviews1.nim @@ -6,7 +6,8 @@ discard """ 2 3 3 -15''' +15 +(oa: [1, 3, 4])''' targets: "c cpp" """ @@ -38,3 +39,13 @@ var y: var int = foo(x) y = 15 echo foo(x) # bug #16132 + +# bug #18690 + +type + F = object + oa: openarray[int] + +let s1 = @[1,3,4,5,6] +var test = F(oa: toOpenArray(s1, 0, 2)) +echo test From 4adada0d8016016104d6259a1167a7f8b37f1611 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 24 Sep 2021 22:34:12 +0800 Subject: [PATCH 1241/2472] fix a typo (rename `temporal` to `temporary` (#18892) [backport] --- lib/pure/strutils.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 3179c73d2..873949ca6 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -2039,7 +2039,7 @@ func countLines*(s: string): int {.rtl, extern: "nsuCountLines".} = ## Returns the number of lines in the string `s`. ## ## This is the same as `len(splitLines(s))`, but much more efficient - ## because it doesn't modify the string creating temporal objects. Every + ## because it doesn't modify the string creating temporary objects. Every ## `character literal `_ ## newline combination (CR, LF, CR-LF) is supported. ## From 8bdb98539570fdf3c41ee5509d5b90913ac1803e Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 25 Sep 2021 19:22:00 +0800 Subject: [PATCH 1242/2472] fix wrong name (rnimsyn => renderer; pnimsyn => parser; scanner => lexer) (#18895) * fix wrong module name * rephrase more word --- compiler/filter_tmpl.nim | 2 +- compiler/lexer.nim | 4 ++-- compiler/nimconf.nim | 2 +- compiler/renderer.nim | 4 ++-- doc/manual.rst | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/compiler/filter_tmpl.nim b/compiler/filter_tmpl.nim index 6165ff2f3..84ed99164 100644 --- a/compiler/filter_tmpl.nim +++ b/compiler/filter_tmpl.nim @@ -22,7 +22,7 @@ type info: TLineInfo indent, emitPar: int x: string # the current input line - outp: PLLStream # the output will be parsed by pnimsyn + outp: PLLStream # the output will be parsed by parser subsChar, nimDirective: char emit, conc, toStr: string curly, bracket, par: int diff --git a/compiler/lexer.nim b/compiler/lexer.nim index ede16fdf1..506b0e924 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -7,12 +7,12 @@ # distribution, for details about the copyright. # -# This scanner is handwritten for efficiency. I used an elegant buffering +# This lexer is handwritten for efficiency. I used an elegant buffering # scheme which I have not seen anywhere else: # We guarantee that a whole line is in the buffer. Thus only when scanning # the \n or \r character we have to check whether we need to read in the next # chunk. (\n or \r already need special handling for incrementing the line -# counter; choosing both \n and \r allows the scanner to properly read Unix, +# counter; choosing both \n and \r allows the lexer to properly read Unix, # DOS or Macintosh text files, even when it is not the native format. import diff --git a/compiler/nimconf.nim b/compiler/nimconf.nim index 94c93a283..1cf22e20a 100644 --- a/compiler/nimconf.nim +++ b/compiler/nimconf.nim @@ -14,7 +14,7 @@ import options, idents, wordrecg, strtabs, lineinfos, pathutils, scriptconfig # ---------------- configuration file parser ----------------------------- -# we use Nim's scanner here to save space and work +# we use Nim's lexer here to save space and work proc ppGetTok(L: var Lexer, tok: var Token) = # simple filter diff --git a/compiler/renderer.nim b/compiler/renderer.nim index 393f35289..22a2d4cbd 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -410,7 +410,7 @@ proc atom(g: TSrcGen; n: PNode): string = if (n.typ != nil) and (n.typ.sym != nil): result = n.typ.sym.name.s else: result = "[type node]" else: - internalError(g.config, "rnimsyn.atom " & $n.kind) + internalError(g.config, "renderer.atom " & $n.kind) result = "" proc lcomma(g: TSrcGen; n: PNode, start: int = 0, theEnd: int = - 1): int = @@ -1699,7 +1699,7 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext, fromStmtList = false) = gsub(g, n[0], c) else: #nkNone, nkExplicitTypeListCall: - internalError(g.config, n.info, "rnimsyn.gsub(" & $n.kind & ')') + internalError(g.config, n.info, "renderer.gsub(" & $n.kind & ')') proc renderTree*(n: PNode, renderFlags: TRenderFlags = {}): string = if n == nil: return "" diff --git a/doc/manual.rst b/doc/manual.rst index 09826441c..dfcf0ba54 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -216,7 +216,7 @@ comment: .. code-block:: nim i = 0 # This is a single comment over multiple lines. - # The scanner merges these two pieces. + # The lexer merges these two pieces. # The comment continues here. From e3b19cbe52e0578de315d259fbb066bfe991a60f Mon Sep 17 00:00:00 2001 From: Aditya Siram Date: Sat, 25 Sep 2021 07:17:41 -0500 Subject: [PATCH 1243/2472] fixes #18878 (#18883) --- lib/core/typeinfo.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/core/typeinfo.nim b/lib/core/typeinfo.nim index d66dc1898..18d2091b9 100644 --- a/lib/core/typeinfo.nim +++ b/lib/core/typeinfo.nim @@ -216,7 +216,8 @@ proc extendSeq*(x: Any) = when defined(gcDestructors): var s = cast[ptr NimSeqV2Reimpl](x.value) let elem = x.rawType.base - s.p = cast[ptr NimSeqPayloadReimpl](prepareSeqAdd(s.len, s.p, 1, elem.size, elem.align)) + if s.p == nil or s.p.cap < s.len+1: + s.p = cast[ptr NimSeqPayloadReimpl](prepareSeqAdd(s.len, s.p, 1, elem.size, elem.align)) inc s.len else: var y = cast[ptr PGenSeq](x.value)[] From 57eb763c65c2d517e2b6ca7b2ca2d3bb5ab6c3d5 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 27 Sep 2021 19:13:02 +0800 Subject: [PATCH 1244/2472] activate winim (#18907) `winim` is dependent on windows API. --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 657ddb2d1..0c47104ac 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -159,7 +159,7 @@ pkg "unicodeplus", "nim c -d:release -r tests/tests.nim" pkg "unpack" pkg "weave", "nimble test_gc_arc", allowFailure = true pkg "websocket", "nim c websocket.nim" -pkg "winim", allowFailure = true +pkg "winim", "nim c winim.nim" pkg "with" pkg "ws", allowFailure = true pkg "yaml", "nim c -r test/tserialization.nim" From 5325a366e79b6e34fcd2b2dfe675897212f8ca2d Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Mon, 27 Sep 2021 16:35:14 +0100 Subject: [PATCH 1245/2472] Add armv8l support (Closes: #18898) (#18901) [backport] --- tools/niminst/buildsh.nimf | 2 +- tools/niminst/makefile.nimf | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/niminst/buildsh.nimf b/tools/niminst/buildsh.nimf index 7b30cd86c..9850abd89 100644 --- a/tools/niminst/buildsh.nimf +++ b/tools/niminst/buildsh.nimf @@ -213,7 +213,7 @@ case $ucpu in mycpu="alpha" ;; *aarch64*|*arm64* ) mycpu="arm64" ;; - *arm*|*armv6l*|*armv71* ) + *arm*|*armv6l*|*armv7l*|*armv8l* ) mycpu="arm" ;; *riscv64|riscv* ) mycpu="riscv64" ;; diff --git a/tools/niminst/makefile.nimf b/tools/niminst/makefile.nimf index a0e9a40f7..69e87b837 100644 --- a/tools/niminst/makefile.nimf +++ b/tools/niminst/makefile.nimf @@ -157,6 +157,9 @@ endif ifeq ($(ucpu),armv7hl) mycpu = arm endif +ifeq ($(ucpu),armv8l) + mycpu = arm +endif ifeq ($(ucpu),aarch64) mycpu = arm64 endif From cdf9ac675b4348a7b5239186637c54920bb92619 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 27 Sep 2021 19:25:00 +0200 Subject: [PATCH 1246/2472] this ensures libp2p continues to compile [backport] (#18908) --- compiler/cgen.nim | 11 ++++++++--- compiler/guards.nim | 10 ++++++++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index a06a87172..43410dc60 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -855,10 +855,15 @@ proc closureSetup(p: BProc, prc: PSym) = [rdLoc(env.loc), getTypeDesc(p.module, env.typ)]) proc containsResult(n: PNode): bool = - if n.kind == nkSym and n.sym.kind == skResult: - result = true + result = false + case n.kind + of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, nkFormalParams: + discard + of nkSym: + if n.sym.kind == skResult: + result = true else: - for i in 0.. Date: Mon, 27 Sep 2021 22:23:31 +0200 Subject: [PATCH 1247/2472] fixes 'lent T' inside object constructor [backport] (#18911) * fixes 'lent T' inside object constructor [backport] * progress --- compiler/sem.nim | 3 ++- compiler/semobjconstr.nim | 2 +- compiler/semstmts.nim | 2 +- tests/views/tviews1.nim | 17 +++++++++++++++++ 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/compiler/sem.nim b/compiler/sem.nim index bdecbe602..70c57864c 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -107,12 +107,13 @@ proc fitNode(c: PContext, formal: PType, arg: PNode; info: TLineInfo): PNode = else: result = fitNodePostMatch(c, formal, result) -proc fitNodeForLocalVar(c: PContext, formal: PType, arg: PNode; info: TLineInfo): PNode = +proc fitNodeConsiderViewType(c: PContext, formal: PType, arg: PNode; info: TLineInfo): PNode = let a = fitNode(c, formal, arg, info) if formal.kind in {tyVar, tyLent}: #classifyViewType(formal) != noView: result = newNodeIT(nkHiddenAddr, a.info, formal) result.add a + formal.flags.incl tfVarIsPtr else: result = a diff --git a/compiler/semobjconstr.nim b/compiler/semobjconstr.nim index 1563e5c16..001956ede 100644 --- a/compiler/semobjconstr.nim +++ b/compiler/semobjconstr.nim @@ -79,7 +79,7 @@ proc semConstrField(c: PContext, flags: TExprFlags, var initValue = semExprFlagDispatched(c, assignment[1], flags) if initValue != nil: - initValue = fitNode(c, field.typ, initValue, assignment.info) + initValue = fitNodeConsiderViewType(c, field.typ, initValue, assignment.info) assignment[0] = newSymNode(field) assignment[1] = initValue assignment.flags.incl nfSem diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 3b699f6c8..4607e857a 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -545,7 +545,7 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = else: # BUGFIX: ``fitNode`` is needed here! # check type compatibility between def.typ and typ - def = fitNodeForLocalVar(c, typ, def, def.info) + def = fitNodeConsiderViewType(c, typ, def, def.info) #changeType(def.skipConv, typ, check=true) else: typ = def.typ.skipTypes({tyStatic, tySink}).skipIntLit(c.idgen) diff --git a/tests/views/tviews1.nim b/tests/views/tviews1.nim index 3dbf664fc..ced487ce8 100644 --- a/tests/views/tviews1.nim +++ b/tests/views/tviews1.nim @@ -49,3 +49,20 @@ type let s1 = @[1,3,4,5,6] var test = F(oa: toOpenArray(s1, 0, 2)) echo test + +type + Foo = object + x: string + y: seq[int] + data: array[10000, byte] + + View[T] = object + x: lent T + +proc mainB = + let f = Foo(y: @[1, 2, 3]) + let foo = View[Foo](x: f) + assert foo.x.x == "" + assert foo.x.y == @[1, 2, 3] + +mainB() From 5eb357f5a2db81d7027eaf5430cda74370e45a67 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 28 Sep 2021 12:45:25 +0800 Subject: [PATCH 1248/2472] change comment (#18913) --- lib/system.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/system.nim b/lib/system.nim index a5356cb54..358aae938 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -160,7 +160,7 @@ when defined(nimHashOrdinalFixed): ## as well as their subtypes. See also ## `SomeOrdinal`. else: - # bootstrap <= 0.20.0 + # bootstrap < 1.2.0 type OrdinalImpl[T] {.magic: Ordinal.} Ordinal* = OrdinalImpl | uint | uint64 From 56bd5cbb877b69f5c4a49470845621267a316f91 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 28 Sep 2021 15:05:22 +0800 Subject: [PATCH 1249/2472] fix a typo (canonilization => canonicalization) (#18914) --- compiler/patterns.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/patterns.nim b/compiler/patterns.nim index 1ef903161..4b39de3ba 100644 --- a/compiler/patterns.nim +++ b/compiler/patterns.nim @@ -36,7 +36,7 @@ proc putLazy(c: PPatternContext, sym: PSym, n: PNode) = proc matches(c: PPatternContext, p, n: PNode): bool proc canonKind(n: PNode): TNodeKind = - ## nodekind canonilization for pattern matching + ## nodekind canonicalization for pattern matching result = n.kind case result of nkCallKinds: result = nkCall From 08cf4cb1e58fad35be3846e98a308500ac2cfdb0 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 28 Sep 2021 17:35:04 +0800 Subject: [PATCH 1250/2472] test ord (#18909) --- compiler/semmagic.nim | 4 ---- 1 file changed, 4 deletions(-) diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim index 87f7da815..874098294 100644 --- a/compiler/semmagic.nim +++ b/compiler/semmagic.nim @@ -216,10 +216,6 @@ proc semOrd(c: PContext, n: PNode): PNode = let parType = n[1].typ if isOrdinalType(parType, allowEnumWithHoles=true): discard - elif parType.kind == tySet: - let a = toInt64(firstOrd(c.config, parType)) - let b = toInt64(lastOrd(c.config, parType)) - result.typ = makeRangeType(c, a, b, n.info) else: localError(c.config, n.info, errOrdinalTypeExpected) result.typ = errorType(c) From f061971a9be5222f0f158f605220f3bda42f7488 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 29 Sep 2021 00:32:39 -0700 Subject: [PATCH 1251/2472] envPairs works in vm, nims (#18615) * envPairs works in vm, nims * fixup --- compiler/vmops.nim | 14 +- lib/pure/includes/osenv.nim | 349 ++++++++++++++-------------- lib/pure/os.nim | 3 +- testament/lib/stdtest/testutils.nim | 7 +- tests/js/tos.nim | 28 +-- tests/stdlib/tosenv.nim | 18 +- tests/test_nimscript.nims | 4 + 7 files changed, 213 insertions(+), 210 deletions(-) diff --git a/compiler/vmops.nim b/compiler/vmops.nim index e287dd41d..b9801234d 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -13,7 +13,7 @@ from std/math import sqrt, ln, log10, log2, exp, round, arccos, arcsin, arctan, arctan2, cos, cosh, hypot, sinh, sin, tan, tanh, pow, trunc, floor, ceil, `mod`, cbrt, arcsinh, arccosh, arctanh, erf, erfc, gamma, lgamma - +from std/sequtils import toSeq when declared(math.copySign): # pending bug #18762, avoid renaming math from std/math as math2 import copySign @@ -22,8 +22,8 @@ when declared(math.signbit): # ditto from std/math as math3 import signbit -from std/os import getEnv, existsEnv, delEnv, putEnv, dirExists, fileExists, walkDir, - getAppFilename, raiseOSError, osLastError +from std/os import getEnv, existsEnv, delEnv, putEnv, envPairs, + dirExists, fileExists, walkDir, getAppFilename, raiseOSError, osLastError from std/md5 import getMD5 from std/times import cpuTime @@ -156,6 +156,12 @@ proc stackTrace2(c: PCtx, msg: string, n: PNode) = stackTrace(c, PStackFrame(prc: c.prc.sym, comesFrom: 0, next: nil), c.exceptionInstr, msg, n.info) proc registerAdditionalOps*(c: PCtx) = + + template wrapIterator(fqname: string, iter: untyped) = + registerCallback c, fqname, proc(a: VmArgs) = + setResult(a, toLit(toSeq(iter))) + + proc gorgeExWrapper(a: VmArgs) = let ret = opGorge(getString(a, 0), getString(a, 1), getString(a, 2), a.currentLineInfo, c.config) @@ -341,3 +347,5 @@ proc registerAdditionalOps*(c: PCtx) = let p = a.getVar(0) let x = a.getFloat(1) addFloatSprintf(p.strVal, x) + + wrapIterator("stdlib.os.envPairsImplSeq"): envPairs() diff --git a/lib/pure/includes/osenv.nim b/lib/pure/includes/osenv.nim index 6aaafbfda..00a82327c 100644 --- a/lib/pure/includes/osenv.nim +++ b/lib/pure/includes/osenv.nim @@ -3,189 +3,200 @@ when not declared(os) and not declared(ospaths): {.error: "This is an include file for os.nim!".} -when defined(nodejs): - proc getEnv*(key: string, default = ""): string {.tags: [ReadEnvEffect].} = - var ret = default.cstring - let key2 = key.cstring - {.emit: "const value = process.env[`key2`];".} - {.emit: "if (value !== undefined) { `ret` = value };".} - result = $ret +when not defined(nimscript): + when defined(nodejs): + proc getEnv*(key: string, default = ""): string {.tags: [ReadEnvEffect].} = + var ret = default.cstring + let key2 = key.cstring + {.emit: "const value = process.env[`key2`];".} + {.emit: "if (value !== undefined) { `ret` = value };".} + result = $ret - proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = - var key2 = key.cstring - var ret: bool - {.emit: "`ret` = `key2` in process.env;".} - result = ret + proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = + var key2 = key.cstring + var ret: bool + {.emit: "`ret` = `key2` in process.env;".} + result = ret - proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = - var key2 = key.cstring - var val2 = val.cstring - {.emit: "process.env[`key2`] = `val2`;".} + proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = + var key2 = key.cstring + var val2 = val.cstring + {.emit: "process.env[`key2`] = `val2`;".} - proc delEnv*(key: string) {.tags: [WriteEnvEffect].} = - var key2 = key.cstring - {.emit: "delete process.env[`key2`];".} + proc delEnv*(key: string) {.tags: [WriteEnvEffect].} = + var key2 = key.cstring + {.emit: "delete process.env[`key2`];".} - iterator envPairs*(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = - var num: int - var keys: RootObj - {.emit: "`keys` = Object.keys(process.env); `num` = `keys`.length;".} - for i in 0..".} - when defined(windows): - proc c_putenv_s(envname: cstring, envval: cstring): cint {.importc: "_putenv_s", header: "".} - from std/private/win_setenv import setEnvImpl else: - proc c_setenv(envname: cstring, envval: cstring, overwrite: cint): cint {.importc: "setenv", header: "".} - proc c_unsetenv(env: cstring): cint {.importc: "unsetenv", header: "".} - proc getEnv*(key: string, default = ""): string {.tags: [ReadEnvEffect].} = - ## Returns the value of the `environment variable`:idx: named `key`. - ## - ## If the variable does not exist, `""` is returned. To distinguish - ## whether a variable exists or it's value is just `""`, call - ## `existsEnv(key) proc <#existsEnv,string>`_. - ## - ## See also: - ## * `existsEnv proc <#existsEnv,string>`_ - ## * `putEnv proc <#putEnv,string,string>`_ - ## * `delEnv proc <#delEnv,string>`_ - ## * `envPairs iterator <#envPairs.i>`_ - runnableExamples: - assert getEnv("unknownEnv") == "" - assert getEnv("unknownEnv", "doesn't exist") == "doesn't exist" - - let env = c_getenv(key) - if env == nil: return default - result = $env - - proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = - ## Checks whether the environment variable named `key` exists. - ## Returns true if it exists, false otherwise. - ## - ## See also: - ## * `getEnv proc <#getEnv,string,string>`_ - ## * `putEnv proc <#putEnv,string,string>`_ - ## * `delEnv proc <#delEnv,string>`_ - ## * `envPairs iterator <#envPairs.i>`_ - runnableExamples: - assert not existsEnv("unknownEnv") - - return c_getenv(key) != nil - - proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = - ## Sets the value of the `environment variable`:idx: named `key` to `val`. - ## If an error occurs, `OSError` is raised. - ## - ## See also: - ## * `getEnv proc <#getEnv,string,string>`_ - ## * `existsEnv proc <#existsEnv,string>`_ - ## * `delEnv proc <#delEnv,string>`_ - ## * `envPairs iterator <#envPairs.i>`_ + proc c_getenv(env: cstring): cstring {. + importc: "getenv", header: "".} when defined(windows): - if key.len == 0 or '=' in key: - raise newException(OSError, "invalid key, got: " & $(key, val)) - if setEnvImpl(key, val, 1'i32) != 0'i32: - raiseOSError(osLastError(), $(key, val)) + proc c_putenv_s(envname: cstring, envval: cstring): cint {.importc: "_putenv_s", header: "".} + from std/private/win_setenv import setEnvImpl else: - if c_setenv(key, val, 1'i32) != 0'i32: - raiseOSError(osLastError(), $(key, val)) + proc c_setenv(envname: cstring, envval: cstring, overwrite: cint): cint {.importc: "setenv", header: "".} + proc c_unsetenv(env: cstring): cint {.importc: "unsetenv", header: "".} - proc delEnv*(key: string) {.tags: [WriteEnvEffect].} = - ## Deletes the `environment variable`:idx: named `key`. - ## If an error occurs, `OSError` is raised. - ## - ## See also:ven - ## * `getEnv proc <#getEnv,string,string>`_ - ## * `existsEnv proc <#existsEnv,string>`_ - ## * `putEnv proc <#putEnv,string,string>`_ - ## * `envPairs iterator <#envPairs.i>`_ - template bail = raiseOSError(osLastError(), key) - when defined(windows): - #[ - # https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/putenv-s-wputenv-s?view=msvc-160 - > You can remove a variable from the environment by specifying an empty string (that is, "") for value_string - note that nil is not legal - ]# - if key.len == 0 or '=' in key: - raise newException(OSError, "invalid key, got: " & key) - if c_putenv_s(key, "") != 0'i32: bail - else: - if c_unsetenv(key) != 0'i32: bail + proc getEnv*(key: string, default = ""): string {.tags: [ReadEnvEffect].} = + ## Returns the value of the `environment variable`:idx: named `key`. + ## + ## If the variable does not exist, `""` is returned. To distinguish + ## whether a variable exists or it's value is just `""`, call + ## `existsEnv(key) proc <#existsEnv,string>`_. + ## + ## See also: + ## * `existsEnv proc <#existsEnv,string>`_ + ## * `putEnv proc <#putEnv,string,string>`_ + ## * `delEnv proc <#delEnv,string>`_ + ## * `envPairs iterator <#envPairs.i>`_ + runnableExamples: + assert getEnv("unknownEnv") == "" + assert getEnv("unknownEnv", "doesn't exist") == "doesn't exist" - when defined(windows): - when useWinUnicode: - when defined(cpp): - proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.importcpp: "(NI16*)wcschr((const wchar_t *)#, #)", - header: "".} + let env = c_getenv(key) + if env == nil: return default + result = $env + + proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = + ## Checks whether the environment variable named `key` exists. + ## Returns true if it exists, false otherwise. + ## + ## See also: + ## * `getEnv proc <#getEnv,string,string>`_ + ## * `putEnv proc <#putEnv,string,string>`_ + ## * `delEnv proc <#delEnv,string>`_ + ## * `envPairs iterator <#envPairs.i>`_ + runnableExamples: + assert not existsEnv("unknownEnv") + + return c_getenv(key) != nil + + proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = + ## Sets the value of the `environment variable`:idx: named `key` to `val`. + ## If an error occurs, `OSError` is raised. + ## + ## See also: + ## * `getEnv proc <#getEnv,string,string>`_ + ## * `existsEnv proc <#existsEnv,string>`_ + ## * `delEnv proc <#delEnv,string>`_ + ## * `envPairs iterator <#envPairs.i>`_ + when defined(windows): + if key.len == 0 or '=' in key: + raise newException(OSError, "invalid key, got: " & $(key, val)) + if setEnvImpl(key, val, 1'i32) != 0'i32: + raiseOSError(osLastError(), $(key, val)) else: - proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.importc: "wcschr", - header: "".} - else: - proc strEnd(cstr: cstring, c = 0'i32): cstring {.importc: "strchr", - header: "".} - elif defined(macosx) and not defined(ios) and not defined(emscripten): - # From the manual: - # Shared libraries and bundles don't have direct access to environ, - # which is only available to the loader ld(1) when a complete program - # is being linked. - # The environment routines can still be used, but if direct access to - # environ is needed, the _NSGetEnviron() routine, defined in - # , can be used to retrieve the address of environ - # at runtime. - proc NSGetEnviron(): ptr cstringArray {.importc: "_NSGetEnviron", - header: "".} - elif defined(haiku): - var gEnv {.importc: "environ", header: "".}: cstringArray - else: - var gEnv {.importc: "environ".}: cstringArray + if c_setenv(key, val, 1'i32) != 0'i32: + raiseOSError(osLastError(), $(key, val)) + + proc delEnv*(key: string) {.tags: [WriteEnvEffect].} = + ## Deletes the `environment variable`:idx: named `key`. + ## If an error occurs, `OSError` is raised. + ## + ## See also:ven + ## * `getEnv proc <#getEnv,string,string>`_ + ## * `existsEnv proc <#existsEnv,string>`_ + ## * `putEnv proc <#putEnv,string,string>`_ + ## * `envPairs iterator <#envPairs.i>`_ + template bail = raiseOSError(osLastError(), key) + when defined(windows): + #[ + # https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/putenv-s-wputenv-s?view=msvc-160 + > You can remove a variable from the environment by specifying an empty string (that is, "") for value_string + note that nil is not legal + ]# + if key.len == 0 or '=' in key: + raise newException(OSError, "invalid key, got: " & key) + if c_putenv_s(key, "") != 0'i32: bail + else: + if c_unsetenv(key) != 0'i32: bail - iterator envPairs*(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = - ## Iterate over all `environments variables`:idx:. - ## - ## In the first component of the tuple is the name of the current variable stored, - ## in the second its value. - ## - ## See also: - ## * `getEnv proc <#getEnv,string,string>`_ - ## * `existsEnv proc <#existsEnv,string>`_ - ## * `putEnv proc <#putEnv,string,string>`_ - ## * `delEnv proc <#delEnv,string>`_ when defined(windows): - block: - template impl(get_fun, typ, size, zero, free_fun) = - let env = get_fun() - var e = env - if e == nil: break - while true: - let eend = strEnd(e) - let kv = $e - let p = find(kv, '=') - yield (substr(kv, 0, p-1), substr(kv, p+1)) - e = cast[typ](cast[ByteAddress](eend)+size) - if typeof(zero)(eend[1]) == zero: break - discard free_fun(env) - when useWinUnicode: - impl(getEnvironmentStringsW, WideCString, 2, 0, freeEnvironmentStringsW) + when useWinUnicode: + when defined(cpp): + proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.importcpp: "(NI16*)wcschr((const wchar_t *)#, #)", + header: "".} else: - impl(getEnvironmentStringsA, cstring, 1, '\0', freeEnvironmentStringsA) + proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.importc: "wcschr", + header: "".} + else: + proc strEnd(cstr: cstring, c = 0'i32): cstring {.importc: "strchr", + header: "".} + elif defined(macosx) and not defined(ios) and not defined(emscripten): + # From the manual: + # Shared libraries and bundles don't have direct access to environ, + # which is only available to the loader ld(1) when a complete program + # is being linked. + # The environment routines can still be used, but if direct access to + # environ is needed, the _NSGetEnviron() routine, defined in + # , can be used to retrieve the address of environ + # at runtime. + proc NSGetEnviron(): ptr cstringArray {.importc: "_NSGetEnviron", + header: "".} + elif defined(haiku): + var gEnv {.importc: "environ", header: "".}: cstringArray else: - var i = 0 - when defined(macosx) and not defined(ios) and not defined(emscripten): - var gEnv = NSGetEnviron()[] - while gEnv[i] != nil: - let kv = $gEnv[i] - inc(i) - let p = find(kv, '=') - yield (substr(kv, 0, p-1), substr(kv, p+1)) + var gEnv {.importc: "environ".}: cstringArray + + iterator envPairsImpl(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = + when defined(windows): + block: + template impl(get_fun, typ, size, zero, free_fun) = + let env = get_fun() + var e = env + if e == nil: break + while true: + let eend = strEnd(e) + let kv = $e + let p = find(kv, '=') + yield (substr(kv, 0, p-1), substr(kv, p+1)) + e = cast[typ](cast[ByteAddress](eend)+size) + if typeof(zero)(eend[1]) == zero: break + discard free_fun(env) + when useWinUnicode: + impl(getEnvironmentStringsW, WideCString, 2, 0, freeEnvironmentStringsW) + else: + impl(getEnvironmentStringsA, cstring, 1, '\0', freeEnvironmentStringsA) + else: + var i = 0 + when defined(macosx) and not defined(ios) and not defined(emscripten): + var gEnv = NSGetEnviron()[] + while gEnv[i] != nil: + let kv = $gEnv[i] + inc(i) + let p = find(kv, '=') + yield (substr(kv, 0, p-1), substr(kv, p+1)) + +proc envPairsImplSeq(): seq[tuple[key, value: string]] = discard # vmops + +iterator envPairs*(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = + ## Iterate over all `environments variables`:idx:. + ## + ## In the first component of the tuple is the name of the current variable stored, + ## in the second its value. + ## + ## Works in native backends, nodejs and vm, like the following APIs: + ## * `getEnv proc <#getEnv,string,string>`_ + ## * `existsEnv proc <#existsEnv,string>`_ + ## * `putEnv proc <#putEnv,string,string>`_ + ## * `delEnv proc <#delEnv,string>`_ + when nimvm: + for ai in envPairsImplSeq(): yield ai + else: + when defined(nimscript): discard + else: + for ai in envPairsImpl(): yield ai diff --git a/lib/pure/os.nim b/lib/pure/os.nim index dd1f9f0b6..e3b8e9f1c 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -871,8 +871,7 @@ proc unixToNativePath*(path: string, drive=""): string {. inc(i) include "includes/oserr" -when not defined(nimscript): - include "includes/osenv" +include "includes/osenv" proc getHomeDir*(): string {.rtl, extern: "nos$1", tags: [ReadEnvEffect, ReadIOEffect].} = diff --git a/testament/lib/stdtest/testutils.nim b/testament/lib/stdtest/testutils.nim index a5476b8d7..12e6a56ab 100644 --- a/testament/lib/stdtest/testutils.nim +++ b/testament/lib/stdtest/testutils.nim @@ -1,5 +1,8 @@ import std/private/miscdollars -from std/os import getEnv +when defined(nimscript): + import std/os # xxx investigate why needed +else: + from std/os import getEnv import std/[macros, genasts] template flakyAssert*(cond: untyped, msg = "", notifySuccess = true) = @@ -26,7 +29,7 @@ template flakyAssert*(cond: untyped, msg = "", notifySuccess = true) = msg2.add $expr & " " & msg echo msg2 -when not defined(js): +when not defined(js) and not defined(nimscript): import std/strutils proc greedyOrderedSubsetLines*(lhs, rhs: string, allowPrefixMatch = false): bool = diff --git a/tests/js/tos.nim b/tests/js/tos.nim index 07eb3aaa3..bfe3cd9b4 100644 --- a/tests/js/tos.nim +++ b/tests/js/tos.nim @@ -1,3 +1,5 @@ +# xxx consider merging this in tests/stdlib/tos.nim for increased coverage (with selecting disabling) + static: doAssert defined(nodejs) import os @@ -19,29 +21,3 @@ block: if not isWindows: doAssert cwd.isAbsolute doAssert relativePath(getCurrentDir() / "foo", "bar") == "../foo" - -import std/sequtils - -template main = - putEnv("foo", "bar") - doAssert getEnv("foo") == "bar" - doAssert existsEnv("foo") - - putEnv("foo", "") - doAssert existsEnv("foo") - putEnv("foo", "bar2") - doAssert getEnv("foo") == "bar2" - - when nimvm: - discard - else: - # need support in vmops: envPairs, delEnv - let s = toSeq(envPairs()) - doAssert ("foo", "bar2") in s - doAssert ("foo", "bar") notin s - - delEnv("foo") - doAssert not existsEnv("foo") - -static: main() -main() diff --git a/tests/stdlib/tosenv.nim b/tests/stdlib/tosenv.nim index df5759d0c..0a50031a1 100644 --- a/tests/stdlib/tosenv.nim +++ b/tests/stdlib/tosenv.nim @@ -13,16 +13,18 @@ template main = for val in ["val", ""]: # ensures empty val works too const key = "NIM_TESTS_TOSENV_KEY" doAssert not existsEnv(key) - putEnv(key, val) + + putEnv(key, "tempval") + doAssert existsEnv(key) + doAssert getEnv(key) == "tempval" + + putEnv(key, val) # change a key that already exists doAssert existsEnv(key) doAssert getEnv(key) == val - when nimvm: discard - else: - doAssert (key, val) in toSeq(envPairs()) + + doAssert (key, val) in toSeq(envPairs()) delEnv(key) - when nimvm: discard - else: - doAssert (key, val) notin toSeq(envPairs()) + doAssert (key, val) notin toSeq(envPairs()) doAssert not existsEnv(key) delEnv(key) # deleting an already deleted env var doAssert not existsEnv(key) @@ -43,7 +45,7 @@ template main = static: main() main() -when not defined(js): +when not defined(js) and not defined(nimscript): block: # bug #18533 proc c_getenv(env: cstring): cstring {.importc: "getenv", header: "".} var thr: Thread[void] diff --git a/tests/test_nimscript.nims b/tests/test_nimscript.nims index 90b6181aa..ecb409677 100644 --- a/tests/test_nimscript.nims +++ b/tests/test_nimscript.nims @@ -71,7 +71,11 @@ import std/[ decls, compilesettings, with, wrapnils ] +# non-std imports +import stdtest/testutils +# tests (increase coverage via code reuse) import stdlib/trandom +import stdlib/tosenv echo "Nimscript imports are successful." From b74b914e4f36199797b77e98bd2d771b4056a0d6 Mon Sep 17 00:00:00 2001 From: Miran Date: Wed, 29 Sep 2021 11:14:47 +0200 Subject: [PATCH 1252/2472] [backport] use old style hints in .cfg files (#18917) refs #18085 --- compiler/nim.cfg | 4 ++-- compiler/nimfix/nimfix.nim.cfg | 2 +- config/nim.cfg | 4 ++-- drnim/nim.cfg | 2 +- nimsuggest/nimsuggest.nim.cfg | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/compiler/nim.cfg b/compiler/nim.cfg index d6ac6a937..4a0287cb5 100644 --- a/compiler/nim.cfg +++ b/compiler/nim.cfg @@ -1,6 +1,6 @@ # Special configuration file for the Nim project -hint:XDeclaredButNotUsed:off +hint[XDeclaredButNotUsed]:off define:booting define:nimcore @@ -23,7 +23,7 @@ define:useStdoutAsStdmsg #gc:markAndSweep @if nimHasWarningObservableStores: - warning:ObservableStores: off + warning[ObservableStores]: off @end @if nimHasEffectsOf: diff --git a/compiler/nimfix/nimfix.nim.cfg b/compiler/nimfix/nimfix.nim.cfg index 27ebf10b9..0d9dbfa4b 100644 --- a/compiler/nimfix/nimfix.nim.cfg +++ b/compiler/nimfix/nimfix.nim.cfg @@ -1,7 +1,7 @@ # Special configuration file for the Nim project # gc:markAndSweep -hint:XDeclaredButNotUsed:off +hint[XDeclaredButNotUsed]:off path:"$projectPath/.." path:"$lib/packages/docutils" diff --git a/config/nim.cfg b/config/nim.cfg index c0a844e03..3b964d124 100644 --- a/config/nim.cfg +++ b/config/nim.cfg @@ -13,8 +13,8 @@ cc = gcc # additional options always passed to the compiler: --parallel_build: "0" # 0 to auto-detect number of processors -hint:LineTooLong:off -#hint:XDeclaredButNotUsed:off +hint[LineTooLong]=off +#hint[XDeclaredButNotUsed]=off # Examples of how to setup a cross-compiler: diff --git a/drnim/nim.cfg b/drnim/nim.cfg index 57dca974c..58c1725d9 100644 --- a/drnim/nim.cfg +++ b/drnim/nim.cfg @@ -1,6 +1,6 @@ # Special configuration file for the Nim project -hint:XDeclaredButNotUsed:off +hint[XDeclaredButNotUsed]:off define:booting define:nimcore diff --git a/nimsuggest/nimsuggest.nim.cfg b/nimsuggest/nimsuggest.nim.cfg index 6e5ee8c71..394449740 100644 --- a/nimsuggest/nimsuggest.nim.cfg +++ b/nimsuggest/nimsuggest.nim.cfg @@ -2,7 +2,7 @@ gc:markAndSweep -hint:XDeclaredButNotUsed:off +hint[XDeclaredButNotUsed]:off path:"$lib/packages/docutils" From f35e9a5752bb4cd4e30105cd27b6bc56cbf22329 Mon Sep 17 00:00:00 2001 From: Jonas Schubert Erlandsson Date: Wed, 29 Sep 2021 19:14:31 +0300 Subject: [PATCH 1253/2472] Adds source to single tags list (#18920) --- lib/pure/htmlparser.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/htmlparser.nim b/lib/pure/htmlparser.nim index c572146a7..21dff69ff 100644 --- a/lib/pure/htmlparser.nim +++ b/lib/pure/htmlparser.nim @@ -215,7 +215,7 @@ const tagMenu, tagNoframes} SingleTags* = {tagArea, tagBase, tagBasefont, tagBr, tagCol, tagFrame, tagHr, tagImg, tagIsindex, - tagLink, tagMeta, tagParam, tagWbr} + tagLink, tagMeta, tagParam, tagWbr, tagSource} proc allLower(s: string): bool = for c in s: From 6bb32da4aecbe80a9cc0cce90322c0e741ee8b85 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 30 Sep 2021 13:46:30 +0800 Subject: [PATCH 1254/2472] [std/tasks] more gcsafe stuffs [backport] (#18926) --- lib/std/tasks.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/std/tasks.nim b/lib/std/tasks.nim index 97c4df3f7..e2ea5377f 100644 --- a/lib/std/tasks.nim +++ b/lib/std/tasks.nim @@ -61,12 +61,12 @@ type Task* = object ## `Task` contains the callback and its arguments. callback: proc (args: pointer) {.nimcall, gcsafe.} args: pointer - destroy: proc (args: pointer) {.nimcall.} + destroy: proc (args: pointer) {.nimcall, gcsafe.} proc `=copy`*(x: var Task, y: Task) {.error.} -proc `=destroy`*(t: var Task) {.inline.} = +proc `=destroy`*(t: var Task) {.inline, gcsafe.} = ## Frees the resources allocated for a `Task`. if t.args != nil: if t.destroy != nil: @@ -219,7 +219,7 @@ macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkC let `objTemp` = cast[ptr `scratchObjType`](args) `functionStmtList` - proc `destroyName`(args: pointer) {.nimcall.} = + proc `destroyName`(args: pointer) {.gcsafe, nimcall.} = let `objTemp2` = cast[ptr `scratchObjType`](args) `tempNode` From c3dd53bb2cf492029ae256f64f64d2577621b995 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 30 Sep 2021 11:35:42 +0200 Subject: [PATCH 1255/2472] typos [backport] (#18929) --- doc/destructors.rst | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/doc/destructors.rst b/doc/destructors.rst index 28d8fa5f9..195dc8f97 100644 --- a/doc/destructors.rst +++ b/doc/destructors.rst @@ -145,7 +145,7 @@ not free the resources afterward by setting the object to its default value default value is written as `wasMoved(x)`. When not provided the compiler is using a combination of `=destroy` and `copyMem` instead. This is efficient hence users rarely need to implement their own `=sink` operator, it is enough to -provide `=destroy` and `=copy`, compiler will take care of the rest. +provide `=destroy` and `=copy`, the compiler will take care of the rest. The prototype of this hook for a type `T` needs to be: @@ -169,7 +169,7 @@ How self-assignments are handled is explained later in this document. `=copy` hook ---------------- +------------ The ordinary assignment in Nim conceptually copies the values. The `=copy` hook is called for assignments that couldn't be transformed into `=sink` @@ -206,11 +206,11 @@ by the compiler. Notice that there is no `=` before the `{.error.}` pragma. `=trace` hook ---------------- +------------- A custom **container** type can support Nim's cycle collector `--gc:orc` via the `=trace` hook. If the container does not implement `=trace`, cyclic data -structure which are constructed with the help of the container might leak +structures which are constructed with the help of the container might leak memory or resources, but memory safety is not compromised. The prototype of this hook for a type `T` needs to be: @@ -225,10 +225,10 @@ to calls of the built-in `=trace` operation. Usually there will only be a need for a custom `=trace` when a custom `=destroy` that deallocates manually allocated resources is also used, and then only when there is a chance of cyclic references from items within the manually allocated resources when it is desired that `--gc:orc` -be able to break and collect these cyclic referenced resources. Currently however, there is a +is able to break and collect these cyclic referenced resources. Currently however, there is a mutual use problem in that whichever of `=destroy`/`=trace` is used first will automatically create a version of the other which will then conflict with the creation of the second of the -pair. The work around for this problem is to forward declare the second of the "hooks" to +pair. The workaround for this problem is to forward declare the second of the "hooks" to prevent the automatic creation. The general pattern in using `=destroy` with `=trace` looks like: @@ -250,8 +250,9 @@ The general pattern in using `=destroy` with `=trace` looks like: dest.arr.dealloc proc `=trace`[T](dest: var Test[T]; env: pointer) = - if dest.arr != nil: # trace the `T`'s which may be cyclic - for i in 0 ..< dest.size: dest.arr[i].`=trace` env + if dest.arr != nil: + # trace the `T`'s which may be cyclic + for i in 0 ..< dest.size: `=trace`(dest.arr[i], env) # following may be other custom "hooks" as required... @@ -265,7 +266,7 @@ Move semantics A "move" can be regarded as an optimized copy operation. If the source of the copy operation is not used afterward, the copy can be replaced by a move. This document uses the notation `lastReadOf(x)` to describe that `x` is not -used afterwards. This property is computed by a static control flow analysis +used afterward. This property is computed by a static control flow analysis but can also be enforced by using `system.move` explicitly. @@ -601,7 +602,7 @@ a form of copy elision. To see how and when we can do that, think about this question: In `dest = src` when do we really have to *materialize* the full copy? - Only if `dest` or `src` are mutated -afterwards. If `dest` is a local variable that is simple to analyze. And if `src` is a +afterward. If `dest` is a local variable that is simple to analyze. And if `src` is a location derived from a formal parameter, we also know it is not mutated! In other words, we do a compile-time copy-on-write analysis. From c38ab3e257fbb08622f9ec6f0068a0fd3a5d82d7 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 30 Sep 2021 12:09:42 +0200 Subject: [PATCH 1256/2472] fixes #18921 [backport] (#18930) --- compiler/commands.nim | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index 9b7e35791..a8caad916 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -482,18 +482,20 @@ proc setCommandEarly*(conf: ConfigRef, command: string) = else: conf.foreignPackageNotes = foreignPackageNotesDefault -proc specialDefine(conf: ConfigRef, key: string) = +proc specialDefine(conf: ConfigRef, key: string; pass: TCmdLinePass) = # Keep this syncronized with the default config/nim.cfg! if cmpIgnoreStyle(key, "nimQuirky") == 0: conf.exc = excQuirky elif cmpIgnoreStyle(key, "release") == 0 or cmpIgnoreStyle(key, "danger") == 0: - conf.options.excl {optStackTrace, optLineTrace, optLineDir, optOptimizeSize} - conf.globalOptions.excl {optExcessiveStackTrace, optCDebug} - conf.options.incl optOptimizeSpeed + if pass in {passCmd1, passPP}: + conf.options.excl {optStackTrace, optLineTrace, optLineDir, optOptimizeSize} + conf.globalOptions.excl {optExcessiveStackTrace, optCDebug} + conf.options.incl optOptimizeSpeed if cmpIgnoreStyle(key, "danger") == 0 or cmpIgnoreStyle(key, "quick") == 0: - conf.options.excl {optObjCheck, optFieldCheck, optRangeCheck, optBoundsCheck, - optOverflowCheck, optAssert, optStackTrace, optLineTrace, optLineDir} - conf.globalOptions.excl {optCDebug} + if pass in {passCmd1, passPP}: + conf.options.excl {optObjCheck, optFieldCheck, optRangeCheck, optBoundsCheck, + optOverflowCheck, optAssert, optStackTrace, optLineTrace, optLineDir} + conf.globalOptions.excl {optCDebug} proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; conf: ConfigRef) = @@ -565,10 +567,10 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; expectArg(conf, switch, arg, pass, info) if {':', '='} in arg: splitSwitch(conf, arg, key, val, pass, info) - specialDefine(conf, key) + specialDefine(conf, key, pass) defineSymbol(conf.symbols, key, val) else: - specialDefine(conf, arg) + specialDefine(conf, arg, pass) defineSymbol(conf.symbols, arg) of "undef", "u": expectArg(conf, switch, arg, pass, info) From f915b3aa86929d87e162b6ddd7589e6337a30397 Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Thu, 30 Sep 2021 08:55:43 -0600 Subject: [PATCH 1257/2472] Alternative to #18928 (#18931) * fixed #18841 * Added test --- compiler/semtypinst.nim | 12 +++---- tests/generics/trecursivegenerics.nim | 52 ++++++++++++++++++++++++++- 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim index 2ca355e7d..0bb53e04d 100644 --- a/compiler/semtypinst.nim +++ b/compiler/semtypinst.nim @@ -344,11 +344,7 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType = x = lookupTypeVar(cl, x) if x != nil: if header == t: header = instCopyType(cl, t) - header[i] = - if x.kind == tyGenericInst: - t[i] - else: - x + header[i] = x propagateToOwner(header, x) else: propagateToOwner(header, x) @@ -382,7 +378,11 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType = cl.typeMap = newTypeMapLayer(cl) for i in 1.. Date: Thu, 30 Sep 2021 22:57:06 +0800 Subject: [PATCH 1258/2472] alternative to #18918 (#18927) * fix #16558 * add testcase --- compiler/semexprs.nim | 2 ++ tests/arc/t16558.nim | 9 +++++++++ 2 files changed, 11 insertions(+) create mode 100644 tests/arc/t16558.nim diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 9134c1301..342b8b05f 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -217,6 +217,8 @@ proc isCastable(c: PContext; dst, src: PType): bool = let s = skipTypes(src, abstractInst) if d.kind == tyRef and s.kind == tyRef and s[0].isFinal != d[0].isFinal: return false + elif d.kind in IntegralTypes and s.kind in {tyString, tySequence}: + return false var dstSize, srcSize: BiggestInt dstSize = computeSize(conf, dst) diff --git a/tests/arc/t16558.nim b/tests/arc/t16558.nim new file mode 100644 index 000000000..7b6eb4669 --- /dev/null +++ b/tests/arc/t16558.nim @@ -0,0 +1,9 @@ +discard """ + matrix: "--gc:arc" + errormsg: "expression cannot be cast to int" +""" + +block: # bug #16558 + var value = "hi there" + var keepInt: int + keepInt = cast[int](value) From 4b764f4c7cff02b1f698d90d4605ae8fa85c3ee6 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 1 Oct 2021 16:19:02 +0800 Subject: [PATCH 1259/2472] close #17319; add testcase (#18934) --- tests/arc/tarcmisc.nim | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/arc/tarcmisc.nim b/tests/arc/tarcmisc.nim index 7daea62c8..d259e1197 100644 --- a/tests/arc/tarcmisc.nim +++ b/tests/arc/tarcmisc.nim @@ -463,3 +463,22 @@ proc putValue[T](n: T) = echo b.n useForward() + + +# bug #17319 +type + BrokenObject = ref object + brokenType: seq[int] + +proc use(obj: BrokenObject) = + discard + +method testMethod(self: BrokenObject) {.base.} = + iterator testMethodIter() {.closure.} = + use(self) + + var nameIterVar = testMethodIter + nameIterVar() + +let mikasa = BrokenObject() +mikasa.testMethod() From 2aeac26f088c5170f7f0c4d8e73d099a32e8895a Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 1 Oct 2021 19:57:06 +0800 Subject: [PATCH 1260/2472] correct licence header (#18935) * rename licence * spaces --- lib/pure/lexbase.nim | 2 +- lib/std/compilesettings.nim | 2 +- lib/std/setutils.nim | 2 +- lib/std/sha1.nim | 2 +- lib/std/strbasics.nim | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/pure/lexbase.nim b/lib/pure/lexbase.nim index 166d8286b..bbc0a38ae 100644 --- a/lib/pure/lexbase.nim +++ b/lib/pure/lexbase.nim @@ -1,6 +1,6 @@ # # -# The Nim Compiler +# Nim's Runtime Library # (c) Copyright 2009 Andreas Rumpf # # See the file "copying.txt", included in this diff --git a/lib/std/compilesettings.nim b/lib/std/compilesettings.nim index e1ffc954a..5bb0bd673 100644 --- a/lib/std/compilesettings.nim +++ b/lib/std/compilesettings.nim @@ -1,6 +1,6 @@ # # -# The Nim Compiler +# Nim's Runtime Library # (c) Copyright 2020 Nim Contributors # # See the file "copying.txt", included in this diff --git a/lib/std/setutils.nim b/lib/std/setutils.nim index 84cc93623..4664d6dcc 100644 --- a/lib/std/setutils.nim +++ b/lib/std/setutils.nim @@ -1,6 +1,6 @@ # # -# The Nim Compiler +# Nim's Runtime Library # (c) Copyright 2020 Nim Contributors # # See the file "copying.txt", included in this diff --git a/lib/std/sha1.nim b/lib/std/sha1.nim index 3cc8a0249..120a81702 100644 --- a/lib/std/sha1.nim +++ b/lib/std/sha1.nim @@ -1,6 +1,6 @@ # # -# The Nim Compiler +# Nim's Runtime Library # (c) Copyright 2015 Nim Contributors # # See the file "copying.txt", included in this diff --git a/lib/std/strbasics.nim b/lib/std/strbasics.nim index 9232204ab..0d4c96f44 100644 --- a/lib/std/strbasics.nim +++ b/lib/std/strbasics.nim @@ -1,6 +1,6 @@ # # -# The Nim Compiler +# Nim's Runtime Library # (c) Copyright 2021 Nim Contributors # # See the file "copying.txt", included in this From 7577ea9e4ce4d9966edc1f1cc27bde1b2ec9f369 Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 2 Oct 2021 02:14:10 +0800 Subject: [PATCH 1261/2472] [std/tempfiles] docs improvement (#18936) * unify comments * more --- lib/std/tempfiles.nim | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/std/tempfiles.nim b/lib/std/tempfiles.nim index 26786d463..ce84c2a37 100644 --- a/lib/std/tempfiles.nim +++ b/lib/std/tempfiles.nim @@ -122,23 +122,24 @@ proc getTempDirImpl(dir: string): string {.inline.} = proc genTempPath*(prefix, suffix: string, dir = ""): string = ## Generates a path name in `dir`. ## - ## If `dir` is empty, (`getTempDir `_) will be used. ## The path begins with `prefix` and ends with `suffix`. + ## + ## .. note:: `dir` must exist (empty `dir` will resolve to `getTempDir `_). let dir = getTempDirImpl(dir) result = dir / (prefix & randomPathName(nimTempPathLength) & suffix) proc createTempFile*(prefix, suffix: string, dir = ""): tuple[cfile: File, path: string] = ## Creates a new temporary file in the directory `dir`. - ## + ## ## This generates a path name using `genTempPath(prefix, suffix, dir)` and ## returns a file handle to an open file and the path of that file, possibly after ## retrying to ensure it doesn't already exist. - ## + ## ## If failing to create a temporary file, `OSError` will be raised. ## ## .. note:: It is the caller's responsibility to close `result.cfile` and ## remove `result.file` when no longer needed. - ## .. note:: `dir` must exist (empty `dir` will resolve to `getTempDir()`). + ## .. note:: `dir` must exist (empty `dir` will resolve to `getTempDir `_). runnableExamples: import std/os doAssertRaises(OSError): discard createTempFile("", "", "nonexistent") @@ -170,7 +171,7 @@ proc createTempDir*(prefix, suffix: string, dir = ""): string = ## If failing to create a temporary directory, `OSError` will be raised. ## ## .. note:: It is the caller's responsibility to remove the directory when no longer needed. - ## .. note:: `dir` must exist (empty `dir` will resolve to `getTempDir()`). + ## .. note:: `dir` must exist (empty `dir` will resolve to `getTempDir `_). runnableExamples: import std/os doAssertRaises(OSError): discard createTempDir("", "", "nonexistent") From 5c4692fad4c807a010d58d553e92360128699197 Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 2 Oct 2021 02:26:29 +0800 Subject: [PATCH 1262/2472] rename nimLegacyJsonutilsHoleyEnum [backport] (#18938) * enable nimPreviewJsonutilsHoleyEnum [backport] * docuement nimPreviewJsonutilsHoleyEnum --- changelog.md | 3 +-- lib/std/jsonutils.nim | 6 +++++- tests/config.nims | 4 +++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/changelog.md b/changelog.md index ab037c328..38ace19ed 100644 --- a/changelog.md +++ b/changelog.md @@ -77,8 +77,7 @@ initialization with a small (< 30000) seed. Use `-d:nimLegacyRandomInitRand` to restore previous behavior for a transition time, see PR #17467. -- `jsonutils` now serializes/deserializes holey enums as regular enums (via `ord`) instead of as strings. - Use `-d:nimLegacyJsonutilsHoleyEnum` for a transition period. `toJson` now serializes `JsonNode` +- With `-d:nimPreviewJsonutilsHoleyEnum`, `jsonutils` now can serialize/deserialize holey enums as regular enums (via `ord`) instead of as strings. It is expected that this behavior becomes the new default in upcoming versions. `toJson` now serializes `JsonNode` as is via reference (without a deep copy) instead of treating `JsonNode` as a regular ref object, this can be customized via `jsonNodeMode`. diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim index 727a75288..19384b5d1 100644 --- a/lib/std/jsonutils.nim +++ b/lib/std/jsonutils.nim @@ -298,6 +298,10 @@ proc jsonTo*(b: JsonNode, T: typedesc, opt = Joptions()): T = proc toJson*[T](a: T, opt = initToJsonOptions()): JsonNode = ## serializes `a` to json; uses `toJsonHook(a: T)` if it's in scope to ## customize serialization, see strtabs.toJsonHook for an example. + ## + ## .. note:: With `-d:nimPreviewJsonutilsHoleyEnum`, `toJson` now can + ## serialize/deserialize holey enums as regular enums (via `ord`) instead of as strings. + ## It is expected that this behavior becomes the new default in upcoming versions. when compiles(toJsonHook(a)): result = toJsonHook(a) elif T is object | tuple: when T is object or isNamedTuple(T): @@ -328,7 +332,7 @@ proc toJson*[T](a: T, opt = initToJsonOptions()): JsonNode = elif T is enum: case opt.enumMode of joptEnumOrd: - when T is Ordinal or not defined(nimLegacyJsonutilsHoleyEnum): %(a.ord) + when T is Ordinal or defined(nimPreviewJsonutilsHoleyEnum): %(a.ord) else: toJson($a, opt) of joptEnumSymbol: when T is OrdinalEnum: diff --git a/tests/config.nims b/tests/config.nims index 86787db1d..ed2a1b7e9 100644 --- a/tests/config.nims +++ b/tests/config.nims @@ -30,10 +30,12 @@ hint("Processing", off) # switch("hint", "ConvFromXtoItselfNotNeeded") # switch("warningAsError", "InheritFromException") # would require fixing a few tests -# experimental API's are enabled in testament, refs https://github.com/timotheecour/Nim/issues/575 +# experimental APIs are enabled in testament, refs https://github.com/timotheecour/Nim/issues/575 # sync with `kochdocs.docDefines` or refactor. switch("define", "nimExperimentalAsyncjsThen") switch("define", "nimExperimentalLinenoiseExtra") +# preview APIs are expected to be the new default in upcoming versions switch("define", "nimPreviewFloatRoundtrip") switch("define", "nimPreviewDotLikeOps") +switch("define", "nimPreviewJsonutilsHoleyEnum") From e9268b52d9a34e7e851105d569bccced4b7261fd Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 3 Oct 2021 07:21:08 +0200 Subject: [PATCH 1263/2472] with this patch :idx: can be used for the index generation for LaTeX (#18946) --- config/nimdoc.tex.cfg | 6 ++++-- lib/packages/docutils/rstgen.nim | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/config/nimdoc.tex.cfg b/config/nimdoc.tex.cfg index 75382ce26..4aff9b379 100644 --- a/config/nimdoc.tex.cfg +++ b/config/nimdoc.tex.cfg @@ -3,7 +3,7 @@ # (c) 2012 Andreas Rumpf # Feel free to edit the templates as you need. -split.item.toc = "20" +split.item.toc = "20" # too long entries in the table of contents wrap around # after this number of characters @@ -97,12 +97,14 @@ doc.file = """ \usepackage{parskip} % paragraphs delimited by vertical space, no indent \usepackage{graphicx} +\newcommand{\nimindexterm}[2]{#2\label{#1}} + \usepackage{dingbat} % for \carriagereturn, etc \usepackage{fvextra} % for code blocks (works better than original fancyvrb) \fvset{ breaklines, breakafter={=}:|\_\{\}[](){,}.;+-*/'", - breaksymbolleft=\color{red}{\ensuremath{\hookrightarrow}}, + breaksymbolleft=\color{red}{\ensuremath{\hookrightarrow}}, breaksymbolright=\color{red}{\small\carriagereturn} } \fvinlineset{% diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index 3734392c3..7d7eb4fb8 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -406,7 +406,7 @@ proc renderIndexTerm*(d: PDoc, n: PRstNode, result: var string) = var term = "" renderAux(d, n, term) setIndexTerm(d, changeFileExt(extractFilename(d.filename), HtmlExt), id, term, d.currentSection) - dispA(d.target, result, "$2", "$2\\label{$1}", + dispA(d.target, result, "$2", "\\nimindexterm{$2}{$1}", [id, term]) type From 45c4332c81521224a4db90fe9bf46efc0f93aaf7 Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 3 Oct 2021 13:22:53 +0800 Subject: [PATCH 1264/2472] document -d:nimStrictDelete [backport] (#18939) --- lib/system.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/system.nim b/lib/system.nim index 358aae938..caa036f37 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2142,6 +2142,10 @@ proc delete*[T](x: var seq[T], i: Natural) {.noSideEffect, auditDelete.} = ## ## This is an `O(n)` operation. ## + ## .. note:: With `-d:nimStrictDelete`, an index error is produced when the index passed + ## to it was out of bounds. `-d:nimStrictDelete` will become the default + ## in upcoming versions. + ## ## See also: ## * `del <#del,seq[T],Natural>`_ for O(1) operation ## From 6b0db5be1f21cec1b75822c021ccee79035a6f74 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sun, 3 Oct 2021 06:25:15 +0100 Subject: [PATCH 1265/2472] Fixes build_all.sh failing on Nimble builds (#18945) When running `build_all.sh` I was getting: ``` bin/nim c -o:bin/nimsuggest -d:danger --skipUserCfg --skipParentCfg --hints:off nimsuggest/nimsuggest.nim bin/nim c -o:bin/nimgrep -d:release --skipUserCfg --skipParentCfg --hints:off tools/nimgrep.nim bin/nim c -o:bin/nimpretty -d:release --skipUserCfg --skipParentCfg --hints:off nimpretty/nimpretty.nim bin/nim c -o:bin/testament -d:release --skipUserCfg --skipParentCfg --hints:off testament/testament.nim bin/nim c -o:bin/nim_dbg --opt:speed --stacktrace -d:debug --stacktraceMsgs -d:nimCompilerStacktraceHints --skipUserCfg --skipParentCfg --hints:off compiler/nim.nim bin/nim c -o:bin/atlas -d:release --skipUserCfg --skipParentCfg --hints:off tools/atlas/atlas.nim /home/dom/.choosenim/toolchains/nim-#devel/koch.nim(722) koch /home/dom/.choosenim/toolchains/nim-#devel/koch.nim(149) bundleNimbleExe /home/dom/.choosenim/toolchains/nim-#devel/tools/deps.nim(32) cloneDependency /home/dom/.choosenim/toolchains/nim-#devel/lib/pure/os.nim(1438) setCurrentDir /home/dom/.choosenim/toolchains/nim-#devel/lib/pure/includes/oserr.nim(95) raiseOSError Error: unhandled exception: No such file or directory Additional info: '/home/dom/.choosenim/toolchains/nim-#devel/dist/nimble' [OSError] ``` With this patch it builds. --- tools/deps.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/deps.nim b/tools/deps.nim index 95b2916a1..59872070d 100644 --- a/tools/deps.nim +++ b/tools/deps.nim @@ -22,14 +22,14 @@ proc cloneDependency*(destDirBase: string, url: string, commit = commitHead, let name = p.splitFile.name var destDir = destDirBase if appendRepoName: destDir = destDir / name - let destDir2 = destDir.quoteShell + let quotedDestDir = destDir.quoteShell if not dirExists(destDir): # note: old code used `destDir / .git` but that wouldn't prevent git clone # from failing - execRetry fmt"git clone -q {url} {destDir2}" + execRetry fmt"git clone -q {url} {quotedDestDir}" if isGitRepo(destDir): let oldDir = getCurrentDir() - setCurrentDir(destDir2) + setCurrentDir(destDir) try: execRetry "git fetch -q" exec fmt"git checkout -q {commit}" From 97c24dd54892a1f256707bc73230fe5443be36ce Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 4 Oct 2021 15:19:20 +0800 Subject: [PATCH 1266/2472] correct changelog [backport] (#18940) --- changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 38ace19ed..f8dc9087c 100644 --- a/changelog.md +++ b/changelog.md @@ -103,7 +103,7 @@ - In `std/dom`, `Interval` is now a `ref object`, same as `Timeout`. Definitions of `setTimeout`, `clearTimeout`, `setInterval`, `clearInterval` were updated. -- With `-d:nimPreviewDotLikeOps` (default in devel), dot-like operators (operators starting with `.`, but not with `..`) +- With `-d:nimPreviewDotLikeOps`, dot-like operators (operators starting with `.`, but not with `..`) now have the same precedence as `.`, so that `a.?b.c` is now parsed as `(a.?b).c` instead of `a.?(b.c)`. A warning is generated when a dot-like operator is used without `-d:nimPreviewDotLikeOps`. From 500d5f0f38dc59895908f25f2c43c2a4c61d7874 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 4 Oct 2021 12:34:29 +0200 Subject: [PATCH 1267/2472] typo [backport] (#18948) --- doc/manual.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual.rst b/doc/manual.rst index dfcf0ba54..4b212fe19 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -1201,7 +1201,7 @@ The only operations that are affected by the `floatChecks` pragma are the `+`, `-`, `*`, `/` operators for floating-point types. An implementation should always use the maximum precision available to evaluate -floating pointer values during semantic analysis; this means expressions like +floating-point values during semantic analysis; this means expressions like `0.09'f32 + 0.01'f32 == 0.09'f64 + 0.01'f64` that are evaluating during constant folding are true. From f017eadc97b041e678196cb2b72c3c66767e7ab0 Mon Sep 17 00:00:00 2001 From: quantimnot <54247259+quantimnot@users.noreply.github.com> Date: Mon, 4 Oct 2021 15:27:13 -0400 Subject: [PATCH 1268/2472] Add support for `strictEffects` to `std/pegs` (#18949) * Add support for `strictEffects` to `std/pegs` * Fixed support of older Nim versions Co-authored-by: quantimnot --- lib/pure/pegs.nim | 5 ++++- tests/stdlib/tpegs.nim | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index b6b05cdc1..11270e34c 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -1285,9 +1285,12 @@ proc parallelReplace*(s: string, subs: varargs[ # copy the rest: add(result, substr(s, i)) +when not defined(nimHasEffectsOf): + {.pragma: effectsOf.} + proc replace*(s: string, sub: Peg, cb: proc( match: int, cnt: int, caps: openArray[string]): string): string {. - rtl, extern: "npegs$1cb".} = + rtl, extern: "npegs$1cb", effectsOf: cb.} = ## Replaces `sub` in `s` by the resulting strings from the callback. ## The callback proc receives the index of the current match (starting with 0), ## the count of captures and an open array with the captures of each match. Examples: diff --git a/tests/stdlib/tpegs.nim b/tests/stdlib/tpegs.nim index 1261d55b8..c3d8942cf 100644 --- a/tests/stdlib/tpegs.nim +++ b/tests/stdlib/tpegs.nim @@ -51,6 +51,9 @@ Event parser output ''' """ +when defined(nimHasEffectsOf): + {.experimental: "strictEffects".} + import std/[strutils, streams, pegs] const From e7bac9177335c75b4eeed18ec5f8a0b0489c0561 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 5 Oct 2021 03:28:13 +0800 Subject: [PATCH 1269/2472] rename nimEnableHashRef [backport] (#18941) * rename nimEnableHashRef [backport] * Apply suggestions from code review Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- changelog.md | 3 ++- lib/pure/hashes.nim | 18 +++++++++++++----- tests/collections/ttables.nim | 1 - tests/config.nims | 1 + tests/stdlib/thashes.nim | 1 - 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/changelog.md b/changelog.md index f8dc9087c..1925878bf 100644 --- a/changelog.md +++ b/changelog.md @@ -64,7 +64,8 @@ for previous behavior. - `hashes.hash` can now support `object` and `ref` (can be overloaded in user code), - if `-d:nimEnableHashRef` is used. + if `-d:nimPreviewHashRef` is used. It is expected that this behavior + becomes the new default in upcoming versions. - `hashes.hash(proc|ptr|ref|pointer)` now calls `hash(int)` and honors `-d:nimIntHash1`, `hashes.hash(closure)` has also been improved. diff --git a/lib/pure/hashes.nim b/lib/pure/hashes.nim index 1c49b0317..d9f534670 100644 --- a/lib/pure/hashes.nim +++ b/lib/pure/hashes.nim @@ -51,8 +51,12 @@ runnableExamples: h = h !& hash(x.bar) result = !$h -## **Note:** If the type has a `==` operator, the following must hold: -## If two values compare equal, their hashes must also be equal. +## .. important:: Use `-d:nimPreviewHashRef` to +## enable hashing `ref`s. It is expected that this behavior +## becomes the new default in upcoming versions. +## +## .. note:: If the type has a `==` operator, the following must hold: +## If two values compare equal, their hashes must also be equal. ## ## See also ## ======== @@ -236,10 +240,14 @@ proc hash*[T](x: ptr[T]): Hash {.inline.} = assert cast[pointer](a[0].addr).hash == a[0].addr.hash hash(cast[pointer](x)) -when defined(nimEnableHashRef): +when defined(nimPreviewHashRef) or defined(nimdoc): proc hash*[T](x: ref[T]): Hash {.inline.} = ## Efficient `hash` overload. - runnableExamples: + ## + ## .. important:: Use `-d:nimPreviewHashRef` to + ## enable hashing `ref`s. It is expected that this behavior + ## becomes the new default in upcoming versions. + runnableExamples("-d:nimPreviewHashRef"): type A = ref object x: int let a = A(x: 3) @@ -247,7 +255,7 @@ when defined(nimEnableHashRef): assert ha != A(x: 3).hash # A(x: 3) is a different ref object from `a`. a.x = 4 assert ha == a.hash # the hash only depends on the address - runnableExamples: + runnableExamples("-d:nimPreviewHashRef"): # you can overload `hash` if you want to customize semantics type A[T] = ref object x, y: T diff --git a/tests/collections/ttables.nim b/tests/collections/ttables.nim index 751509062..638f4241b 100644 --- a/tests/collections/ttables.nim +++ b/tests/collections/ttables.nim @@ -8,7 +8,6 @@ And we get here ''' joinable: false targets: "c cpp js" -matrix: "-d:nimEnableHashRef" """ # xxx wrap in a template to test in VM, see https://github.com/timotheecour/Nim/issues/534#issuecomment-769565033 diff --git a/tests/config.nims b/tests/config.nims index ed2a1b7e9..894c4bea0 100644 --- a/tests/config.nims +++ b/tests/config.nims @@ -39,3 +39,4 @@ switch("define", "nimExperimentalLinenoiseExtra") switch("define", "nimPreviewFloatRoundtrip") switch("define", "nimPreviewDotLikeOps") switch("define", "nimPreviewJsonutilsHoleyEnum") +switch("define", "nimPreviewHashRef") diff --git a/tests/stdlib/thashes.nim b/tests/stdlib/thashes.nim index 1e9b02d0c..46576ef12 100644 --- a/tests/stdlib/thashes.nim +++ b/tests/stdlib/thashes.nim @@ -1,6 +1,5 @@ discard """ targets: "c cpp js" - matrix: "-d:nimEnableHashRef" """ import std/hashes From f1f1e85ec673b9ea2ea5d3a97aa27993be153602 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 5 Oct 2021 12:29:59 +0200 Subject: [PATCH 1270/2472] fixes #18954 (#18955) --- lib/pure/sugar.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/sugar.nim b/lib/pure/sugar.nim index f82e1706f..d7beef8d9 100644 --- a/lib/pure/sugar.nim +++ b/lib/pure/sugar.nim @@ -121,9 +121,9 @@ macro `=>`*(p, b: untyped): untyped = else: error("Incorrect procedure parameter.", c) params.add(identDefs) - of nnkIdent: + of nnkIdent, nnkOpenSymChoice, nnkClosedSymChoice, nnkSym: var identDefs = newNimNode(nnkIdentDefs) - identDefs.add(p) + identDefs.add(ident $p) identDefs.add(ident"auto") identDefs.add(newEmptyNode()) params.add(identDefs) From 9fc2918314a75fd1923af049966e4c4e9d8cf710 Mon Sep 17 00:00:00 2001 From: quantimnot <54247259+quantimnot@users.noreply.github.com> Date: Wed, 6 Oct 2021 03:41:42 -0400 Subject: [PATCH 1271/2472] Document how to compile a reproducible build (#18953) * Make compiler build more reproducible Removed the compile time from the version output. Added Git commit's author's datetime in UTC timezone. Fixes #18508 See https://reproducible-builds.org/docs/timestamps/ See https://reproducible-builds.org/docs/source-date-epoch/ * Revert "Make compiler build more reproducible" This reverts commit 2f9359ae89622c2d95ef243d13251a310e94a199. Documented how to use compile reproducible builds with the `SOURCE_DATE_EPOCH` environment variable. * Corrected `readme.md` formatting Co-authored-by: quantimnot --- doc/intern.rst | 11 +++++++++++ doc/packaging.rst | 4 ++++ readme.md | 3 ++- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/doc/intern.rst b/doc/intern.rst index ab74c4a3a..fe0d333fd 100644 --- a/doc/intern.rst +++ b/doc/intern.rst @@ -68,6 +68,17 @@ More information about its options can be found in the `koch `_ documentation. +Reproducible builds +------------------- + +Set the compilation timestamp with the `SOURCE_DATE_EPOCH` environment variable. + +.. code:: cmd + + export SOURCE_DATE_EPOCH=$(git log -n 1 --format=%at) + koch boot # or `./build_all.sh` + + Developing the compiler ======================= diff --git a/doc/packaging.rst b/doc/packaging.rst index fcdc4ae04..7976dfe92 100644 --- a/doc/packaging.rst +++ b/doc/packaging.rst @@ -6,6 +6,9 @@ This page provide hints on distributing Nim using OS packages. See `distros `_ for tools to detect Linux distribution at runtime. +See `here `_ for how to +compile reproducible builds. + Supported architectures ----------------------- @@ -71,3 +74,4 @@ What to install: - Optionally: manpages, documentation, shell completion - When installing documentation, .idx files are not required - The "compiler" directory contains compiler sources and should not be part of the compiler binary package + diff --git a/readme.md b/readme.md index b28c25ec1..7f333d5d4 100644 --- a/readme.md +++ b/readme.md @@ -71,7 +71,6 @@ the installation instructions on the website to do so: https://nim-lang.org/inst For package maintainers: see [packaging guidelines](https://nim-lang.github.io/Nim/packaging.html). - First, get Nim from github: ``` @@ -89,6 +88,8 @@ should add the ``bin`` directory to your PATH. See also [rebuilding the compiler](doc/intern.rst#rebuilding-the-compiler). +See also [reproducible builds](doc/intern.rst#reproducible-builds). + ## Koch ``koch`` is the build tool used to build various parts of Nim and to generate From cb30dc51701f0d4bc5dedc2615de68e4c1e5ba71 Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 6 Oct 2021 15:42:35 +0800 Subject: [PATCH 1272/2472] enable tests for #2710 (#18961) --- tests/metatype/ttypedesc3.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/metatype/ttypedesc3.nim b/tests/metatype/ttypedesc3.nim index b691d3488..98a59f613 100644 --- a/tests/metatype/ttypedesc3.nim +++ b/tests/metatype/ttypedesc3.nim @@ -3,6 +3,7 @@ output: ''' proc Base proc Child method Base +method Child yield Base yield Child 12 @@ -24,8 +25,7 @@ Base.pr Child.pr Base.me -when false: - Child.me #<- bug #2710 +Child.me #<- bug #2710 for s in Base.it: echo s for s in Child.it: echo s #<- bug #2662 From b2873f0f63f7741ee995363a6bb225a228d03d18 Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 6 Oct 2021 18:47:46 +0800 Subject: [PATCH 1273/2472] [tools] use the right parameter [backport:1.0] (#18957) --- tools/finish.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/finish.nim b/tools/finish.nim index 4b5131045..69838829c 100644 --- a/tools/finish.nim +++ b/tools/finish.nim @@ -41,7 +41,7 @@ proc downloadMingw(): DownloadResult = let curl = findExe"curl" var cmd: string if curl.len > 0: - cmd = quoteShell(curl) & " --out " & "dist" / mingw & " " & url + cmd = quoteShell(curl) & " --output " & "dist" / mingw & " " & url elif fileExists"bin/nimgrab.exe": cmd = r"bin\nimgrab.exe " & url & " dist" / mingw if cmd.len > 0: From f03872d99ec2d25de829415a75162f9d8297bb19 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Wed, 6 Oct 2021 14:53:01 +0300 Subject: [PATCH 1274/2472] rst: minor fixes (#18960) --- lib/packages/docutils/rst.nim | 1 + lib/packages/docutils/rstast.nim | 4 ++-- lib/packages/docutils/rstgen.nim | 2 +- tests/stdlib/trst.nim | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index abea026dd..29234f28b 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -172,6 +172,7 @@ ## ``header``, ``footer``, ``meta``, ``class`` ## - no ``role`` directives and no custom interpreted text roles ## - some standard roles are not supported (check `RST roles list`_) +## - no generic admonition support ## * inline markup ## - no simple-inline-markup ## - no embedded aliases diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim index fa0620f44..e1ed7c099 100644 --- a/lib/packages/docutils/rstast.nim +++ b/lib/packages/docutils/rstast.nim @@ -381,7 +381,7 @@ proc renderRstToText*(node: PRstNode): string = result.add renderRstToText(node.sons[i]) if node.kind in code: result.add "`" -proc renderRstToStr*(node: PRstNode, indent=0): string = +proc treeRepr*(node: PRstNode, indent=0): string = ## Writes the parsed RST `node` into an AST tree with compact string ## representation in the format (one line per every sub-node): ## ``indent - kind - [text|level|order|adType] - anchor (if non-zero)`` @@ -411,4 +411,4 @@ proc renderRstToStr*(node: PRstNode, indent=0): string = result.add (if node.anchor == "": "" else: " anchor='" & node.anchor & "'") result.add "\n" for son in node.sons: - result.add renderRstToStr(son, indent=indent+2) + result.add treeRepr(son, indent=indent+2) diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index 7d7eb4fb8..8eac37307 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -1164,7 +1164,7 @@ proc renderAdmonition(d: PDoc, n: PRstNode, result: var string) = case n.adType of "hint", "note", "tip": htmlCls = "admonition-info"; texSz = "\\normalsize"; texColor = "green" - of "attention", "admonition", "important", "warning": + of "attention", "admonition", "important", "warning", "caution": htmlCls = "admonition-warning"; texSz = "\\large"; texColor = "orange" of "danger", "error": htmlCls = "admonition-error"; texSz = "\\Large"; texColor = "red" diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index bc11f219c..e9f67324c 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -55,7 +55,7 @@ proc toAst(input: string, var (rst, _, _) = rstParse(input, filen, line=LineRstInit, column=ColRstInit, rstOptions, myFindFile, testMsgHandler) - result = renderRstToStr(rst) + result = treeRepr(rst) except EParseError as e: if e.msg != "": result = e.msg From eede2bfb5de30f7f411d9437e19f253f8dac629b Mon Sep 17 00:00:00 2001 From: Miran Date: Thu, 7 Oct 2021 10:48:09 +0200 Subject: [PATCH 1275/2472] update csources hash (#18969) --- config/build_config.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/build_config.txt b/config/build_config.txt index aee15e695..15f44bb57 100644 --- a/config/build_config.txt +++ b/config/build_config.txt @@ -2,4 +2,4 @@ nim_comment="key-value pairs for windows/posix bootstrapping build scripts" nim_csourcesDir=csources_v1 nim_csourcesUrl=https://github.com/nim-lang/csources_v1.git nim_csourcesBranch=master -nim_csourcesHash=5e9be0383f842c9c22ae86e41946a896c43e95d0 +nim_csourcesHash=561b417c65791cd8356b5f73620914ceff845d10 From 8eef55715725a9dba2ff0afbdbb564383f3a29a6 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 7 Oct 2021 12:01:06 +0200 Subject: [PATCH 1276/2472] fixes a 'mixin' statement handling regression [backport:1.2] (#18968) --- compiler/semtempl.nim | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim index eda4ca6cc..c10917bcb 100644 --- a/compiler/semtempl.nim +++ b/compiler/semtempl.nim @@ -110,9 +110,14 @@ proc semBindStmt(c: PContext, n: PNode, toBind: var IntSet): PNode = proc semMixinStmt(c: PContext, n: PNode, toMixin: var IntSet): PNode = result = copyNode(n) + var count = 0 for i in 0.. Date: Thu, 7 Oct 2021 18:01:27 +0800 Subject: [PATCH 1277/2472] typo [backport] (#18967) --- lib/pure/strscans.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/strscans.nim b/lib/pure/strscans.nim index 291af7408..99442075a 100644 --- a/lib/pure/strscans.nim +++ b/lib/pure/strscans.nim @@ -35,7 +35,7 @@ substrings starting with ``$``. These constructions are available: ``$o`` Matches an octal integer. This uses ``parseutils.parseOct``. ``$i`` Matches a decimal integer. This uses ``parseutils.parseInt``. ``$h`` Matches a hex integer. This uses ``parseutils.parseHex``. -``$f`` Matches a floating pointer number. Uses ``parseFloat``. +``$f`` Matches a floating-point number. Uses ``parseFloat``. ``$w`` Matches an ASCII identifier: ``[A-Za-z_][A-Za-z_0-9]*``. ``$c`` Matches a single ASCII character. ``$s`` Skips optional whitespace. From 19774a72e7fcc7c5b7649bb956b149362c05ea15 Mon Sep 17 00:00:00 2001 From: quantimnot <54247259+quantimnot@users.noreply.github.com> Date: Thu, 7 Oct 2021 06:16:58 -0400 Subject: [PATCH 1278/2472] Fixed `strictFuncs` support for `std/pegs` (#18951) * Fixed `strictFuncs` support for `std/pegs` Enabled `std/pegs` in the `strictFuncs` import test. Fixes #18057 Fixes #16892 See #18111 * Rebased from `devel` * Conditionally compile `std/pegs` in `koch` This is for supporting `csources` bootstrap. Co-authored-by: quantimnot --- lib/pure/pegs.nim | 280 ++++++++++++------------ tests/effects/tstrict_funcs_imports.nim | 2 +- tools/kochdocs.nim | 11 +- 3 files changed, 151 insertions(+), 142 deletions(-) diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index 11270e34c..bac8b1a0e 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -87,26 +87,26 @@ type else: sons: seq[Peg] NonTerminal* = ref NonTerminalObj -proc kind*(p: Peg): PegKind = p.kind +func kind*(p: Peg): PegKind = p.kind ## Returns the *PegKind* of a given *Peg* object. -proc term*(p: Peg): string = p.term +func term*(p: Peg): string = p.term ## Returns the *string* representation of a given *Peg* variant object ## where present. -proc ch*(p: Peg): char = p.ch +func ch*(p: Peg): char = p.ch ## Returns the *char* representation of a given *Peg* variant object ## where present. -proc charChoice*(p: Peg): ref set[char] = p.charChoice +func charChoice*(p: Peg): ref set[char] = p.charChoice ## Returns the *charChoice* field of a given *Peg* variant object ## where present. -proc nt*(p: Peg): NonTerminal = p.nt +func nt*(p: Peg): NonTerminal = p.nt ## Returns the *NonTerminal* object of a given *Peg* variant object ## where present. -proc index*(p: Peg): range[-MaxSubpatterns..MaxSubpatterns-1] = p.index +func index*(p: Peg): range[-MaxSubpatterns..MaxSubpatterns-1] = p.index ## Returns the back-reference index of a captured sub-pattern in the ## *Captures* object for a given *Peg* variant object where present. @@ -120,59 +120,59 @@ iterator pairs*(p: Peg): (int, Peg) {.inline.} = for i in 0 ..< p.sons.len: yield (i, p.sons[i]) -proc name*(nt: NonTerminal): string = nt.name +func name*(nt: NonTerminal): string = nt.name ## Gets the name of the symbol represented by the parent *Peg* object variant ## of a given *NonTerminal*. -proc line*(nt: NonTerminal): int = nt.line +func line*(nt: NonTerminal): int = nt.line ## Gets the line number of the definition of the parent *Peg* object variant ## of a given *NonTerminal*. -proc col*(nt: NonTerminal): int = nt.col +func col*(nt: NonTerminal): int = nt.col ## Gets the column number of the definition of the parent *Peg* object variant ## of a given *NonTerminal*. -proc flags*(nt: NonTerminal): set[NonTerminalFlag] = nt.flags +func flags*(nt: NonTerminal): set[NonTerminalFlag] = nt.flags ## Gets the *NonTerminalFlag*-typed flags field of the parent *Peg* variant ## object of a given *NonTerminal*. -proc rule*(nt: NonTerminal): Peg = nt.rule +func rule*(nt: NonTerminal): Peg = nt.rule ## Gets the *Peg* object representing the rule definition of the parent *Peg* ## object variant of a given *NonTerminal*. -proc term*(t: string): Peg {.noSideEffect, rtl, extern: "npegs$1Str".} = +func term*(t: string): Peg {.rtl, extern: "npegs$1Str".} = ## constructs a PEG from a terminal string if t.len != 1: result = Peg(kind: pkTerminal, term: t) else: result = Peg(kind: pkChar, ch: t[0]) -proc termIgnoreCase*(t: string): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func termIgnoreCase*(t: string): Peg {. + rtl, extern: "npegs$1".} = ## constructs a PEG from a terminal string; ignore case for matching result = Peg(kind: pkTerminalIgnoreCase, term: t) -proc termIgnoreStyle*(t: string): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func termIgnoreStyle*(t: string): Peg {. + rtl, extern: "npegs$1".} = ## constructs a PEG from a terminal string; ignore style for matching result = Peg(kind: pkTerminalIgnoreStyle, term: t) -proc term*(t: char): Peg {.noSideEffect, rtl, extern: "npegs$1Char".} = +func term*(t: char): Peg {.rtl, extern: "npegs$1Char".} = ## constructs a PEG from a terminal char assert t != '\0' result = Peg(kind: pkChar, ch: t) -proc charSet*(s: set[char]): Peg {.noSideEffect, rtl, extern: "npegs$1".} = +func charSet*(s: set[char]): Peg {.rtl, extern: "npegs$1".} = ## constructs a PEG from a character set `s` assert '\0' notin s result = Peg(kind: pkCharChoice) new(result.charChoice) result.charChoice[] = s -proc len(a: Peg): int {.inline.} = return a.sons.len -proc add(d: var Peg, s: Peg) {.inline.} = add(d.sons, s) +func len(a: Peg): int {.inline.} = return a.sons.len +func add(d: var Peg, s: Peg) {.inline.} = add(d.sons, s) -proc addChoice(dest: var Peg, elem: Peg) = +func addChoice(dest: var Peg, elem: Peg) = var L = dest.len-1 if L >= 0 and dest.sons[L].kind == pkCharChoice: # caution! Do not introduce false aliasing here! @@ -195,12 +195,12 @@ template multipleOp(k: PegKind, localOpt: untyped) = if result.len == 1: result = result.sons[0] -proc `/`*(a: varargs[Peg]): Peg {. - noSideEffect, rtl, extern: "npegsOrderedChoice".} = +func `/`*(a: varargs[Peg]): Peg {. + rtl, extern: "npegsOrderedChoice".} = ## constructs an ordered choice with the PEGs in `a` multipleOp(pkOrderedChoice, addChoice) -proc addSequence(dest: var Peg, elem: Peg) = +func addSequence(dest: var Peg, elem: Peg) = var L = dest.len-1 if L >= 0 and dest.sons[L].kind == pkTerminal: # caution! Do not introduce false aliasing here! @@ -212,12 +212,12 @@ proc addSequence(dest: var Peg, elem: Peg) = else: add(dest, elem) else: add(dest, elem) -proc sequence*(a: varargs[Peg]): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func sequence*(a: varargs[Peg]): Peg {. + rtl, extern: "npegs$1".} = ## constructs a sequence with all the PEGs from `a` multipleOp(pkSequence, addSequence) -proc `?`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsOptional".} = +func `?`*(a: Peg): Peg {.rtl, extern: "npegsOptional".} = ## constructs an optional for the PEG `a` if a.kind in {pkOption, pkGreedyRep, pkGreedyAny, pkGreedyRepChar, pkGreedyRepSet}: @@ -227,7 +227,7 @@ proc `?`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsOptional".} = else: result = Peg(kind: pkOption, sons: @[a]) -proc `*`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsGreedyRep".} = +func `*`*(a: Peg): Peg {.rtl, extern: "npegsGreedyRep".} = ## constructs a "greedy repetition" for the PEG `a` case a.kind of pkGreedyRep, pkGreedyRepChar, pkGreedyRepSet, pkGreedyAny, pkOption: @@ -242,94 +242,94 @@ proc `*`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsGreedyRep".} = else: result = Peg(kind: pkGreedyRep, sons: @[a]) -proc `!*`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsSearch".} = +func `!*`*(a: Peg): Peg {.rtl, extern: "npegsSearch".} = ## constructs a "search" for the PEG `a` result = Peg(kind: pkSearch, sons: @[a]) -proc `!*\`*(a: Peg): Peg {.noSideEffect, rtl, +func `!*\`*(a: Peg): Peg {.rtl, extern: "npgegsCapturedSearch".} = ## constructs a "captured search" for the PEG `a` result = Peg(kind: pkCapturedSearch, sons: @[a]) -proc `+`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsGreedyPosRep".} = +func `+`*(a: Peg): Peg {.rtl, extern: "npegsGreedyPosRep".} = ## constructs a "greedy positive repetition" with the PEG `a` return sequence(a, *a) -proc `&`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsAndPredicate".} = +func `&`*(a: Peg): Peg {.rtl, extern: "npegsAndPredicate".} = ## constructs an "and predicate" with the PEG `a` result = Peg(kind: pkAndPredicate, sons: @[a]) -proc `!`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsNotPredicate".} = +func `!`*(a: Peg): Peg {.rtl, extern: "npegsNotPredicate".} = ## constructs a "not predicate" with the PEG `a` result = Peg(kind: pkNotPredicate, sons: @[a]) -proc any*: Peg {.inline.} = +func any*: Peg {.inline.} = ## constructs the PEG `any character`:idx: (``.``) result = Peg(kind: pkAny) -proc anyRune*: Peg {.inline.} = +func anyRune*: Peg {.inline.} = ## constructs the PEG `any rune`:idx: (``_``) result = Peg(kind: pkAnyRune) -proc newLine*: Peg {.inline.} = +func newLine*: Peg {.inline.} = ## constructs the PEG `newline`:idx: (``\n``) result = Peg(kind: pkNewLine) -proc unicodeLetter*: Peg {.inline.} = +func unicodeLetter*: Peg {.inline.} = ## constructs the PEG ``\letter`` which matches any Unicode letter. result = Peg(kind: pkLetter) -proc unicodeLower*: Peg {.inline.} = +func unicodeLower*: Peg {.inline.} = ## constructs the PEG ``\lower`` which matches any Unicode lowercase letter. result = Peg(kind: pkLower) -proc unicodeUpper*: Peg {.inline.} = +func unicodeUpper*: Peg {.inline.} = ## constructs the PEG ``\upper`` which matches any Unicode uppercase letter. result = Peg(kind: pkUpper) -proc unicodeTitle*: Peg {.inline.} = +func unicodeTitle*: Peg {.inline.} = ## constructs the PEG ``\title`` which matches any Unicode title letter. result = Peg(kind: pkTitle) -proc unicodeWhitespace*: Peg {.inline.} = +func unicodeWhitespace*: Peg {.inline.} = ## constructs the PEG ``\white`` which matches any Unicode ## whitespace character. result = Peg(kind: pkWhitespace) -proc startAnchor*: Peg {.inline.} = +func startAnchor*: Peg {.inline.} = ## constructs the PEG ``^`` which matches the start of the input. result = Peg(kind: pkStartAnchor) -proc endAnchor*: Peg {.inline.} = +func endAnchor*: Peg {.inline.} = ## constructs the PEG ``$`` which matches the end of the input. result = !any() -proc capture*(a: Peg = Peg(kind: pkEmpty)): Peg {.noSideEffect, rtl, extern: "npegsCapture".} = +func capture*(a: Peg = Peg(kind: pkEmpty)): Peg {.rtl, extern: "npegsCapture".} = ## constructs a capture with the PEG `a` result = Peg(kind: pkCapture, sons: @[a]) -proc backref*(index: range[1..MaxSubpatterns], reverse: bool = false): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func backref*(index: range[1..MaxSubpatterns], reverse: bool = false): Peg {. + rtl, extern: "npegs$1".} = ## constructs a back reference of the given `index`. `index` starts counting ## from 1. `reverse` specifies wether indexing starts from the end of the ## capture list. result = Peg(kind: pkBackRef, index: (if reverse: -index else: index - 1)) -proc backrefIgnoreCase*(index: range[1..MaxSubpatterns], reverse: bool = false): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func backrefIgnoreCase*(index: range[1..MaxSubpatterns], reverse: bool = false): Peg {. + rtl, extern: "npegs$1".} = ## constructs a back reference of the given `index`. `index` starts counting ## from 1. `reverse` specifies wether indexing starts from the end of the ## capture list. Ignores case for matching. result = Peg(kind: pkBackRefIgnoreCase, index: (if reverse: -index else: index - 1)) -proc backrefIgnoreStyle*(index: range[1..MaxSubpatterns], reverse: bool = false): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func backrefIgnoreStyle*(index: range[1..MaxSubpatterns], reverse: bool = false): Peg {. + rtl, extern: "npegs$1".} = ## constructs a back reference of the given `index`. `index` starts counting ## from 1. `reverse` specifies wether indexing starts from the end of the ## capture list. Ignores style for matching. result = Peg(kind: pkBackRefIgnoreStyle, index: (if reverse: -index else: index - 1)) -proc spaceCost(n: Peg): int = +func spaceCost(n: Peg): int = case n.kind of pkEmpty: discard of pkTerminal, pkTerminalIgnoreCase, pkTerminalIgnoreStyle, pkChar, @@ -344,8 +344,8 @@ proc spaceCost(n: Peg): int = inc(result, spaceCost(n.sons[i])) if result >= InlineThreshold: break -proc nonterminal*(n: NonTerminal): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func nonterminal*(n: NonTerminal): Peg {. + rtl, extern: "npegs$1".} = ## constructs a PEG that consists of the nonterminal symbol assert n != nil if ntDeclared in n.flags and spaceCost(n.rule) < InlineThreshold: @@ -354,8 +354,8 @@ proc nonterminal*(n: NonTerminal): Peg {. else: result = Peg(kind: pkNonTerminal, nt: n) -proc newNonTerminal*(name: string, line, column: int): NonTerminal {. - noSideEffect, rtl, extern: "npegs$1".} = +func newNonTerminal*(name: string, line, column: int): NonTerminal {. + rtl, extern: "npegs$1".} = ## constructs a nonterminal symbol result = NonTerminal(name: name, line: line, col: column) @@ -390,7 +390,7 @@ template natural*: Peg = # ------------------------- debugging ----------------------------------------- -proc esc(c: char, reserved = {'\0'..'\255'}): string = +func esc(c: char, reserved = {'\0'..'\255'}): string = case c of '\b': result = "\\b" of '\t': result = "\\t" @@ -406,14 +406,14 @@ proc esc(c: char, reserved = {'\0'..'\255'}): string = elif c in reserved: result = '\\' & c else: result = $c -proc singleQuoteEsc(c: char): string = return "'" & esc(c, {'\''}) & "'" +func singleQuoteEsc(c: char): string = return "'" & esc(c, {'\''}) & "'" -proc singleQuoteEsc(str: string): string = +func singleQuoteEsc(str: string): string = result = "'" for c in items(str): add result, esc(c, {'\''}) add result, '\'' -proc charSetEscAux(cc: set[char]): string = +func charSetEscAux(cc: set[char]): string = const reserved = {'^', '-', ']'} result = "" var c1 = 0 @@ -430,13 +430,13 @@ proc charSetEscAux(cc: set[char]): string = c1 = c2 inc(c1) -proc charSetEsc(cc: set[char]): string = +func charSetEsc(cc: set[char]): string = if card(cc) >= 128+64: result = "[^" & charSetEscAux({'\1'..'\xFF'} - cc) & ']' else: result = '[' & charSetEscAux(cc) & ']' -proc toStrAux(r: Peg, res: var string) = +func toStrAux(r: Peg, res: var string) = case r.kind of pkEmpty: add(res, "()") of pkAny: add(res, '.') @@ -522,7 +522,7 @@ proc toStrAux(r: Peg, res: var string) = of pkStartAnchor: add(res, '^') -proc `$` *(r: Peg): string {.noSideEffect, rtl, extern: "npegsToString".} = +func `$` *(r: Peg): string {.rtl, extern: "npegsToString".} = ## converts a PEG to its string representation result = "" toStrAux(r, result) @@ -535,7 +535,7 @@ type ml: int origStart: int -proc bounds*(c: Captures, +func bounds*(c: Captures, i: range[0..MaxSubpatterns-1]): tuple[first, last: int] = ## returns the bounds ``[first..last]`` of the `i`'th capture. result = c.matches[i] @@ -548,11 +548,11 @@ when not useUnicode: inc(i) template runeLenAt(s, i): untyped = 1 - proc isAlpha(a: char): bool {.inline.} = return a in {'a'..'z', 'A'..'Z'} - proc isUpper(a: char): bool {.inline.} = return a in {'A'..'Z'} - proc isLower(a: char): bool {.inline.} = return a in {'a'..'z'} - proc isTitle(a: char): bool {.inline.} = return false - proc isWhiteSpace(a: char): bool {.inline.} = return a in {' ', '\9'..'\13'} + func isAlpha(a: char): bool {.inline.} = return a in {'a'..'z', 'A'..'Z'} + func isUpper(a: char): bool {.inline.} = return a in {'A'..'Z'} + func isLower(a: char): bool {.inline.} = return a in {'a'..'z'} + func isTitle(a: char): bool {.inline.} = return false + func isWhiteSpace(a: char): bool {.inline.} = return a in {' ', '\9'..'\13'} template matchOrParse(mopProc: untyped) = # Used to make the main matcher proc *rawMatch* as well as event parser @@ -860,8 +860,8 @@ template matchOrParse(mopProc: untyped) = leave(pkStartAnchor, s, p, start, result) of pkRule, pkList: assert false -proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int - {.noSideEffect, rtl, extern: "npegs$1".} = +func rawMatch*(s: string, p: Peg, start: int, c: var Captures): int + {.rtl, extern: "npegs$1".} = ## low-level matching proc that implements the PEG interpreter. Use this ## for maximum efficiency (every other PEG operation ends up calling this ## proc). @@ -873,7 +873,11 @@ proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int template leave(pk, s, p, start, length) = discard matchOrParse(matchIt) - result = matchIt(s, p, start, c) + {.cast(noSideEffect).}: + # This cast is allowed because the `matchOrParse` template is used for + # both matching and parsing, but side effects are only possible when it's + # used by `eventParser`. + result = matchIt(s, p, start, c) macro mkHandlerTplts(handlers: untyped): untyped = # Transforms the handler spec in *handlers* into handler templates. @@ -900,7 +904,7 @@ macro mkHandlerTplts(handlers: untyped): untyped = # StmtList # # ... - proc mkEnter(hdName, body: NimNode): NimNode = + func mkEnter(hdName, body: NimNode): NimNode = template helper(hdName, body) {.dirty.} = template hdName(s, p, start) = let s {.inject.} = s @@ -1069,8 +1073,8 @@ template fillMatches(s, caps, c) = else: caps[k] = "" -proc matchLen*(s: string, pattern: Peg, matches: var openArray[string], - start = 0): int {.noSideEffect, rtl, extern: "npegs$1Capture".} = +func matchLen*(s: string, pattern: Peg, matches: var openArray[string], + start = 0): int {.rtl, extern: "npegs$1Capture".} = ## the same as ``match``, but it returns the length of the match, ## if there is no match, -1 is returned. Note that a match length ## of zero can happen. It's possible that a suffix of `s` remains @@ -1080,8 +1084,8 @@ proc matchLen*(s: string, pattern: Peg, matches: var openArray[string], result = rawMatch(s, pattern, start, c) if result >= 0: fillMatches(s, matches, c) -proc matchLen*(s: string, pattern: Peg, - start = 0): int {.noSideEffect, rtl, extern: "npegs$1".} = +func matchLen*(s: string, pattern: Peg, + start = 0): int {.rtl, extern: "npegs$1".} = ## the same as ``match``, but it returns the length of the match, ## if there is no match, -1 is returned. Note that a match length ## of zero can happen. It's possible that a suffix of `s` remains @@ -1090,22 +1094,22 @@ proc matchLen*(s: string, pattern: Peg, c.origStart = start result = rawMatch(s, pattern, start, c) -proc match*(s: string, pattern: Peg, matches: var openArray[string], - start = 0): bool {.noSideEffect, rtl, extern: "npegs$1Capture".} = +func match*(s: string, pattern: Peg, matches: var openArray[string], + start = 0): bool {.rtl, extern: "npegs$1Capture".} = ## returns ``true`` if ``s[start..]`` matches the ``pattern`` and ## the captured substrings in the array ``matches``. If it does not ## match, nothing is written into ``matches`` and ``false`` is ## returned. result = matchLen(s, pattern, matches, start) != -1 -proc match*(s: string, pattern: Peg, - start = 0): bool {.noSideEffect, rtl, extern: "npegs$1".} = +func match*(s: string, pattern: Peg, + start = 0): bool {.rtl, extern: "npegs$1".} = ## returns ``true`` if ``s`` matches the ``pattern`` beginning from ``start``. result = matchLen(s, pattern, start) != -1 -proc find*(s: string, pattern: Peg, matches: var openArray[string], - start = 0): int {.noSideEffect, rtl, extern: "npegs$1Capture".} = +func find*(s: string, pattern: Peg, matches: var openArray[string], + start = 0): int {.rtl, extern: "npegs$1Capture".} = ## returns the starting position of ``pattern`` in ``s`` and the captured ## substrings in the array ``matches``. If it does not match, nothing ## is written into ``matches`` and -1 is returned. @@ -1119,9 +1123,9 @@ proc find*(s: string, pattern: Peg, matches: var openArray[string], return -1 # could also use the pattern here: (!P .)* P -proc findBounds*(s: string, pattern: Peg, matches: var openArray[string], +func findBounds*(s: string, pattern: Peg, matches: var openArray[string], start = 0): tuple[first, last: int] {. - noSideEffect, rtl, extern: "npegs$1Capture".} = + rtl, extern: "npegs$1Capture".} = ## returns the starting position and end position of ``pattern`` in ``s`` ## and the captured ## substrings in the array ``matches``. If it does not match, nothing @@ -1136,8 +1140,8 @@ proc findBounds*(s: string, pattern: Peg, matches: var openArray[string], return (i, i+L-1) return (-1, 0) -proc find*(s: string, pattern: Peg, - start = 0): int {.noSideEffect, rtl, extern: "npegs$1".} = +func find*(s: string, pattern: Peg, + start = 0): int {.rtl, extern: "npegs$1".} = ## returns the starting position of ``pattern`` in ``s``. If it does not ## match, -1 is returned. var c: Captures @@ -1160,8 +1164,8 @@ iterator findAll*(s: string, pattern: Peg, start = 0): string = yield substr(s, i, i+L-1) inc(i, L) -proc findAll*(s: string, pattern: Peg, start = 0): seq[string] {. - noSideEffect, rtl, extern: "npegs$1".} = +func findAll*(s: string, pattern: Peg, start = 0): seq[string] {. + rtl, extern: "npegs$1".} = ## returns all matching *substrings* of `s` that match `pattern`. ## If it does not match, @[] is returned. result = @[] @@ -1192,31 +1196,31 @@ template `=~`*(s: string, pattern: Peg): bool = # ------------------------- more string handling ------------------------------ -proc contains*(s: string, pattern: Peg, start = 0): bool {. - noSideEffect, rtl, extern: "npegs$1".} = +func contains*(s: string, pattern: Peg, start = 0): bool {. + rtl, extern: "npegs$1".} = ## same as ``find(s, pattern, start) >= 0`` return find(s, pattern, start) >= 0 -proc contains*(s: string, pattern: Peg, matches: var openArray[string], - start = 0): bool {.noSideEffect, rtl, extern: "npegs$1Capture".} = +func contains*(s: string, pattern: Peg, matches: var openArray[string], + start = 0): bool {.rtl, extern: "npegs$1Capture".} = ## same as ``find(s, pattern, matches, start) >= 0`` return find(s, pattern, matches, start) >= 0 -proc startsWith*(s: string, prefix: Peg, start = 0): bool {. - noSideEffect, rtl, extern: "npegs$1".} = +func startsWith*(s: string, prefix: Peg, start = 0): bool {. + rtl, extern: "npegs$1".} = ## returns true if `s` starts with the pattern `prefix` result = matchLen(s, prefix, start) >= 0 -proc endsWith*(s: string, suffix: Peg, start = 0): bool {. - noSideEffect, rtl, extern: "npegs$1".} = +func endsWith*(s: string, suffix: Peg, start = 0): bool {. + rtl, extern: "npegs$1".} = ## returns true if `s` ends with the pattern `suffix` var c: Captures c.origStart = start for i in start .. s.len-1: if rawMatch(s, suffix, i, c) == s.len - i: return true -proc replacef*(s: string, sub: Peg, by: string): string {. - noSideEffect, rtl, extern: "npegs$1".} = +func replacef*(s: string, sub: Peg, by: string): string {. + rtl, extern: "npegs$1".} = ## Replaces `sub` in `s` by the string `by`. Captures can be accessed in `by` ## with the notation ``$i`` and ``$#`` (see strutils.`%`). Examples: ## @@ -1244,8 +1248,8 @@ proc replacef*(s: string, sub: Peg, by: string): string {. inc(i, x) add(result, substr(s, i)) -proc replace*(s: string, sub: Peg, by = ""): string {. - noSideEffect, rtl, extern: "npegs$1".} = +func replace*(s: string, sub: Peg, by = ""): string {. + rtl, extern: "npegs$1".} = ## Replaces `sub` in `s` by the string `by`. Captures cannot be accessed ## in `by`. result = "" @@ -1261,9 +1265,9 @@ proc replace*(s: string, sub: Peg, by = ""): string {. inc(i, x) add(result, substr(s, i)) -proc parallelReplace*(s: string, subs: varargs[ +func parallelReplace*(s: string, subs: varargs[ tuple[pattern: Peg, repl: string]]): string {. - noSideEffect, rtl, extern: "npegs$1".} = + rtl, extern: "npegs$1".} = ## Returns a modified copy of `s` with the substitutions in `subs` ## applied in parallel. result = "" @@ -1288,7 +1292,7 @@ proc parallelReplace*(s: string, subs: varargs[ when not defined(nimHasEffectsOf): {.pragma: effectsOf.} -proc replace*(s: string, sub: Peg, cb: proc( +func replace*(s: string, sub: Peg, cb: proc( match: int, cnt: int, caps: openArray[string]): string): string {. rtl, extern: "npegs$1cb", effectsOf: cb.} = ## Replaces `sub` in `s` by the resulting strings from the callback. @@ -1297,7 +1301,7 @@ proc replace*(s: string, sub: Peg, cb: proc( ## ## .. code-block:: nim ## - ## proc handleMatches*(m: int, n: int, c: openArray[string]): string = + ## func handleMatches*(m: int, n: int, c: openArray[string]): string = ## result = "" ## if m > 0: ## result.add ", " @@ -1380,8 +1384,8 @@ iterator split*(s: string, sep: Peg): string = if first < last: yield substr(s, first, last-1) -proc split*(s: string, sep: Peg): seq[string] {. - noSideEffect, rtl, extern: "npegs$1".} = +func split*(s: string, sep: Peg): seq[string] {. + rtl, extern: "npegs$1".} = ## Splits the string `s` into substrings. result = @[] for it in split(s, sep): result.add it @@ -1445,20 +1449,20 @@ const "@", "built-in", "escaped", "$", "$", "^" ] -proc handleCR(L: var PegLexer, pos: int): int = +func handleCR(L: var PegLexer, pos: int): int = assert(L.buf[pos] == '\c') inc(L.lineNumber) result = pos+1 if result < L.buf.len and L.buf[result] == '\L': inc(result) L.lineStart = result -proc handleLF(L: var PegLexer, pos: int): int = +func handleLF(L: var PegLexer, pos: int): int = assert(L.buf[pos] == '\L') inc(L.lineNumber) result = pos+1 L.lineStart = result -proc init(L: var PegLexer, input, filename: string, line = 1, col = 0) = +func init(L: var PegLexer, input, filename: string, line = 1, col = 0) = L.buf = input L.bufpos = 0 L.lineNumber = line @@ -1466,18 +1470,18 @@ proc init(L: var PegLexer, input, filename: string, line = 1, col = 0) = L.lineStart = 0 L.filename = filename -proc getColumn(L: PegLexer): int {.inline.} = +func getColumn(L: PegLexer): int {.inline.} = result = abs(L.bufpos - L.lineStart) + L.colOffset -proc getLine(L: PegLexer): int {.inline.} = +func getLine(L: PegLexer): int {.inline.} = result = L.lineNumber -proc errorStr(L: PegLexer, msg: string, line = -1, col = -1): string = +func errorStr(L: PegLexer, msg: string, line = -1, col = -1): string = var line = if line < 0: getLine(L) else: line var col = if col < 0: getColumn(L) else: col result = "$1($2, $3) Error: $4" % [L.filename, $line, $col, msg] -proc getEscapedChar(c: var PegLexer, tok: var Token) = +func getEscapedChar(c: var PegLexer, tok: var Token) = inc(c.bufpos) if c.bufpos >= len(c.buf): tok.kind = tkInvalid @@ -1537,7 +1541,7 @@ proc getEscapedChar(c: var PegLexer, tok: var Token) = add(tok.literal, c.buf[c.bufpos]) inc(c.bufpos) -proc skip(c: var PegLexer) = +func skip(c: var PegLexer) = var pos = c.bufpos while pos < c.buf.len: case c.buf[pos] @@ -1554,7 +1558,7 @@ proc skip(c: var PegLexer) = break # EndOfFile also leaves the loop c.bufpos = pos -proc getString(c: var PegLexer, tok: var Token) = +func getString(c: var PegLexer, tok: var Token) = tok.kind = tkStringLit var pos = c.bufpos + 1 var quote = c.buf[pos-1] @@ -1575,7 +1579,7 @@ proc getString(c: var PegLexer, tok: var Token) = inc(pos) c.bufpos = pos -proc getDollar(c: var PegLexer, tok: var Token) = +func getDollar(c: var PegLexer, tok: var Token) = var pos = c.bufpos + 1 var neg = false if pos < c.buf.len and c.buf[pos] == '^': @@ -1595,7 +1599,7 @@ proc getDollar(c: var PegLexer, tok: var Token) = tok.kind = tkDollar c.bufpos = pos -proc getCharSet(c: var PegLexer, tok: var Token) = +func getCharSet(c: var PegLexer, tok: var Token) = tok.kind = tkCharSet tok.charset = {} var pos = c.bufpos + 1 @@ -1652,7 +1656,7 @@ proc getCharSet(c: var PegLexer, tok: var Token) = c.bufpos = pos if caret: tok.charset = {'\1'..'\xFF'} - tok.charset -proc getSymbol(c: var PegLexer, tok: var Token) = +func getSymbol(c: var PegLexer, tok: var Token) = var pos = c.bufpos while pos < c.buf.len: add(tok.literal, c.buf[pos]) @@ -1661,7 +1665,7 @@ proc getSymbol(c: var PegLexer, tok: var Token) = c.bufpos = pos tok.kind = tkIdentifier -proc getBuiltin(c: var PegLexer, tok: var Token) = +func getBuiltin(c: var PegLexer, tok: var Token) = if c.bufpos+1 < c.buf.len and c.buf[c.bufpos+1] in strutils.Letters: inc(c.bufpos) getSymbol(c, tok) @@ -1670,7 +1674,7 @@ proc getBuiltin(c: var PegLexer, tok: var Token) = tok.kind = tkEscaped getEscapedChar(c, tok) # may set tok.kind to tkInvalid -proc getTok(c: var PegLexer, tok: var Token) = +func getTok(c: var PegLexer, tok: var Token) = tok.kind = tkInvalid tok.modifier = modNone setLen(tok.literal, 0) @@ -1792,7 +1796,7 @@ proc getTok(c: var PegLexer, tok: var Token) = add(tok.literal, c.buf[c.bufpos]) inc(c.bufpos) -proc arrowIsNextTok(c: PegLexer): bool = +func arrowIsNextTok(c: PegLexer): bool = # the only look ahead we need var pos = c.bufpos while pos < c.buf.len and c.buf[pos] in {'\t', ' '}: inc(pos) @@ -1813,23 +1817,23 @@ type identIsVerbatim: bool skip: Peg -proc pegError(p: PegParser, msg: string, line = -1, col = -1) = +func pegError(p: PegParser, msg: string, line = -1, col = -1) = var e: ref EInvalidPeg new(e) e.msg = errorStr(p, msg, line, col) raise e -proc getTok(p: var PegParser) = +func getTok(p: var PegParser) = getTok(p, p.tok) if p.tok.kind == tkInvalid: pegError(p, "'" & p.tok.literal & "' is invalid token") -proc eat(p: var PegParser, kind: TokKind) = +func eat(p: var PegParser, kind: TokKind) = if p.tok.kind == kind: getTok(p) else: pegError(p, tokKindToStr[kind] & " expected") -proc parseExpr(p: var PegParser): Peg {.gcsafe.} +func parseExpr(p: var PegParser): Peg {.gcsafe.} -proc getNonTerminal(p: var PegParser, name: string): NonTerminal = +func getNonTerminal(p: var PegParser, name: string): NonTerminal = for i in 0..high(p.nonterms): result = p.nonterms[i] if cmpIgnoreStyle(result.name, name) == 0: return @@ -1837,13 +1841,13 @@ proc getNonTerminal(p: var PegParser, name: string): NonTerminal = result = newNonTerminal(name, getLine(p), getColumn(p)) add(p.nonterms, result) -proc modifiedTerm(s: string, m: Modifier): Peg = +func modifiedTerm(s: string, m: Modifier): Peg = case m of modNone, modVerbatim: result = term(s) of modIgnoreCase: result = termIgnoreCase(s) of modIgnoreStyle: result = termIgnoreStyle(s) -proc modifiedBackref(s: int, m: Modifier): Peg = +func modifiedBackref(s: int, m: Modifier): Peg = var reverse = s < 0 index = if reverse: -s else: s @@ -1852,7 +1856,7 @@ proc modifiedBackref(s: int, m: Modifier): Peg = of modIgnoreCase: result = backrefIgnoreCase(index, reverse) of modIgnoreStyle: result = backrefIgnoreStyle(index, reverse) -proc builtin(p: var PegParser): Peg = +func builtin(p: var PegParser): Peg = # do not use "y", "skip" or "i" as these would be ambiguous case p.tok.literal of "n": result = newLine() @@ -1872,11 +1876,11 @@ proc builtin(p: var PegParser): Peg = of "white": result = unicodeWhitespace() else: pegError(p, "unknown built-in: " & p.tok.literal) -proc token(terminal: Peg, p: PegParser): Peg = +func token(terminal: Peg, p: PegParser): Peg = if p.skip.kind == pkEmpty: result = terminal else: result = sequence(p.skip, terminal) -proc primary(p: var PegParser): Peg = +func primary(p: var PegParser): Peg = case p.tok.kind of tkAmp: getTok(p) @@ -1968,7 +1972,7 @@ proc primary(p: var PegParser): Peg = getTok(p) else: break -proc seqExpr(p: var PegParser): Peg = +func seqExpr(p: var PegParser): Peg = result = primary(p) while true: case p.tok.kind @@ -1982,13 +1986,13 @@ proc seqExpr(p: var PegParser): Peg = else: break else: break -proc parseExpr(p: var PegParser): Peg = +func parseExpr(p: var PegParser): Peg = result = seqExpr(p) while p.tok.kind == tkBar: getTok(p) result = result / seqExpr(p) -proc parseRule(p: var PegParser): NonTerminal = +func parseRule(p: var PegParser): NonTerminal = if p.tok.kind == tkIdentifier and arrowIsNextTok(p): result = getNonTerminal(p, p.tok.literal) if ntDeclared in result.flags: @@ -2002,7 +2006,7 @@ proc parseRule(p: var PegParser): NonTerminal = else: pegError(p, "rule expected, but found: " & p.tok.literal) -proc rawParse(p: var PegParser): Peg = +func rawParse(p: var PegParser): Peg = ## parses a rule or a PEG expression while p.tok.kind == tkBuiltin: case p.tok.literal @@ -2032,7 +2036,7 @@ proc rawParse(p: var PegParser): Peg = elif ntUsed notin nt.flags and i > 0: pegError(p, "unused rule: " & nt.name, nt.line, nt.col) -proc parsePeg*(pattern: string, filename = "pattern", line = 1, col = 0): Peg = +func parsePeg*(pattern: string, filename = "pattern", line = 1, col = 0): Peg = ## constructs a Peg object from `pattern`. `filename`, `line`, `col` are ## used for error messages, but they only provide start offsets. `parsePeg` ## keeps track of line and column numbers within `pattern`. @@ -2047,14 +2051,14 @@ proc parsePeg*(pattern: string, filename = "pattern", line = 1, col = 0): Peg = getTok(p) result = rawParse(p) -proc peg*(pattern: string): Peg = +func peg*(pattern: string): Peg = ## constructs a Peg object from the `pattern`. The short name has been ## chosen to encourage its use as a raw string modifier:: ## ## peg"{\ident} \s* '=' \s* {.*}" result = parsePeg(pattern, "pattern") -proc escapePeg*(s: string): string = +func escapePeg*(s: string): string = ## escapes `s` so that it is matched verbatim when used as a peg. result = "" var inQuote = false diff --git a/tests/effects/tstrict_funcs_imports.nim b/tests/effects/tstrict_funcs_imports.nim index 4e9b9fe66..264771a1a 100644 --- a/tests/effects/tstrict_funcs_imports.nim +++ b/tests/effects/tstrict_funcs_imports.nim @@ -84,7 +84,7 @@ import parseutils, parsexml, pathnorm, - # pegs, + pegs, posix_utils, prelude, punycode, diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index fe3335c0f..0c5c8f4a3 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -1,6 +1,9 @@ ## Part of 'koch' responsible for the documentation generation. -import os, strutils, osproc, sets, pathnorm, pegs, sequtils +import os, strutils, osproc, sets, pathnorm, sequtils +# XXX: Remove this feature check once the csources supports it. +when defined(nimHasCastPragmaBlocks): + import std/pegs from std/private/globs import nativeToUnixPath, walkDirRecFilter, PathEntry import "../compiler/nimpaths" @@ -338,7 +341,9 @@ proc buildDocs*(args: string, localOnly = false, localOutDir = "") = if not localOnly: buildDocsDir(args, webUploadOutput / NimVersion) - let gaFilter = peg"@( y'--doc.googleAnalytics:' @(\s / $) )" - args = args.replace(gaFilter) + # XXX: Remove this feature check once the csources supports it. + when defined(nimHasCastPragmaBlocks): + let gaFilter = peg"@( y'--doc.googleAnalytics:' @(\s / $) )" + args = args.replace(gaFilter) buildDocsDir(args, localOutDir) From 6f15af41a7ea7aeb09b9c9b9c1d7ca3b505107c9 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 7 Oct 2021 15:07:24 +0200 Subject: [PATCH 1279/2472] fixes a regression caused by overloadable enums even though they're opt-in (#18970) --- compiler/semtempl.nim | 10 +++++++--- tests/enum/mregression.nim | 4 ++++ tests/enum/tregression.nim | 13 +++++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 tests/enum/mregression.nim create mode 100644 tests/enum/tregression.nim diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim index c10917bcb..49ce55a64 100644 --- a/compiler/semtempl.nim +++ b/compiler/semtempl.nim @@ -250,7 +250,7 @@ proc semTemplSymbol(c: PContext, n: PNode, s: PSym; isField: bool): PNode = of skUnknown: # Introduced in this pass! Leave it as an identifier. result = n - of OverloadableSyms: + of OverloadableSyms-{skEnumField}: result = symChoice(c, n, s, scOpen, isField) of skGenericParam: if isField and sfGenSym in s.flags: result = n @@ -261,8 +261,12 @@ proc semTemplSymbol(c: PContext, n: PNode, s: PSym; isField: bool): PNode = if isField and sfGenSym in s.flags: result = n else: result = newSymNodeTypeDesc(s, c.idgen, n.info) else: - if isField and sfGenSym in s.flags: result = n - else: result = newSymNode(s, n.info) + if s.kind == skEnumField and overloadableEnums in c.features: + result = symChoice(c, n, s, scOpen, isField) + elif isField and sfGenSym in s.flags: + result = n + else: + result = newSymNode(s, n.info) # Issue #12832 when defined(nimsuggest): suggestSym(c.graph, n.info, s, c.graph.usageSym, false) diff --git a/tests/enum/mregression.nim b/tests/enum/mregression.nim new file mode 100644 index 000000000..9c3e3cf8b --- /dev/null +++ b/tests/enum/mregression.nim @@ -0,0 +1,4 @@ + +type + OtherEnum* = enum + Success, Failed, More diff --git a/tests/enum/tregression.nim b/tests/enum/tregression.nim new file mode 100644 index 000000000..597f0c80a --- /dev/null +++ b/tests/enum/tregression.nim @@ -0,0 +1,13 @@ +discard """ + action: "compile" +""" +import options, mregression + +type + MyEnum = enum + Success + +template t = + echo some(Success) + +t() From 08107c2192e6aebb386df2711f2935731ee599fd Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 8 Oct 2021 00:54:21 +0800 Subject: [PATCH 1280/2472] [minor] give more friendly description (#18973) --- nimdoc/tester.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimdoc/tester.nim b/nimdoc/tester.nim index 1e438b97b..32dce0709 100644 --- a/nimdoc/tester.nim +++ b/nimdoc/tester.nim @@ -80,4 +80,4 @@ let testNimDoc(test2Dir, test2DocsDir, test2Switches, fixup) if failures > 0: - quit "$# failures occurred; see note in tester.nim regarding -d:nimTestsNimdocFixup" % $failures + quit "$# failures occurred; see note in nimdoc/tester.nim regarding -d:nimTestsNimdocFixup" % $failures From 787def271b1cabc6f898caa42f892125de9fa908 Mon Sep 17 00:00:00 2001 From: antonl05 <88133270+antonl05@users.noreply.github.com> Date: Sat, 9 Oct 2021 14:17:07 +0530 Subject: [PATCH 1281/2472] add OpenIndiana to list (#18972) * add OpenIndiana support * point sunos to solaris --- tools/niminst/buildsh.nimf | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tools/niminst/buildsh.nimf b/tools/niminst/buildsh.nimf index 9850abd89..5f68a6a17 100644 --- a/tools/niminst/buildsh.nimf +++ b/tools/niminst/buildsh.nimf @@ -139,6 +139,11 @@ case $uos in myos="solaris" LINK_FLAGS="$LINK_FLAGS -ldl -lm -lsocket -lnsl" ;; + *SunOS* ) + myos="solaris" + LINK_FLAGS="$LINK_FLAGS -ldl -lm -lsocket -lnsl" + isOpenIndiana="yes" + ;; *haiku* ) myos="haiku" LINK_FLAGS="$LINK_FLAGS -lroot -lnetwork" @@ -159,7 +164,12 @@ esac case $ucpu in *i386* | *i486* | *i586* | *i686* | *bepc* | *i86pc* ) - mycpu="i386" ;; + if [ isOpenIndiana -eq "yes" ] ; then + mycpu="amd64" + else + mycpu="i386" + fi + ;; *amd*64* | *x86-64* | *x86_64* ) mycpu="amd64" ;; *sparc*|*sun* ) From 83128f217f63045974a48e61b65386abbfc97352 Mon Sep 17 00:00:00 2001 From: narimiran Date: Mon, 11 Oct 2021 14:38:59 +0200 Subject: [PATCH 1282/2472] disable testing of `fidget`, to make CIs green --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 0c47104ac..e037cc609 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -63,7 +63,7 @@ pkg "delaunay" pkg "docopt" pkg "easygl", "nim c -o:egl -r src/easygl.nim", "https://github.com/jackmott/easygl" pkg "elvis" -pkg "fidget" +pkg "fidget", allowFailure = true pkg "fragments", "nim c -r fragments/dsl.nim" pkg "fusion" pkg "gara" From 0ae2d1ea88fdc7b6910ddbc960c7db086be383e4 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Tue, 12 Oct 2021 06:41:30 -0700 Subject: [PATCH 1283/2472] Improve error message when NodeJS is not installed and 'nim js -r' is run (#18978) --- compiler/nodejs.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compiler/nodejs.nim b/compiler/nodejs.nim index 781035bb7..283643e8d 100644 --- a/compiler/nodejs.nim +++ b/compiler/nodejs.nim @@ -3,5 +3,8 @@ import os proc findNodeJs*(): string {.inline.} = ## Find NodeJS executable and return it as a string. result = findExe("nodejs") - if result == "": + if result.len == 0: result = findExe("node") + if result.len == 0: + echo "Please install NodeJS first, see https://nodejs.org/en/download" + raise newException(IOError, "NodeJS not found in PATH: " & result) From 2aa97a228aa3a34b48f1af9901f29379fb131a31 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Wed, 13 Oct 2021 10:09:45 +0100 Subject: [PATCH 1284/2472] Removes deprecated {.injectStmt.}. Fixes #18666 (#18984) --- compiler/ccgcalls.nim | 1 - compiler/ccgstmts.nim | 5 ---- compiler/cgen.nim | 3 --- compiler/cgendata.nim | 1 - compiler/pragmas.nim | 8 +----- compiler/wordrecg.nim | 2 +- lib/system/gc_ms.nim | 10 ------- tests/pragmas/tinjectstmt.nim | 51 ----------------------------------- 8 files changed, 2 insertions(+), 79 deletions(-) delete mode 100644 tests/pragmas/tinjectstmt.nim diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index f6cc6da74..12f366a53 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -801,6 +801,5 @@ proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) = genNamedParamCall(p, ri, d) else: genPrefixCall(p, le, ri, d) - postStmtActions(p) proc genCall(p: BProc, e: PNode, d: var TLoc) = genAsgnCall(p, nil, e, d) diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index 6cbff6ee9..559849f0d 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -1509,11 +1509,6 @@ proc genPragma(p: BProc, n: PNode) = for it in n.sons: case whichPragma(it) of wEmit: genEmit(p, it) - of wInjectStmt: - var p = newProc(nil, p.module) - p.options.excl {optLineTrace, optStackTrace} - genStmts(p, it[1]) - p.module.injectStmt = p.s(cpsStmts) else: discard diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 43410dc60..412a428da 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -271,9 +271,6 @@ proc genLineDir(p: BProc, t: PNode) = linefmt(p, cpsStmts, "nimln_($1, $2);$n", [line, quotedFilename(p.config, t.info)]) -proc postStmtActions(p: BProc) {.inline.} = - p.s(cpsStmts).add(p.module.injectStmt) - proc accessThreadLocalVar(p: BProc, s: PSym) proc emulatedThreadVars(conf: ConfigRef): bool {.inline.} proc genProc(m: BModule, prc: PSym) diff --git a/compiler/cgendata.nim b/compiler/cgendata.nim index 3678adacf..88b8d4090 100644 --- a/compiler/cgendata.nim +++ b/compiler/cgendata.nim @@ -170,7 +170,6 @@ type labels*: Natural # for generating unique module-scope names extensionLoaders*: array['0'..'9', Rope] # special procs for the # OpenGL wrapper - injectStmt*: Rope sigConflicts*: CountTable[SigHash] g*: BModuleList ndi*: NdiFile diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index e9f52c71f..a6e6131d7 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -55,7 +55,7 @@ const wDeprecated, wFloatChecks, wInfChecks, wNanChecks, wPragma, wEmit, wUnroll, wLinearScanEnd, wPatterns, wTrMacros, wEffects, wNoForward, wReorder, wComputedGoto, - wInjectStmt, wExperimental, wThis, wUsed, wInvariant, wAssume, wAssert} + wExperimental, wThis, wUsed, wInvariant, wAssume, wAssert} lambdaPragmas* = {FirstCallConv..LastCallConv, wNoSideEffect, wSideEffect, wNoreturn, wNosinks, wDynlib, wHeader, wThread, wAsmNoStackFrame, @@ -1204,12 +1204,6 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, of wExportNims: if sym == nil: invalidPragma(c, it) else: magicsys.registerNimScriptSymbol(c.graph, sym) - of wInjectStmt: - warningDeprecated(c.config, it.info, "'.injectStmt' pragma is deprecated") - if it.kind notin nkPragmaCallKinds or it.len != 2: - localError(c.config, it.info, "expression expected") - else: - it[1] = c.semExpr(c, it[1]) of wExperimental: if not isTopLevel(c): localError(c.config, n.info, "'experimental' pragma only valid as toplevel statement or in a 'push' environment") diff --git a/compiler/wordrecg.nim b/compiler/wordrecg.nim index 4ba807518..ce06effab 100644 --- a/compiler/wordrecg.nim +++ b/compiler/wordrecg.nim @@ -80,7 +80,7 @@ type wLocalPassc = "localPassC", wBorrow = "borrow", wDiscardable = "discardable", wFieldChecks = "fieldChecks", wSubsChar = "subschar", wAcyclic = "acyclic", wShallow = "shallow", wUnroll = "unroll", wLinearScanEnd = "linearScanEnd", - wComputedGoto = "computedGoto", wInjectStmt = "injectStmt", wExperimental = "experimental", + wComputedGoto = "computedGoto", wExperimental = "experimental", wWrite = "write", wGensym = "gensym", wInject = "inject", wDirty = "dirty", wInheritable = "inheritable", wThreadVar = "threadvar", wEmit = "emit", wAsmNoStackFrame = "asmNoStackFrame", wImplicitStatic = "implicitStatic", diff --git a/lib/system/gc_ms.nim b/lib/system/gc_ms.nim index 9e306c497..0675b9f2e 100644 --- a/lib/system/gc_ms.nim +++ b/lib/system/gc_ms.nim @@ -428,16 +428,6 @@ proc sweep(gch: var GcHeap) = if c.refcount == rcBlack: c.refcount = rcWhite else: freeCyclicCell(gch, c) -when false: - # meant to be used with the now-deprected `.injectStmt`: {.injectStmt: newGcInvariant().} - proc newGcInvariant*() = - for x in allObjects(gch.region): - if isCell(x): - var c = cast[PCell](x) - if c.typ == nil: - writeStackTrace() - quit 1 - proc markGlobals(gch: var GcHeap) = if gch.gcThreadId == 0: when defined(nimTracing): diff --git a/tests/pragmas/tinjectstmt.nim b/tests/pragmas/tinjectstmt.nim deleted file mode 100644 index c6256bda6..000000000 --- a/tests/pragmas/tinjectstmt.nim +++ /dev/null @@ -1,51 +0,0 @@ -discard """ - joinable: false - output:''' -onInject: 1 -onInject: 2 -ok0 -ok1 -onInject: 3 -onInject: 4 -onInject: 5 -0 -onInject: 6 -onInject: 7 -onInject: 8 -1 -onInject: 9 -onInject: 10 -onInject: 11 -2 -ok2 -onInject: 12 -''' -""" - -# test {.injectStmt.} - -#[ -{.injectStmt.} pragma can be used to inject a statement before every -other statement in the current module. It's now undocumented and may be removed -in the future and replaced with something more general and without its limitations. -(e.g. doesn't work in VM or js backends). -]# - -from system/ansi_c import c_printf - -var count = 0 -proc onInject*() = - count.inc - # echo count # xxx would fail, probably infinite recursion - c_printf("onInject: %d\n", cast[int](count)) - -{.injectStmt: onInject().} -echo "ok0" -proc main()= - echo "ok1" - for a in 0..<3: - echo a - echo "ok2" - -static: main() # xxx injectStmt not honored in VM -main() From e645be4d0c6c53cd9483d4fde24c4ddc7bd1bf96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9E=97=E4=BA=A6=E6=81=A9?= Date: Wed, 13 Oct 2021 17:35:47 +0800 Subject: [PATCH 1285/2472] add ghci like type annotation buildEchoStmt (1049) (#18875) * add ghci like type annotation buildEchoStmt (1049) * Update compiler/semexprs.nim * Update compiler/semexprs.nim Co-authored-by: flywind Co-authored-by: Andreas Rumpf Co-authored-by: flywind --- compiler/semexprs.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 342b8b05f..c11cecfa9 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1057,6 +1057,7 @@ proc buildEchoStmt(c: PContext, n: PNode): PNode = else: result.add localErrorNode(c, n, "system needs: echo") result.add(n) + result.add(newStrNode(nkStrLit, ": " & n.typ.typeToString)) result = semExpr(c, result) proc semExprNoType(c: PContext, n: PNode): PNode = From 2ac3ba713b61e4e6b70c98da59dc05925f8b5e6b Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 13 Oct 2021 20:57:25 +0800 Subject: [PATCH 1286/2472] fix #18985 (#18988) --- compiler/semtypes.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 9f7a17a53..d03fa88a8 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -2076,7 +2076,10 @@ proc processMagicType(c: PContext, m: PSym) = setMagicType(c.config, m, tySequence, szUncomputedSize) if optSeqDestructors in c.config.globalOptions: incl m.typ.flags, tfHasAsgn - assert c.graph.sysTypes[tySequence] == nil + if defined(nimsuggest) or c.config.cmd == cmdCheck: # bug #18985 + discard + else: + assert c.graph.sysTypes[tySequence] == nil c.graph.sysTypes[tySequence] = m.typ of mOrdinal: setMagicIntegral(c.config, m, tyOrdinal, szUncomputedSize) From e2b19bb2deba0932cf324c0127ffa7f2cbed8b5d Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 13 Oct 2021 20:58:31 +0800 Subject: [PATCH 1287/2472] remove deprecated stuffs from std/times (#18937) --- lib/pure/times.nim | 50 +++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index b41bb28b5..113f73d2a 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -488,7 +488,7 @@ proc fromEpochDay(epochday: int64): proc getDayOfYear*(monthday: MonthdayRange, month: Month, year: int): YeardayRange {.tags: [], raises: [], benign.} = ## Returns the day of the year. - ## Equivalent with `initDateTime(monthday, month, year, 0, 0, 0).yearday`. + ## Equivalent with `dateTime(year, month, monthday, 0, 0, 0, 0).yearday`. runnableExamples: doAssert getDayOfYear(1, mJan, 2000) == 0 doAssert getDayOfYear(10, mJan, 2000) == 9 @@ -508,7 +508,7 @@ proc getDayOfYear*(monthday: MonthdayRange, month: Month, year: int): proc getDayOfWeek*(monthday: MonthdayRange, month: Month, year: int): WeekDay {.tags: [], raises: [], benign.} = ## Returns the day of the week enum from day, month and year. - ## Equivalent with `initDateTime(monthday, month, year, 0, 0, 0).weekday`. + ## Equivalent with `dateTime(year, month, monthday, 0, 0, 0, 0).weekday`. runnableExamples: doAssert getDayOfWeek(13, mJun, 1990) == dWed doAssert $getDayOfWeek(13, mJun, 1990) == "Wednesday" @@ -1060,13 +1060,13 @@ proc isLeapDay*(dt: DateTime): bool {.since: (1, 1).} = ## Returns whether `t` is a leap day, i.e. Feb 29 in a leap year. This matters ## as it affects time offset calculations. runnableExamples: - let dt = initDateTime(29, mFeb, 2020, 00, 00, 00, utc()) + let dt = dateTime(2020, mFeb, 29, 00, 00, 00, 00, utc()) doAssert dt.isLeapDay doAssert dt+1.years-1.years != dt - let dt2 = initDateTime(28, mFeb, 2020, 00, 00, 00, utc()) + let dt2 = dateTime(2020, mFeb, 28, 00, 00, 00, 00, utc()) doAssert not dt2.isLeapDay doAssert dt2+1.years-1.years == dt2 - doAssertRaises(Exception): discard initDateTime(29, mFeb, 2021, 00, 00, 00, utc()) + doAssertRaises(Exception): discard dateTime(2021, mFeb, 29, 00, 00, 00, 00, utc()) assertDateTimeInitialized dt dt.year.isLeapYear and dt.month == mFeb and dt.monthday == 29 @@ -1372,7 +1372,7 @@ proc initDateTime*(monthday: MonthdayRange, month: Month, year: int, proc `+`*(dt: DateTime, dur: Duration): DateTime = runnableExamples: - let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) + let dt = dateTime(2017, mMar, 30, 00, 00, 00, 00, utc()) let dur = initDuration(hours = 5) doAssert $(dt + dur) == "2017-03-30T05:00:00Z" @@ -1380,7 +1380,7 @@ proc `+`*(dt: DateTime, dur: Duration): DateTime = proc `-`*(dt: DateTime, dur: Duration): DateTime = runnableExamples: - let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) + let dt = dateTime(2017, mMar, 30, 00, 00, 00, 00, utc()) let dur = initDuration(days = 5) doAssert $(dt - dur) == "2017-03-25T00:00:00Z" @@ -1389,8 +1389,8 @@ proc `-`*(dt: DateTime, dur: Duration): DateTime = proc `-`*(dt1, dt2: DateTime): Duration = ## Compute the duration between `dt1` and `dt2`. runnableExamples: - let dt1 = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) - let dt2 = initDateTime(25, mMar, 2017, 00, 00, 00, utc()) + let dt1 = dateTime(2017, mMar, 30, 00, 00, 00, 00, utc()) + let dt2 = dateTime(2017, mMar, 25, 00, 00, 00, 00, utc()) doAssert dt1 - dt2 == initDuration(days = 5) @@ -1998,10 +1998,10 @@ proc toDateTime(p: ParsedTime, zone: Timezone, f: TimeFormat, if p.utcOffset.isNone: # No timezone parsed - assume timezone is `zone` - result = initDateTime(monthday, month, year, hour, minute, second, nanosecond, zone) + result = dateTime(year, month, monthday, hour, minute, second, nanosecond, zone) else: # Otherwise convert to `zone` - result = (initDateTime(monthday, month, year, hour, minute, second, nanosecond, utc()).toTime + + result = (dateTime(year, month, monthday, hour, minute, second, nanosecond, utc()).toTime + initDuration(seconds = p.utcOffset.get())).inZone(zone) proc format*(dt: DateTime, f: TimeFormat, @@ -2009,7 +2009,7 @@ proc format*(dt: DateTime, f: TimeFormat, ## Format `dt` using the format specified by `f`. runnableExamples: let f = initTimeFormat("yyyy-MM-dd") - let dt = initDateTime(01, mJan, 2000, 00, 00, 00, utc()) + let dt = dateTime(2000, mJan, 01, 00, 00, 00, 00, utc()) doAssert "2000-01-01" == dt.format(f) assertDateTimeInitialized dt result = "" @@ -2034,7 +2034,7 @@ proc format*(dt: DateTime, f: string, loc: DateTimeLocale = DefaultLocale): stri ## See `Parsing and formatting dates`_ for documentation of the ## `format` argument. runnableExamples: - let dt = initDateTime(01, mJan, 2000, 00, 00, 00, utc()) + let dt = dateTime(2000, mJan, 01, 00, 00, 00, 00, utc()) doAssert "2000-01-01" == format(dt, "yyyy-MM-dd") let dtFormat = initTimeFormat(f) result = dt.format(dtFormat, loc) @@ -2057,7 +2057,7 @@ proc format*(time: Time, f: string, zone: Timezone = local()): string ## See `Parsing and formatting dates`_ for documentation of the ## `f` argument. runnableExamples: - var dt = initDateTime(01, mJan, 1970, 00, 00, 00, utc()) + var dt = dateTime(1970, mJan, 01, 00, 00, 00, 00, utc()) var tm = dt.toTime() doAssert format(tm, "yyyy-MM-dd'T'HH:mm:ss", utc()) == "1970-01-01T00:00:00" time.inZone(zone).format(f) @@ -2083,7 +2083,7 @@ proc parse*(input: string, f: TimeFormat, zone: Timezone = local(), ## Month and day names from the passed in `loc` are used. runnableExamples: let f = initTimeFormat("yyyy-MM-dd") - let dt = initDateTime(01, mJan, 2000, 00, 00, 00, utc()) + let dt = dateTime(2000, mJan, 01, 00, 00, 00, 00, utc()) doAssert dt == "2000-01-01".parse(f, utc()) var inpIdx = 0 # Input index var patIdx = 0 # Pattern index @@ -2125,7 +2125,7 @@ proc parse*(input, f: string, tz: Timezone = local(), ## See `Parsing and formatting dates`_ for documentation of the ## `f` argument. runnableExamples: - let dt = initDateTime(01, mJan, 2000, 00, 00, 00, utc()) + let dt = dateTime(2000, mJan, 01, 00, 00, 00, 00, utc()) doAssert dt == parse("2000-01-01", "yyyy-MM-dd", utc()) let dtFormat = initTimeFormat(f) result = input.parse(dtFormat, tz, loc = loc) @@ -2159,7 +2159,7 @@ proc `$`*(dt: DateTime): string {.tags: [], raises: [], benign.} = ## Converts a `DateTime` object to a string representation. ## It uses the format `yyyy-MM-dd'T'HH:mm:sszzz`. runnableExamples: - let dt = initDateTime(01, mJan, 2000, 12, 00, 00, utc()) + let dt = dateTime(2000, mJan, 01, 12, 00, 00, 00, utc()) doAssert $dt == "2000-01-01T12:00:00Z" doAssert $default(DateTime) == "Uninitialized DateTime" if not dt.isInitialized: @@ -2171,7 +2171,7 @@ proc `$`*(time: Time): string {.tags: [], raises: [], benign.} = ## Converts a `Time` value to a string representation. It will use the local ## time zone and use the format `yyyy-MM-dd'T'HH:mm:sszzz`. runnableExamples: - let dt = initDateTime(01, mJan, 1970, 00, 00, 00, local()) + let dt = dateTime(1970, mJan, 01, 00, 00, 00, 00, local()) let tm = dt.toTime() doAssert $tm == "1970-01-01T00:00:00" & format(dt, "zzz") $time.local @@ -2193,7 +2193,7 @@ proc initTimeInterval*(nanoseconds, microseconds, milliseconds, ## `seconds`, `minutes`, `hours`, `days`, `months`, and `years`. runnableExamples: let day = initTimeInterval(hours = 24) - let dt = initDateTime(01, mJan, 2000, 12, 00, 00, utc()) + let dt = dateTime(2000, mJan, 01, 12, 00, 00, 00, utc()) doAssert $(dt + day) == "2000-01-02T12:00:00Z" doAssert initTimeInterval(hours = 24) != initTimeInterval(days = 1) result.nanoseconds = nanoseconds @@ -2279,8 +2279,8 @@ proc between*(startDt, endDt: DateTime): TimeInterval = ## - If `startDt.timezone != endDt.timezone`, then the result will be ## equivalent to `between(startDt.utc, endDt.utc)`. runnableExamples: - var a = initDateTime(25, mMar, 2015, 12, 0, 0, utc()) - var b = initDateTime(1, mApr, 2017, 15, 0, 15, utc()) + var a = dateTime(2015, mMar, 25, 12, 0, 0, 00, utc()) + var b = dateTime(2017, mApr, 1, 15, 0, 15, 00, utc()) var ti = initTimeInterval(years = 2, weeks = 1, hours = 3, seconds = 15) doAssert between(a, b) == ti doAssert between(a, b) == -between(b, a) @@ -2360,8 +2360,8 @@ proc between*(startDt, endDt: DateTime): TimeInterval = startDate = endDate # Handle hours, minutes, seconds, milliseconds, microseconds and nanoseconds - let newStartDt = initDateTime(startDate.monthday, startDate.month.Month, - startDate.year, startDt.hour, startDt.minute, startDt.second, + let newStartDt = dateTime(startDate.year, startDate.month.Month, + startDate.monthday, startDt.hour, startDt.minute, startDt.second, startDt.nanosecond, startDt.timezone) let dur = endDt - newStartDt let parts = toParts(dur) @@ -2512,7 +2512,7 @@ proc `+`*(dt: DateTime, interval: TimeInterval): DateTime = ## So adding one month to `31 October` will result in `31 November`, which ## will overflow and result in `1 December`. runnableExamples: - let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) + let dt = dateTime(2017, mMar, 30, 00, 00, 00, 00, utc()) doAssert $(dt + 1.months) == "2017-04-30T00:00:00Z" # This is correct and happens due to monthday overflow. doAssert $(dt - 1.months) == "2017-03-02T00:00:00Z" @@ -2535,7 +2535,7 @@ proc `-`*(dt: DateTime, interval: TimeInterval): DateTime = ## then the `months` component and so on. The returned `DateTime` will ## have the same timezone as the input. runnableExamples: - let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) + let dt = dateTime(2017, mMar, 30, 00, 00, 00, 00, utc()) doAssert $(dt - 5.days) == "2017-03-25T00:00:00Z" dt + (-interval) From f93bfc0a329f193c8b83062e10140c2fb55a7873 Mon Sep 17 00:00:00 2001 From: Miran Date: Wed, 13 Oct 2021 20:31:04 +0200 Subject: [PATCH 1288/2472] [backport] add v1.6 changelog (#18932) --- changelog.md | 538 +------------------- changelogs/changelog_1_6_0.md | 927 ++++++++++++++++++++++++++++++++++ 2 files changed, 928 insertions(+), 537 deletions(-) create mode 100644 changelogs/changelog_1_6_0.md diff --git a/changelog.md b/changelog.md index 1925878bf..ab63bd8b5 100644 --- a/changelog.md +++ b/changelog.md @@ -1,560 +1,24 @@ -# v1.6.x - yyyy-mm-dd +# v1.8.x - yyyy-mm-dd ## Changes affecting backward compatibility -- Deprecated `std/mersenne`. -- `system.delete` had a most surprising behavior when the index passed to it was out of - bounds (it would delete the last entry then). Compile with `-d:nimStrictDelete` so - that an index error is produced instead. But be aware that your code might depend on - this quirky behavior so a review process is required on your part before you can - use `-d:nimStrictDelete`. To make this review easier, use the `-d:nimAuditDelete` - switch, it pretends that `system.delete` is deprecated so that it is easier to see - where it was used in your code. - - `-d:nimStrictDelete` will become the default in upcoming versions. - - -- `cuchar` is now deprecated as it aliased `char` where arguably it should have aliased `uint8`. - Please use `char` or `uint8` instead. - -- `repr` now doesn't insert trailing newline; previous behavior was very inconsistent, - see #16034. Use `-d:nimLegacyReprWithNewline` for previous behavior. - -- A type conversion from one enum type to another now produces an `[EnumConv]` warning. - You should use `ord` (or `cast`, but the compiler won't help, if you misuse it) instead. - ``` - type A = enum a1, a2 - type B = enum b1, b2 - echo a1.B # produces a warning - echo a1.ord.B # produces no warning - ``` - -- A dangerous implicit conversion to `cstring` now triggers a `[CStringConv]` warning. - This warning will become an error in future versions! Use an explicit conversion - like `cstring(x)` in order to silence the warning. - -- There is a new warning for *any* type conversion to enum that can be enabled via - `.warning[AnyEnumConv]:on` or `--warning:AnyEnumConv:on`. - -- Type mismatch errors now show more context, use `-d:nimLegacyTypeMismatch` for previous - behavior. - -- `math.round` now is rounded "away from zero" in JS backend which is consistent - with other backends. See #9125. Use `-d:nimLegacyJsRound` for previous behavior. - -- Changed the behavior of `uri.decodeQuery` when there are unencoded `=` - characters in the decoded values. Prior versions would raise an error. This is - no longer the case to comply with the HTML spec and other languages - implementations. Old behavior can be obtained with - `-d:nimLegacyParseQueryStrict`. `cgi.decodeData` which uses the same - underlying code is also updated the same way. - -- On POSIX systems, the default signal handlers used for Nim programs (it's - used for printing the stacktrace on fatal signals) will now re-raise the - signal for the OS default handlers to handle. - - This lets the OS perform its default actions, which might include core - dumping (on select signals) and notifying the parent process about the cause - of termination. - -- On POSIX systems, we now ignore `SIGPIPE` signals, use `-d:nimLegacySigpipeHandler` - for previous behavior. - -- `hashes.hash` can now support `object` and `ref` (can be overloaded in user code), - if `-d:nimPreviewHashRef` is used. It is expected that this behavior - becomes the new default in upcoming versions. - -- `hashes.hash(proc|ptr|ref|pointer)` now calls `hash(int)` and honors `-d:nimIntHash1`, - `hashes.hash(closure)` has also been improved. - -- The unary slice `..b` was deprecated, use `0..b` instead. - -- Removed `.travis.yml`, `appveyor.yml.disabled`, `.github/workflows/ci.yml.disabled`. - -- `random.initRand(seed)` now produces non-skewed values for the 1st call to `rand()` after - initialization with a small (< 30000) seed. Use `-d:nimLegacyRandomInitRand` to restore - previous behavior for a transition time, see PR #17467. - -- With `-d:nimPreviewJsonutilsHoleyEnum`, `jsonutils` now can serialize/deserialize holey enums as regular enums (via `ord`) instead of as strings. It is expected that this behavior becomes the new default in upcoming versions. `toJson` now serializes `JsonNode` - as is via reference (without a deep copy) instead of treating `JsonNode` as a regular ref object, - this can be customized via `jsonNodeMode`. - -- `json` and `jsonutils` now serialize NaN, Inf, -Inf as strings, so that - `%[NaN, -Inf]` is the string `["nan","-inf"]` instead of `[nan,-inf]` which was invalid json. - - -- `strformat` is now part of `include std/prelude`. - -- Deprecated `proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T]` in `std/algorithm`. - -- In `std/macros`, `treeRepr,lispRepr,astGenRepr` now represent SymChoice nodes in a collapsed way, - use `-d:nimLegacyMacrosCollapseSymChoice` to get previous behavior. - -- The configuration subsystem now allows for `-d:release` and `-d:danger` to work as expected. - The downside is that these defines now have custom logic that doesn't apply for - other defines. - -- `std/os`: `putEnv` now raises if the 1st argument contains a `=` - -- Renamed `-d:nimCompilerStackraceHints` to `-d:nimCompilerStacktraceHints`. - -- In `std/dom`, `Interval` is now a `ref object`, same as `Timeout`. Definitions of `setTimeout`, - `clearTimeout`, `setInterval`, `clearInterval` were updated. - -- With `-d:nimPreviewDotLikeOps`, dot-like operators (operators starting with `.`, but not with `..`) - now have the same precedence as `.`, so that `a.?b.c` is now parsed as `(a.?b).c` instead of `a.?(b.c)`. - A warning is generated when a dot-like operator is used without `-d:nimPreviewDotLikeOps`. - -- The allocator for Nintendo Switch, which was nonfunctional because - of breaking changes in libnx, was removed, in favour of the new `-d:nimAllocPagesViaMalloc` option. - -- `net.parseIpAddress` now only allows IPv4 addresses in strict form as defined - in [RFC 6943](https://www.rfc-editor.org/rfc/rfc6943#section-3.1.1). - Specifically, octal numbers in IPv4 addresses are no longer accepted (before - they were parsed as decimal numbers). ## Standard library additions and changes -- `strformat`: - - added support for parenthesized expressions. - - added support for const string's instead of just string literals - -- `os.copyFile` is now 2.5x faster on OSX, by using `copyfile` from `copyfile.h`; - use `-d:nimLegacyCopyFile` for OSX < 10.5. - -- `system.addFloat` and `system.$` now can produce string representations of floating point numbers - that are minimal in size and that "roundtrip" (via the "Dragonbox" algorithm). This currently has - to be enabled via `-d:nimPreviewFloatRoundtrip`. It is expected that this behavior becomes the new default - in upcoming versions, as with other `nimPreviewX` define flags. - -- Fixed buffer overflow bugs in `net`. - -- Exported `sslHandle` from `net` and `asyncnet`. - -- Added `sections` iterator in `parsecfg`. - -- Make custom op in macros.quote work for all statements. - -- On Windows the SSL library now checks for valid certificates. - It uses the `cacert.pem` file for this purpose which was extracted - from `https://curl.se/ca/cacert.pem`. Besides - the OpenSSL DLLs (e.g. libssl-1_1-x64.dll, libcrypto-1_1-x64.dll) you - now also need to ship `cacert.pem` with your `.exe` file. - -- `typetraits`: - `distinctBase` now is identity instead of error for non distinct types. - Added `enumLen` to return the number of elements in an enum. - Added `HoleyEnum` for enums with holes, `OrdinalEnum` for enums without holes. - Added `hasClosure`. - Added `pointerBase` to return `T` for `ref T | ptr T`. - -- `prelude` now works with the JavaScript target. - Added `sequtils` import to `prelude`. - `prelude` can now be used via `include std/prelude`, but `include prelude` still works. - -- Added `almostEqual` in `math` for comparing two float values using a machine epsilon. - -- Added `clamp` in `math` which allows using a `Slice` to clamp to a value. - -- Added `ceilDiv` in `math` for round up integer division. - -- The JSON module can now handle integer literals and floating point literals of - arbitrary length and precision. - Numbers that do not fit the underlying `BiggestInt` or `BiggestFloat` fields are - kept as string literals and one can use external BigNum libraries to handle these. - The `parseFloat` family of functions also has now optional `rawIntegers` and - `rawFloats` parameters that can be used to enforce that all integer or float - literals remain in the "raw" string form so that client code can easily treat - small and large numbers uniformly. - -- Added `BackwardsIndex` overload for `JsonNode`. - -- `json.%`,`json.to`, `jsonutils.formJson`,`jsonutils.toJson` now work with `uint|uint64` - instead of raising (as in 1.4) or giving wrong results (as in 1.2). - -- `jsonutils` now handles `cstring` (including as Table key), and `set`. - -- added `jsonutils.jsonTo` overload with `opt = Joptions()` param. - -- `jsonutils.toJson` now supports customization via `ToJsonOptions`. - -- Added an overload for the `collect` macro that inferes the container type based - on the syntax of the last expression. Works with std seqs, tables and sets. - -- Added `randState` template that exposes the default random number generator. - Useful for library authors. - -- Added `random.initRand()` overload with no argument which uses the current time as a seed. - -- `random.initRand(seed)` now allows `seed == 0`. - -- Added `std/sysrand` module to get random numbers from a secure source - provided by the operating system. - -- Added `std/enumutils` module. Added `genEnumCaseStmt` macro that generates - case statement to parse string to enum. -- Added `items` for enums with holes. -- Added `symbolName` to return the enum symbol name ignoring the human readable name. -- Added `symbolRank` to return the index in which an enum member is listed in an enum. - -- Removed deprecated `iup` module from stdlib, it has already moved to - [nimble](https://github.com/nim-lang/iup). - -- various functions in `httpclient` now accept `url` of type `Uri`. Moreover `request` function's - `httpMethod` argument of type `string` was deprecated in favor of `HttpMethod` enum type. - -- `nodejs` backend now supports osenv: `getEnv`, `putEnv`, `envPairs`, `delEnv`, `existsEnv`. - -- Added `cmpMem` to `system`. - -- `doAssertRaises` now correctly handles foreign exceptions. - -- Added `asyncdispatch.activeDescriptors` that returns the number of currently - active async event handles/file descriptors. - -- Added `getPort` to `asynchttpserver`. - -- `--gc:orc` is now 10% faster than previously for common workloads. If - you have trouble with its changed behavior, compile with `-d:nimOldOrc`. - -- `os.FileInfo` (returned by `getFileInfo`) now contains `blockSize`, - determining preferred I/O block size for this file object. - -- Added `os.getCacheDir()` to return platform specific cache directory. - -- Added a simpler to use `io.readChars` overload. - -- Added `**` to jsffi. - -- `writeStackTrace` is available in JS backend now. - -- Added `decodeQuery` to `std/uri`. - -- `strscans.scanf` now supports parsing single characters. - -- `strscans.scanTuple` added which uses `strscans.scanf` internally, - returning a tuple which can be unpacked for easier usage of `scanf`. - -- Added `setutils.toSet` that can take any iterable and convert it to a built-in `set`, - if the iterable yields a built-in settable type. - -- Added `setutils.fullSet` which returns a full built-in `set` for a valid type. - -- Added `setutils.complement` which returns the complement of a built-in `set`. - -- Added `setutils.[]=`. - -- Added `math.isNaN`. - -- Added `jsbigints` module, arbitrary precision integers for JavaScript target. - -- Added `math.copySign`. - -- Added new operations for singly- and doubly linked lists: `lists.toSinglyLinkedList` - and `lists.toDoublyLinkedList` convert from `openArray`s; `lists.copy` implements - shallow copying; `lists.add` concatenates two lists - an O(1) variation that consumes - its argument, `addMoved`, is also supplied. - -- Added `euclDiv` and `euclMod` to `math`. - -- Added `httpcore.is1xx` and missing HTTP codes. - -- Added `jsconsole.jsAssert` for JavaScript target. - -- Added `posix_utils.osReleaseFile` to get system identification from `os-release` file on Linux and the BSDs. - https://www.freedesktop.org/software/systemd/man/os-release.html - -- Added `socketstream` module that wraps sockets in the stream interface - -- Added `sugar.dumpToString` which improves on `sugar.dump`. - -- Added `math.signbit`. - -- Removed the optional `longestMatch` parameter of the `critbits._WithPrefix` iterators (it never worked reliably) - -- In `lists`: renamed `append` to `add` and retained `append` as an alias; - added `prepend` and `prependMoved` analogously to `add` and `addMoved`; - added `remove` for `SinglyLinkedList`s. - -- Deprecated `any`. See https://github.com/nim-lang/RFCs/issues/281 - -- Added optional `options` argument to `copyFile`, `copyFileToDir`, and - `copyFileWithPermissions`. By default, on non-Windows OSes, symlinks are - followed (copy files symlinks point to); on Windows, `options` argument is - ignored and symlinks are skipped. - -- On non-Windows OSes, `copyDir` and `copyDirWithPermissions` copy symlinks as - symlinks (instead of skipping them as it was before); on Windows symlinks are - skipped. - -- On non-Windows OSes, `moveFile` and `moveDir` move symlinks as symlinks - (instead of skipping them sometimes as it was before). - -- Added optional `followSymlinks` argument to `setFilePermissions`. - -- Added `os.isAdmin` to tell whether the caller's process is a member of the - Administrators local group (on Windows) or a root (on POSIX). - -- Added experimental `linenoise.readLineStatus` to get line and status (e.g. ctrl-D or ctrl-C). - -- Added `compilesettings.SingleValueSetting.libPath`. - -- `std/wrapnils` doesn't use `experimental:dotOperators` anymore, avoiding - issues like https://github.com/nim-lang/Nim/issues/13063 (which affected error messages) - for modules importing `std/wrapnils`. - Added `??.` macro which returns an `Option`. - `std/wrapnils` can now be used to protect against `FieldDefect` errors in - case objects, generates optimal code (no overhead compared to manual - if-else branches), and preserves lvalue semantics which allows modifying - an expression. - -- Added `math.frexp` overload procs. Deprecated `c_frexp`, use `frexp` instead. - -- `parseopt.initOptParser` has been made available and `parseopt` has been - added back to `prelude` for all backends. Previously `initOptParser` was - unavailable if the `os` module did not have `paramCount` or `paramStr`, - but the use of these in `initOptParser` were conditionally to the runtime - arguments passed to it, so `initOptParser` has been changed to raise - `ValueError` when the real command line is not available. `parseopt` was - previously excluded from `prelude` for JS, as it could not be imported. - -- Added `system.prepareStrMutation` for better support of low - level `moveMem`, `copyMem` operations for Orc's copy-on-write string - implementation. - -- Added `std/strbasics` for high performance string operations. - Added `strip`, `setSlice`, `add(a: var string, b: openArray[char])`. - -- `std/options` changed `$some(3)` to `"some(3)"` instead of `"Some(3)"` - and `$none(int)` to `"none(int)"` instead of `"None[int]"`. - -- Added `algorithm.merge`. - -- Added `std/jsfetch` module [Fetch](https://developer.mozilla.org/docs/Web/API/Fetch_API) wrapper for JavaScript target. - -- Added `std/jsheaders` module [Headers](https://developer.mozilla.org/en-US/docs/Web/API/Headers) wrapper for JavaScript target. - -- Added `std/jsformdata` module [FormData](https://developer.mozilla.org/en-US/docs/Web/API/FormData) wrapper for JavaScript target. - -- `system.addEscapedChar` now renders `\r` as `\r` instead of `\c`, to be compatible - with most other languages. - -- Removed support for named procs in `sugar.=>`. - -- Added `jscore.debugger` to [call any available debugging functionality, such as breakpoints.](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/debugger). - -- Added `htmlgen.portal` for [making "SPA style" pages using HTML only](https://web.dev/hands-on-portals). - -- `std/times`: - Added `ZZZ` and `ZZZZ` patterns to `times.nim` `DateTime` parsing, to match time - zone offsets without colons, e.g. `UTC+7 -> +0700`. - - Added `dateTime` and deprecated `initDateTime`. - -- Added `jsconsole.dir`, `jsconsole.dirxml`, `jsconsole.timeStamp`. - -- Added dollar `$` and `len` for `jsre.RegExp`. - -- Added `std/tasks`. - -- Added `hasDataBuffered` to `asyncnet`. - -- Added `std/tempfiles`. - -- Added `genasts.genAst` that avoids the problems inherent with `quote do` and can - be used as a replacement. - -- Added `copyWithin` [for `seq` and `array` for JavaScript targets](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/copyWithin). - -- Fixed premature garbage collection in asyncdispatch, when a stack trace override is in place. - -- Added setCurrentException for JS backend. - -- Added `dom.scrollIntoView` proc with options - -- Added `dom.setInterval`, `dom.clearInterval` overloads. - -- Merged `dom_extensions` module into `dom` module, - it was a module with a single line, see https://github.com/nim-lang/RFCs/issues/413 - -- Allow reading parameters when compiling for Nintendo Switch. - -- Deprecated `sequtils.delete` and added an overload taking a `Slice` that raises a defect - if the slice is out of bounds, likewise with `strutils.delete`. ## Language changes -- The `cstring` doesn't support `[]=` operator in JS backend. - -- nil dereference is not allowed at compile time. `cast[ptr int](nil)[]` is rejected at compile time. - -- The required name of case statement macros for the experimental - `caseStmtMacros` feature has changed from `match` to `` `case` ``. - -- `typedesc[Foo]` now renders as such instead of `type Foo` in compiler messages. - -- The unary minus in `-1` is now part of the integer literal, it is now parsed as a single token. - This implies that edge cases like `-128'i8` finally work correctly. - -- Custom numeric literals (e.g. `-128'bignum`) are now supported. - -- Tuple expressions are now parsed consistently as - `nnkTupleConstr` node. Will affect macros expecting nodes to be of `nnkPar`. - -- Added `iterable[T]` type class to match called iterators, which enables writing: - `template fn(a: iterable)` instead of `template fn(a: untyped)` - -- A new import syntax `import foo {.all.}` now allows to import all symbols (public or private) - from `foo`. This can be useful for testing purposes. - -- Added a new module `std/importutils`, and an API `privateAccess`, which allows access to private fields - for an object type in the current scope. - -- `typeof(voidStmt)` now works and returns `void`. - -- The `gc:orc` algorithm was refined so that custom container types can participate in the - cycle collection process. See the documentation of `=trace` for more details. - -- On embedded devices `malloc` can now be used instead of `mmap` via `-d:nimAllocPagesViaMalloc`. - This is only supported for `--gc:orc` or `--gc:arc`. - -- The effect system was refined and there is a new `.effectsOf` annotation that does - explicitly what was previously done implicitly. See the manual for details. - To write code that is portable with older Nim versions, use this idiom: - -```nim - -when defined(nimHasEffectsOf): - {.experimental: "strictEffects".} -else: - {.pragma: effectsOf.} - -proc mysort(s: seq; cmp: proc(a, b: T): int) {.effectsOf: cmp.} - -``` - - To enable the new effect system, use --experimental:strictEffects. - - -- Nim now supports a small subset of Unicode operators as operator symbols. - The supported symbols are: "∙ ∘ × ★ ⊗ ⊘ ⊙ ⊛ ⊠ ⊡ ∩ ∧ ⊓ ± ⊕ ⊖ ⊞ ⊟ ∪ ∨ ⊔". - To enable this feature, use `--experimental:unicodeOperators`. Note that due - to parser limitations you **cannot** enable this feature via a - pragma `{.experimental: "unicodeOperators".}` reliably, you need to enable - it via the command line or in a configuration file. - -- There is a new `cast` section `{.cast(uncheckedAssign).}: body` that disables some - compiler checks regarding `case objects`. This allows serialization libraries - to avoid ugly, non-portable solutions. See https://github.com/nim-lang/RFCs/issues/407 - for more details. - -- Enum values can now be overloaded. This needs to be enabled - via `{.experimental: "overloadableEnums".}`. We hope that this feature allows for the - development of more fluent (less ugly) APIs. See https://github.com/nim-lang/RFCs/issues/373 - for more details. ## Compiler changes -- Added `--declaredLocs` to show symbol declaration location in messages. - -- You can now enable/disable VM tracing in user code via `vmutils.vmTrace`. - -- Deprecated `TaintedString` and `--taintmode`. - -- Deprecated `--nilseqs` which is now a noop. - -- Added `--spellSuggest` to show spelling suggestions on typos. - -- Added `--filenames:abs|canonical|legacyRelProj` which replaces --listFullPaths:on|off - -- Added `--processing:dots|filenames|off` which customizes `hintProcessing` - -- Added `--unitsep:on|off` to control whether to add ASCII unit separator `\31` before a newline - for every generated message (potentially multiline), so tooling can tell when messages start and end. - -- Source+Edit links now appear on top of every docgen'd page when - `nim doc --git.url:url ...` is given. - -- Added `nim --eval:cmd` to evaluate a command directly, see `nim --help`. - -- VM now supports `addr(mystring[ind])` (index + index assignment) - -- Added `--hintAsError` with similar semantics as `--warningAsError`. - -- TLS: OSX now uses native TLS (`--tlsEmulation:off`), TLS now works with importcpp non-POD types, - such types must use `.cppNonPod` and `--tlsEmulation:off`should be used. - -- Now array literals(JS backend) uses JS typed arrays when the corresponding js typed array exists, - for example `[byte(1), 2, 3]` generates `new Uint8Array([1, 2, 3])`. - -- docgen: rst files can now use single backticks instead of double backticks and correctly render - in both rst2html (as before) as well as common tools rendering rst directly (e.g. github), by - adding: `default-role:: code` directive inside the rst file, which is now handled by rst2html. - -- Added `-d:nimStrictMode` in CI in several places to ensure code doesn't have certain hints/warnings - -- Added `then`, `catch` to `asyncjs`, for now hidden behind `-d:nimExperimentalAsyncjsThen`. - -- `--newruntime` and `--refchecks` are deprecated. - -- Added `unsafeIsolate` and `extract` to `std/isolation`. - -- `--hint:CC` now goes to stderr (like all other hints) instead of stdout. - -- `--hint:all:on|off` is now supported to select or deselect all hints; it - differs from `--hints:on|off` which acts as a (reversible) gate. - Likewise with `--warning:all:on|off`. - -- json build instructions are now generated in `$nimcache/outFileBasename.json` - instead of `$nimcache/projectName.json`. This allows avoiding recompiling a given project - compiled with different options if the output file differs. - -- `--usenimcache` (implied by `nim r main`) now generates an output file that includes a hash of - some of the compilation options, which allows caching generated binaries: - nim r main # recompiles - nim r -d:foo main # recompiles - nim r main # uses cached binary - nim r main arg1 arg2 # ditto (runtime arguments are irrelevant) - -- `nim r` now supports cross compilation from unix to windows when specifying `-d:mingw` by using wine, - e.g.: `nim r --eval:'import os; echo "a" / "b"'` prints `a\b` - -- `nim` can compile version 1.4.0 as follows: `nim c --lib:lib --stylecheck:off -d:nimVersion140 compiler/nim`. - `-d:nimVersion140` is not needed for bootstrapping, only for building 1.4.0 from devel. - -- The style checking of the compiler now supports a `--styleCheck:usages` switch. This switch - enforces that every symbol is written as it was declared, not enforcing - the official Nim style guide. To be enabled, this has to be combined either - with `--styleCheck:error` or `--styleCheck:hint`. ## Tool changes -- `nimscript` now handles `except Exception as e`. -- `nim e` now accepts arbitrary file extensions for the nimscript file, - although `.nims` is still the preferred extension in general. -- Latex doc generation is revised: output `.tex` files should be compiled - by `xelatex` (not by `pdflatex` as before). Now default Latex settings - provide support for Unicode and do better job for avoiding margin overflows. - -- Implemented `doc2tex` compiler command which converts documentation in - `.nim` files to Latex. - -- The rst parser now supports markdown table syntax. - Known limitations: - - cell alignment is not supported, i.e. alignment annotations in a delimiter - row (`:---`, `:--:`, `---:`) are ignored, - - every table row must start with `|`, e.g. `| cell 1 | cell 2 |`. - -- `fusion` is now un-bundled from nim, `./koch fusion` will - install it via nimble at a fixed hash. - -- testament: added `nimoutFull: bool` spec to compare full output of compiler - instead of a subset. diff --git a/changelogs/changelog_1_6_0.md b/changelogs/changelog_1_6_0.md new file mode 100644 index 000000000..1ce6c2093 --- /dev/null +++ b/changelogs/changelog_1_6_0.md @@ -0,0 +1,927 @@ +# v1.6.0 - 2021-10-14 + + + +# Major new features + +With so many new features, pinpointing the most salient ones is a subjective exercise, +but here are a select few: + + +## `iterable[T]` + +The `iterable[T]` type class was added to match called iterators, +which solves a number of long-standing issues related to iterators. +Example: + +```nim +iterator iota(n: int): int = + for i in 0.. response.json()) + .then((json: JsObject) => console.log(json)) + .catch((err: Error) => console.log("Request Failed", err)) +discard fn() +``` + + +## New `std/tasks` module + +Provides basic primitives for creating parallel programs. +Example: +```nim +import std/tasks +var num = 0 +proc hello(a: int) = num+=a + +let b = toTask hello(13) # arguments must be isolated, see `std/isolation` +b.invoke() +assert num == 13 +b.invoke() # can be called again +assert num == 26 +``` + + +## New module: `std/genasts` + +Provides an API `genAst` that avoids the problems inherent with `quote do` and can +be used as a replacement. +Example showing how this could be used for writing a simplified version of `unittest.check`: +```nim +import std/[genasts, macros, strutils] +macro check2(cond: bool): untyped = + assert cond.kind == nnkInfix, "$# not implemented" % $cond.kind + result = genAst(cond, s = repr(cond), lhs = cond[1], rhs = cond[2]): + # each local symbol we access must be explicitly captured + if not cond: + doAssert false, "'$#'' failed: lhs: '$#', rhs: '$#'" % [s, $lhs, $rhs] +let a = 3 +check2 a*2 == a+3 +if false: check2 a*2 < a+1 # would error with: 'a * 2 < a + 1'' failed: lhs: '6', rhs: '4' +``` + +See PR [#17426](https://github.com/nim-lang/Nim/pull/17426) for details. + + +## New module: `std/setutils` + +- Added `setutils.toSet` that can take any iterable and convert it to a built-in `set`, + if the iterable yields a built-in settable type. +- Added `setutils.fullSet` which returns a full built-in `set` for a valid type. +- Added `setutils.complement` which returns the complement of a built-in `set`. +- Added `setutils.[]=`. + + +## New module: `std/enumutils` + +- Added `genEnumCaseStmt` macro that generates + case statement to parse string to enum. +- Added `items` for enums with holes. +- Added `symbolName` to return the `enum` symbol name ignoring the human-readable name. +- Added `symbolRank` to return the index in which an `enum` member is listed in an enum. + + +## `system` + +- Added `system.prepareMutation` for better support of low + level `moveMem`, `copyMem` operations for `gc:orc`'s copy-on-write string + implementation. +- `system.addEscapedChar` now renders `\r` as `\r` instead of `\c`, to be compatible + with most other languages. +- Added `cmpMem` to `system`. +- `doAssertRaises` now correctly handles foreign exceptions. +- `addInt` now supports unsigned integers. + +Compatibility notes: +- `system.delete` had surprising behavior when the index passed to it was out of + bounds (it would delete the last entry then). Compile with `-d:nimStrictDelete` so + that an index error is produced instead. Be aware however that your code might depend on + this quirky behavior so a review process is required on your part before you can + use `-d:nimStrictDelete`. To make this review easier, use the `-d:nimAuditDelete` + switch, which pretends that `system.delete` is deprecated so that it is easier + to see where it was used in your code. + `-d:nimStrictDelete` will become the default in upcoming versions. +- `cuchar` is now deprecated as it aliased `char` where arguably it should have aliased `uint8`. + Please use `char` or `uint8` instead. +- `repr` now doesn't insert trailing newlines; the previous behavior was very inconsistent, + see [#16034](https://github.com/nim-lang/Nim/pull/16034). + Use `-d:nimLegacyReprWithNewline` for the previous behavior. + `repr` now also renders ASTs correctly for user defined literals, setters, `do`, etc. +- Deprecated `any`. See RFC [#281](https://github.com/nim-lang/RFCs/issues/281). +- The unary slice `..b` was deprecated, use `0..b` instead. + + +## `std/math` + +- Added `almostEqual` for comparing two float values using a machine epsilon. +- Added `clamp` which allows using a `Slice` to clamp to a value. +- Added `ceilDiv` for integer division that rounds up. +- Added `isNaN`. +- Added `copySign`. +- Added `euclDiv` and `euclMod`. +- Added `signbit`. +- Added `frexp` overload procs. Deprecated `c_frexp`, use `frexp` instead. + +Compatibility notes: +- `math.round` now rounds "away from zero" in the JS backend, which is consistent + with other backends. See [#9125](https://github.com/nim-lang/Nim/pull/9125). + Use `-d:nimLegacyJsRound` for the previous behavior. + + +## Random number generators: `std/random`, `std/sysrand`, `std/oids` + +- Added `std/sysrand` module (see details above). +- Added `randState` template that exposes the default random number generator. + Useful for library authors. +- Added `initRand()` overload with no argument which uses the current time as a seed. +- `initRand(seed)` now allows `seed == 0`. +- Fixed overflow bugs. +- Fix `initRand` to avoid random number sequences overlapping, refs + [#18744](https://github.com/nim-lang/Nim/pull/18744). +- `std/oids` now uses `std/random`. + +Compatibility notes: +- Deprecated `std/mersenne`. +- `random.initRand(seed)` now produces non-skewed values for the first call to + `rand()` after initialization with a small (< 30000) seed. + Use `-d:nimLegacyRandomInitRand` to restore previous behavior for a transition + time, see PR [#17467](https://github.com/nim-lang/Nim/pull/17467). + + +## `std/json`, `std/jsonutils` + +- With `-d:nimPreviewJsonutilsHoleyEnum`, `jsonutils` now can serialize/deserialize + holey enums as regular enums (via `ord`) instead of as strings. + It is expected that this behavior becomes the new default in upcoming versions. + `toJson` now serializes `JsonNode` as is via reference (without a deep copy) + instead of treating `JsonNode` as a regular ref object, + this can be customized via `jsonNodeMode`. +- `std/json` and `std/jsonutils` now serialize `NaN`, `Inf`, `-Inf` as strings, + so that `%[NaN, -Inf]` is the string `["nan","-inf"]` instead of `[nan,-inf]` + which was invalid JSON. +- `std/json` can now handle integer literals and floating point literals of + arbitrary length and precision. + Numbers that do not fit the underlying `BiggestInt` or `BiggestFloat` fields are + kept as string literals and one can use external BigNum libraries to handle these. + The `parseFloat` family of functions also has now optional `rawIntegers` and + `rawFloats` parameters that can be used to enforce that all integer or float + literals remain in the "raw" string form so that client code can easily treat + small and large numbers uniformly. +- Added `BackwardsIndex` overload for `JsonNode`. +- `json.%`,`json.to`, `jsonutils.fromJson`,`jsonutils.toJson` now work with `uint|uint64` + instead of raising (as in 1.4) or giving wrong results (as in 1.2). +- `std/jsonutils` now handles `cstring` (including as Table key), and `set`. +- Added `jsonutils.jsonTo` overload with `opt = Joptions()` param. +- `jsonutils.toJson` now supports customization via `ToJsonOptions`. +- `std/json`, `std/jsonutils` now support round-trip serialization when + `-d:nimPreviewFloatRoundtrip` is used. + + +## `std/typetraits`, `std/compilesettings` + +- `distinctBase` now is identity instead of error for non distinct types. +- `distinctBase` now allows controlling whether to be recursive or not. +- Added `enumLen` to return the number of elements in an enum. +- Added `HoleyEnum` for enums with holes, `OrdinalEnum` for enums without holes. +- Added `hasClosure`. +- Added `pointerBase` to return `T` for `ref T | ptr T`. +- Added `compilesettings.SingleValueSetting.libPath`. + + +## networking: `std/net`, `std/asyncnet`, `std/htmlgen`, `std/httpclient`, `std/asyncdispatch`, `std/asynchttpserver`, `std/httpcore` + +- Fixed buffer overflow bugs in `std/net`. +- Exported `sslHandle` from `std/net` and `std/asyncnet`. +- Added `hasDataBuffered` to `std/asyncnet`. +- Various functions in `std/httpclient` now accept `url` of type `Uri`. + Moreover, the `request` function's `httpMethod` argument of type `string` was + deprecated in favor of `HttpMethod` `enum` type; see + [#15919](https://github.com/nim-lang/Nim/pull/15919). +- Added `asyncdispatch.activeDescriptors` that returns the number of currently + active async event handles/file descriptors. +- Added `getPort` to `std/asynchttpserver` to resolve OS-assigned `Port(0)`; + this is usually recommended instead of hardcoding ports which can lead to + "Address already in use" errors. +- Fixed premature garbage collection in `std/asyncdispatch`, when a stacktrace + override is in place. +- Added `httpcore.is1xx` and missing HTTP codes. +- Added `htmlgen.portal` for [making "SPA style" pages using HTML only](https://web.dev/hands-on-portals). + +Compatibility notes: +- On Windows, the SSL library now checks for valid certificates. + For this purpose it uses the `cacert.pem` file, which was extracted + from `https://curl.se/ca/cacert.pem`. Besides + the OpenSSL DLLs (e.g. `libssl-1_1-x64.dll`, `libcrypto-1_1-x64.dll`) you + now also need to ship `cacert.pem` with your `.exe` file. + + +## `std/hashes` + +- `hashes.hash` can now support `object` and `ref` (can be overloaded in user code), + if `-d:nimPreviewHashRef` is used. It is expected that this behavior + becomes the new default in upcoming versions. +- `hashes.hash(proc|ptr|ref|pointer)` now calls `hash(int)` and honors `-d:nimIntHash1`. + `hashes.hash(closure)` has also been improved. + + +## OS: `std/os`, `std/io`, `std/socketstream`, `std/linenoise`, `std/tempfiles` + +- `os.FileInfo` (returned by `getFileInfo`) now contains `blockSize`, + determining preferred I/O block size for this file object. +- Added `os.getCacheDir()` to return platform specific cache directory. +- Improved `os.getTempDir()`, see PR [#16914](https://github.com/nim-lang/Nim/pull/16914). +- Added `os.isAdmin` to tell whether the caller's process is a member of the + Administrators local group (on Windows) or a root (on POSIX). +- Added optional `options` argument to `copyFile`, `copyFileToDir`, and + `copyFileWithPermissions`. By default, on non-Windows OSes, symlinks are + followed (copy files symlinks point to); on Windows, `options` argument is + ignored and symlinks are skipped. +- On non-Windows OSes, `copyDir` and `copyDirWithPermissions` copy symlinks as + symlinks (instead of skipping them as it was before); on Windows symlinks are + skipped. +- On non-Windows OSes, `moveFile` and `moveDir` move symlinks as symlinks + (instead of skipping them sometimes as it was before). +- Added optional `followSymlinks` argument to `setFilePermissions`. +- Added a simpler to use `io.readChars` overload. +- Added `socketstream` module that wraps sockets in the stream interface. +- Added experimental `linenoise.readLineStatus` to get line and status (e.g. ctrl-D or ctrl-C). + + +## Environment variable handling + +- Empty environment variable values are now supported across OS's and backends. +- Environment variable APIs now work in multithreaded scenarios, by delegating to + direct OS calls instead of trying to keep track of the environment. +- `putEnv`, `delEnv` now work at compile time. +- NodeJS backend now supports osenv: `getEnv`, `putEnv`, `envPairs`, `delEnv`, `existsEnv`. + +Compatibility notes: +- `std/os`: `putEnv` now raises if the first argument contains a `=`. + + +## POSIX + +- On POSIX systems, the default signal handlers used for Nim programs (it's + used for printing the stacktrace on fatal signals) will now re-raise the + signal for the OS default handlers to handle. + This lets the OS perform its default actions, which might include core + dumping (on select signals) and notifying the parent process about the cause + of termination. +- On POSIX systems, we now ignore `SIGPIPE` signals, use `-d:nimLegacySigpipeHandler` + for previous behavior. +- Added `posix_utils.osReleaseFile` to get system identification from `os-release` + file on Linux and the BSDs. + ([link](https://www.freedesktop.org/software/systemd/man/os-release.html)) +- Removed undefined behavior for `posix.open`. + + +## `std/prelude` + +- `std/strformat` is now part of `include std/prelude`. +- Added `std/sequtils` import to `std/prelude`. +- `std/prelude` now works with the JS target. +- `std/prelude` can now be used via `include std/prelude`, but `include prelude` still works. + + +## String manipulation: `std/strformat`, `std/strbasics` + +- Added support for parenthesized expressions. +- Added support for const strings instead of just string literals. +- Added `std/strbasics` for high-performance string operations. +- Added `strip`, `setSlice`, `add(a: var string, b: openArray[char])`. + + +## `std/wrapnils` + +- `std/wrapnils` doesn't use `experimental:dotOperators` anymore, avoiding + issues like bug [#13063](https://github.com/nim-lang/Nim/issues/13063) + (which affected error messages) for modules importing `std/wrapnils`. +- Added `??.` macro which returns an `Option`. +- `std/wrapnils` can now be used to protect against `FieldDefect` errors in + case objects, generates optimal code (no overhead compared to manual + if-else branches), and preserves lvalue semantics which allows modifying + an expression. + + +## Containers: `std/algorithm`, `std/lists`, `std/sequtils`, `std/options`, `std/packedsets` + +- Removed the optional `longestMatch` parameter of the `critbits._WithPrefix` + iterators (it never worked reliably). +- Added `algorithm.merge`. +- In `std/lists`: renamed `append` to `add` and retained `append` as an alias; + added `prepend` and `prependMoved` analogously to `add` and `addMoved`; + added `remove` for `SinglyLinkedList`s. +- Added new operations for singly- and doubly linked lists: `lists.toSinglyLinkedList` + and `lists.toDoublyLinkedList` convert from `openArray`s; `lists.copy` implements + shallow copying; `lists.add` concatenates two lists - an O(1) variation that consumes + its argument, `addMoved`, is also supplied. + See PRs [#16362](https://github.com/nim-lang/Nim/pull/16362), + [#16536](https://github.com/nim-lang/Nim/pull/16536). +- New module: `std/packedsets`. + Generalizes `std/intsets`, see PR [#15564](https://github.com/nim-lang/Nim/pull/15564). + +Compatibility notes: +- Deprecated `sequtils.delete` and added an overload taking a `Slice` that raises + a defect if the slice is out of bounds, likewise with `strutils.delete`. +- Deprecated `proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T]` + in `std/algorithm`. +- `std/options` changed `$some(3)` to `"some(3)"` instead of `"Some(3)"` + and `$none(int)` to `"none(int)"` instead of `"None[int]"`. + + +## `std/times` + +- Added `ZZZ` and `ZZZZ` patterns to `times.nim` `DateTime` parsing, to match time + zone offsets without colons, e.g. `UTC+7 -> +0700`. +- Added `dateTime` and deprecated `initDateTime`. + + +## `std/macros` and AST + +- New module `std/genasts`, see description above. +- The required name of case statement macros for the experimental + `caseStmtMacros` feature has changed from `match` to `` `case` ``. +- Tuple expressions are now parsed consistently as + `nnkTupleConstr` node. Will affect macros expecting nodes to be of `nnkPar`. +- In `std/macros`, `treeRepr,lispRepr,astGenRepr` now represent SymChoice nodes + in a collapsed way. + Use `-d:nimLegacyMacrosCollapseSymChoice` to get the previous behavior. +- Made custom op in `macros.quote` work for all statements. + + +## `std/sugar` + +- Added `sugar.dumpToString` which improves on `sugar.dump`. +- Added an overload for the `collect` macro that infers the container type based + on the syntax of the last expression. Works with std seqs, tables and sets. + +Compatibility notes: +- Removed support for named procs in `sugar.=>`. + + +## Parsing: `std/parsecfg`, `std/strscans`, `std/uri` + +- Added `sections` iterator in `parsecfg`. +- `strscans.scanf` now supports parsing single characters. +- Added `strscans.scanTuple` which uses `strscans.scanf` internally, + returning a tuple which can be unpacked for easier usage of `scanf`. +- Added `decodeQuery` to `std/uri`. +- `parseopt.initOptParser` has been made available and `parseopt` has been + added back to `std/prelude` for all backends. Previously `initOptParser` was + unavailable if the `std/os` module did not have `paramCount` or `paramStr`, + but the use of these in `initOptParser` were conditionally to the runtime + arguments passed to it, so `initOptParser` has been changed to raise + `ValueError` when the real command line is not available. `parseopt` was + previously excluded from `std/prelude` for JS, as it could not be imported. + +Compatibility notes: +- Changed the behavior of `uri.decodeQuery` when there are unencoded `=` + characters in the decoded values. Prior versions would raise an error. This is + no longer the case to comply with the HTML spec and other languages' + implementations. Old behavior can be obtained with + `-d:nimLegacyParseQueryStrict`. `cgi.decodeData` which uses the same + underlying code is also updated the same way. + + +## JS stdlib changes + +- Added `std/jsbigints` module, which provides arbitrary precision integers for the JS target. +- Added `setCurrentException` for the JS backend. +- `writeStackTrace` is available in the JS backend now. +- Added `then`, `catch` to `std/asyncjs` for promise pipelining, for now hidden + behind `-d:nimExperimentalAsyncjsThen`. +- Added `std/jsfetch` module [Fetch](https://developer.mozilla.org/docs/Web/API/Fetch_API) + wrapper for the JS target. +- Added `std/jsheaders` module [Headers](https://developer.mozilla.org/en-US/docs/Web/API/Headers) + wrapper for the JS target. +- Added `std/jsformdata` module [FormData](https://developer.mozilla.org/en-US/docs/Web/API/FormData) + wrapper for the JS target. +- Added `jscore.debugger` to [call any available debugging functionality, such as breakpoints](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/debugger). +- Added `jsconsole.dir`, `jsconsole.dirxml`, `jsconsole.timeStamp`. +- Added dollar `$` and `len` for `jsre.RegExp`. +- Added `jsconsole.jsAssert` for the JS target. +- Added `**` to `std/jsffi`. +- Added `copyWithin` [for `seq` and `array` for JS targets](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/copyWithin). +- In `std/dom`, `Interval` is now a `ref object`, same as `Timeout`. + Definitions of `setTimeout`, `clearTimeout`, `setInterval`, `clearInterval` were updated. +- Added `dom.scrollIntoView` proc with options. +- Added `dom.setInterval`, `dom.clearInterval` overloads. +- Merged `std/dom_extensions` into the `std/dom` module, + as it was a module with a single line, see RFC [#413](https://github.com/nim-lang/RFCs/issues/413). +- `$` now gives more correct results on the JS backend. + + +## JS compiler changes + +- `cstring` doesn't support the `[]=` operator anymore in the JS backend. +- Array literals now use JS typed arrays when the corresponding JS typed array exists, + for example `[byte(1), 2, 3]` generates `new Uint8Array([1, 2, 3])`. + + +## VM and nimscript backend + +- VM now supports `addr(mystring[ind])` (index + index assignment). +- `nimscript` now handles `except Exception as e`. +- `nil` dereference is not allowed at compile time. `cast[ptr int](nil)[]` is rejected at compile time. +- `static[T]` now works better, refs [#17590](https://github.com/nim-lang/Nim/pull/17590), + [#15853](https://github.com/nim-lang/Nim/pull/15853). +- `distinct T` conversions now work in VM. +- `items(cstring)` now works in VM. +- Fix `addr`, `len`, `high` in VM ([#16002](https://github.com/nim-lang/Nim/pull/16002), + [#16610](https://github.com/nim-lang/Nim/pull/16610)). +- `std/cstrutils` now works in VM. + + +## OS/platform-specific notes + +- Support for Apple silicon/M1. +- Support for 32-bit RISC-V, refs [#16231](https://github.com/nim-lang/Nim/pull/16231). +- Support for armv8l, refs [#18901](https://github.com/nim-lang/Nim/pull/18901). +- Support for CROSSOS, refs [#18889](https://github.com/nim-lang/Nim/pull/18889). +- The allocator for Nintendo Switch, which was nonfunctional because + of breaking changes in libnx, was removed, in favor of the new `-d:nimAllocPagesViaMalloc` option. +- Allow reading parameters when compiling for Nintendo Switch. +- `--nimcache` now correctly works in a cross-compilation setting. +- Cross compilation targeting Windows was improved. +- This now works from macOS/Linux: `nim r -d:mingw main` + + + +## Performance / memory optimizations + +- The comment field in PNode AST was moved to a side channel, reducing overall + memory usage during compilation by a factor 1.25x +- `std/jsonutils` deserialization is now up to 20x faster. +- `os.copyFile` is now 2.5x faster on macOS, by using `copyfile` from `copyfile.h`; + use `-d:nimLegacyCopyFile` for macOS < 10.5. +- Float to string conversion is now 10x faster thanks to the Dragonbox algorithm, + with `-d:nimPreviewFloatRoundtrip`. +- `newSeqWith` is 3x faster. +- CI now supports batching (making Windows CI 2.3X faster). +- Sets now uses the optimized `countSetBits` proc, see PR + [#17334](https://github.com/nim-lang/Nim/pull/17334). + + +## Debugging + +- You can now enable/disable VM tracing in user code via `vmutils.vmTrace`. +- `koch tools` now builds `bin/nim_dbg` which allows easy access to a debug version + of Nim without recompiling. +- Added new module `compiler/debugutils` to help with debugging Nim compiler. +- Renamed `-d:nimCompilerStackraceHints` to `-d:nimCompilerStacktraceHints` and + used it in more contexts; this flag which works in tandem with `--stackTraceMsgs` + to show user code context in compiler stacktraces. + + +## Type system + +- `typeof(voidStmt)` now works and returns `void`. +- `enum` values can now be overloaded. This needs to be enabled + via `{.experimental: "overloadableEnums".}`. We hope that this feature allows + for the development of more fluent (less ugly) APIs. + See RFC [#373](https://github.com/nim-lang/RFCs/issues/373) for more details. +- A type conversion from one `enum` type to another now produces an `[EnumConv]` warning. + You should use `ord` (or `cast`, but the compiler won't help, if you misuse it) instead. + ```nim + type A = enum a1, a2 + type B = enum b1, b2 + echo a1.B # produces a warning + echo a1.ord.B # produces no warning + ``` +- A dangerous implicit conversion to `cstring` now triggers a `[CStringConv]` warning. + This warning will become an error in future versions! Use an explicit conversion + like `cstring(x)` in order to silence the warning. +- There is a new warning for *any* type conversion to `enum` that can be enabled via + `.warning[AnyEnumConv]:on` or `--warning:AnyEnumConv:on`. +- Reusing a type name in a different scope now works, refs [#17710](https://github.com/nim-lang/Nim/pull/17710). +- Fixed implicit and explicit generics in procedures, refs [#18808](https://github.com/nim-lang/Nim/pull/18808). + + +## New-style concepts + +Example: +```nim +type + Comparable = concept # no T, an atom + proc cmp(a, b: Self): int +``` +The new design does not rely on `system.compiles` and may compile faster. +See PR [#15251](https://github.com/nim-lang/Nim/pull/15251) +and RFC [#168](https://github.com/nim-lang/RFCs/issues/168) for details. + + +## Lexical / syntactic + +- Nim now supports a small subset of Unicode operators as operator symbols. + The supported symbols are: "∙ ∘ × ★ ⊗ ⊘ ⊙ ⊛ ⊠ ⊡ ∩ ∧ ⊓ ± ⊕ ⊖ ⊞ ⊟ ∪ ∨ ⊔". + To enable this feature, use `--experimental:unicodeOperators`. Note that due + to parser limitations you **cannot** enable this feature via a + pragma `{.experimental: "unicodeOperators".}` reliably, you need to enable + it via the command line or in a configuration file. +- `var a {.foo.} = expr` now works inside templates (except when `foo` is overloaded). + + +## Compiler messages, error messages, hints, warnings + +- Significant improvement to error messages involving effect mismatches, + see PRs [#18384](https://github.com/nim-lang/Nim/pull/18384), + [#18418](https://github.com/nim-lang/Nim/pull/18418). +- Added `--declaredLocs` to show symbol declaration location in error messages. +- Added `--spellSuggest` to show spelling suggestions on typos. +- Added `--processing:dots|filenames|off` which customizes `hintProcessing`; + `--processing:filenames` shows which include/import modules are being compiled as an import stack. +- `FieldDefect` messages now shows discriminant value + lineinfo, in all backends (C, JS, VM) +- Added `--hintAsError` with similar semantics as `--warningAsError`. +- Added `--unitsep:on|off` to control whether to add ASCII unit separator `\31` + before a newline for every generated message (potentially multiline), + so tooling can tell when messages start and end. +- Added `--filenames:abs|canonical|legacyRelProj` which replaces `--listFullPaths:on|off` +- `--hint:all:on|off` is now supported to select or deselect all hints; it + differs from `--hints:on|off` which acts as a (reversible) gate. + Likewise with `--warning:all:on|off`. +- The style checking of the compiler now supports a `--styleCheck:usages` switch. + This switch enforces that every symbol is written as it was declared, not enforcing + the official Nim style guide. To be enabled, this has to be combined either + with `--styleCheck:error` or `--styleCheck:hint`. +- Type mismatch errors now show more context, use `-d:nimLegacyTypeMismatch` for + previous behavior. +- `typedesc[Foo]` now renders as such instead of `type Foo` in compiler messages. +- `runnableExamples` now show originating location in stacktraces on failure. +- `SuccessX` message now shows more useful information. +- New `DuplicateModuleImport` warning; improved `UnusedImport` and + `XDeclaredButNotUsed` accuracy. + +Compatibility notes: +- `--hint:CC` now prints to stderr (like all other hints) instead of stdout. + + +## Building and running Nim programs, configuration system + +- JSON build instructions are now generated in `$nimcache/outFileBasename.json` + instead of `$nimcache/projectName.json`. This allows avoiding recompiling a + given project compiled with different options if the output file differs. +- `--usenimcache` (implied by `nim r main`) now generates an output file that includes + a hash of some of the compilation options, which allows caching generated binaries: + ```bash + nim r main # recompiles + nim r -d:foo main # recompiles + nim r main # uses cached binary + nim r main arg1 arg2 # likewise (runtime arguments are irrelevant) + ``` +- `nim r` now supports cross compilation from unix to windows when specifying + `-d:mingw` by using Wine, e.g.: + `nim r --eval:'import os; echo "a" / "b"'` prints `a\b`. +- `nim` can compile version 1.4.0 as follows: + `nim c --lib:lib --stylecheck:off -d:nimVersion140 compiler/nim`. + `-d:nimVersion140` is not needed for bootstrapping, only for building 1.4.0 from devel. +- `nim e` now accepts arbitrary file extensions for the nimscript file, + although `.nims` is still the preferred extension in general. +- The configuration subsystem now allows for `-d:release` and `-d:danger` to work as expected. + The downside is that these defines now have custom logic that doesn't apply for + other defines. + + +## Multithreading + +- TLS: macOS now uses native TLS (`--tlsEmulation:off`). TLS now works with + `importcpp` non-POD types; such types must use `.cppNonPod` and + `--tlsEmulation:off`should be used. +- Added `unsafeIsolate` and `extract` to `std/isolation`. +- Added `std/tasks`, see description above. + + +## Memory management + +- `--gc:arc` now bootstraps (PR [#17342](https://github.com/nim-lang/Nim/pull/17342)). +- Lots of improvements to `gc:arc`, `gc:orc`, see PR + [#15697](https://github.com/nim-lang/Nim/pull/15697), + [#16849](https://github.com/nim-lang/Nim/pull/16849), + [#17993](https://github.com/nim-lang/Nim/pull/17993). +- `--gc:orc` is now 10% faster than previously for common workloads. + If you have trouble with its changed behavior, compile with `-d:nimOldOrc`. +- The `--gc:orc` algorithm was refined so that custom container types can participate in the + cycle collection process. See the documentation of `=trace` for more details. +- On embedded devices `malloc` can now be used instead of `mmap` via `-d:nimAllocPagesViaMalloc`. + This is only supported for `--gc:orc` or `--gc:arc`. + +Compatibility notes: +- `--newruntime` and `--refchecks` are deprecated, + use `--gc:arc`, `--gc:orc`, or `--gc:none` as appropriate instead. + + +## Docgen + +- docgen: RST files can now use single backticks instead of double backticks and + correctly render in both `nim rst2html` (as before) as well as common tools rendering + RST directly (e.g. GitHub). + This is done by adding the `default-role:: code` directive inside the RST file + (which is now handled by `nim rst2html`). +- Source+Edit links now appear on top of every docgen'd page when + `nim doc --git.url:url ...` is given. +- Latex doc generation is revised: output `.tex` files should be compiled + by `xelatex` (not by `pdflatex` as before). Now default Latex settings + provide support for Unicode and better avoid margin overflows. + The minimum required version is TeXLive 2018 (or an equivalent MikTeX version). +- The RST parser now supports footnotes, citations, admonitions, and short style + references with symbols. +- The RST parser now supports Markdown table syntax. + Known limitations: + - cell alignment is not supported, i.e. alignment annotations in a delimiter + row (`:---`, `:--:`, `---:`) are ignored + - every table row must start with `|`, e.g. `| cell 1 | cell 2 |`. +- Implemented `doc2tex` compiler command which converts documentation in + `.nim` files to Latex. +- docgen now supports syntax highlighting for inline code. +- docgen now supports same-line doc comments: + ```nim + func fn*(a: int): int = 42 ## Doc comment + ``` +- docgen now renders `deprecated` and other pragmas. +- `runnableExamples` now works with templates and nested templates. +- `runnableExamples: "-r:off"` now works for examples that should compile but not run. +- `runnableExamples` now renders code verbatim, and produces correct code in all cases. +- docgen now shows correct, canonical import paths in docs. +- docgen now shows all routines in sidebar, and the proc signature is now shown in sidebar. + + +## Effects and checks + +- Significant improvement to error messages involving effect mismatches +- There is a new `cast` section `{.cast(uncheckedAssign).}: body` that disables some + compiler checks regarding `case objects`. This allows serialization libraries + to avoid ugly, non-portable solutions. See RFC + [#407](https://github.com/nim-lang/RFCs/issues/407) for more details. + + +## Tools + +- Major improvements to `nimgrep`, see PR [#15612 +](https://github.com/nim-lang/Nim/pull/15612). +- `fusion` is now un-bundled from Nim, `./koch fusion` will + install it via Nimble at a fixed hash. +- `testament`: added `nimoutFull: bool` spec to compare full output of compiler + instead of a subset; many bugfixes to testament. + + +## Misc/cleanups + +- Deprecated `TaintedString` and `--taintmode`. +- Deprecated `--nilseqs` which is now a noop. +- Added `-d:nimStrictMode` in CI in several places to ensure code doesn't have + certain hints/warnings. +- Removed `.travis.yml`, `appveyor.yml.disabled`, `.github/workflows/ci.yml.disabled`. +- `[skip ci]` now works in azure and CI pipelines, see detail in PR + [#17561](https://github.com/nim-lang/Nim/pull/17561). From c4c9876785a17f9b43bb1e0748eff67a71a7487f Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 14 Oct 2021 15:23:01 +0800 Subject: [PATCH 1289/2472] fix a potential bug (#18900) --- lib/std/private/digitsutils.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/private/digitsutils.nim b/lib/std/private/digitsutils.nim index 268a3ba7d..9be2ab3ef 100644 --- a/lib/std/private/digitsutils.nim +++ b/lib/std/private/digitsutils.nim @@ -95,7 +95,7 @@ proc addInt*(result: var string; x: int64) = var num: uint64 if x < 0: if x == low(int64): - num = uint64(x) + num = cast[uint64](x) else: num = uint64(-x) let base = result.len From 3493783d0f3f90b2a90c87b93c92360168272f34 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 14 Oct 2021 15:23:36 +0800 Subject: [PATCH 1290/2472] test whether it is unnecessary (#18893) --- compiler/semtypinst.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim index 0bb53e04d..5d300fc58 100644 --- a/compiler/semtypinst.nim +++ b/compiler/semtypinst.nim @@ -35,7 +35,7 @@ proc checkConstructedType*(conf: ConfigRef; info: TLineInfo, typ: PType) = proc searchInstTypes*(g: ModuleGraph; key: PType): PType = let genericTyp = key[0] if not (genericTyp.kind == tyGenericBody and - key[0] == genericTyp and genericTyp.sym != nil): return + genericTyp.sym != nil): return for inst in typeInstCacheItems(g, genericTyp.sym): if inst.id == key.id: return inst From 8a4eeba218b22d4258f95a279771851082a2e197 Mon Sep 17 00:00:00 2001 From: Miran Date: Thu, 14 Oct 2021 09:56:58 +0200 Subject: [PATCH 1291/2472] try to fix broken ssl test (#18991) --- tests/untestable/thttpclient_ssl_remotenetwork.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/untestable/thttpclient_ssl_remotenetwork.nim b/tests/untestable/thttpclient_ssl_remotenetwork.nim index f56fe274b..d8137e516 100644 --- a/tests/untestable/thttpclient_ssl_remotenetwork.nim +++ b/tests/untestable/thttpclient_ssl_remotenetwork.nim @@ -48,7 +48,7 @@ when enableRemoteNetworking and (defined(nimTestsEnableFlaky) or not defined(win ("https://sha256.badssl.com/", good, "sha256"), ("https://sha384.badssl.com/", good, "sha384"), ("https://sha512.badssl.com/", good, "sha512"), - ("https://1000-sans.badssl.com/", good, "1000-sans"), + ("https://1000-sans.badssl.com/", bad, "1000-sans"), ("https://10000-sans.badssl.com/", good_broken, "10000-sans"), ("https://ecc256.badssl.com/", good, "ecc256"), ("https://ecc384.badssl.com/", good, "ecc384"), From 73330711a3d753d25b073fb1dc24cb16e559724d Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 14 Oct 2021 23:55:05 +0800 Subject: [PATCH 1292/2472] make choosenim work on windows [backport] (#18993) --- bin/nim-gdb.bash | 1 - compiler/installer.ini | 1 - 2 files changed, 2 deletions(-) delete mode 120000 bin/nim-gdb.bash diff --git a/bin/nim-gdb.bash b/bin/nim-gdb.bash deleted file mode 120000 index f6eba4b0c..000000000 --- a/bin/nim-gdb.bash +++ /dev/null @@ -1 +0,0 @@ -nim-gdb \ No newline at end of file diff --git a/compiler/installer.ini b/compiler/installer.ini index 9e31b4e3f..41eee09f1 100644 --- a/compiler/installer.ini +++ b/compiler/installer.ini @@ -120,7 +120,6 @@ Files: "bin/nim" InstallScript: "yes" UninstallScript: "yes" Files: "bin/nim-gdb" -Files: "bin/nim-gdb.bash" [InnoSetup] From f4525efcf3b915be8e17c9730e9e0ffd0afe3a0f Mon Sep 17 00:00:00 2001 From: ee7 <45465154+ee7@users.noreply.github.com> Date: Fri, 15 Oct 2021 09:10:27 +0200 Subject: [PATCH 1293/2472] changelog_1_6_0: mention breaking change in effect tracking (#18995) --- changelogs/changelog_1_6_0.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/changelogs/changelog_1_6_0.md b/changelogs/changelog_1_6_0.md index 1ce6c2093..4a1047d20 100644 --- a/changelogs/changelog_1_6_0.md +++ b/changelogs/changelog_1_6_0.md @@ -905,6 +905,36 @@ Compatibility notes: to avoid ugly, non-portable solutions. See RFC [#407](https://github.com/nim-lang/RFCs/issues/407) for more details. +Compatibility notes: +- Fixed effect tracking for borrowed procs (see [#18882](https://github.com/nim-lang/Nim/pull/18882)). + One consequence is that, under some circumstances, Nim could previously permit a procedure with side effects to be written with `func` - you may need to change some occurrences of `func` to `proc`. + To illustrate, Nim versions before 1.6.0 compile the below without error + ```nim + proc print(s: string) = + echo s + + type + MyString = distinct string + + proc print(s: MyString) {.borrow.} + + func foo(s: MyString) = + print(s) + ``` + but Nim 1.6.0 produces the error + ``` + Error: 'foo' can have side effects + ``` + similar to how we expect that + ```nim + func print(s: string) = + echo s + ``` + produces + ``` + Error: 'print' can have side effects + ``` + ## Tools From 3b1a601fe1ad6ac27a21b5f28442474d47c60a2f Mon Sep 17 00:00:00 2001 From: ee7 <45465154+ee7@users.noreply.github.com> Date: Sat, 16 Oct 2021 11:25:05 +0200 Subject: [PATCH 1294/2472] sequtils: fix errors from `strictFuncs` use (#18998) Nim 1.4.x compiled the below code without error when using `--experimental:strictFuncs` import std/sequtils type Foo = ref object let foo1 = Foo() let foo2 = Foo() let foos = @[foo1, foo2] let fooTuples = @[(foo1, 1), (foo2, 2)] discard repeat(foo1, 3) discard zip(foos, foos) discard unzip(fooTuples) However, since 2020-12-09, devel Nim produced errors like /tmp/bar.nim(11, 15) template/generic instantiation of `repeat` from here /foo/nim/pure/collections/sequtils.nim(172, 6) Error: 'repeat' can have side effects an object reachable from 'x' is potentially mutated /foo/nim/pure/collections/sequtils.nim(183, 15) the mutation is here /foo/nim/pure/collections/sequtils.nim(183, 15) is the statement that connected the mutation to the parameter This commit reverts some `proc` to `func` changes so that code that: - calls `repeat`, `zip`, or `unzip` - and instantiates them with types containing `ref` can once again be compiled with `strictFuncs`. Otherwise, a user might be forced to drop or alter their `strictFuncs` use when upgrading from Nim 1.4.x, or when writing new code that uses these procedures (at least for now, with the current `strictFuncs` implementation). This commit also adds tests to assert that the remaining funcs in this module can be compiled with `strictFuncs` when used with types containing `ref`. The original batch of `proc` to `func` changes in `sequtils.nim` was in commit 6f57ebae349f, which was partially reverted in 38eb021f8158. See also: https://github.com/nim-lang/Nim/issues/16305 --- lib/pure/collections/sequtils.nim | 6 +++--- tests/stdlib/tsequtils.nim | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index 12b961371..64a7be7a9 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -169,7 +169,7 @@ func cycle*[T](s: openArray[T], n: Natural): seq[T] = result[o] = e inc o -func repeat*[T](x: T, n: Natural): seq[T] = +proc repeat*[T](x: T, n: Natural): seq[T] = ## Returns a new sequence with the item `x` repeated `n` times. ## `n` must be a non-negative number (zero or more). ## @@ -246,7 +246,7 @@ func maxIndex*[T](s: openArray[T]): int {.since: (1, 1).} = template zipImpl(s1, s2, retType: untyped): untyped = - func zip*[S, T](s1: openArray[S], s2: openArray[T]): retType = + proc zip*[S, T](s1: openArray[S], s2: openArray[T]): retType = ## Returns a new sequence with a combination of the two input containers. ## ## The input containers can be of different types. @@ -289,7 +289,7 @@ when (NimMajor, NimMinor) <= (1, 0): else: zipImpl(s1, s2, seq[(S, T)]) -func unzip*[S, T](s: openArray[(S, T)]): (seq[S], seq[T]) {.since: (1, 1).} = +proc unzip*[S, T](s: openArray[(S, T)]): (seq[S], seq[T]) {.since: (1, 1).} = ## Returns a tuple of two sequences split out from a sequence of 2-field tuples. runnableExamples: let diff --git a/tests/stdlib/tsequtils.nim b/tests/stdlib/tsequtils.nim index 338473407..179f619f0 100644 --- a/tests/stdlib/tsequtils.nim +++ b/tests/stdlib/tsequtils.nim @@ -10,6 +10,7 @@ from algorithm import sorted {.experimental: "strictEffects".} {.push warningAsError[Effect]: on.} +{.experimental: "strictFuncs".} # helper for testing double substitution side effects which are handled # by `evalOnceAs` @@ -456,6 +457,31 @@ block: # xxx: obscure CT error: basic_types.nim(16, 16) Error: internal error: symbol has no generated name: true doAssert: iter(3).mapIt(2*it).foldl(a + b) == 6 +block: # strictFuncs tests with ref object + type Foo = ref object + + let foo1 = Foo() + let foo2 = Foo() + let foos = @[foo1, foo2] + + # Procedures that are `func` + discard concat(foos, foos) + discard count(foos, foo1) + discard cycle(foos, 3) + discard deduplicate(foos) + discard minIndex(foos) + discard maxIndex(foos) + discard distribute(foos, 2) + var mutableFoos = foos + mutableFoos.delete(0..1) + mutableFoos.insert(foos) + + # Some procedures that are `proc`, but were reverted from `func` + discard repeat(foo1, 3) + discard zip(foos, foos) + let fooTuples = @[(foo1, 1), (foo2, 2)] + discard unzip(fooTuples) + template main = # xxx move all tests here block: # delete tests From 162b07d72cbe79bad1ddf99413ba508f07a831b1 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Sun, 17 Oct 2021 06:38:07 +0300 Subject: [PATCH 1295/2472] fix parsesql.treeRepr when nil is in input (#18974) * fix parsesql.treeRepr when nil is in input * return newNode(nkNone) as default value, not nil --- lib/pure/parsesql.nim | 6 +++--- tests/stdlib/tparsesql.nim | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/lib/pure/parsesql.nim b/lib/pure/parsesql.nim index eb5d7c2cc..9d74d08d0 100644 --- a/lib/pure/parsesql.nim +++ b/lib/pure/parsesql.nim @@ -982,7 +982,7 @@ proc parseInsert(p: var SqlParser): SqlNode = parseParIdentList(p, n) result.add n else: - result.add(nil) + result.add(newNode(nkNone)) if isKeyw(p, "default"): getTok(p) eat(p, "values") @@ -1017,7 +1017,7 @@ proc parseUpdate(p: var SqlParser): SqlNode = if isKeyw(p, "where"): result.add(parseWhere(p)) else: - result.add(nil) + result.add(newNode(nkNone)) proc parseDelete(p: var SqlParser): SqlNode = getTok(p) @@ -1029,7 +1029,7 @@ proc parseDelete(p: var SqlParser): SqlNode = if isKeyw(p, "where"): result.add(parseWhere(p)) else: - result.add(nil) + result.add(newNode(nkNone)) proc parseSelect(p: var SqlParser): SqlNode = getTok(p) diff --git a/tests/stdlib/tparsesql.nim b/tests/stdlib/tparsesql.nim index ba9e601a1..b440dc90f 100644 --- a/tests/stdlib/tparsesql.nim +++ b/tests/stdlib/tparsesql.nim @@ -3,6 +3,17 @@ discard """ """ import parsesql +doAssert treeRepr(parseSql("INSERT INTO STATS VALUES (10, 5.5); ") +) == """ + +nkStmtList + nkInsert + nkIdent STATS + nkNone + nkValueList + nkIntegerLit 10 + nkNumericLit 5.5""" + doAssert $parseSQL("SELECT foo FROM table;") == "select foo from table;" doAssert $parseSQL(""" SELECT @@ -183,8 +194,31 @@ SET ContactName = 'Alfred Schmidt', City= 'Frankfurt' WHERE CustomerID = 1; """) == "update Customers set ContactName = 'Alfred Schmidt' , City = 'Frankfurt' where CustomerID = 1;" +doAssert treeRepr(parseSql("""UPDATE Customers + SET ContactName = 'Alice', City= 'Frankfurt';""") +) == """ + +nkStmtList + nkUpdate + nkIdent Customers + nkAsgn + nkIdent ContactName + nkStringLit Alice + nkAsgn + nkIdent City + nkStringLit Frankfurt + nkNone""" + doAssert $parseSQL("DELETE FROM table_name;") == "delete from table_name;" +doAssert treeRepr(parseSQL("DELETE FROM table_name;") +) == """ + +nkStmtList + nkDelete + nkIdent table_name + nkNone""" + doAssert $parseSQL("DELETE * FROM table_name;") == "delete from table_name;" doAssert $parseSQL(""" From f77dea01fd980734a3ed4acb812575827540aff9 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 16 Oct 2021 23:37:52 -0700 Subject: [PATCH 1296/2472] define `nimVersion` automatically and avoid needing -d:nimVersion140 (#18726) * define `nimVersion` and avoid needing -d:nimVersion140 * fix changelog --- changelog.md | 3 ++- compiler/ast.nim | 2 +- compiler/ccgexprs.nim | 4 ++-- compiler/options.nim | 18 ++++++++++++++++++ config/config.nims | 2 ++ 5 files changed, 25 insertions(+), 4 deletions(-) diff --git a/changelog.md b/changelog.md index ab63bd8b5..fbda70582 100644 --- a/changelog.md +++ b/changelog.md @@ -1,7 +1,6 @@ # v1.8.x - yyyy-mm-dd - ## Changes affecting backward compatibility @@ -16,6 +15,8 @@ ## Compiler changes +- `nim` can now compile version 1.4.0 as follows: `nim c --lib:lib --stylecheck:off compiler/nim`, + without requiring `-d:nimVersion140` which is now a noop. ## Tool changes diff --git a/compiler/ast.nim b/compiler/ast.nim index 0a3b1b72d..abd2ff01e 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -656,7 +656,7 @@ type mUnaryPlusI, mBitnotI, mUnaryPlusF64, mUnaryMinusF64, mCharToStr, mBoolToStr, - mIntToStr, mInt64ToStr, mFloatToStr, # for -d:nimVersion140 + mIntToStr, mInt64ToStr, mFloatToStr, # for compiling nimStdlibVersion < 1.5.1 (not bootstrapping) mCStrToStr, mStrToStr, mEnumToStr, mAnd, mOr, diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index ac4a26bd6..fc09fefc3 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -894,7 +894,7 @@ proc genFieldCheck(p: BProc, e: PNode, obj: Rope, field: PSym) = let discIndex = rdSetElemLoc(p.config, v, u.t) if optTinyRtti in p.config.globalOptions: # not sure how to use `genEnumToStr` here - if p.config.isDefined("nimVersion140"): + if p.config.getStdlibVersion < (1,5,1): const code = "{ #raiseFieldError($1); $2} $n" linefmt(p, cpsStmts, code, [strLit, raiseInstr(p)]) else: @@ -905,7 +905,7 @@ proc genFieldCheck(p: BProc, e: PNode, obj: Rope, field: PSym) = let first = p.config.firstOrd(disc.sym.typ) let firstLit = int64Literal(cast[int](first)) let discName = genTypeInfo(p.config, p.module, disc.sym.typ, e.info) - if p.config.isDefined("nimVersion140"): + if p.config.getStdlibVersion < (1,5,1): const code = "{ #raiseFieldError($1); $2} $n" linefmt(p, cpsStmts, code, [strLit, raiseInstr(p)]) else: diff --git a/compiler/options.nim b/compiler/options.nim index eafcd816d..d3f868316 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -165,6 +165,7 @@ const cmdCtags, cmdBuildindex} type + NimVer* = tuple[major: int, minor: int, patch: int] TStringSeq* = seq[string] TGCMode* = enum # the selected GC gcUnselected = "unselected" @@ -346,6 +347,7 @@ type outDir*: AbsoluteDir jsonBuildFile*: AbsoluteFile prefixDir*, libpath*, nimcacheDir*: AbsoluteDir + nimStdlibVersion*: NimVer dllOverrides, moduleOverrides*, cfileSpecificOptions*: StringTableRef projectName*: string # holds a name like 'nim' projectPath*: AbsoluteDir # holds a path like /home/alice/projects/nim/compiler/ @@ -389,6 +391,16 @@ type cppCustomNamespace*: string vmProfileData*: ProfileData +proc parseNimVersion*(a: string): NimVer = + # could be moved somewhere reusable + if a.len > 0: + let b = a.split(".") + assert b.len == 3, a + template fn(i) = result[i] = b[i].parseInt # could be optimized if needed + fn(0) + fn(1) + fn(2) + proc assignIfDefault*[T](result: var T, val: T, def = default(T)) = ## if `result` was already assigned to a value (that wasn't `def`), this is a noop. if result == def: result = val @@ -560,6 +572,12 @@ proc newPartialConfigRef*(): ConfigRef = proc cppDefine*(c: ConfigRef; define: string) = c.cppDefines.incl define +proc getStdlibVersion*(conf: ConfigRef): NimVer = + if conf.nimStdlibVersion == (0,0,0): + let s = conf.symbols.getOrDefault("nimVersion", "") + conf.nimStdlibVersion = s.parseNimVersion + result = conf.nimStdlibVersion + proc isDefined*(conf: ConfigRef; symbol: string): bool = if conf.symbols.hasKey(symbol): result = true diff --git a/config/config.nims b/config/config.nims index 2ae86012c..893d6960f 100644 --- a/config/config.nims +++ b/config/config.nims @@ -14,3 +14,5 @@ when defined(nimStrictMode): # switch("hint", "ConvFromXtoItselfNotNeeded") switch("hintAsError", "ConvFromXtoItselfNotNeeded") # future work: XDeclaredButNotUsed + +switch("define", "nimVersion:" & NimVersion) From f0af4a36b932af3932b611c1f728ebfbfe3a2749 Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Sun, 17 Oct 2021 08:24:43 -0600 Subject: [PATCH 1297/2472] Added setGlobalValue to VM api (#19007) --- compiler/nimeval.nim | 4 ++++ compiler/vm.nim | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/compiler/nimeval.nim b/compiler/nimeval.nim index 577c7b586..82e2f0812 100644 --- a/compiler/nimeval.nim +++ b/compiler/nimeval.nim @@ -57,6 +57,10 @@ proc callRoutine*(i: Interpreter; routine: PSym; args: openArray[PNode]): PNode proc getGlobalValue*(i: Interpreter; letOrVar: PSym): PNode = result = vm.getGlobalValue(PCtx i.graph.vm, letOrVar) +proc setGlobalValue*(i: Interpreter; letOrVar: PSym, val: PNode) = + ## Sets a global value to a given PNode, does not do any type checking. + vm.setGlobalValue(PCtx i.graph.vm, letOrVar, val) + proc implementRoutine*(i: Interpreter; pkg, module, name: string; impl: proc (a: VmArgs) {.closure, gcsafe.}) = assert i != nil diff --git a/compiler/vm.nim b/compiler/vm.nim index 9fa25efbe..1b443aff1 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -2162,6 +2162,11 @@ proc getGlobalValue*(c: PCtx; s: PSym): PNode = internalAssert c.config, s.kind in {skLet, skVar} and sfGlobal in s.flags result = c.globals[s.position-1] +proc setGlobalValue*(c: PCtx; s: PSym, val: PNode) = + ## Does not do type checking so ensure the `val` matches the `s.typ` + internalAssert c.config, s.kind in {skLet, skVar} and sfGlobal in s.flags + c.globals[s.position-1] = val + include vmops proc setupGlobalCtx*(module: PSym; graph: ModuleGraph; idgen: IdGenerator) = From c239db4817586f769327a6d27c244682111f903a Mon Sep 17 00:00:00 2001 From: narimiran Date: Tue, 19 Oct 2021 17:01:56 +0200 Subject: [PATCH 1298/2472] bump NimVersion to 1.7.1 --- lib/system.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/system.nim b/lib/system.nim index caa036f37..012aefe6a 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2121,7 +2121,7 @@ const ## when (NimMajor, NimMinor, NimPatch) >= (1, 3, 1): discard # see also std/private/since - NimMinor* {.intdefine.}: int = 5 + NimMinor* {.intdefine.}: int = 7 ## is the minor number of Nim's version. ## Odd for devel, even for releases. From 62701cd3b92677c2014cf70b1cf5a5ba6e0468bf Mon Sep 17 00:00:00 2001 From: John Novak Date: Thu, 21 Oct 2021 01:06:00 +1000 Subject: [PATCH 1299/2472] Fix isInvalidFilename & always operate on the full passed in string (#19004) * Fix isInvalidFilename segfault & deprecate the method * Update lib/pure/os.nim Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Update lib/pure/os.nim Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- lib/pure/os.nim | 91 +++++++++++++++++++++++++------------------------ 1 file changed, 46 insertions(+), 45 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index e3b8e9f1c..5320aa87e 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -39,8 +39,8 @@ const weirdTarget = defined(nimscript) or defined(js) since (1, 1): const invalidFilenameChars* = {'/', '\\', ':', '*', '?', '"', '<', '>', '|', '^', '\0'} ## \ - ## Characters that may produce invalid filenames across Linux, Windows, Mac, etc. - ## You can check if your filename contains these char and strip them for safety. + ## Characters that may produce invalid filenames across Linux, Windows and Mac. + ## You can check if your filename contains any of these chars and strip them for safety. ## Mac bans ``':'``, Linux bans ``'/'``, Windows bans all others. invalidFilenames* = [ "CON", "PRN", "AUX", "NUL", @@ -297,7 +297,7 @@ proc isAbsolute*(path: string): bool {.rtl, noSideEffect, extern: "nos$1", raise result = path[0] == '$' elif defined(posix) or defined(js): # `or defined(js)` wouldn't be needed pending https://github.com/nim-lang/Nim/issues/13469 - # This works around the problem for posix, but windows is still broken with nim js -d:nodejs + # This works around the problem for posix, but Windows is still broken with nim js -d:nodejs result = path[0] == '/' else: doAssert false # if ever hits here, adapt as needed @@ -321,7 +321,7 @@ when doslikeFileSystem: proc sameRoot(path1, path2: string): bool {.noSideEffect, raises: [].} = ## Return true if path1 and path2 have a same root. ## - ## Detail of windows path formats: + ## Detail of Windows path formats: ## https://docs.microsoft.com/en-us/dotnet/standard/io/file-path-formats assert(isAbsolute(path1)) @@ -370,7 +370,7 @@ proc relativePath*(path, base: string, sep = DirSep): string {. ## this can be useful to ensure the relative path only contains `'/'` ## so that it can be used for URL constructions. ## - ## On windows, if a root of `path` and a root of `base` are different, + ## On Windows, if a root of `path` and a root of `base` are different, ## returns `path` as is because it is impossible to make a relative path. ## That means an absolute path can be returned. ## @@ -558,27 +558,25 @@ iterator parentDirs*(path: string, fromRoot=false, inclusive=true): string = ## See also: ## * `parentDir proc <#parentDir,string>`_ ## - ## **Examples:** - ## - ## .. code-block:: - ## let g = "a/b/c" - ## - ## for p in g.parentDirs: - ## echo p - ## # a/b/c - ## # a/b - ## # a - ## - ## for p in g.parentDirs(fromRoot=true): - ## echo p - ## # a/ - ## # a/b/ - ## # a/b/c - ## - ## for p in g.parentDirs(inclusive=false): - ## echo p - ## # a/b - ## # a + runnableExamples: + let g = "a/b/c" + + for p in g.parentDirs: + echo p + # a/b/c + # a/b + # a + + for p in g.parentDirs(fromRoot=true): + echo p + # a/ + # a/b/ + # a/b/c + + for p in g.parentDirs(inclusive=false): + echo p + # a/b + # a if not fromRoot: var current = path @@ -944,8 +942,7 @@ proc getCacheDir*(): string = proc getCacheDir*(app: string): string = ## Returns the cache directory for an application `app`. ## - ## * On windows, this uses: `getCacheDir() / app / "cache"` - ## + ## * On Windows, this uses: `getCacheDir() / app / "cache"` ## * On other platforms, this uses: `getCacheDir() / app` when defined(windows): getCacheDir() / app / "cache" @@ -1988,7 +1985,7 @@ proc removeFile*(file: string) {.rtl, extern: "nos$1", tags: [WriteDirEffect], n proc tryMoveFSObject(source, dest: string, isDir: bool): bool {.noWeirdTarget.} = ## Moves a file (or directory if `isDir` is true) from `source` to `dest`. ## - ## Returns false in case of `EXDEV` error or `AccessDeniedError` on windows (if `isDir` is true). + ## Returns false in case of `EXDEV` error or `AccessDeniedError` on Windows (if `isDir` is true). ## In case of other errors `OSError` is raised. ## Returns true in case of success. when defined(windows): @@ -2144,7 +2141,7 @@ iterator walkPattern*(pattern: string): string {.tags: [ReadDirEffect], noWeirdT ## * `walkDirRec iterator <#walkDirRec.i,string>`_ runnableExamples: import std/sequtils - let paths = toSeq(walkPattern("lib/pure/*")) # works on windows too + let paths = toSeq(walkPattern("lib/pure/*")) # works on Windows too assert "lib/pure/concurrency".unixToNativePath in paths assert "lib/pure/os.nim".unixToNativePath in paths walkCommon(pattern, defaultWalkFilter) @@ -2163,7 +2160,7 @@ iterator walkFiles*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTar ## * `walkDirRec iterator <#walkDirRec.i,string>`_ runnableExamples: import std/sequtils - assert "lib/pure/os.nim".unixToNativePath in toSeq(walkFiles("lib/pure/*.nim")) # works on windows too + assert "lib/pure/os.nim".unixToNativePath in toSeq(walkFiles("lib/pure/*.nim")) # works on Windows too walkCommon(pattern, isFile) iterator walkDirs*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTarget.} = @@ -2180,7 +2177,7 @@ iterator walkDirs*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTarg ## * `walkDirRec iterator <#walkDirRec.i,string>`_ runnableExamples: import std/sequtils - let paths = toSeq(walkDirs("lib/pure/*")) # works on windows too + let paths = toSeq(walkDirs("lib/pure/*")) # works on Windows too assert "lib/pure/concurrency".unixToNativePath in paths walkCommon(pattern, isDir) @@ -2278,7 +2275,9 @@ iterator walkDir*(dir: string; relative = false, checkDir = false): ## If `checkDir` is true, `OSError` is raised when `dir` ## doesn't exist. ## - ## Example: This directory structure:: + ## **Example:** + ## + ## This directory structure:: ## dirA / dirB / fileB1.txt ## / dirC ## / fileA1.txt @@ -2291,7 +2290,6 @@ iterator walkDir*(dir: string; relative = false, checkDir = false): # this also works at compile time assert collect(for k in walkDir("dirA"): k.path).join(" ") == "dirA/dirB dirA/dirC dirA/fileA2.txt dirA/fileA1.txt" - ## ## See also: ## * `walkPattern iterator <#walkPattern.i,string>`_ ## * `walkFiles iterator <#walkFiles.i,string>`_ @@ -3271,7 +3269,7 @@ template rawToFormalFileInfo(rawInfo, path, formalInfo): untyped = formalInfo.lastAccessTime = fromWinTime(rdFileTime(rawInfo.ftLastAccessTime)) formalInfo.lastWriteTime = fromWinTime(rdFileTime(rawInfo.ftLastWriteTime)) formalInfo.creationTime = fromWinTime(rdFileTime(rawInfo.ftCreationTime)) - formalInfo.blockSize = 8192 # xxx use windows API instead of hardcoding + formalInfo.blockSize = 8192 # xxx use Windows API instead of hardcoding # Retrieve basic permissions if (rawInfo.dwFileAttributes and FILE_ATTRIBUTE_READONLY) != 0'i32: @@ -3494,27 +3492,30 @@ proc setLastModificationTime*(file: string, t: times.Time) {.noWeirdTarget.} = discard h.closeHandle if res == 0'i32: raiseOSError(osLastError(), file) -func isValidFilename*(filename: string, maxLen = 259.Positive): bool {.since: (1, 1).} = +func isValidFilename*(filename: string, maxLen = 259.Positive): bool {.since: (1, 1), deprecated: "Deprecated since v1.5.1".} = ## Returns true if ``filename`` is valid for crossplatform use. ## ## This is useful if you want to copy or save files across Windows, Linux, Mac, etc. ## You can pass full paths as argument too, but func only checks filenames. - ## It uses ``invalidFilenameChars``, ``invalidFilenames`` and ``maxLen`` to verify the specified ``filename``. ## - ## .. code-block:: nim - ## assert not isValidFilename(" foo") ## Leading white space - ## assert not isValidFilename("foo ") ## Trailing white space - ## assert not isValidFilename("foo.") ## Ends with Dot - ## assert not isValidFilename("con.txt") ## "CON" is invalid (Windows) - ## assert not isValidFilename("OwO:UwU") ## ":" is invalid (Mac) - ## assert not isValidFilename("aux.bat") ## "AUX" is invalid (Windows) + ## It uses `invalidFilenameChars`, `invalidFilenames` and `maxLen` to verify the specified `filename`. ## + runnableExamples: + assert not isValidFilename(" foo") # Leading white space + assert not isValidFilename("foo ") # Trailing white space + assert not isValidFilename("foo.") # Ends with dot + assert not isValidFilename("con.txt") # "CON" is invalid (Windows) + assert not isValidFilename("OwO:UwU") # ":" is invalid (Mac) + assert not isValidFilename("aux.bat") # "AUX" is invalid (Windows) + assert not isValidFilename("") # Empty string + assert not isValidFilename("foo/") # Filename is empty + # https://docs.microsoft.com/en-us/dotnet/api/system.io.pathtoolongexception # https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file # https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247%28v=vs.85%29.aspx result = true let f = filename.splitFile() - if unlikely(f.name.len + f.ext.len > maxLen or + if unlikely(f.name.len + f.ext.len > maxLen or f.name.len == 0 or f.name[0] == ' ' or f.name[^1] == ' ' or f.name[^1] == '.' or find(f.name, invalidFilenameChars) != -1): return false for invalid in invalidFilenames: From 1a45da9150c6cf5da324e754f49dc4cf957f1180 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 21 Oct 2021 16:53:15 +0200 Subject: [PATCH 1300/2472] use two underscores for easy demangling [backport:1.6] (#19028) --- compiler/ccgtypes.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 492e1f181..23e1a5867 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -39,7 +39,7 @@ proc mangleName(m: BModule; s: PSym): Rope = result = s.loc.r if result == nil: result = s.name.s.mangle.rope - result.add "_" + result.add "__" result.add m.g.graph.ifaces[s.itemId.module].uniqueName result.add "_" result.add rope s.itemId.item From 5e2b9341f343c3f80f92cfd160a0186f23521a42 Mon Sep 17 00:00:00 2001 From: Danil Yarantsev Date: Thu, 21 Oct 2021 18:01:48 +0300 Subject: [PATCH 1301/2472] Add Elbrus 2000 architecture (#19024) * Add Elbrus 2000 architecture * Add e2k to niminst * Update compiler/installer.ini Co-authored-by: Andreas Rumpf --- compiler/platform.nim | 6 ++++-- lib/system/platforms.nim | 4 +++- tools/nim.zsh-completion | 1 + tools/niminst/buildsh.nimf | 2 ++ tools/niminst/makefile.nimf | 3 +++ 5 files changed, 13 insertions(+), 3 deletions(-) diff --git a/compiler/platform.nim b/compiler/platform.nim index 1bc00b629..18d43f006 100644 --- a/compiler/platform.nim +++ b/compiler/platform.nim @@ -197,7 +197,8 @@ type cpuNone, cpuI386, cpuM68k, cpuAlpha, cpuPowerpc, cpuPowerpc64, cpuPowerpc64el, cpuSparc, cpuVm, cpuHppa, cpuIa64, cpuAmd64, cpuMips, cpuMipsel, cpuArm, cpuArm64, cpuJS, cpuNimVM, cpuAVR, cpuMSP430, - cpuSparc64, cpuMips64, cpuMips64el, cpuRiscV32, cpuRiscV64, cpuEsp, cpuWasm32 + cpuSparc64, cpuMips64, cpuMips64el, cpuRiscV32, cpuRiscV64, cpuEsp, cpuWasm32, + cpuE2k type TInfoCPU* = tuple[name: string, intSize: int, endian: Endianness, @@ -232,7 +233,8 @@ const (name: "riscv32", intSize: 32, endian: littleEndian, floatSize: 64, bit: 32), (name: "riscv64", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64), (name: "esp", intSize: 32, endian: littleEndian, floatSize: 64, bit: 32), - (name: "wasm32", intSize: 32, endian: littleEndian, floatSize: 64, bit: 32)] + (name: "wasm32", intSize: 32, endian: littleEndian, floatSize: 64, bit: 32), + (name: "e2k", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64)] type Target* = object diff --git a/lib/system/platforms.nim b/lib/system/platforms.nim index bd6541b6a..5bca8cb1c 100644 --- a/lib/system/platforms.nim +++ b/lib/system/platforms.nim @@ -35,7 +35,8 @@ type msp430, ## TI MSP430 microcontroller riscv32, ## RISC-V 32-bit processor riscv64, ## RISC-V 64-bit processor - wasm32 ## WASM, 32-bit + wasm32, ## WASM, 32-bit + e2k ## MCST Elbrus 2000 OsPlatform* {.pure.} = enum ## the OS this program will run on. none, dos, windows, os2, linux, morphos, skyos, solaris, @@ -93,5 +94,6 @@ const elif defined(riscv32): CpuPlatform.riscv32 elif defined(riscv64): CpuPlatform.riscv64 elif defined(wasm32): CpuPlatform.wasm32 + elif defined(e2k): CpuPlatform.e2k else: CpuPlatform.none ## the CPU this program will run on. diff --git a/tools/nim.zsh-completion b/tools/nim.zsh-completion index f7aeb72c6..07a221e9c 100644 --- a/tools/nim.zsh-completion +++ b/tools/nim.zsh-completion @@ -74,6 +74,7 @@ _nim() { '*--cpu=arm64[compile for ARM64 architecture]' \ '*--cpu=avr[compile for AVR architecture]' \ '*--cpu=esp[compile for ESP architecture]' \ + '*--cpu=e2k[compile for Elbrus 2000 architecture]' \ '*--cpu=hppa[compile for HPPA architecture]' \ '*--cpu=i386[compile for i386 architecture]' \ '*--cpu=ia64[compile for ia64 architecture]' \ diff --git a/tools/niminst/buildsh.nimf b/tools/niminst/buildsh.nimf index 5f68a6a17..b81ac731f 100644 --- a/tools/niminst/buildsh.nimf +++ b/tools/niminst/buildsh.nimf @@ -227,6 +227,8 @@ case $ucpu in mycpu="arm" ;; *riscv64|riscv* ) mycpu="riscv64" ;; + *e2k* ) + mycpu="e2k" ;; *) echo 2>&1 "Error: unknown processor: $ucpu" exit 1 diff --git a/tools/niminst/makefile.nimf b/tools/niminst/makefile.nimf index 69e87b837..c4f2f0e79 100644 --- a/tools/niminst/makefile.nimf +++ b/tools/niminst/makefile.nimf @@ -166,6 +166,9 @@ endif ifeq ($(ucpu),riscv64) mycpu = riscv64 endif +ifeq ($(ucpu),e2k) + mycpu = e2k +endif ifndef mycpu $(error unknown CPU architecture: $(ucpu) See makefile.nimf) endif From d6345874236e03d21f4f6acbad34546ce27052b7 Mon Sep 17 00:00:00 2001 From: Abishek PY <43115551+vj-abishek@users.noreply.github.com> Date: Fri, 22 Oct 2021 15:40:32 +0530 Subject: [PATCH 1302/2472] feat: copy to clipboard (#18963) * feat: copy to clipboard * fix: CI failure related issue * fix: CI failure issue * fix: copy to clipboard button bug * feat: copy pragmadots value to clipboard --- doc/nimdoc.css | 32 ++++++++++++- nimdoc/testproject/expected/nimdoc.out.css | 32 ++++++++++++- tools/dochack/dochack.nim | 56 ++++++++++++++++++++++ 3 files changed, 118 insertions(+), 2 deletions(-) diff --git a/doc/nimdoc.css b/doc/nimdoc.css index 4abea9ce0..3353f1cda 100644 --- a/doc/nimdoc.css +++ b/doc/nimdoc.css @@ -38,6 +38,10 @@ Modified by Boyd Greenfield and narimiran --program: #6060c0; --option: #508000; --raw-data: #a4255b; + + --clipboard-image-normal: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' style='color: black' fill='none' viewBox='0 0 24 24' stroke='currentColor'%3E %3Cpath stroke-linecap='round' stroke-linejoin='round' stroke-width='2' d='M9 5H7a2 2 0 00-2 2v12a2 2 0 002 2h10a2 2 0 002-2V7a2 2 0 00-2-2h-2M9 5a2 2 0 002 2h2a2 2 0 002-2M9 5a2 2 0 012-2h2a2 2 0 012 2' /%3E %3C/svg%3E"); + --clipboard-image-selected: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' style='color: black' viewBox='0 0 20 20' fill='currentColor'%3E %3Cpath d='M8 3a1 1 0 011-1h2a1 1 0 110 2H9a1 1 0 01-1-1z' /%3E %3Cpath d='M6 3a2 2 0 00-2 2v11a2 2 0 002 2h8a2 2 0 002-2V5a2 2 0 00-2-2 3 3 0 01-3 3H9a3 3 0 01-3-3z' /%3E %3C/svg%3E"); + --clipboard-image: var(--clipboard-image-normal) } [data-theme="dark"] { @@ -68,6 +72,10 @@ Modified by Boyd Greenfield and narimiran --program: #9090c0; --option: #90b010; --raw-data: #8be9fd; + + --clipboard-image-normal: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' style='color: lightgray' fill='none' viewBox='0 0 24 24' stroke='currentColor'%3E %3Cpath stroke-linecap='round' stroke-linejoin='round' stroke-width='2' d='M9 5H7a2 2 0 00-2 2v12a2 2 0 002 2h10a2 2 0 002-2V7a2 2 0 00-2-2h-2M9 5a2 2 0 002 2h2a2 2 0 002-2M9 5a2 2 0 012-2h2a2 2 0 012 2' /%3E %3C/svg%3E"); + --clipboard-image-selected: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' style='color: lightgray' viewBox='0 0 20 20' fill='currentColor'%3E %3Cpath d='M8 3a1 1 0 011-1h2a1 1 0 110 2H9a1 1 0 01-1-1z' /%3E %3Cpath d='M6 3a2 2 0 00-2 2v11a2 2 0 002 2h8a2 2 0 002-2V5a2 2 0 00-2-2 3 3 0 01-3 3H9a3 3 0 01-3-3z' /%3E %3C/svg%3E"); + --clipboard-image: var(--clipboard-image-normal); } .theme-switch-wrapper { @@ -572,6 +580,10 @@ span.tok { margin-right: 0.2em; } +.copyToClipBoard { + position: relative; +} + pre { font-family: "Source Code Pro", Monaco, Menlo, Consolas, "Courier New", monospace; color: var(--text); @@ -590,7 +602,25 @@ pre { border: 1px solid var(--border); -webkit-border-radius: 6px; -moz-border-radius: 6px; - border-radius: 6px; } + border-radius: 6px; +} + +.copyToClipBoardBtn { + visibility: hidden; + position: absolute; + width: 24px; + border-radius: 4px; + background-image: var(--clipboard-image); + right: 5px; + top: 13px; + background-color: var(--secondary-background); + padding: 11px; + border: 0; +} + +.copyToClipBoard:hover .copyToClipBoardBtn { + visibility: visible; +} .pre-scrollable { max-height: 340px; diff --git a/nimdoc/testproject/expected/nimdoc.out.css b/nimdoc/testproject/expected/nimdoc.out.css index 4abea9ce0..3353f1cda 100644 --- a/nimdoc/testproject/expected/nimdoc.out.css +++ b/nimdoc/testproject/expected/nimdoc.out.css @@ -38,6 +38,10 @@ Modified by Boyd Greenfield and narimiran --program: #6060c0; --option: #508000; --raw-data: #a4255b; + + --clipboard-image-normal: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' style='color: black' fill='none' viewBox='0 0 24 24' stroke='currentColor'%3E %3Cpath stroke-linecap='round' stroke-linejoin='round' stroke-width='2' d='M9 5H7a2 2 0 00-2 2v12a2 2 0 002 2h10a2 2 0 002-2V7a2 2 0 00-2-2h-2M9 5a2 2 0 002 2h2a2 2 0 002-2M9 5a2 2 0 012-2h2a2 2 0 012 2' /%3E %3C/svg%3E"); + --clipboard-image-selected: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' style='color: black' viewBox='0 0 20 20' fill='currentColor'%3E %3Cpath d='M8 3a1 1 0 011-1h2a1 1 0 110 2H9a1 1 0 01-1-1z' /%3E %3Cpath d='M6 3a2 2 0 00-2 2v11a2 2 0 002 2h8a2 2 0 002-2V5a2 2 0 00-2-2 3 3 0 01-3 3H9a3 3 0 01-3-3z' /%3E %3C/svg%3E"); + --clipboard-image: var(--clipboard-image-normal) } [data-theme="dark"] { @@ -68,6 +72,10 @@ Modified by Boyd Greenfield and narimiran --program: #9090c0; --option: #90b010; --raw-data: #8be9fd; + + --clipboard-image-normal: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' style='color: lightgray' fill='none' viewBox='0 0 24 24' stroke='currentColor'%3E %3Cpath stroke-linecap='round' stroke-linejoin='round' stroke-width='2' d='M9 5H7a2 2 0 00-2 2v12a2 2 0 002 2h10a2 2 0 002-2V7a2 2 0 00-2-2h-2M9 5a2 2 0 002 2h2a2 2 0 002-2M9 5a2 2 0 012-2h2a2 2 0 012 2' /%3E %3C/svg%3E"); + --clipboard-image-selected: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' style='color: lightgray' viewBox='0 0 20 20' fill='currentColor'%3E %3Cpath d='M8 3a1 1 0 011-1h2a1 1 0 110 2H9a1 1 0 01-1-1z' /%3E %3Cpath d='M6 3a2 2 0 00-2 2v11a2 2 0 002 2h8a2 2 0 002-2V5a2 2 0 00-2-2 3 3 0 01-3 3H9a3 3 0 01-3-3z' /%3E %3C/svg%3E"); + --clipboard-image: var(--clipboard-image-normal); } .theme-switch-wrapper { @@ -572,6 +580,10 @@ span.tok { margin-right: 0.2em; } +.copyToClipBoard { + position: relative; +} + pre { font-family: "Source Code Pro", Monaco, Menlo, Consolas, "Courier New", monospace; color: var(--text); @@ -590,7 +602,25 @@ pre { border: 1px solid var(--border); -webkit-border-radius: 6px; -moz-border-radius: 6px; - border-radius: 6px; } + border-radius: 6px; +} + +.copyToClipBoardBtn { + visibility: hidden; + position: absolute; + width: 24px; + border-radius: 4px; + background-image: var(--clipboard-image); + right: 5px; + top: 13px; + background-color: var(--secondary-background); + padding: 11px; + border: 0; +} + +.copyToClipBoard:hover .copyToClipBoardBtn { + visibility: visible; +} .pre-scrollable { max-height: 340px; diff --git a/tools/dochack/dochack.nim b/tools/dochack/dochack.nim index 4a491cf88..83e81f1c0 100644 --- a/tools/dochack/dochack.nim +++ b/tools/dochack/dochack.nim @@ -341,3 +341,59 @@ proc search*() {.exportc.} = if timer != nil: clearTimeout(timer) timer = setTimeout(wrapper, 400) + +proc copyToClipboard*() {.exportc.} = + {.emit: """ + + function updatePreTags() { + + const allPreTags = document.querySelectorAll("pre") + + allPreTags.forEach((e) => { + + const div = document.createElement("div") + div.classList.add("copyToClipBoard") + + const preTag = document.createElement("pre") + preTag.innerHTML = e.innerHTML + + const button = document.createElement("button") + button.value = e.textContent.replace('...', '') + button.classList.add("copyToClipBoardBtn") + + div.appendChild(preTag) + div.appendChild(button) + + e.outerHTML = div.outerHTML + + }) + } + + + function copyTextToClipboard(e) { + const clipBoardContent = e.target.value + navigator.clipboard.writeText(clipBoardContent).then(function() { + e.target.style.setProperty("--clipboard-image", "var(--clipboard-image-selected)") + }, function(err) { + console.error("Could not copy text: ", err); + }); + } + + window.addEventListener("click", (e) => { + if (e.target.classList.contains("copyToClipBoardBtn")) { + copyTextToClipboard(e) + } + }) + + window.addEventListener("mouseover", (e) => { + if (e.target.nodeName === "PRE") { + e.target.nextElementSibling.style.setProperty("--clipboard-image", "var(--clipboard-image-normal)") + } + }) + + window.addEventListener("DOMContentLoaded", updatePreTags) + + """ + .} + +copyToClipboard() \ No newline at end of file From 4d8108046b71b6e40d88cde7496e687e68116877 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 22 Oct 2021 18:15:08 +0800 Subject: [PATCH 1303/2472] remove exception (#18906) --- compiler/semfold.nim | 8 +------- tests/system/tdollars.nim | 7 +++++++ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/compiler/semfold.nim b/compiler/semfold.nim index 1f8efc4de..5ae91d306 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -292,13 +292,7 @@ proc evalOp(m: TMagic, n, a, b, c: PNode; idgen: IdGenerator; g: ModuleGraph): P else: result = newStrNodeT("true", n, g) of mFloatToStr: result = newStrNodeT($getFloat(a), n, g) of mCStrToStr, mCharToStr: - if a.kind == nkBracket: - var s = "" - for b in a.sons: - s.add b.getStrOrChar - result = newStrNodeT(s, n, g) - else: - result = newStrNodeT(getStrOrChar(a), n, g) + result = newStrNodeT(getStrOrChar(a), n, g) of mStrToStr: result = newStrNodeT(getStrOrChar(a), n, g) of mEnumToStr: result = newStrNodeT(ordinalValToString(a, g), n, g) of mArrToSeq: diff --git a/tests/system/tdollars.nim b/tests/system/tdollars.nim index 17d195e76..93fa5cb9e 100644 --- a/tests/system/tdollars.nim +++ b/tests/system/tdollars.nim @@ -187,5 +187,12 @@ proc main()= test2 high(int8), "127" test2 low(int8), "-128" + block: + const + a: array[3, char] = ['N', 'i', 'm'] + aStr = $(a) + + doAssert aStr == """['N', 'i', 'm']""" + static: main() main() From 490c4226a55e8abd7cb504d4e24fcb714bff23f1 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 22 Oct 2021 17:32:02 +0200 Subject: [PATCH 1304/2472] allow converting static vars to `openArray` (#19035) When assigning constant output to a seq, and then passing that static seq to other functions that take `openArray`, the compiler may end up producing errors, as it does not know how to convert `static[seq[T]]` to `openArray[T]`. By ignoring the `static` wrapper on the type for the purpose of determining data memory location and length, this gets resolved cleanly. Unfortunately, it is relatively tricky to come up with a minimal example, as there are followup problems from the failing conversion, e.g., this may lead to `internal error: inconsistent environment type`, instead of the relevant `openArrayLoc` error message. --- compiler/ccgcalls.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index 12f366a53..fe0516f9a 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -214,7 +214,7 @@ proc openArrayLoc(p: BProc, formalType: PType, n: PNode): Rope = else: var a: TLoc initLocExpr(p, if n.kind == nkHiddenStdConv: n[1] else: n, a) - case skipTypes(a.t, abstractVar).kind + case skipTypes(a.t, abstractVar+{tyStatic}).kind of tyOpenArray, tyVarargs: if reifiedOpenArray(n): if a.t.kind in {tyVar, tyLent}: From 6621e1a3ead7166fa116bfbb8f0d2acb4e318d81 Mon Sep 17 00:00:00 2001 From: BarrOff <58253563+BarrOff@users.noreply.github.com> Date: Fri, 22 Oct 2021 23:42:54 +0000 Subject: [PATCH 1305/2472] use the correct header for TIOCGWINSZ on Solaris (#19037) --- lib/posix/termios.nim | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/posix/termios.nim b/lib/posix/termios.nim index b03ce74f3..f755c720d 100644 --- a/lib/posix/termios.nim +++ b/lib/posix/termios.nim @@ -237,8 +237,11 @@ proc tcFlow*(fd: cint; action: cint): cint {.importc: "tcflow", header: "".} # Get process group ID for session leader for controlling terminal FD. -# Window size ioctl. Should work on on any Unix that xterm has been ported to. -var TIOCGWINSZ*{.importc, header: "".}: culong +# Window size ioctl. Solaris based systems have an uncommen place for this. +when defined(solaris) or defined(sunos): + var TIOCGWINSZ*{.importc, header: "".}: culong +else: + var TIOCGWINSZ*{.importc, header: "".}: culong when defined(nimHasStyleChecks): {.push styleChecks: off.} From 41d99185918923063bd6ae34898c8ac18a941169 Mon Sep 17 00:00:00 2001 From: BarrOff <58253563+BarrOff@users.noreply.github.com> Date: Sun, 24 Oct 2021 05:24:31 +0000 Subject: [PATCH 1306/2472] use `gmake` in the buildscript on Solaris systems (#19036) --- ci/funs.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/funs.sh b/ci/funs.sh index 1638931c6..8e7f8faf7 100644 --- a/ci/funs.sh +++ b/ci/funs.sh @@ -76,6 +76,8 @@ _nimBuildCsourcesIfNeeded(){ makeX=gmake elif [ "$unamestr" = 'CROSSOS' ]; then makeX=gmake + elif [ "$unamestr" = 'SunOS' ]; then + makeX=gmake else makeX=make fi From 141b76e36519219915ada9086d1c9b1d0b465659 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Sun, 24 Oct 2021 09:51:57 +0000 Subject: [PATCH 1307/2472] Add Zephyr Support (#19003) * Porting Nim to run on Zephyr. Includes changes to `std/net`. Squashed commit of the following: tweaking more memory / malloc things revert back bitmasks tweaking nim to use kernel heap as C malloc doesn't work fixing socket polling on zephyr cleanup getting maximum sockets for process or for rtos'es reorganizing and fixing net for async / system merge netlite changes back into nativesockets merge netlite changes back into nativesockets reverting native sockets back tweaking nim / zephyr network adding option to run 'net-lite' from linux bridging zephyr's max connections fixing net errors fixing compilation with getAddrString fixing compilation with getAddrString experimenting with a nativesockets_lite ... getAddrString experimenting with a nativesockets_lite ... getAddrString experimenting with a nativesockets_lite ... getLocalAddr experimenting with a nativesockets_lite ... getLocalAddr experimenting with a nativesockets_lite ... add note regarding incorrect FreeRTOS Sockadd_in fields changing to NIM_STATIC_ASSERT cleaning up the static_assert error messages cleaning up the static_assert error messages setting up static assert ftw! testing compile time asserts reworking Sockaddr objects to more closely match various platforms reworking Sockaddr objects to more closely match various platforms reworking Sockaddr objects to more closely match various platforms finding missing items (issue #18684) fixup posix constants (issue #18684) adding plumbing for zephyr os (issue #18684) adding plumbing for zephyr os (issue #18684) * fixing constant capitalizations * remove extra debug prints and fix TSa_Family/cint issue * remove extra debug prints and fix TSa_Family/cint issue * Porting Nim to run on Zephyr. Includes changes to `std/net`. Squashed commit of the following: tweaking more memory / malloc things revert back bitmasks tweaking nim to use kernel heap as C malloc doesn't work fixing socket polling on zephyr cleanup getting maximum sockets for process or for rtos'es reorganizing and fixing net for async / system merge netlite changes back into nativesockets merge netlite changes back into nativesockets reverting native sockets back tweaking nim / zephyr network adding option to run 'net-lite' from linux bridging zephyr's max connections fixing net errors fixing compilation with getAddrString fixing compilation with getAddrString experimenting with a nativesockets_lite ... getAddrString experimenting with a nativesockets_lite ... getAddrString experimenting with a nativesockets_lite ... getLocalAddr experimenting with a nativesockets_lite ... getLocalAddr experimenting with a nativesockets_lite ... add note regarding incorrect FreeRTOS Sockadd_in fields changing to NIM_STATIC_ASSERT cleaning up the static_assert error messages cleaning up the static_assert error messages setting up static assert ftw! testing compile time asserts reworking Sockaddr objects to more closely match various platforms reworking Sockaddr objects to more closely match various platforms reworking Sockaddr objects to more closely match various platforms finding missing items (issue #18684) fixup posix constants (issue #18684) adding plumbing for zephyr os (issue #18684) adding plumbing for zephyr os (issue #18684) * fixing constant capitalizations * remove extra debug prints and fix TSa_Family/cint issue * remove extra debug prints and fix TSa_Family/cint issue * fixing PR issues * Porting Nim to run on Zephyr. Includes changes to `std/net`. Squashed commit of the following: tweaking more memory / malloc things revert back bitmasks tweaking nim to use kernel heap as C malloc doesn't work fixing socket polling on zephyr cleanup getting maximum sockets for process or for rtos'es reorganizing and fixing net for async / system merge netlite changes back into nativesockets merge netlite changes back into nativesockets reverting native sockets back tweaking nim / zephyr network adding option to run 'net-lite' from linux bridging zephyr's max connections fixing net errors fixing compilation with getAddrString fixing compilation with getAddrString experimenting with a nativesockets_lite ... getAddrString experimenting with a nativesockets_lite ... getAddrString experimenting with a nativesockets_lite ... getLocalAddr experimenting with a nativesockets_lite ... getLocalAddr experimenting with a nativesockets_lite ... add note regarding incorrect FreeRTOS Sockadd_in fields changing to NIM_STATIC_ASSERT cleaning up the static_assert error messages cleaning up the static_assert error messages setting up static assert ftw! testing compile time asserts reworking Sockaddr objects to more closely match various platforms reworking Sockaddr objects to more closely match various platforms reworking Sockaddr objects to more closely match various platforms finding missing items (issue #18684) fixup posix constants (issue #18684) adding plumbing for zephyr os (issue #18684) adding plumbing for zephyr os (issue #18684) * fixing constant capitalizations * remove extra debug prints and fix TSa_Family/cint issue * remove extra debug prints and fix TSa_Family/cint issue * Remerge * fixing constant capitalizations * remove extra debug prints and fix TSa_Family/cint issue * remove extra debug prints and fix TSa_Family/cint issue * fixing PR issues * fix maxDescriptors on zephyr/freertos * move maxDescriptors to selector.nim -- fixes compile issue * change realloc impl on zephyr to match ansi c behavior * change realloc impl on zephyr to match ansi c behavior * force compileOnly mode for tlwip Co-authored-by: Jaremy J. Creechley Co-authored-by: Jaremy Creechley --- compiler/options.nim | 4 +- compiler/platform.nim | 6 +- lib/posix/posix.nim | 6 +- lib/posix/posix_freertos_consts.nim | 2 + lib/posix/posix_other.nim | 94 ++-- lib/posix/posix_other_consts.nim | 12 +- lib/pure/asyncdispatch.nim | 7 +- lib/pure/asyncnet.nim | 14 +- lib/pure/ioselects/ioselectors_epoll.nim | 5 +- lib/pure/ioselects/ioselectors_poll.nim | 5 +- lib/pure/ioselects/ioselectors_select.nim | 2 +- lib/pure/nativesockets.nim | 615 ++++++++++++---------- lib/pure/net.nim | 17 +- lib/pure/selectors.nim | 19 + lib/system/ansi_c.nim | 31 +- lib/system/bitmasks.nim | 1 + lib/system/dyncalls.nim | 2 +- lib/system/io.nim | 9 +- lib/system/mm/malloc.nim | 15 +- tests/stdlib/tlwip.nim | 2 +- 20 files changed, 522 insertions(+), 346 deletions(-) diff --git a/compiler/options.nim b/compiler/options.nim index d3f868316..2feaa2d43 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -595,7 +595,7 @@ proc isDefined*(conf: ConfigRef; symbol: string): bool = osQnx, osAtari, osAix, osHaiku, osVxWorks, osSolaris, osNetbsd, osFreebsd, osOpenbsd, osDragonfly, osMacosx, osIos, - osAndroid, osNintendoSwitch, osFreeRTOS, osCrossos} + osAndroid, osNintendoSwitch, osFreeRTOS, osCrossos, osZephyr} of "linux": result = conf.target.targetOS in {osLinux, osAndroid} of "bsd": @@ -615,6 +615,8 @@ proc isDefined*(conf: ConfigRef; symbol: string): bool = result = conf.target.targetOS == osNintendoSwitch of "freertos", "lwip": result = conf.target.targetOS == osFreeRTOS + of "zephyr": + result = conf.target.targetOS == osZephyr of "littleendian": result = CPU[conf.target.targetCPU].endian == littleEndian of "bigendian": result = CPU[conf.target.targetCPU].endian == bigEndian of "cpu8": result = CPU[conf.target.targetCPU].bit == 8 diff --git a/compiler/platform.nim b/compiler/platform.nim index 18d43f006..eb1398677 100644 --- a/compiler/platform.nim +++ b/compiler/platform.nim @@ -22,7 +22,7 @@ type osNone, osDos, osWindows, osOs2, osLinux, osMorphos, osSkyos, osSolaris, osIrix, osNetbsd, osFreebsd, osOpenbsd, osDragonfly, osCrossos, osAix, osPalmos, osQnx, osAmiga, osAtari, osNetware, osMacos, osMacosx, osIos, osHaiku, osAndroid, osVxWorks - osGenode, osJS, osNimVM, osStandalone, osNintendoSwitch, osFreeRTOS, osAny + osGenode, osJS, osNimVM, osStandalone, osNintendoSwitch, osFreeRTOS, osZephyr, osAny type TInfoOSProp* = enum @@ -185,6 +185,10 @@ const objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/", scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".", props: {ospPosix}), + (name: "Zephyr", parDir: "..", dllFrmt: "lib$1.so", altDirSep: "/", + objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/", + scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".", + props: {ospPosix}), (name: "Any", parDir: "..", dllFrmt: "lib$1.so", altDirSep: "/", objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/", scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".", diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 45fe00d5d..c2504f994 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -151,8 +151,10 @@ proc htons*(a1: uint16): uint16 {.importc, header: "".} proc ntohl*(a1: uint32): uint32 {.importc, header: "".} proc ntohs*(a1: uint16): uint16 {.importc, header: "".} -proc inet_addr*(a1: cstring): InAddrT {.importc, header: "".} -proc inet_ntoa*(a1: InAddr): cstring {.importc, header: "".} +when not defined(zephyr): + proc inet_addr*(a1: cstring): InAddrT {.importc, header: "".} + proc inet_ntoa*(a1: InAddr): cstring {.importc, header: "".} + proc inet_ntop*(a1: cint, a2: pointer, a3: cstring, a4: int32): cstring {. importc:"(char *)$1", header: "".} proc inet_pton*(a1: cint, a2: cstring, a3: pointer): cint {. diff --git a/lib/posix/posix_freertos_consts.nim b/lib/posix/posix_freertos_consts.nim index efe87aacf..ca500534a 100644 --- a/lib/posix/posix_freertos_consts.nim +++ b/lib/posix/posix_freertos_consts.nim @@ -363,6 +363,8 @@ var SEM_FAILED* {.importc: "SEM_FAILED", header: "".}: pointer # # var RLIMIT_NOFILE* {.importc: "RLIMIT_NOFILE", header: "".}: cint +var FD_MAX* {.importc: "CONFIG_LWIP_MAX_SOCKETS", header: "".}: cint + # var FD_SETSIZE* {.importc: "FD_SETSIZE", header: "".}: cint diff --git a/lib/posix/posix_other.nim b/lib/posix/posix_other.nim index 6584bfab2..688867b98 100644 --- a/lib/posix/posix_other.nim +++ b/lib/posix/posix_other.nim @@ -10,7 +10,7 @@ when defined(nimHasStyleChecks): {.push styleChecks: off.} -when defined(freertos): +when defined(freertos) or defined(zephyr): const hasSpawnH = false # should exist for every Posix system nowadays hasAioH = false @@ -391,9 +391,29 @@ when hasSpawnH: header: "", final, pure.} = object when defined(linux): + const Sockaddr_max_length* = 255 # from sys/un.h const Sockaddr_un_path_length* = 108 +elif defined(zephyr): + when defined(net_ipv6): + const Sockaddr_max_length* = 24 + elif defined(net_raw): + const Sockaddr_max_length* = 20 + else: + const Sockaddr_max_length* = 8 + const Sockaddr_un_path_length* = Sockaddr_max_length + # Zephyr is heavily customizable so it's easy to get to a state + # where Nim & Zephyr IPv6 settings are out of sync, causing painful runtime failures. + {.emit: ["NIM_STATIC_ASSERT(NET_SOCKADDR_MAX_SIZE == ", + Sockaddr_max_length, + ",\"NET_SOCKADDR_MAX_SIZE and Sockaddr_max_length size mismatch!", + " Check that Nim and Zephyr IPv4/IPv6 settings match.", + " Try adding -d:net_ipv6 to enable IPv6 for Nim on Zephyr.\" );"].} +elif defined(freertos) or defined(lwip): + const Sockaddr_max_length* = 14 + const Sockaddr_un_path_length* = 108 else: + const Sockaddr_max_length* = 255 # according to http://pubs.opengroup.org/onlinepubs/009604499/basedefs/sys/un.h.html # this is >=92 const Sockaddr_un_path_length* = 92 @@ -408,48 +428,50 @@ when defined(lwip): pure, final.} = object ## struct sockaddr sa_len*: uint8 ## Address family. sa_family*: TSa_Family ## Address family. - sa_data*: array[0..255, char] ## Socket address (variable-length data). + sa_data*: array[0..Sockaddr_max_length-sizeof(uint8)-sizeof(TSa_Family), char] ## Socket address (variable-length data). + + Sockaddr_storage* {.importc: "struct sockaddr_storage", + header: "", + pure, final.} = object ## struct sockaddr_storage + s2_len*: uint8 ## Address family. + ss_family*: TSa_Family ## Address family. + s2_data1*: array[2, char] ## Address family. + s2_data2*: array[3, uint32] ## Address family. + when defined(lwip6) or defined(net_ipv6): + s2_data3*: array[3, uint32] ## Address family. +elif defined(zephyr): + type + SockAddr* {.importc: "struct sockaddr", header: "", + pure, final.} = object ## struct sockaddr + sa_family*: TSa_Family ## Address family. + data*: array[0..Sockaddr_max_length-sizeof(TSa_Family), char] ## Socket address (variable-length data). + + Sockaddr_storage* {.importc: "struct sockaddr_storage", + header: "", + pure, final.} = object ## struct sockaddr_storage + ss_family*: TSa_Family ## Address family. + data*: array[0..Sockaddr_max_length-sizeof(TSa_Family), char] ## Socket address (variable-length data). + {.emit: ["NIM_STATIC_ASSERT(sizeof(struct sockaddr) == ", sizeof(Sockaddr), ",\"struct size mismatch\" );"].} + {.emit: ["NIM_STATIC_ASSERT(sizeof(struct sockaddr_storage) == ", sizeof(Sockaddr_storage), ",\"struct size mismatch\" );"].} else: type SockAddr* {.importc: "struct sockaddr", header: "", pure, final.} = object ## struct sockaddr sa_family*: TSa_Family ## Address family. - sa_data*: array[0..255, char] ## Socket address (variable-length data). + sa_data*: array[0..Sockaddr_max_length-sizeof(TSa_Family), char] ## Socket address (variable-length data). + + Sockaddr_storage* {.importc: "struct sockaddr_storage", + header: "", + pure, final.} = object ## struct sockaddr_storage + ss_family*: TSa_Family ## Address family. type Sockaddr_un* {.importc: "struct sockaddr_un", header: "", pure, final.} = object ## struct sockaddr_un sun_family*: TSa_Family ## Address family. - sun_path*: array[0..Sockaddr_un_path_length-1, char] ## Socket path + sun_path*: array[0..Sockaddr_un_path_length-sizeof(TSa_Family), char] ## Socket path -when defined(lwip): - when not defined(lwip6): - type - Sockaddr_storage* {.importc: "struct sockaddr_storage", - header: "", - pure, final.} = object ## struct sockaddr_storage - s2_len*: uint8 ## Address family. - ss_family*: TSa_Family ## Address family. - s2_data1*: array[2, char] ## Address family. - s2_data2*: array[3, uint32] ## Address family. - else: - type - Sockaddr_storage* {.importc: "struct sockaddr_storage", - header: "", - pure, final.} = object ## struct sockaddr_storage - s2_len*: uint8 ## Address family. - ss_family*: TSa_Family ## Address family. - s2_data1*: array[2, char] ## Address family. - s2_data2*: array[3, uint32] ## Address family. - s2_data3*: array[3, uint32] ## Address family. -else: - type - Sockaddr_storage* {.importc: "struct sockaddr_storage", - header: "", - pure, final.} = object ## struct sockaddr_storage - ss_family*: TSa_Family ## Address family. - type Tif_nameindex* {.importc: "struct if_nameindex", final, pure, header: "".} = object ## struct if_nameindex @@ -494,6 +516,7 @@ type header: "".} = object ## struct in_addr s_addr*: InAddrScalar + # TODO: Fixme for FreeRTOS/LwIP, these are incorrect Sockaddr_in* {.importc: "struct sockaddr_in", pure, final, header: "".} = object ## struct sockaddr_in sin_family*: TSa_Family ## AF_INET. @@ -577,7 +600,12 @@ when not defined(lwip): events*: cshort ## The input event flags (see below). revents*: cshort ## The output event flags (see below). - Tnfds* {.importc: "nfds_t", header: "".} = cint + when defined(zephyr): + type + Tnfds* = distinct cint + else: + type + Tnfds* {.importc: "nfds_t", header: "".} = cint var errno* {.importc, header: "".}: cint ## error variable @@ -625,7 +653,7 @@ elif defined(solaris): # Solaris doesn't have MSG_NOSIGNAL const MSG_NOSIGNAL* = 0'i32 -elif defined(freertos) or defined(lwip): +elif defined(zephyr) or defined(freertos) or defined(lwip): # LwIP/FreeRTOS doesn't have MSG_NOSIGNAL const MSG_NOSIGNAL* = 0x20'i32 diff --git a/lib/posix/posix_other_consts.nim b/lib/posix/posix_other_consts.nim index f43407b40..59ac6f9ed 100644 --- a/lib/posix/posix_other_consts.nim +++ b/lib/posix/posix_other_consts.nim @@ -462,6 +462,9 @@ var RLIMIT_NOFILE* {.importc: "RLIMIT_NOFILE", header: "".}: cin # var FD_SETSIZE* {.importc: "FD_SETSIZE", header: "".}: cint +when defined(zephyr): + # Zephyr specific hardcoded value + var FD_MAX* {.importc: "CONFIG_POSIX_MAX_FDS ", header: "".}: cint # var MSG_CTRUNC* {.importc: "MSG_CTRUNC", header: "".}: cint @@ -487,10 +490,15 @@ var SO_SNDTIMEO* {.importc: "SO_SNDTIMEO", header: "".}: cint var SO_TYPE* {.importc: "SO_TYPE", header: "".}: cint var SOCK_DGRAM* {.importc: "SOCK_DGRAM", header: "".}: cint var SOCK_RAW* {.importc: "SOCK_RAW", header: "".}: cint -var SOCK_SEQPACKET* {.importc: "SOCK_SEQPACKET", header: "".}: cint +when defined(zephyr): + const SOCK_SEQPACKET* = cint(5) + var SOMAXCONN* {.importc: "CONFIG_NET_SOCKETS_POLL_MAX", header: "".}: cint +else: + var SOCK_SEQPACKET* {.importc: "SOCK_SEQPACKET", header: "".}: cint + var SOMAXCONN* {.importc: "SOMAXCONN", header: "".}: cint + var SOCK_STREAM* {.importc: "SOCK_STREAM", header: "".}: cint var SOL_SOCKET* {.importc: "SOL_SOCKET", header: "".}: cint -var SOMAXCONN* {.importc: "SOMAXCONN", header: "".}: cint var MSG_PEEK* {.importc: "MSG_PEEK", header: "".}: cint var MSG_TRUNC* {.importc: "MSG_TRUNC", header: "".}: cint var MSG_WAITALL* {.importc: "MSG_WAITALL", header: "".}: cint diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index d87e2360f..e15fb0851 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -1973,15 +1973,18 @@ proc activeDescriptors*(): int {.inline.} = when defined(posix): import posix -when defined(linux) or defined(windows) or defined(macosx) or defined(bsd): +when defined(linux) or defined(windows) or defined(macosx) or defined(bsd) or + defined(zephyr) or defined(freertos): proc maxDescriptors*(): int {.raises: OSError.} = ## Returns the maximum number of active file descriptors for the current ## process. This involves a system call. For now `maxDescriptors` is ## supported on the following OSes: Windows, Linux, OSX, BSD. when defined(windows): result = 16_700_000 + elif defined(zephyr) or defined(freertos): + result = FD_MAX else: var fdLim: RLimit if getrlimit(RLIMIT_NOFILE, fdLim) < 0: raiseOSError(osLastError()) - result = int(fdLim.rlim_cur) - 1 + result = int(fdLim.rlim_cur) - 1 \ No newline at end of file diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim index 5523fa10e..abb4dd075 100644 --- a/lib/pure/asyncnet.nim +++ b/lib/pure/asyncnet.nim @@ -103,6 +103,7 @@ export SOBool # TODO: Remove duplication introduced by PR #4683. const defineSsl = defined(ssl) or defined(nimdoc) +const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) when defineSsl: import openssl @@ -178,11 +179,12 @@ proc getLocalAddr*(socket: AsyncSocket): (string, Port) = ## This is high-level interface for `getsockname`:idx:. getLocalAddr(socket.fd, socket.domain) -proc getPeerAddr*(socket: AsyncSocket): (string, Port) = - ## Get the socket's peer address and port number. - ## - ## This is high-level interface for `getpeername`:idx:. - getPeerAddr(socket.fd, socket.domain) +when not useNimNetLite: + proc getPeerAddr*(socket: AsyncSocket): (string, Port) = + ## Get the socket's peer address and port number. + ## + ## This is high-level interface for `getpeername`:idx:. + getPeerAddr(socket.fd, socket.domain) proc newAsyncSocket*(domain, sockType, protocol: cint, buffered = true, @@ -655,7 +657,7 @@ proc hasDataBuffered*(s: AsyncSocket): bool {.since: (1, 5).} = # xxx dedup with std/net s.isBuffered and s.bufLen > 0 and s.currPos != s.bufLen -when defined(posix): +when defined(posix) and not useNimNetLite: proc connectUnix*(socket: AsyncSocket, path: string): owned(Future[void]) = ## Binds Unix socket to `path`. diff --git a/lib/pure/ioselects/ioselectors_epoll.nim b/lib/pure/ioselects/ioselectors_epoll.nim index b62b4c2db..05d9d19a8 100644 --- a/lib/pure/ioselects/ioselectors_epoll.nim +++ b/lib/pure/ioselects/ioselectors_epoll.nim @@ -73,10 +73,7 @@ type proc newSelector*[T](): Selector[T] = # Retrieve the maximum fd count (for current OS) via getrlimit() - var a = RLimit() - if getrlimit(posix.RLIMIT_NOFILE, a) != 0: - raiseOSError(osLastError()) - var maxFD = int(a.rlim_max) + var maxFD = maxDescriptors() doAssert(maxFD > 0) # Start with a reasonable size, checkFd() will grow this on demand const numFD = 1024 diff --git a/lib/pure/ioselects/ioselectors_poll.nim b/lib/pure/ioselects/ioselectors_poll.nim index 6b10e19ae..0d8fef78a 100644 --- a/lib/pure/ioselects/ioselectors_poll.nim +++ b/lib/pure/ioselects/ioselectors_poll.nim @@ -53,10 +53,7 @@ else: body proc newSelector*[T](): Selector[T] = - var a = RLimit() - if getrlimit(posix.RLIMIT_NOFILE, a) != 0: - raiseIOSelectorsError(osLastError()) - var maxFD = int(a.rlim_max) + var maxFD = maxDescriptors() when hasThreadSupport: result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) diff --git a/lib/pure/ioselects/ioselectors_select.nim b/lib/pure/ioselects/ioselectors_select.nim index 6f216ac85..88cbaf28c 100644 --- a/lib/pure/ioselects/ioselectors_select.nim +++ b/lib/pure/ioselects/ioselectors_select.nim @@ -313,7 +313,7 @@ proc selectInto*[T](s: Selector[T], timeout: int, verifySelectParams(timeout) if timeout != -1: - when defined(genode) or defined(freertos): + when defined(genode) or defined(freertos) or defined(zephyr): tv.tv_sec = Time(timeout div 1_000) else: tv.tv_sec = timeout.int32 div 1_000 diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index 13c08dd92..4037d7181 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -21,6 +21,7 @@ when hostOS == "solaris": {.passl: "-lsocket -lnsl".} const useWinVersion = defined(windows) or defined(nimdoc) +const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) when useWinVersion: import winlean @@ -35,9 +36,12 @@ else: export SocketHandle, Sockaddr_in, Addrinfo, INADDR_ANY, SockAddr, SockLen, Sockaddr_in6, Sockaddr_storage, - inet_ntoa, recv, `==`, connect, send, accept, recvfrom, sendto, + recv, `==`, connect, send, accept, recvfrom, sendto, freeAddrInfo +when not useNimNetLite: + export inet_ntoa + export SO_ERROR, SOL_SOCKET, @@ -332,125 +336,6 @@ template htons*(x: uint16): untyped = ## order, this is a no-op; otherwise, it performs a 2-byte swap operation. nativesockets.ntohs(x) -proc getServByName*(name, proto: string): Servent {.tags: [ReadIOEffect].} = - ## Searches the database from the beginning and finds the first entry for - ## which the service name specified by `name` matches the s_name member - ## and the protocol name specified by `proto` matches the s_proto member. - ## - ## On posix this will search through the `/etc/services` file. - when useWinVersion: - var s = winlean.getservbyname(name, proto) - else: - var s = posix.getservbyname(name, proto) - if s == nil: raiseOSError(osLastError(), "Service not found.") - result.name = $s.s_name - result.aliases = cstringArrayToSeq(s.s_aliases) - result.port = Port(s.s_port) - result.proto = $s.s_proto - -proc getServByPort*(port: Port, proto: string): Servent {.tags: [ReadIOEffect].} = - ## Searches the database from the beginning and finds the first entry for - ## which the port specified by `port` matches the s_port member and the - ## protocol name specified by `proto` matches the s_proto member. - ## - ## On posix this will search through the `/etc/services` file. - when useWinVersion: - var s = winlean.getservbyport(ze(int16(port)).cint, proto) - else: - var s = posix.getservbyport(ze(int16(port)).cint, proto) - if s == nil: raiseOSError(osLastError(), "Service not found.") - result.name = $s.s_name - result.aliases = cstringArrayToSeq(s.s_aliases) - result.port = Port(s.s_port) - result.proto = $s.s_proto - -proc getHostByAddr*(ip: string): Hostent {.tags: [ReadIOEffect].} = - ## This function will lookup the hostname of an IP Address. - var myaddr: InAddr - myaddr.s_addr = inet_addr(ip) - - when useWinVersion: - var s = winlean.gethostbyaddr(addr(myaddr), sizeof(myaddr).cuint, - cint(AF_INET)) - if s == nil: raiseOSError(osLastError()) - else: - var s = - when defined(android4): - posix.gethostbyaddr(cast[cstring](addr(myaddr)), sizeof(myaddr).cint, - cint(posix.AF_INET)) - else: - posix.gethostbyaddr(addr(myaddr), sizeof(myaddr).SockLen, - cint(posix.AF_INET)) - if s == nil: - raiseOSError(osLastError(), $hstrerror(h_errno)) - - result.name = $s.h_name - result.aliases = cstringArrayToSeq(s.h_aliases) - when useWinVersion: - result.addrtype = Domain(s.h_addrtype) - else: - if s.h_addrtype == posix.AF_INET: - result.addrtype = AF_INET - elif s.h_addrtype == posix.AF_INET6: - result.addrtype = AF_INET6 - else: - raiseOSError(osLastError(), "unknown h_addrtype") - if result.addrtype == AF_INET: - result.addrList = @[] - var i = 0 - while not isNil(s.h_addr_list[i]): - var inaddrPtr = cast[ptr InAddr](s.h_addr_list[i]) - result.addrList.add($inet_ntoa(inaddrPtr[])) - inc(i) - else: - result.addrList = cstringArrayToSeq(s.h_addr_list) - result.length = int(s.h_length) - -proc getHostByName*(name: string): Hostent {.tags: [ReadIOEffect].} = - ## This function will lookup the IP address of a hostname. - when useWinVersion: - var s = winlean.gethostbyname(name) - else: - var s = posix.gethostbyname(name) - if s == nil: raiseOSError(osLastError()) - result.name = $s.h_name - result.aliases = cstringArrayToSeq(s.h_aliases) - when useWinVersion: - result.addrtype = Domain(s.h_addrtype) - else: - if s.h_addrtype == posix.AF_INET: - result.addrtype = AF_INET - elif s.h_addrtype == posix.AF_INET6: - result.addrtype = AF_INET6 - else: - raiseOSError(osLastError(), "unknown h_addrtype") - if result.addrtype == AF_INET: - result.addrList = @[] - var i = 0 - while not isNil(s.h_addr_list[i]): - var inaddrPtr = cast[ptr InAddr](s.h_addr_list[i]) - result.addrList.add($inet_ntoa(inaddrPtr[])) - inc(i) - else: - result.addrList = cstringArrayToSeq(s.h_addr_list) - result.length = int(s.h_length) - -proc getHostname*(): string {.tags: [ReadIOEffect].} = - ## Returns the local hostname (not the FQDN) - # https://tools.ietf.org/html/rfc1035#section-2.3.1 - # https://tools.ietf.org/html/rfc2181#section-11 - const size = 256 - result = newString(size) - when useWinVersion: - let success = winlean.gethostname(result, size) - else: - # Posix - let success = posix.gethostname(result, size) - if success != 0.cint: - raiseOSError(osLastError()) - let x = len(cstring(result)) - result.setLen(x) - proc getSockDomain*(socket: SocketHandle): Domain = ## Returns the socket's domain (AF_INET or AF_INET6). var name: Sockaddr_in6 @@ -464,162 +349,354 @@ proc getSockDomain*(socket: SocketHandle): Domain = else: raise newException(IOError, "Unknown socket family in getSockDomain") -proc getAddrString*(sockAddr: ptr SockAddr): string = - ## Returns the string representation of address within sockAddr - if sockAddr.sa_family.cint == nativeAfInet: - result = $inet_ntoa(cast[ptr Sockaddr_in](sockAddr).sin_addr) - elif sockAddr.sa_family.cint == nativeAfInet6: - let addrLen = when not useWinVersion: posix.INET6_ADDRSTRLEN.int - else: 46 # it's actually 46 in both cases - result = newString(addrLen) - let addr6 = addr cast[ptr Sockaddr_in6](sockAddr).sin6_addr - when not useWinVersion: - if posix.inet_ntop(posix.AF_INET6, addr6, addr result[0], - result.len.int32) == nil: - raiseOSError(osLastError()) - if posix.IN6_IS_ADDR_V4MAPPED(addr6) != 0: - result.setSlice("::ffff:".len..= Sockaddr_un_path_length: + raise newException(ValueError, "socket path too long") + copyMem(addr result.sun_path, path.cstring, path.len + 1) + + proc getSockName*(socket: SocketHandle): Port = + ## Returns the socket's associated port number. + var name: Sockaddr_in + when useWinVersion: + name.sin_family = uint16(ord(AF_INET)) + else: + name.sin_family = TSa_Family(posix.AF_INET) + #name.sin_port = htons(cint16(port)) + #name.sin_addr.s_addr = htonl(INADDR_ANY) + var namelen = sizeof(name).SockLen + if getsockname(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + result = Port(nativesockets.ntohs(name.sin_port)) + + proc getLocalAddr*(socket: SocketHandle, domain: Domain): (string, Port) = + ## Returns the socket's local address and port number. + ## + ## Similar to POSIX's `getsockname`:idx:. + case domain + of AF_INET: + var name: Sockaddr_in + when useWinVersion: + name.sin_family = uint16(ord(AF_INET)) + else: + name.sin_family = TSa_Family(posix.AF_INET) + var namelen = sizeof(name).SockLen + if getsockname(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: raiseOSError(osLastError()) - else: - raise newException(IOError, "Unknown socket family in getAddrString") - setLen(strAddress, len(cstring(strAddress))) + result = ($inet_ntoa(name.sin_addr), + Port(nativesockets.ntohs(name.sin_port))) + of AF_INET6: + var name: Sockaddr_in6 + when useWinVersion: + name.sin6_family = uint16(ord(AF_INET6)) + else: + name.sin6_family = TSa_Family(posix.AF_INET6) + var namelen = sizeof(name).SockLen + if getsockname(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + # Cannot use INET6_ADDRSTRLEN here, because it's a C define. + result[0] = newString(64) + if inet_ntop(name.sin6_family.cint, + addr name.sin6_addr, addr result[0][0], (result[0].len+1).int32).isNil: + raiseOSError(osLastError()) + setLen(result[0], result[0].cstring.len) + result[1] = Port(nativesockets.ntohs(name.sin6_port)) + else: + raiseOSError(OSErrorCode(-1), "invalid socket family in getLocalAddr") -when defined(posix) and not defined(nimdoc): - proc makeUnixAddr*(path: string): Sockaddr_un = - result.sun_family = AF_UNIX.TSa_Family - if path.len >= Sockaddr_un_path_length: - raise newException(ValueError, "socket path too long") - copyMem(addr result.sun_path, path.cstring, path.len + 1) + proc getPeerAddr*(socket: SocketHandle, domain: Domain): (string, Port) = + ## Returns the socket's peer address and port number. + ## + ## Similar to POSIX's `getpeername`:idx: + case domain + of AF_INET: + var name: Sockaddr_in + when useWinVersion: + name.sin_family = uint16(ord(AF_INET)) + else: + name.sin_family = TSa_Family(posix.AF_INET) + var namelen = sizeof(name).SockLen + if getpeername(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + result = ($inet_ntoa(name.sin_addr), + Port(nativesockets.ntohs(name.sin_port))) + of AF_INET6: + var name: Sockaddr_in6 + when useWinVersion: + name.sin6_family = uint16(ord(AF_INET6)) + else: + name.sin6_family = TSa_Family(posix.AF_INET6) + var namelen = sizeof(name).SockLen + if getpeername(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + # Cannot use INET6_ADDRSTRLEN here, because it's a C define. + result[0] = newString(64) + if inet_ntop(name.sin6_family.cint, + addr name.sin6_addr, addr result[0][0], (result[0].len+1).int32).isNil: + raiseOSError(osLastError()) + setLen(result[0], result[0].cstring.len) + result[1] = Port(nativesockets.ntohs(name.sin6_port)) + else: + raiseOSError(OSErrorCode(-1), "invalid socket family in getLocalAddr") + +when useNimNetLite: -proc getSockName*(socket: SocketHandle): Port = - ## Returns the socket's associated port number. - var name: Sockaddr_in when useWinVersion: - name.sin_family = uint16(ord(AF_INET)) - else: - name.sin_family = TSa_Family(posix.AF_INET) - #name.sin_port = htons(cint16(port)) - #name.sin_addr.s_addr = htonl(INADDR_ANY) - var namelen = sizeof(name).SockLen - if getsockname(socket, cast[ptr SockAddr](addr(name)), - addr(namelen)) == -1'i32: - raiseOSError(osLastError()) - result = Port(nativesockets.ntohs(name.sin_port)) + const + INET_ADDRSTRLEN = 16 + INET6_ADDRSTRLEN = 46 # it's actually 46 in both cases -proc getLocalAddr*(socket: SocketHandle, domain: Domain): (string, Port) = - ## Returns the socket's local address and port number. - ## - ## Similar to POSIX's `getsockname`:idx:. - case domain - of AF_INET: - var name: Sockaddr_in - when useWinVersion: - name.sin_family = uint16(ord(AF_INET)) - else: - name.sin_family = TSa_Family(posix.AF_INET) - var namelen = sizeof(name).SockLen - if getsockname(socket, cast[ptr SockAddr](addr(name)), - addr(namelen)) == -1'i32: - raiseOSError(osLastError()) - result = ($inet_ntoa(name.sin_addr), - Port(nativesockets.ntohs(name.sin_port))) - of AF_INET6: - var name: Sockaddr_in6 - when useWinVersion: - name.sin6_family = uint16(ord(AF_INET6)) - else: - name.sin6_family = TSa_Family(posix.AF_INET6) - var namelen = sizeof(name).SockLen - if getsockname(socket, cast[ptr SockAddr](addr(name)), - addr(namelen)) == -1'i32: - raiseOSError(osLastError()) - # Cannot use INET6_ADDRSTRLEN here, because it's a C define. - result[0] = newString(64) - if inet_ntop(name.sin6_family.cint, - addr name.sin6_addr, addr result[0][0], (result[0].len+1).int32).isNil: - raiseOSError(osLastError()) - setLen(result[0], result[0].cstring.len) - result[1] = Port(nativesockets.ntohs(name.sin6_port)) - else: - raiseOSError(OSErrorCode(-1), "invalid socket family in getLocalAddr") + proc sockAddrToStr(sa: ptr Sockaddr): string {.noinit.} = + let af_family = sa.sa_family + var nl, v4Slice: cint + var si_addr: ptr InAddr -proc getPeerAddr*(socket: SocketHandle, domain: Domain): (string, Port) = - ## Returns the socket's peer address and port number. - ## - ## Similar to POSIX's `getpeername`:idx: - case domain - of AF_INET: - var name: Sockaddr_in - when useWinVersion: - name.sin_family = uint16(ord(AF_INET)) + if af_family == AF_INET.TSa_Family: + nl = INET_ADDRSTRLEN + si_addr = cast[ptr Sockaddr_in](sa).sin_addr.addr() + elif af_family == AF_INET6.TSa_Family: + nl = INET6_ADDRSTRLEN + let si6_addr = cast[ptr Sockaddr_in6](sa).sin6_addr.addr() + si_addr = cast[ptr InAddr](si6_addr) # let's us reuse logic below + when defined(posix) and not defined(nimdoc) and not defined(zephyr): + if posix.IN6_IS_ADDR_V4MAPPED(si6_addr) != 0: + v4Slice = "::ffff:".len() else: - name.sin_family = TSa_Family(posix.AF_INET) - var namelen = sizeof(name).SockLen - if getpeername(socket, cast[ptr SockAddr](addr(name)), - addr(namelen)) == -1'i32: - raiseOSError(osLastError()) - result = ($inet_ntoa(name.sin_addr), - Port(nativesockets.ntohs(name.sin_port))) - of AF_INET6: - var name: Sockaddr_in6 - when useWinVersion: - name.sin6_family = uint16(ord(AF_INET6)) + when defined(posix) and not defined(nimdoc): + if af_family.cint == nativeAfUnix: + return "unix" + return "" + + result = newString(nl) + let namePtr = result.cstring() + if namePtr == inet_ntop(af_family.cint, si_addr, namePtr, nl): + result.setLen(len(namePtr)) + if v4Slice > 0: result.setSlice(v4Slice.int ..< nl.int) else: - name.sin6_family = TSa_Family(posix.AF_INET6) - var namelen = sizeof(name).SockLen - if getpeername(socket, cast[ptr SockAddr](addr(name)), - addr(namelen)) == -1'i32: + return "" + + proc sockAddrToStr(sa: var Sockaddr_in | var Sockaddr_in6): string = + result = sockAddrToStr(cast[ptr SockAddr](unsafeAddr(sa))) + + proc getAddrString*(sockAddr: ptr SockAddr): string = + result = sockAddrToStr(sockAddr) + if result.len() == 0: raiseOSError(osLastError()) - # Cannot use INET6_ADDRSTRLEN here, because it's a C define. - result[0] = newString(64) - if inet_ntop(name.sin6_family.cint, - addr name.sin6_addr, addr result[0][0], (result[0].len+1).int32).isNil: - raiseOSError(osLastError()) - setLen(result[0], result[0].cstring.len) - result[1] = Port(nativesockets.ntohs(name.sin6_port)) - else: - raiseOSError(OSErrorCode(-1), "invalid socket family in getLocalAddr") + + proc getAddrString*(sockAddr: ptr SockAddr, strAddress: var string) {.noinit.} = + strAddress = getAddrString(sockAddr) + + proc getLocalAddr*(socket: SocketHandle, domain: Domain): (string, Port) = + ## Returns the socket's local address and port number. + ## + ## Similar to POSIX's `getsockname`:idx:. + template sockGetNameOrRaiseError(socket: untyped, name: untyped) = + var namelen = sizeof(socket).SockLen + if getsockname(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + + case domain + of AF_INET: + var name = Sockaddr_in(sin_family: TSa_Family(posix.AF_INET)) + sockGetNameOrRaiseError(socket, name) + result = (sockAddrToStr(name), + Port(nativesockets.ntohs(name.sin_port))) + of AF_INET6: + var name = Sockaddr_in6(sin6_family: TSa_Family(posix.AF_INET6)) + sockGetNameOrRaiseError(socket, name) + result = (sockAddrToStr(name), + Port(nativesockets.ntohs(name.sin6_port))) + else: + raiseOSError(OSErrorCode(-1), "invalid socket family in getLocalAddr") + proc getSockOptInt*(socket: SocketHandle, level, optname: int): int {. tags: [ReadIOEffect].} = @@ -733,14 +810,14 @@ proc accept*(fd: SocketHandle, inheritable = defined(nimInheritHandles)): (Socke ## child processes. ## ## Returns (osInvalidSocket, "") if an error occurred. - var sockAddress: Sockaddr_in + var sockAddress: Sockaddr var addrLen = sizeof(sockAddress).SockLen var sock = when (defined(linux) or defined(bsd)) and not defined(nimdoc): - accept4(fd, cast[ptr SockAddr](addr(sockAddress)), addr(addrLen), + accept4(fd, addr(sockAddress), addr(addrLen), if inheritable: 0 else: SOCK_CLOEXEC) else: - accept(fd, cast[ptr SockAddr](addr(sockAddress)), addr(addrLen)) + accept(fd, addr(sockAddress), addr(addrLen)) when declared(setInheritable) and not (defined(linux) or defined(bsd)): if not setInheritable(sock, inheritable): close sock @@ -748,7 +825,11 @@ proc accept*(fd: SocketHandle, inheritable = defined(nimInheritHandles)): (Socke if sock == osInvalidSocket: return (osInvalidSocket, "") else: - return (sock, $inet_ntoa(sockAddress.sin_addr)) + when useNimNetLite: + var name = sockAddrToStr(addr sockAddress) + return (sock, name) + else: + return (sock, $inet_ntoa(cast[Sockaddr_in](sockAddress).sin_addr)) when defined(windows): var wsa: WSAData diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 09cdc8afb..7b0ff78e7 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -85,12 +85,14 @@ runnableExamples("-r:off"): import std/private/since -import nativesockets, os, strutils, times, sets, options, std/monotimes +import nativesockets +import os, strutils, times, sets, options, std/monotimes import ssl_config export nativesockets.Port, nativesockets.`$`, nativesockets.`==` export Domain, SockType, Protocol const useWinVersion = defined(windows) or defined(nimdoc) +const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) const defineSsl = defined(ssl) or defined(nimdoc) when useWinVersion: @@ -1243,11 +1245,12 @@ proc getLocalAddr*(socket: Socket): (string, Port) = ## This is high-level interface for `getsockname`:idx:. getLocalAddr(socket.fd, socket.domain) -proc getPeerAddr*(socket: Socket): (string, Port) = - ## Get the socket's peer address and port number. - ## - ## This is high-level interface for `getpeername`:idx:. - getPeerAddr(socket.fd, socket.domain) +when not useNimNetLite: + proc getPeerAddr*(socket: Socket): (string, Port) = + ## Get the socket's peer address and port number. + ## + ## This is high-level interface for `getpeername`:idx:. + getPeerAddr(socket.fd, socket.domain) proc setSockOpt*(socket: Socket, opt: SOBool, value: bool, level = SOL_SOCKET) {.tags: [WriteIOEffect].} = @@ -1259,7 +1262,7 @@ proc setSockOpt*(socket: Socket, opt: SOBool, value: bool, var valuei = cint(if value: 1 else: 0) setSockOptInt(socket.fd, cint(level), toCInt(opt), valuei) -when defined(posix) or defined(nimdoc): +when defined(nimdoc) or (defined(posix) and not useNimNetLite): proc connectUnix*(socket: Socket, path: string) = ## Connects to Unix socket on `path`. ## This only works on Unix-style systems: Mac OS X, BSD and Linux diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim index 82550e09b..ec441f6da 100644 --- a/lib/pure/selectors.nim +++ b/lib/pure/selectors.nim @@ -323,6 +323,23 @@ else: # Anything higher is the time to wait in milliseconds. doAssert(timeout >= -1, "Cannot select with a negative value, got: " & $timeout) + when defined(linux) or defined(windows) or defined(macosx) or defined(bsd) or + defined(zephyr) or defined(freertos): + template maxDescriptors*(): int = + ## Returns the maximum number of active file descriptors for the current + ## process. This involves a system call. For now `maxDescriptors` is + ## supported on the following OSes: Windows, Linux, OSX, BSD. + when defined(windows): + 16_700_000 + elif defined(zephyr) or defined(freertos): + FD_MAX + else: + var fdLim: RLimit + var res = int(getrlimit(RLIMIT_NOFILE, fdLim)) + if res >= 0: + res = int(fdLim.rlim_cur) - 1 + res + when defined(linux) and not defined(emscripten): include ioselects/ioselectors_epoll elif bsdPlatform: @@ -337,5 +354,7 @@ else: include ioselects/ioselectors_select elif defined(freertos) or defined(lwip): include ioselects/ioselectors_select + elif defined(zephyr): + include ioselects/ioselectors_poll else: include ioselects/ioselectors_poll diff --git a/lib/system/ansi_c.nim b/lib/system/ansi_c.nim index e5f26e070..259d36633 100644 --- a/lib/system/ansi_c.nim +++ b/lib/system/ansi_c.nim @@ -142,14 +142,29 @@ proc c_sprintf*(buf, frmt: cstring): cint {. importc: "sprintf", header: "", varargs, noSideEffect.} # we use it only in a way that cannot lead to security issues -proc c_malloc*(size: csize_t): pointer {. - importc: "malloc", header: "".} -proc c_calloc*(nmemb, size: csize_t): pointer {. - importc: "calloc", header: "".} -proc c_free*(p: pointer) {. - importc: "free", header: "".} -proc c_realloc*(p: pointer, newsize: csize_t): pointer {. - importc: "realloc", header: "".} +when defined(zephyr) and not defined(zephyrUseLibcMalloc): + proc c_malloc*(size: csize_t): pointer {. + importc: "k_malloc", header: "".} + proc c_calloc*(nmemb, size: csize_t): pointer {. + importc: "k_calloc", header: "".} + proc c_free*(p: pointer) {. + importc: "k_free", header: "".} + proc c_realloc*(p: pointer, newsize: csize_t): pointer = + # Zephyr's kernel malloc doesn't support realloc + result = c_malloc(newSize) + # match the ansi c behavior + if not result.isNil(): + copyMem(result, p, newSize) + c_free(p) +else: + proc c_malloc*(size: csize_t): pointer {. + importc: "malloc", header: "".} + proc c_calloc*(nmemb, size: csize_t): pointer {. + importc: "calloc", header: "".} + proc c_free*(p: pointer) {. + importc: "free", header: "".} + proc c_realloc*(p: pointer, newsize: csize_t): pointer {. + importc: "realloc", header: "".} proc c_fwrite*(buf: pointer, size, n: csize_t, f: CFilePtr): cint {. importc: "fwrite", header: "".} diff --git a/lib/system/bitmasks.nim b/lib/system/bitmasks.nim index d7c55a4d9..caf86568a 100644 --- a/lib/system/bitmasks.nim +++ b/lib/system/bitmasks.nim @@ -15,6 +15,7 @@ const PageSize = 1 shl PageShift PageMask = PageSize-1 + MemAlign = # also minimal allocatable memory block when defined(useMalloc): when defined(amd64): 16 diff --git a/lib/system/dyncalls.nim b/lib/system/dyncalls.nim index c14fcf31e..36c2c5fe1 100644 --- a/lib/system/dyncalls.nim +++ b/lib/system/dyncalls.nim @@ -176,7 +176,7 @@ elif defined(genode): proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr = raiseAssert("nimGetProcAddr not implemented") -elif defined(nintendoswitch) or defined(freertos): +elif defined(nintendoswitch) or defined(freertos) or defined(zephyr): proc nimUnloadLibrary(lib: LibHandle) = cstderr.rawWrite("nimUnLoadLibrary not implemented") cstderr.rawWrite("\n") diff --git a/lib/system/io.nim b/lib/system/io.nim index 59f070f73..6f4accc2a 100644 --- a/lib/system/io.nim +++ b/lib/system/io.nim @@ -347,7 +347,7 @@ when defined(nimdoc) or (defined(posix) and not defined(nimscript)) or defined(w ## availability with `declared() `. when SupportIoctlInheritCtl: result = c_ioctl(f, if inheritable: FIONCLEX else: FIOCLEX) != -1 - elif defined(freertos): + elif defined(freertos) or defined(zephyr): result = true elif defined(posix): var flags = c_fcntl(f, F_GETFD) @@ -780,8 +780,11 @@ when declared(stdout): var echoLock: SysLock initSysLock echoLock - const stdOutLock = not defined(windows) and not defined(android) and - not defined(nintendoswitch) and not defined(freertos) and + const stdOutLock = not defined(windows) and + not defined(android) and + not defined(nintendoswitch) and + not defined(freertos) and + not defined(zephyr) and hostOS != "any" proc echoBinSafe(args: openArray[string]) {.compilerproc.} = diff --git a/lib/system/mm/malloc.nim b/lib/system/mm/malloc.nim index 2ba5c0fec..d41dce705 100644 --- a/lib/system/mm/malloc.nim +++ b/lib/system/mm/malloc.nim @@ -2,13 +2,22 @@ {.push stackTrace: off.} proc allocImpl(size: Natural): pointer = - c_malloc(size.csize_t) + result = c_malloc(size.csize_t) + when defined(zephyr): + if result == nil: + raiseOutOfMem() proc alloc0Impl(size: Natural): pointer = - c_calloc(size.csize_t, 1) + result = c_calloc(size.csize_t, 1) + when defined(zephyr): + if result == nil: + raiseOutOfMem() proc reallocImpl(p: pointer, newSize: Natural): pointer = - c_realloc(p, newSize.csize_t) + result = c_realloc(p, newSize.csize_t) + when defined(zephyr): + if result == nil: + raiseOutOfMem() proc realloc0Impl(p: pointer, oldsize, newSize: Natural): pointer = result = realloc(p, newSize.csize_t) diff --git a/tests/stdlib/tlwip.nim b/tests/stdlib/tlwip.nim index 7f7822236..fc53be592 100644 --- a/tests/stdlib/tlwip.nim +++ b/tests/stdlib/tlwip.nim @@ -1,6 +1,6 @@ discard """ targets: "c" - cmd: "nim $target --os:freertos --gc:arc $options $file" + cmd: "nim $target --compileOnly --os:freertos --gc:arc $options $file" disabled: "bsd" disabled: "windows" action: compile From 582468da1cc266d0dd922ec5c620713bbc45d02a Mon Sep 17 00:00:00 2001 From: Tail Wag Games Date: Mon, 25 Oct 2021 03:13:30 -0500 Subject: [PATCH 1308/2472] deinitializing locks at program exit (#19043) * deinitializing locks at program exit * deinitLock shouldn't be called for js backend I guess... * I suppose this is the best way to detect the ewruntime option * I guess I need these guards here too... * fixing merge conflict --- lib/std/exitprocs.nim | 3 +++ lib/system/io.nim | 56 ++++++++++++++++++++++++------------------- 2 files changed, 35 insertions(+), 24 deletions(-) diff --git a/lib/std/exitprocs.nim b/lib/std/exitprocs.nim index c6537f7f8..c76583a8c 100644 --- a/lib/std/exitprocs.nim +++ b/lib/std/exitprocs.nim @@ -44,6 +44,9 @@ proc callClosures() {.noconv.} = of kClosure: fun.fun1() of kNoconv: fun.fun2() + when not defined(js) and not defined(nimOwnedEnabled): + deinitLock(gFunsLock) + template fun() = if gFuns.len == 0: addAtExit(callClosures) diff --git a/lib/system/io.nim b/lib/system/io.nim index 6f4accc2a..e7369392a 100644 --- a/lib/system/io.nim +++ b/lib/system/io.nim @@ -20,19 +20,19 @@ type incompleteStruct.} = object File* = ptr CFile ## The type representing a file handle. - FileMode* = enum ## The file mode when opening a file. - fmRead, ## Open the file for read access only. - fmWrite, ## Open the file for write access only. - ## If the file does not exist, it will be - ## created. Existing files will be cleared! - fmReadWrite, ## Open the file for read and write access. - ## If the file does not exist, it will be - ## created. Existing files will be cleared! - fmReadWriteExisting, ## Open the file for read and write access. - ## If the file does not exist, it will not be - ## created. The existing file will not be cleared. - fmAppend ## Open the file for writing only; append data - ## at the end. + FileMode* = enum ## The file mode when opening a file. + fmRead, ## Open the file for read access only. + fmWrite, ## Open the file for write access only. + ## If the file does not exist, it will be + ## created. Existing files will be cleared! + fmReadWrite, ## Open the file for read and write access. + ## If the file does not exist, it will be + ## created. Existing files will be cleared! + fmReadWriteExisting, ## Open the file for read and write access. + ## If the file does not exist, it will not be + ## created. The existing file will not be cleared. + fmAppend ## Open the file for writing only; append data + ## at the end. FileHandle* = cint ## type that represents an OS file handle; this is ## useful for low-level file access @@ -40,7 +40,8 @@ type # text file handling: when not defined(nimscript) and not defined(js): # duplicated between io and ansi_c - const stdioUsesMacros = (defined(osx) or defined(freebsd) or defined(dragonfly)) and not defined(emscripten) + const stdioUsesMacros = (defined(osx) or defined(freebsd) or defined( + dragonfly)) and not defined(emscripten) const stderrName = when stdioUsesMacros: "__stderrp" else: "stderr" const stdoutName = when stdioUsesMacros: "__stdoutp" else: "stdout" const stdinName = when stdioUsesMacros: "__stdinp" else: "stdin" @@ -158,7 +159,7 @@ proc checkErr(f: File) = # shouldn't happen quit(1) -{.push stackTrace:off, profiler:off.} +{.push stackTrace: off, profiler: off.} proc readBuffer*(f: File, buffer: pointer, len: Natural): int {. tags: [ReadIOEffect], benign.} = ## reads `len` bytes into the buffer pointed to by `buffer`. Returns @@ -167,7 +168,8 @@ proc readBuffer*(f: File, buffer: pointer, len: Natural): int {. result = cast[int](c_fread(buffer, 1, cast[csize_t](len), f)) if result != len: checkErr(f) -proc readBytes*(f: File, a: var openArray[int8|uint8], start, len: Natural): int {. +proc readBytes*(f: File, a: var openArray[int8|uint8], start, + len: Natural): int {. tags: [ReadIOEffect], benign.} = ## reads `len` bytes into the buffer `a` starting at `a[start]`. Returns ## the actual number of bytes that have been read which may be less than @@ -418,8 +420,8 @@ proc readLine*(f: File, line: var string): bool {.tags: [ReadIOEffect], numberOfCharsRead -= 2 # handle Ctrl+Z as EOF for i in 0..", final, pure.} = object ## struct stat filler_1: array[24, char] - st_mode: Mode ## Mode of file + st_mode: Mode ## Mode of file filler_2: array[144 - 24 - 4, char] proc modeIsDir(m: Mode): bool = @@ -660,7 +662,7 @@ when defined(posix) and not defined(nimscript): Stat {.importc: "struct stat", header: "", final, pure.} = object ## struct stat - st_mode: Mode ## Mode of file + st_mode: Mode ## Mode of file proc modeIsDir(m: Mode): bool {.importc: "S_ISDIR", header: "".} ## Test for a directory. @@ -727,7 +729,8 @@ proc open*(f: var File, filehandle: FileHandle, ## ## The passed file handle will no longer be inheritable. when not defined(nimInheritHandles) and declared(setInheritable): - let oshandle = when defined(windows): FileHandle getOsfhandle(filehandle) else: filehandle + let oshandle = when defined(windows): FileHandle getOsfhandle( + filehandle) else: filehandle if not setInheritable(oshandle, false): return false f = c_fdopen(filehandle, FormatOpen[mode]) @@ -780,6 +783,10 @@ when declared(stdout): var echoLock: SysLock initSysLock echoLock + when not defined(js) and not defined(nimOwnedEnabled): + import std/exitprocs + addExitProc(proc() {.noconv.} = deinitSys echoLock) + const stdOutLock = not defined(windows) and not defined(android) and not defined(nintendoswitch) and @@ -898,7 +905,8 @@ proc readLines*(filename: string, n: Natural): seq[string] = else: sysFatal(IOError, "cannot open: " & filename) -template readLines*(filename: string): seq[string] {.deprecated: "use readLines with two arguments".} = +template readLines*(filename: string): seq[ + string] {.deprecated: "use readLines with two arguments".} = readLines(filename, 1) iterator lines*(filename: string): string {.tags: [ReadIOEffect].} = @@ -915,7 +923,7 @@ iterator lines*(filename: string): string {.tags: [ReadIOEffect].} = for line in filename.lines: buffer.add(line.replace("a", "0") & '\n') writeFile(filename, buffer) - var f = open(filename, bufSize=8000) + var f = open(filename, bufSize = 8000) try: var res = newStringOfCap(80) while f.readLine(res): yield res From 3ecb369ca1a12754486429bda4ab0bc60f9b86d7 Mon Sep 17 00:00:00 2001 From: quantimnot <54247259+quantimnot@users.noreply.github.com> Date: Tue, 26 Oct 2021 02:33:26 -0400 Subject: [PATCH 1309/2472] Document file changes that may be needed for add a new platform (#19055) Co-authored-by: quantimnot --- doc/intern.rst | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/doc/intern.rst b/doc/intern.rst index fe0d333fd..e8fb4ac86 100644 --- a/doc/intern.rst +++ b/doc/intern.rst @@ -207,6 +207,42 @@ implementation uses C's `setjmp`:c: function to store all registers on the hardware stack. It may be necessary that the new platform needs to replace this generic code by some assembler code. +Files that may need changed for your platform include: + +* `compiler/platform.nim` + Add os/cpu properties. +* `lib/system.nim` + Add os/cpu to the documentation for `system.hostOS` and `system.hostCPU`. +* `compiler/options.nim` + Add special os/cpu property checks in `isDefined`. +* `compiler/installer.ini` + Add os/cpu to `Project.Platforms` field. +* `lib/system/platforms.nim` + Add os/cpu. +* `lib/pure/include/osseps.nim` + Add os specializations. +* `lib/pure/distros.nim` + Add os, package handler. +* `tools/niminst/makefile.nimf` + Add os/cpu compiler/linker flags. +* `tools/niminst/buildsh.nimf` + Add os/cpu compiler/linker flags. + +If the `--os` or `--cpu` options aren't passed to the compiler, then Nim will +determine the current host os, cpu and endianess from `system.cpuEndian`, +`system.hostOS` and `system.hostCPU`. Those values are derived from +`compiler/platform.nim`. + +In order for the new platform to be bootstrapped from the `csources`, it must: + +* have `compiler/platform.nim` updated +* have `compiler/installer.ini` updated +* have `tools/niminst/buildsh.nimf` updated +* have `tools/niminst/makefile.nimf` updated +* be backported to the Nim version used by the `csources` +* the new `csources` must be pushed +* the new `csources` revision must be updated in `config/build_config.txt` + Runtime type information ======================== From 83a2515af7aeb9a1c12015321243399a0d1f4c95 Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Tue, 26 Oct 2021 03:27:11 -0600 Subject: [PATCH 1310/2472] Fixed generic distinct conversions for 'var' (#18837) * SameTypeAux now properly traverses generic distincts * Smarter traversal of distincts * Removed redundant check * Fixed nkConv for jsgen * Added test for non distinct nkConv * using skiptypes for distinct now * Fixed genaddr for nkconv --- compiler/jsgen.nim | 140 +++++++++++++---------- compiler/types.nim | 8 +- tests/converter/texplicit_conversion.nim | 6 + tests/distinct/tdistinct.nim | 17 +++ 4 files changed, 104 insertions(+), 67 deletions(-) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index ff4d2839e..7e8c25173 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -1288,75 +1288,89 @@ template isIndirect(x: PSym): bool = v.kind notin {skProc, skFunc, skConverter, skMethod, skIterator, skConst, skTemp, skLet}) -proc genAddr(p: PProc, n: PNode, r: var TCompRes) = - case n[0].kind - of nkSym: - let s = n[0].sym - if s.loc.r == nil: internalError(p.config, n.info, "genAddr: 3") - case s.kind - of skParam: - r.res = s.loc.r - r.address = nil - r.typ = etyNone - of skVar, skLet, skResult: - r.kind = resExpr - let jsType = mapType(p, n.typ) - if jsType == etyObject: - # make addr() a no-op: - r.typ = etyNone - if isIndirect(s): - r.res = s.loc.r & "[0]" - else: - r.res = s.loc.r - r.address = nil - elif {sfGlobal, sfAddrTaken} * s.flags != {} or jsType == etyBaseIndex: - # for ease of code generation, we do not distinguish between - # sfAddrTaken and sfGlobal. - r.typ = etyBaseIndex - r.address = s.loc.r - r.res = rope("0") +proc genSymAddr(p: PProc, n: PNode, typ: PType, r: var TCompRes) = + let s = n.sym + if s.loc.r == nil: internalError(p.config, n.info, "genAddr: 3") + case s.kind + of skParam: + r.res = s.loc.r + r.address = nil + r.typ = etyNone + of skVar, skLet, skResult: + r.kind = resExpr + let jsType = mapType(p): + if typ.isNil: + n.typ else: - # 'var openArray' for instance produces an 'addr' but this is harmless: - gen(p, n[0], r) - #internalError(p.config, n.info, "genAddr: 4 " & renderTree(n)) - else: internalError(p.config, n.info, $("genAddr: 2", s.kind)) - of nkCheckedFieldExpr: - genCheckedFieldOp(p, n[0], n.typ, r) - of nkDotExpr: - if mapType(p, n.typ) == etyBaseIndex: - genFieldAddr(p, n[0], r) + typ + if jsType == etyObject: + # make addr() a no-op: + r.typ = etyNone + if isIndirect(s): + r.res = s.loc.r & "[0]" + else: + r.res = s.loc.r + r.address = nil + elif {sfGlobal, sfAddrTaken} * s.flags != {} or jsType == etyBaseIndex: + # for ease of code generation, we do not distinguish between + # sfAddrTaken and sfGlobal. + r.typ = etyBaseIndex + r.address = s.loc.r + r.res = rope("0") else: - genFieldAccess(p, n[0], r) - of nkBracketExpr: - var ty = skipTypes(n[0].typ, abstractVarRange) - if ty.kind in MappedToObject: - gen(p, n[0], r) - else: - let kindOfIndexedExpr = skipTypes(n[0][0].typ, abstractVarRange).kind - case kindOfIndexedExpr - of tyArray, tyOpenArray, tySequence, tyString, tyCstring, tyVarargs: - genArrayAddr(p, n[0], r) - of tyTuple: - genFieldAddr(p, n[0], r) - else: internalError(p.config, n[0].info, "expr(nkBracketExpr, " & $kindOfIndexedExpr & ')') - of nkObjDownConv: - gen(p, n[0], r) - of nkHiddenDeref: - gen(p, n[0], r) - of nkHiddenAddr: - gen(p, n[0], r) - of nkStmtListExpr: - if n.len == 1: gen(p, n[0], r) - else: internalError(p.config, n[0].info, "genAddr for complex nkStmtListExpr") - of nkCallKinds: - if n[0].typ.kind == tyOpenArray: # 'var openArray' for instance produces an 'addr' but this is harmless: - # namely toOpenArray(a, 1, 3) + gen(p, n, r) + #internalError(p.config, n.info, "genAddr: 4 " & renderTree(n)) + else: internalError(p.config, n.info, $("genAddr: 2", s.kind)) + +proc genAddr(p: PProc, n: PNode, r: var TCompRes) = + if n.kind == nkSym: + genSymAddr(p, n, nil, r) + else: + case n[0].kind + of nkSym: + genSymAddr(p, n[0], n.typ, r) + of nkCheckedFieldExpr: + genCheckedFieldOp(p, n[0], n.typ, r) + of nkDotExpr: + if mapType(p, n.typ) == etyBaseIndex: + genFieldAddr(p, n[0], r) + else: + genFieldAccess(p, n[0], r) + of nkBracketExpr: + var ty = skipTypes(n[0].typ, abstractVarRange) + if ty.kind in MappedToObject: + gen(p, n[0], r) + else: + let kindOfIndexedExpr = skipTypes(n[0][0].typ, abstractVarRange).kind + case kindOfIndexedExpr + of tyArray, tyOpenArray, tySequence, tyString, tyCstring, tyVarargs: + genArrayAddr(p, n[0], r) + of tyTuple: + genFieldAddr(p, n[0], r) + of tyGenericBody: + genAddr(p, n[^1], r) + else: internalError(p.config, n[0].info, "expr(nkBracketExpr, " & $kindOfIndexedExpr & ')') + of nkObjDownConv: gen(p, n[0], r) + of nkHiddenDeref: + gen(p, n[0], r) + of nkHiddenAddr: + gen(p, n[0], r) + of nkConv: + genAddr(p, n[0], r) + of nkStmtListExpr: + if n.len == 1: gen(p, n[0], r) + else: internalError(p.config, n[0].info, "genAddr for complex nkStmtListExpr") + of nkCallKinds: + if n[0].typ.kind == tyOpenArray: + # 'var openArray' for instance produces an 'addr' but this is harmless: + # namely toOpenArray(a, 1, 3) + gen(p, n[0], r) + else: + internalError(p.config, n[0].info, "genAddr: " & $n[0].kind) else: internalError(p.config, n[0].info, "genAddr: " & $n[0].kind) - else: - internalError(p.config, n[0].info, "genAddr: " & $n[0].kind) proc attachProc(p: PProc; content: Rope; s: PSym) = p.g.code.add(content) diff --git a/compiler/types.nim b/compiler/types.nim index cf6579317..2c7b91d9f 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -1121,15 +1121,15 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool = case c.cmp of dcEq: return false of dcEqIgnoreDistinct: - while a.kind == tyDistinct: a = a[0] - while b.kind == tyDistinct: b = b[0] + a = a.skipTypes({tyDistinct, tyGenericInst}) + b = b.skipTypes({tyDistinct, tyGenericInst}) if a.kind != b.kind: return false of dcEqOrDistinctOf: - while a.kind == tyDistinct: a = a[0] + a = a.skipTypes({tyDistinct, tyGenericInst}) if a.kind != b.kind: return false # this is required by tunique_type but makes no sense really: - if x.kind == tyGenericInst and IgnoreTupleFields notin c.flags: + if tyDistinct notin {x.kind, y.kind} and x.kind == tyGenericInst and IgnoreTupleFields notin c.flags: let lhs = x.skipGenericAlias rhs = y.skipGenericAlias diff --git a/tests/converter/texplicit_conversion.nim b/tests/converter/texplicit_conversion.nim index 6b2e96faf..e36d78ad5 100644 --- a/tests/converter/texplicit_conversion.nim +++ b/tests/converter/texplicit_conversion.nim @@ -11,3 +11,9 @@ converter toInt(s: string): int = let x = (int)"234" echo x + +block: # Test for nkconv + proc foo(o: var int) = + assert o == 0 + var a = 0 + foo(int(a)) \ No newline at end of file diff --git a/tests/distinct/tdistinct.nim b/tests/distinct/tdistinct.nim index fd60b4ac0..dd8237854 100644 --- a/tests/distinct/tdistinct.nim +++ b/tests/distinct/tdistinct.nim @@ -8,6 +8,7 @@ false false false Foo +foo ''' """ @@ -140,6 +141,22 @@ block tRequiresInit: let s = "test" doAssert s == "test" +block: #17322 + type + A[T] = distinct string + + proc foo(a: var A) = + a.string.add "foo" + + type + B = distinct A[int] + + var b: B + foo(A[int](b)) + echo A[int](b).string + b.string.add "bar" + assert b.string == "foobar" + type Foo = distinct string template main() = From 8d5a27518929bd4c54f4beb7e40a5fc382d3dd05 Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Tue, 26 Oct 2021 03:29:07 -0600 Subject: [PATCH 1311/2472] Fixed distinct composite type class proc borrowing (#18904) * Fixed composite type class proc borrowing * Moved borrow search into transf * added borrow check to symbol flag --- compiler/semcall.nim | 13 +++++++++++-- compiler/seminst.nim | 6 +++++- compiler/transf.nim | 8 ++++++++ tests/borrow/typeclassborrow.nim | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 tests/borrow/typeclassborrow.nim diff --git a/compiler/semcall.nim b/compiler/semcall.nim index a3064788e..36658d472 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -703,7 +703,16 @@ proc searchForBorrowProc(c: PContext, startScope: PScope, fn: PSym): PSym = call.add(newIdentNode(fn.name, fn.info)) for i in 1.. Date: Tue, 26 Oct 2021 13:32:52 -0500 Subject: [PATCH 1312/2472] Fix #19052; [backport:1.6.0] (#19053) * Fix #19052; [backport:1.6.0] Adds a compile flag to avoid a getrandom syscall, fixing #19052. This is neccesary when the getrandom syscall is missing, as noted in #19052, particularly in kernel versions < 3.17 when getrandom was introduced. Specifically relevant is this is missing from kernel 3.10, which is the supported kernel throughout RHEL 7 and CentOS 7, which is widely used at many organizations. Without this, versions of nim that include sysrand (i.e. versions >= 1.6.0) will not compile without modification, however with this change a compile flag may be used to fall back using /dev/urandom as done with any unknown Posix OS (preferred here as a fallback since it already supplies a cryptographically secure PRNG and existing code deals with entropy pool init, etc). The change is placed behind a compile flag, as discussed in github ticket #19052 (summed up here): * First, I can't seem to catch that a importc such as SYS_getrandom is declared without using it (the declared proc returns true, but compiler throws an undeclared identifier flag when referencing it). * Second, it seemed preferable to be behaviorally explicit vs implicit when considering this is intended to be a cryptographically secure PRNG. * Third, if I intend to compile on a kernel >= 3.17 while running the binary on at least one system < 3.17, I'll want to be able to target this without relying on a compile time determination if the getrandom syscall is available. * Documenting compile flag for -d:nimNoGetRandom and adding changelog entry Related to #19052 and comments in PR #19053. Also created a new changelog file since none currently exists. Co-authored-by: Timothy Alexander --- changelogs/changelog.md | 31 +++++++++++++++++++++++++++++++ lib/std/sysrand.nim | 7 ++++++- 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 changelogs/changelog.md diff --git a/changelogs/changelog.md b/changelogs/changelog.md new file mode 100644 index 000000000..475f568ec --- /dev/null +++ b/changelogs/changelog.md @@ -0,0 +1,31 @@ +# v1.xx.x - yyyy-mm-dd + +## Changes affecting backward compatibility + +## Standard library additions and changes + +### New compile flag (`-d:nimNoGetRandom`) when building `std/sysrand` to remove dependency on linux `getrandom` syscall + +This compile flag only affects linux builds and is necessary if either compiling on a linux kernel version < 3.17, or if code built will be executing on kernel < 3.17. + +On linux kernels < 3.17 (such as kernel 3.10 in RHEL7 and CentOS7), the `getrandom` syscall was not yet introduced. Without this, the `std/sysrand` module will not build properly, and if code is built on a kernel >= 3.17 without the flag, any usage of the `std/sysrand` module will fail to execute on a kernel < 3.17 (since it attempts to perform a syscall to `getrandom`, which isn't present in the current kernel). A compile flag has been added to force the `std/sysrand` module to use /dev/urandom (available since linux kernel 1.3.30), rather than the `getrandom` syscall. This allows for use of a cryptographically secure PRNG, regardless of kernel support for the `getrandom` syscall. + +When building for RHEL7/CentOS7 for example, the entire build process for nim from a source package would then be: +```sh +$ yum install devtoolset-8 # Install GCC version 8 vs the standard 4.8.5 on RHEL7/CentOS7. Alternatively use -d:nimEmulateOverflowChecks. See issue #13692 for details +$ scl enable devtoolset-8 bash # Run bash shell with default toolchain of gcc 8 +$ sh build.sh # per unix install instructions +$ bin/nim c koch # per unix install instructions +$ ./koch boot -d:release # per unix install instructions +$ ./koch tools -d:nimNoGetRandom # pass the nimNoGetRandom flag to compile std/sysrand without support for getrandom syscall +``` + +This is necessary to pass when building nim on kernel versions < 3.17 in particular to avoid an error of "SYS_getrandom undeclared" during the build process for stdlib (sysrand in particular). + +## Language changes + + +## Compiler changes + + +## Tool changes diff --git a/lib/std/sysrand.nim b/lib/std/sysrand.nim index 75983e3c6..b35f24a72 100644 --- a/lib/std/sysrand.nim +++ b/lib/std/sysrand.nim @@ -38,6 +38,11 @@ ## .. _randomFillSync: https://nodejs.org/api/crypto.html#crypto_crypto_randomfillsync_buffer_offset_size ## .. _/dev/urandom: https://en.wikipedia.org/wiki//dev/random ## +## On a Linux target, a call to the `getrandom` syscall can be avoided (e.g. +## for targets running kernel version < 3.17) by passing a compile flag of +## `-d:nimNoGetRandom`. If this flag is passed, sysrand will use `/dev/urandom` +## as with any other POSIX compliant OS. +## runnableExamples: doAssert urandom(0).len == 0 @@ -159,7 +164,7 @@ elif defined(windows): result = randomBytes(addr dest[0], size) -elif defined(linux): +elif defined(linux) and not defined(nimNoGetRandom): # TODO using let, pending bootstrap >= 1.4.0 var SYS_getrandom {.importc: "SYS_getrandom", header: "".}: clong const syscallHeader = """#include From f50bcf82c3e45921961ee4145bac88701fec62b0 Mon Sep 17 00:00:00 2001 From: Netsu <52031254+WeebNetsu@users.noreply.github.com> Date: Tue, 26 Oct 2021 21:42:22 +0200 Subject: [PATCH 1313/2472] Minor update to terminal docs (#19056) * Update terminal.nim - Added some extra docs to cursorUp/Down/Forward/Backward - I was able to use hideCursor and showCursor without adding stdout, removed the parameter - Added docs to terminalHeight()* and terminalWidth()* * Update lib/pure/terminal.nim Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Update lib/pure/terminal.nim Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Added back f: file to cursor movement * Removed unnecessary comments Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- lib/pure/terminal.nim | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index 25b1a4cdd..c9aafc037 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -170,6 +170,7 @@ when defined(windows): return 0 proc terminalWidth*(): int = + ## Returns the terminal width in columns. var w: int = 0 w = terminalWidthIoctl([getStdHandle(STD_INPUT_HANDLE), getStdHandle(STD_OUTPUT_HANDLE), @@ -178,6 +179,7 @@ when defined(windows): return 80 proc terminalHeight*(): int = + ## Returns the terminal height in rows. var h: int = 0 h = terminalHeightIoctl([getStdHandle(STD_INPUT_HANDLE), getStdHandle(STD_OUTPUT_HANDLE), @@ -389,6 +391,9 @@ when defined(windows): proc cursorUp*(f: File, count = 1) = ## Moves the cursor up by `count` rows. + runnableExamples("-r:off"): + stdout.cursorUp(2) + write(stdout, "Hello World!") # anything written at that location will be erased/replaced with this when defined(windows): let h = conHandle(f) var p = getCursorPos(h) @@ -399,6 +404,9 @@ proc cursorUp*(f: File, count = 1) = proc cursorDown*(f: File, count = 1) = ## Moves the cursor down by `count` rows. + runnableExamples("-r:off"): + stdout.cursorDown(2) + write(stdout, "Hello World!") # anything written at that location will be erased/replaced with this when defined(windows): let h = conHandle(f) var p = getCursorPos(h) @@ -409,6 +417,9 @@ proc cursorDown*(f: File, count = 1) = proc cursorForward*(f: File, count = 1) = ## Moves the cursor forward by `count` columns. + runnableExamples("-r:off"): + stdout.cursorForward(2) + write(stdout, "Hello World!") # anything written at that location will be erased/replaced with this when defined(windows): let h = conHandle(f) var p = getCursorPos(h) @@ -419,6 +430,9 @@ proc cursorForward*(f: File, count = 1) = proc cursorBackward*(f: File, count = 1) = ## Moves the cursor backward by `count` columns. + runnableExamples("-r:off"): + stdout.cursorBackward(2) + write(stdout, "Hello World!") # anything written at that location will be erased/replaced with this when defined(windows): let h = conHandle(f) var p = getCursorPos(h) From ee65767d44cca79ce2677564151be69a60f6a94d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Derek=20=E5=91=86?= <116649+derekdai@users.noreply.github.com> Date: Wed, 27 Oct 2021 17:45:05 +0800 Subject: [PATCH 1314/2472] fix a tiny formating issue in doc/destructors.rst (#19058) --- doc/destructors.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/destructors.rst b/doc/destructors.rst index 195dc8f97..b4575b2e2 100644 --- a/doc/destructors.rst +++ b/doc/destructors.rst @@ -653,7 +653,7 @@ The ability to override a hook leads to a phase ordering problem: discard -The solution is to define `proc `=destroy`[T](f: var Foo[T])` before +The solution is to define ````proc `=destroy`[T](f: var Foo[T])```` before it is used. The compiler generates implicit hooks for all types in *strategic places* so that an explicitly provided hook that comes too "late" can be detected reliably. These *strategic places* From cd2b093d07303aa7bfcb4a007281f56de3755110 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Derek=20=E5=91=86?= <116649+derekdai@users.noreply.github.com> Date: Thu, 28 Oct 2021 15:35:57 +0800 Subject: [PATCH 1315/2472] fix a tiny code snippet formatting issue in `doc/constructors.rst`, again (#19065) --- doc/destructors.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/destructors.rst b/doc/destructors.rst index b4575b2e2..6e4509666 100644 --- a/doc/destructors.rst +++ b/doc/destructors.rst @@ -653,7 +653,7 @@ The ability to override a hook leads to a phase ordering problem: discard -The solution is to define ````proc `=destroy`[T](f: var Foo[T])```` before +The solution is to define ``proc `=destroy`[T](f: var Foo[T])`` before it is used. The compiler generates implicit hooks for all types in *strategic places* so that an explicitly provided hook that comes too "late" can be detected reliably. These *strategic places* From c80e2c173686bd12904e5487752dc0ce20cb8bcb Mon Sep 17 00:00:00 2001 From: ynfle <23086821+ynfle@users.noreply.github.com> Date: Thu, 28 Oct 2021 11:02:18 +0300 Subject: [PATCH 1316/2472] Docs: change clipboard cursor type to `pointer` (#19064) --- tools/dochack/dochack.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/dochack/dochack.nim b/tools/dochack/dochack.nim index 83e81f1c0..3a663808d 100644 --- a/tools/dochack/dochack.nim +++ b/tools/dochack/dochack.nim @@ -360,6 +360,7 @@ proc copyToClipboard*() {.exportc.} = const button = document.createElement("button") button.value = e.textContent.replace('...', '') button.classList.add("copyToClipBoardBtn") + button.style = "cursor: pointer" div.appendChild(preTag) div.appendChild(button) @@ -396,4 +397,4 @@ proc copyToClipboard*() {.exportc.} = """ .} -copyToClipboard() \ No newline at end of file +copyToClipboard() From 7ba2659f733b97db63b7552415ad048e34d4a11a Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Thu, 28 Oct 2021 20:20:52 +0300 Subject: [PATCH 1317/2472] docgen: implement doc link resolution in current module (#18642) --- compiler/docgen.nim | 154 ++++++++- compiler/lineinfos.nim | 2 + compiler/typesrenderer.nim | 66 +++- config/nimdoc.cfg | 11 + config/nimdoc.tex.cfg | 10 +- doc/docgen.rst | 170 ++++++++++ doc/nimdoc.css | 4 + lib/packages/docutils/dochelpers.nim | 267 +++++++++++++++ lib/packages/docutils/rst.nim | 319 ++++++++++++++---- lib/packages/docutils/rstast.nim | 3 + lib/packages/docutils/rstgen.nim | 27 +- lib/pure/strutils.nim | 5 + .../expected/index.html | 4 + nimdoc/testproject/expected/nimdoc.out.css | 4 + .../expected/subdir/subdir_b/utils.html | 297 +++++++++++++++- .../expected/subdir/subdir_b/utils.idx | 14 + nimdoc/testproject/expected/testproject.html | 210 +++++++++++- nimdoc/testproject/expected/theindex.html | 52 ++- nimdoc/testproject/subdir/subdir_b/utils.nim | 65 ++++ .../subdir/subdir_b/utils_helpers.nim | 1 + .../subdir/subdir_b/utils_overview.rst | 8 + tests/stdlib/tdochelpers.nim | 155 +++++++++ tests/stdlib/trst.nim | 109 +++++- tests/stdlib/trstgen.nim | 19 +- 24 files changed, 1833 insertions(+), 143 deletions(-) create mode 100644 lib/packages/docutils/dochelpers.nim create mode 100644 nimdoc/testproject/subdir/subdir_b/utils_helpers.nim create mode 100644 nimdoc/testproject/subdir/subdir_b/utils_overview.rst create mode 100644 tests/stdlib/tdochelpers.nim diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 1acfc7489..2639840d1 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -13,7 +13,7 @@ import ast, strutils, strtabs, algorithm, sequtils, options, msgs, os, idents, wordrecg, syntaxes, renderer, lexer, - packages/docutils/rst, packages/docutils/rstgen, + packages/docutils/[rst, rstgen, dochelpers], json, xmltree, trees, types, typesrenderer, astalgo, lineinfos, intsets, pathutils, tables, nimpaths, renderverbatim, osproc @@ -44,8 +44,14 @@ type ## runnableExamples). substitutions: seq[string] ## Variable names in `doc.item`... sortName: string ## The string used for sorting in output + info: rstast.TLineInfo ## place where symbol was defined (for messages) + anchor: string ## e.g. HTML anchor + name: string ## short name of the symbol, not unique + ## (includes backticks ` if present) + detailedName: string ## longer name like `proc search(x: int): int` ModSection = object ## Section like Procs, Types, etc. - secItems: seq[Item] ## Pre-processed items. + secItems: Table[string, seq[Item]] + ## Map basic name -> pre-processed items. finalMarkup: string ## The items, after RST pass 2 and rendering. ModSections = array[TSymKind, ModSection] TocItem = object ## HTML TOC item @@ -91,12 +97,22 @@ type thisDir*: AbsoluteDir exampleGroups: OrderedTable[string, ExampleGroup] wroteSupportFiles*: bool + nimToRstFid: Table[lineinfos.FileIndex, rstast.FileIndex] + ## map Nim FileIndex -> RST one, it's needed because we keep them separate PDoc* = ref TDocumentor ## Alias to type less. proc add(dest: var ItemPre, rst: PRstNode) = dest.add ItemFragment(isRst: true, rst: rst) proc add(dest: var ItemPre, str: string) = dest.add ItemFragment(isRst: false, str: str) +proc addRstFileIndex(d: PDoc, info: lineinfos.TLineInfo): rstast.FileIndex = + let invalid = rstast.FileIndex(-1) + result = d.nimToRstFid.getOrDefault(info.fileIndex, default = invalid) + if result == invalid: + let fname = toFullPath(d.conf, info) + result = addFilename(d.sharedState, fname) + d.nimToRstFid[info.fileIndex] = result + proc cmpDecimalsIgnoreCase(a, b: string): int = ## For sorting with correct handling of cases like 'uint8' and 'uint16'. ## Also handles leading zeros well (however note that leading zeros are @@ -223,6 +239,7 @@ template declareClosures = of meFootnoteMismatch: k = errRstFootnoteMismatch of mwRedefinitionOfLabel: k = warnRstRedefinitionOfLabel of mwUnknownSubstitution: k = warnRstUnknownSubstitutionX + of mwAmbiguousLink: k = warnRstAmbiguousLink of mwBrokenLink: k = warnRstBrokenLink of mwUnsupportedLanguage: k = warnRstLanguageXNotSupported of mwUnsupportedField: k = warnRstFieldXNotSupported @@ -236,7 +253,7 @@ template declareClosures = result = getCurrentDir() / s if not fileExists(result): result = "" -proc parseRst(text, filename: string, +proc parseRst(text: string, line, column: int, conf: ConfigRef, sharedState: PRstSharedState): PRstNode = declareClosures() @@ -352,7 +369,8 @@ proc getVarIdx(varnames: openArray[string], id: string): int = proc genComment(d: PDoc, n: PNode): PRstNode = if n.comment.len > 0: - result = parseRst(n.comment, toFullPath(d.conf, n.info), + d.sharedState.currFileIdx = addRstFileIndex(d, n.info) + result = parseRst(n.comment, toLinenumber(n.info), toColumn(n.info) + DocColOffset, d.conf, d.sharedState) @@ -885,6 +903,57 @@ proc genSeeSrc(d: PDoc, path: string, line: int): string = "path", path.string, "line", $line, "url", gitUrl, "commit", commit, "devel", develBranch]]) +proc symbolPriority(k: TSymKind): int = + result = case k + of skMacro: -3 + of skTemplate: -2 + of skIterator: -1 + else: 0 # including skProc which have higher priority + # documentation itself has even higher priority 1 + +proc toLangSymbol(k: TSymKind, n: PNode, baseName: string): LangSymbol = + ## Converts symbol info (names/types/parameters) in `n` into format + ## `LangSymbol` convenient for ``rst.nim``/``dochelpers.nim``. + result.name = baseName.nimIdentNormalize + result.symKind = k.toHumanStr + if k in routineKinds: + var + paramTypes: seq[string] + renderParamTypes(paramTypes, n[paramsPos], toNormalize=true) + let paramNames = renderParamNames(n[paramsPos], toNormalize=true) + # In some rare cases (system.typeof) parameter type is not set for default: + doAssert paramTypes.len <= paramNames.len + for i in 0 ..< paramNames.len: + if i < paramTypes.len: + result.parameters.add (paramNames[i], paramTypes[i]) + else: + result.parameters.add (paramNames[i], "") + result.parametersProvided = true + + result.outType = renderOutType(n[paramsPos], toNormalize=true) + + if k in {skProc, skFunc, skType, skIterator}: + # Obtain `result.generics` + # Use `n[miscPos]` since n[genericParamsPos] does not contain constraints + var genNode: PNode = nil + if k == skType: + genNode = n[1] # FIXME: what is index 1? + else: + if n[miscPos].kind != nkEmpty: + genNode = n[miscPos][1] # FIXME: what is index 1? + if genNode != nil: + var literal = "" + var r: TSrcGen + initTokRender(r, genNode, {renderNoBody, renderNoComments, + renderNoPragmas, renderNoProcDefs}) + var kind = tkEof + while true: + getNextTok(r, kind, literal) + if kind == tkEof: + break + if kind != tkSpaces: + result.generics.add(literal.nimIdentNormalize) + proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) = if (docFlags != kForceExport) and not isVisible(d, nameNode): return let @@ -915,6 +984,8 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) = inc(d.id) let plainNameEsc = esc(d.target, plainName.strip) + detailedName = k.toHumanStr & " " & ( + if k in routineKinds: plainName else: name) uniqueName = if k in routineKinds: plainNameEsc else: name sortName = if k in routineKinds: plainName.strip else: name cleanPlainSymbol = renderPlainSymbolName(nameNode) @@ -923,20 +994,32 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) = symbolOrId = d.newUniquePlainSymbol(complexSymbol) symbolOrIdEnc = encodeUrl(symbolOrId, usePlus = false) deprecationMsg = genDeprecationMsg(d, pragmaNode) + rstLangSymbol = toLangSymbol(k, n, cleanPlainSymbol) + + # we generate anchors automatically for subsequent use in doc comments + let lineinfo = rstast.TLineInfo( + line: nameNode.info.line, col: nameNode.info.col, + fileIndex: addRstFileIndex(d, nameNode.info)) + addAnchorNim(d.sharedState, refn = symbolOrId, tooltip = detailedName, + rstLangSymbol, priority = symbolPriority(k), info = lineinfo) nodeToHighlightedHtml(d, n, result, {renderNoBody, renderNoComments, renderDocComments, renderSyms}, symbolOrIdEnc) let seeSrc = genSeeSrc(d, toFullPath(d.conf, n.info), n.info.line.int) - d.section[k].secItems.add Item( + d.section[k].secItems.mgetOrPut(cleanPlainSymbol, newSeq[Item]()).add Item( descRst: comm, sortName: sortName, + info: lineinfo, + anchor: symbolOrId, + detailedName: detailedName, + name: name, substitutions: @[ - "name", name, "uniqueName", uniqueName, + "uniqueName", uniqueName, "header", result, "itemID", $d.id, "header_plain", plainNameEsc, "itemSym", cleanPlainSymbol, - "itemSymOrID", symbolOrId, "itemSymEnc", plainSymbolEnc, + "itemSymEnc", plainSymbolEnc, "itemSymOrIDEnc", symbolOrIdEnc, "seeSrc", seeSrc, "deprecationMsg", deprecationMsg]) @@ -1184,6 +1267,11 @@ proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) = if comm.len != 0: d.modDescPre.add(comm) else: discard +proc overloadGroupName(s: string, k: TSymKind): string = + ## Turns a name like `f` into anchor `f-procs-all` + #s & " " & k.toHumanStr & "s all" + s & "-" & k.toHumanStr & "s-all" + proc finishGenerateDoc*(d: var PDoc) = ## Perform 2nd RST pass for resolution of links/footnotes/headings... # copy file map `filenames` to ``rstgen.nim`` for its warnings @@ -1197,6 +1285,21 @@ proc finishGenerateDoc*(d: var PDoc) = break preparePass2(d.sharedState, firstRst) + # add anchors to overload groups before RST resolution + for k in TSymKind: + if k in routineKinds: + for plainName, overloadChoices in d.section[k].secItems: + if overloadChoices.len > 1: + let refn = overloadGroupName(plainName, k) + let tooltip = "$1 ($2 overloads)" % [ + k.toHumanStr & " " & plainName, $overloadChoices.len] + addAnchorNim(d.sharedState, refn, tooltip, + LangSymbol(symKind: k.toHumanStr, name: plainName, + isGroup: true), + priority = symbolPriority(k), + # select index `0` just to have any meaningful warning: + info = overloadChoices[0].info) + # Finalize fragments of ``.nim`` or ``.rst`` file proc renderItemPre(d: PDoc, fragments: ItemPre, result: var string) = for f in fragments: @@ -1207,14 +1310,33 @@ proc finishGenerateDoc*(d: var PDoc) = of false: result &= f.str proc cmp(x, y: Item): int = cmpDecimalsIgnoreCase(x.sortName, y.sortName) for k in TSymKind: - for item in d.section[k].secItems.sorted(cmp): - var itemDesc: string - renderItemPre(d, item.descRst, itemDesc) - d.section[k].finalMarkup.add( - getConfigVar(d.conf, "doc.item") % ( - item.substitutions & @["desc", itemDesc])) - itemDesc = "" - d.section[k].secItems.setLen 0 + # add symbols to section for each `k`, while optionally wrapping + # overloadable items with the same basic name by ``doc.item2`` + let overloadableNames = toSeq(keys(d.section[k].secItems)) + for plainName in overloadableNames.sorted(cmpDecimalsIgnoreCase): + var overloadChoices = d.section[k].secItems[plainName] + overloadChoices.sort(cmp) + var nameContent = "" + for item in overloadChoices: + var itemDesc: string + renderItemPre(d, item.descRst, itemDesc) + nameContent.add( + getConfigVar(d.conf, "doc.item") % ( + item.substitutions & @[ + "desc", itemDesc, + "name", item.name, + "itemSymOrID", item.anchor])) + if k in routineKinds: + let plainNameEsc1 = esc(d.target, plainName.strip) + let plainNameEsc2 = esc(d.target, plainName.strip, escMode=emUrl) + d.section[k].finalMarkup.add( + getConfigVar(d.conf, "doc.item2") % ( + @["header_plain", plainNameEsc1, + "overloadGroupName", overloadGroupName(plainNameEsc2, k), + "content", nameContent])) + else: + d.section[k].finalMarkup.add(nameContent) + d.section[k].secItems.clear renderItemPre(d, d.modDescPre, d.modDescFinal) d.modDescPre.setLen 0 d.hasToc = d.hasToc or d.sharedState.hasToc @@ -1493,7 +1615,7 @@ proc commandRstAux(cache: IdentCache, conf: ConfigRef; filename: AbsoluteFile, outExt: string) = var filen = addFileExt(filename, "txt") var d = newDocumentor(filen, cache, conf, outExt, isPureRst = true) - let rst = parseRst(readFile(filen.string), filen.string, + let rst = parseRst(readFile(filen.string), line=LineRstInit, column=ColRstInit, conf, d.sharedState) d.modDescPre = @[ItemFragment(isRst: true, rst: rst)] diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index 8bd5a0890..e13387be6 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -50,6 +50,7 @@ type warnSmallLshouldNotBeUsed = "SmallLshouldNotBeUsed", warnUnknownMagic = "UnknownMagic", warnRstRedefinitionOfLabel = "RedefinitionOfLabel", warnRstUnknownSubstitutionX = "UnknownSubstitutionX", + warnRstAmbiguousLink = "AmbiguousLink", warnRstBrokenLink = "BrokenLink", warnRstLanguageXNotSupported = "LanguageXNotSupported", warnRstFieldXNotSupported = "FieldXNotSupported", @@ -123,6 +124,7 @@ const warnUnknownMagic: "unknown magic '$1' might crash the compiler", warnRstRedefinitionOfLabel: "redefinition of label '$1'", warnRstUnknownSubstitutionX: "unknown substitution '$1'", + warnRstAmbiguousLink: "ambiguous doc link $1", warnRstBrokenLink: "broken link '$1'", warnRstLanguageXNotSupported: "language '$1' not supported", warnRstFieldXNotSupported: "field '$1' not supported", diff --git a/compiler/typesrenderer.nim b/compiler/typesrenderer.nim index 0da05d70d..c81ff283e 100644 --- a/compiler/typesrenderer.nim +++ b/compiler/typesrenderer.nim @@ -11,6 +11,12 @@ import renderer, strutils, ast, types const defaultParamSeparator* = "," +template mayNormalize(s: string): string = + if toNormalize: + s.nimIdentNormalize + else: + s + proc renderPlainSymbolName*(n: PNode): string = ## Returns the first non '*' nkIdent node from the tree. ## @@ -30,24 +36,26 @@ proc renderPlainSymbolName*(n: PNode): string = result = "" #internalError(n.info, "renderPlainSymbolName() with " & $n.kind) -proc renderType(n: PNode): string = +proc renderType(n: PNode, toNormalize: bool): string = ## Returns a string with the node type or the empty string. + ## This proc should be kept in sync with `toLangSymbols` from + ## ``lib/packages/docutils/dochelpers.nim``. case n.kind: - of nkIdent: result = n.ident.s - of nkSym: result = typeToString(n.sym.typ) + of nkIdent: result = mayNormalize(n.ident.s) + of nkSym: result = mayNormalize(typeToString(n.sym.typ)) of nkVarTy: if n.len == 1: - result = renderType(n[0]) + result = renderType(n[0], toNormalize) else: result = "var" of nkRefTy: if n.len == 1: - result = "ref." & renderType(n[0]) + result = "ref." & renderType(n[0], toNormalize) else: result = "ref" of nkPtrTy: if n.len == 1: - result = "ptr." & renderType(n[0]) + result = "ptr." & renderType(n[0], toNormalize) else: result = "ptr" of nkProcTy: @@ -57,36 +65,53 @@ proc renderType(n: PNode): string = assert params.kind == nkFormalParams assert params.len > 0 result = "proc(" - for i in 1..= 3 let typePos = n.len - 2 - let typeStr = renderType(n[typePos]) + let typeStr = renderType(n[typePos], toNormalize) result = typeStr for i in 1..= 2 - result = renderType(n[0]) & '[' - for i in 1.. 1: result.add ", " - result.add(renderType(n[i])) + result.add(renderType(n[i], toNormalize)) else: result = "" -proc renderParamTypes(found: var seq[string], n: PNode) = +proc renderParamNames*(n: PNode, toNormalize=false): seq[string] = + ## Returns parameter names of routine `n`. + doAssert n.kind == nkFormalParams + case n.kind + of nkFormalParams: + for i in 1.. 0 - var typeStr = renderType(n[typePos]) + var typeStr = renderType(n[typePos], toNormalize) if typeStr.len < 1 and n[typePos+1].kind != nkEmpty: # Try with the last node, maybe its a default value. let typ = n[typePos+1].typ @@ -111,7 +136,8 @@ proc renderParamTypes(found: var seq[string], n: PNode) = found.add($n) #internalError(n.info, "renderParamTypes(found,n) with " & $n.kind) -proc renderParamTypes*(n: PNode, sep = defaultParamSeparator): string = +proc renderParamTypes*(n: PNode, sep = defaultParamSeparator, + toNormalize=false): string = ## Returns the types contained in `n` joined by `sep`. ## ## This proc expects to be passed as `n` the parameters of any callable. The @@ -120,6 +146,10 @@ proc renderParamTypes*(n: PNode, sep = defaultParamSeparator): string = ## other characters may appear too, like ``[]`` or ``|``. result = "" var found: seq[string] = @[] - renderParamTypes(found, n) + renderParamTypes(found, n, toNormalize) if found.len > 0: result = found.join(sep) + +proc renderOutType*(n: PNode, toNormalize=false): string = + assert n.kind == nkFormalParams + result = renderType(n[0], toNormalize) diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index 162b2b4a3..c1074f344 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -56,6 +56,17 @@ $seeSrc
    """ +# A wrapper of a few overloaded `doc.item`s with the same basic name +# * $header_plain - see above +# * $overloadGroupName - the anchor for this whole group +# * $content - string containing `doc.item`s themselves +doc.item2 = """ + +
    +$content +
    +""" + # Chunk of HTML emitted for each entry in the HTML table of contents. # See doc.item for available substitution variables. diff --git a/config/nimdoc.tex.cfg b/config/nimdoc.tex.cfg index 4aff9b379..d7ab652eb 100644 --- a/config/nimdoc.tex.cfg +++ b/config/nimdoc.tex.cfg @@ -19,7 +19,8 @@ doc.section.toc = "" doc.item = """ \vspace{1em} -\phantomsection\addcontentsline{toc}{subsection}{$uniqueName} +\phantomsection\addcontentsline{toc}{subsubsection}{$uniqueName} +\label{$itemSymOrID}\hypertarget{$itemSymOrID}{} \begin{rstdocitem} $header @@ -30,6 +31,13 @@ $desc \end{addmargin} """ +doc.item2 = """ +\phantomsection\addcontentsline{toc}{subsection}{$header_plain} +\label{$overloadGroupName}\hypertarget{$overloadGroupName}{} + +$content +""" + doc.item.toc = "" doc.toc = r"\tableofcontents \newpage" diff --git a/doc/docgen.rst b/doc/docgen.rst index 2074ee06f..51e7102eb 100644 --- a/doc/docgen.rst +++ b/doc/docgen.rst @@ -229,6 +229,176 @@ Output:: Note that the `jsondoc`:option: command outputs its JSON without pretty-printing it, while `jsondoc0`:option: outputs pretty-printed JSON. + +Referencing Nim symbols: simple documentation links +=================================================== + +You can reference Nim identifiers from Nim documentation comments, currently +only inside their ``.nim`` file (or inside a ``.rst`` file included from +a ``.nim``). The point is that such links will be resolved automatically +by `nim doc`:cmd: (or `nim jsondoc`:cmd: or `nim doc2tex`:cmd:). +This pertains to any exported symbol like `proc`, `const`, `iterator`, etc. +Syntax for referencing is basically a normal RST one: addition of +underscore `_` to a *link text*. +Link text is either one word or a group of words enclosed by backticks `\`` +(for a one word case backticks are usually omitted). +Link text will be displayed *as is* while *link target* will be set to +the anchor [*]_ of Nim symbol that corresponds to link text. + +.. [*] anchors' format is described in `HTML anchor generation`_ section below. + +If you have a constant: + +.. code:: Nim + + const pi* = 3.14 + +then it should be referenced in one of the 2 forms: + +A. non-qualified (no symbol kind specification):: + pi_ +B. qualified (with symbol kind specification):: + `const pi`_ + +For routine kinds there are more options. Consider this definition: + +.. code:: Nim + + proc foo*(a: int, b: float): string + +Generally following syntax is allowed for referencing `foo`: + +* short (without parameters): + + A. non-qualified:: + + foo_ + + B. qualified:: + + `proc foo`_ + +* longer variants (with parameters): + + A. non-qualified: + + 1) specifying parameters names:: + + `foo(a, b)`_ + + 2) specifying parameters types:: + + `foo(int, float)`_ + + 3) specifying both names and types:: + + `foo(a: int, b: float)`_ + + 4) output parameter can also be specified if you wish:: + + `foo(a: int, b: float): string`_ + + B. qualified: all 4 options above are valid. + Particularly you can use the full format:: + + `proc foo(a: int, b: float): string`_ + +.. Tip:: Avoid cluttering your text with extraneous information by using + one of shorter forms:: + + binarySearch_ + `binarySearch(a, key, cmp)`_ + + Brevity is better for reading! If you use a short form and have an + ambiguity problem (see below) then just add some additional info. + +Symbol kind like `proc` can also be specified in the postfix form:: + + `foo proc`_ + `walkDir(d: string) iterator`_ + +.. Warning:: An ambiguity in resolving documentation links may arise because of: + + 1. clash with other RST anchors + * manually setup anchors + * automatically set up, e.g. section names + 2. collision with other Nim symbols: + + * routines with different parameters can exist e.g. for + `proc` and `template`. In this case they are split between their + corresponding sections in output file. Qualified references are + useful in this case -- just disambiguate by referring to these + sections explicitly:: + + See `foo proc`_ and `foo template`_. + + * because in Nim `proc` and `iterator` belong to different namespaces, + so there can be a collision even if parameters are the same. + Use `\`proc foo\`_`:literal: or `\`iterator foo\`_`:literal: then. + + Any ambiguity is always reported with Nim compiler warnings and an anchor + with higher priority is selected. Manual anchors have highest + priority, then go automatic RST anchors; then Nim-generated anchors + (while procs have higher priority than other Nim symbol kinds). + +Generic parameters can also be used. All in all, this long form will be +recognized fine:: + + `proc binarySearch*[T; K](a: openArray[T], key: K, cmp: proc(T, K)): int`_ + +**Limitations**: + +1. The parameters of a nested routine type can be specified only with types + (without parameter names, see form A.2 above). + E.g. for this signature: + + .. code:: Nim + + proc binarySearch*[T, K](a: openArray[T]; key: K; + cmp: proc (x: T; y: K): int {.closure.}): int + ~~ ~~ ~~~~~ + + you cannot use names underlined by `~~` so it must be referenced with + ``cmp: proc(T, K)``. Hence these forms are valid:: + + `binarySearch(a: openArray[T], key: K, cmp: proc(T, K))`_ + `binarySearch(openArray[T], K, proc(T, K))`_ + `binarySearch(a, key, cmp)`_ +2. Default values in routine parameters are not recognized, one needs to + specify the type and/or name instead. E.g. for referencing `proc f(x = 7)` + use one of the mentioned forms:: + + `f(int)`_ or `f(x)`_ or `f(x: int)`_. +3. Generic parameters must be given the same way as in the + definition of referenced symbol. + + * their names should be the same + * parameters list should be given the same way, e.g. without substitutions + between commas (,) and semicolons (;). + +.. Note:: A bit special case is operators + (as their signature is also defined with `\``): + + .. code:: Nim + + func `$`(x: MyType): string + func `[]`*[T](x: openArray[T]): T + + A short form works without additional backticks:: + + `$`_ + `[]`_ + + However for fully-qualified reference copy-pasting backticks (`) into other + backticks will not work in our RST parser (because we use Markdown-like + inline markup rules). You need either to delete backticks or keep + them and escape with backslash \\:: + + no backticks: `func $`_ + escaped: `func \`$\``_ + no backticks: `func [][T](x: openArray[T]): T`_ + escaped: `func \`[]\`[T](x: openArray[T]): T`_ + Related Options =============== diff --git a/doc/nimdoc.css b/doc/nimdoc.css index 3353f1cda..7c819ea30 100644 --- a/doc/nimdoc.css +++ b/doc/nimdoc.css @@ -263,6 +263,10 @@ a.reference-toplevel { font-weight: bold; } +a.nimdoc { + word-spacing: 0.3em; +} + a.toc-backref { text-decoration: none; color: var(--text); } diff --git a/lib/packages/docutils/dochelpers.nim b/lib/packages/docutils/dochelpers.nim new file mode 100644 index 000000000..c488c4d99 --- /dev/null +++ b/lib/packages/docutils/dochelpers.nim @@ -0,0 +1,267 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2021 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Integration helpers between ``docgen.nim`` and ``rst.nim``. +## +## Function `toLangSymbol(linkText)`_ produces a signature `docLink` of +## `type LangSymbol`_ in ``rst.nim``, while `match(generated, docLink)`_ +## matches it with `generated`, produced from `PNode` by ``docgen.rst``. + +import rstast + +type + LangSymbol* = object ## symbol signature in Nim + symKind*: string ## "proc", "const", etc + name*: string ## plain symbol name without any parameters + generics*: string ## generic parameters (without brackets) + isGroup*: bool ## is LangSymbol a group with overloads? + # the following fields are valid iff `isGroup` == false + # (always false when parsed by `toLangSymbol` because link like foo_ + # can point to just a single symbol foo, e.g. proc). + parametersProvided*: bool ## to disambiguate `proc f`_ and `proc f()`_ + parameters*: seq[tuple[name: string, `type`: string]] + ## name-type seq, e.g. for proc + outType*: string ## result type, e.g. for proc + +func nimIdentBackticksNormalize*(s: string): string = + ## Normalizes the string `s` as a Nim identifier. + ## + ## Unlike `nimIdentNormalize` removes spaces and backticks. + ## + ## .. Warning:: No checking (e.g. that identifiers cannot start from + ## digits or '_', or that number of backticks is even) is performed. + runnableExamples: + doAssert nimIdentBackticksNormalize("Foo_bar") == "Foobar" + doAssert nimIdentBackticksNormalize("FoO BAr") == "Foobar" + doAssert nimIdentBackticksNormalize("`Foo BAR`") == "Foobar" + doAssert nimIdentBackticksNormalize("` Foo BAR `") == "Foobar" + # not a valid identifier: + doAssert nimIdentBackticksNormalize("`_x_y`") == "_xy" + result = newString(s.len) + var firstChar = true + var j = 0 + for i in 0..len(s) - 1: + if s[i] in {'A'..'Z'}: + if not firstChar: # to lowercase + result[j] = chr(ord(s[i]) + (ord('a') - ord('A'))) + else: + result[j] = s[i] + firstChar = false + inc j + elif s[i] notin {'_', ' ', '`'}: + result[j] = s[i] + inc j + firstChar = false + elif s[i] == '_' and firstChar: + result[j] = '_' + inc j + firstChar = false + else: discard # just omit '`' or ' ' + if j != s.len: setLen(result, j) + +proc toLangSymbol*(linkText: PRstNode): LangSymbol = + ## Parses `linkText` into a more structured form using a state machine. + ## + ## This proc is designed to allow link syntax with operators even + ## without escaped backticks inside:: + ## + ## `proc *`_ + ## `proc []`_ + ## + ## This proc should be kept in sync with the `renderTypes` proc from + ## ``compiler/typesrenderer.nim``. + assert linkText.kind in {rnRef, rnInner} + + const NimDefs = ["proc", "func", "macro", "method", "iterator", + "template", "converter", "const", "type", "var"] + type + State = enum + inBeginning + afterSymKind + beforeSymbolName # auxiliary state to catch situations like `proc []`_ after space + atSymbolName + afterSymbolName + genericsPar + parameterName + parameterType + outType + var state = inBeginning + var curIdent = "" + template flushIdent() = + if curIdent != "": + case state + of inBeginning: doAssert false, "incorrect state inBeginning" + of afterSymKind: result.symKind = curIdent + of beforeSymbolName: doAssert false, "incorrect state beforeSymbolName" + of atSymbolName: result.name = curIdent.nimIdentBackticksNormalize + of afterSymbolName: doAssert false, "incorrect state afterSymbolName" + of genericsPar: result.generics = curIdent + of parameterName: result.parameters.add (curIdent, "") + of parameterType: + for a in countdown(result.parameters.len - 1, 0): + if result.parameters[a].`type` == "": + result.parameters[a].`type` = curIdent + of outType: result.outType = curIdent + curIdent = "" + var parens = 0 + let L = linkText.sons.len + template s(i: int): string = linkText.sons[i].text + var i = 0 + template nextState = + case s(i) + of " ": + if state == afterSymKind: + flushIdent + state = beforeSymbolName + of "`": + curIdent.add "`" + inc i + while i < L: # add contents between ` ` as a whole + curIdent.add s(i) + if s(i) == "`": + break + inc i + curIdent = curIdent.nimIdentBackticksNormalize + if state in {inBeginning, afterSymKind, beforeSymbolName}: + state = atSymbolName + flushIdent + state = afterSymbolName + of "[": + if state notin {inBeginning, afterSymKind, beforeSymbolName}: + inc parens + if state in {inBeginning, afterSymKind, beforeSymbolName}: + state = atSymbolName + curIdent.add s(i) + elif state in {atSymbolName, afterSymbolName} and parens == 1: + flushIdent + state = genericsPar + curIdent.add s(i) + else: curIdent.add s(i) + of "]": + if state notin {inBeginning, afterSymKind, beforeSymbolName, atSymbolName}: + dec parens + if state == genericsPar and parens == 0: + curIdent.add s(i) + flushIdent + else: curIdent.add s(i) + of "(": + inc parens + if state in {inBeginning, afterSymKind, beforeSymbolName}: + result.parametersProvided = true + state = atSymbolName + flushIdent + state = parameterName + elif state in {atSymbolName, afterSymbolName, genericsPar} and parens == 1: + result.parametersProvided = true + flushIdent + state = parameterName + else: curIdent.add s(i) + of ")": + dec parens + if state in {parameterName, parameterType} and parens == 0: + flushIdent + state = outType + else: curIdent.add s(i) + of "{": # remove pragmas + while i < L: + if s(i) == "}": + break + inc i + of ",", ";": + if state in {parameterName, parameterType} and parens == 1: + flushIdent + state = parameterName + else: curIdent.add s(i) + of "*": # skip export symbol + if state == atSymbolName: + flushIdent + state = afterSymbolName + elif state == afterSymbolName: + discard + else: curIdent.add "*" + of ":": + if state == outType: discard + elif state == parameterName: + flushIdent + state = parameterType + else: curIdent.add ":" + else: + let isPostfixSymKind = i > 0 and i == L - 1 and + result.symKind == "" and s(i) in NimDefs + if isPostfixSymKind: # for links like `foo proc`_ + result.symKind = s(i) + else: + case state + of inBeginning: + if s(i) in NimDefs: + state = afterSymKind + else: + state = atSymbolName + curIdent.add s(i) + of afterSymKind, beforeSymbolName: + state = atSymbolName + curIdent.add s(i) + of parameterType: + case s(i) + of "ref": curIdent.add "ref." + of "ptr": curIdent.add "ptr." + of "var": discard + else: curIdent.add s(i).nimIdentBackticksNormalize + of atSymbolName: + curIdent.add s(i) + else: + curIdent.add s(i).nimIdentBackticksNormalize + while i < L: + nextState + inc i + if state == afterSymKind: # treat `type`_ as link to symbol `type` + state = atSymbolName + flushIdent + result.isGroup = false + +proc match*(generated: LangSymbol, docLink: LangSymbol): bool = + ## Returns true if `generated` can be a target for `docLink`. + ## If `generated` is an overload group then only `symKind` and `name` + ## are compared for success. + result = true + if docLink.symKind != "": + if generated.symKind == "proc": + result = docLink.symKind in ["proc", "func"] + else: + result = generated.symKind == docLink.symKind + if not result: return + result = generated.name == docLink.name + if not result: return + if generated.isGroup: + # if `()` were added then it's not a reference to the whole group: + return not docLink.parametersProvided + if docLink.generics != "": + result = generated.generics == docLink.generics + if not result: return + if docLink.outType != "": + result = generated.outType == docLink.outType + if not result: return + if docLink.parametersProvided: + result = generated.parameters.len == docLink.parameters.len + if not result: return + var onlyType = false + for i in 0 ..< generated.parameters.len: + let g = generated.parameters[i] + let d = docLink.parameters[i] + if i == 0: + if g.`type` == d.name: + onlyType = true # only types, not names, are provided in `docLink` + if onlyType: + result = g.`type` == d.name: + else: + if d.`type` != "": + result = g.`type` == d.`type` + if not result: return + result = g.name == d.name + if not result: return diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 29234f28b..2e908f4e5 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -114,7 +114,7 @@ ## .. _`extra features`: ## ## Optional additional features, turned on by ``options: RstParseOption`` in -## `rstParse proc <#rstParse,string,string,int,int,bool,RstParseOptions,FindFileHandler,MsgHandler>`_: +## `proc rstParse`_: ## ## * emoji / smiley symbols ## * Markdown tables @@ -196,7 +196,7 @@ ## .. _Sphinx roles: https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html import - os, strutils, rstast, std/enumutils, algorithm, lists, sequtils, + os, strutils, rstast, dochelpers, std/enumutils, algorithm, lists, sequtils, std/private/miscdollars, tables from highlite import SourceLanguage, getSourceLanguage @@ -231,6 +231,7 @@ type meFootnoteMismatch = "mismatch in number of footnotes and their refs: $1", mwRedefinitionOfLabel = "redefinition of label '$1'", mwUnknownSubstitution = "unknown substitution '$1'", + mwAmbiguousLink = "ambiguous doc link $1", mwBrokenLink = "broken link '$1'", mwUnsupportedLanguage = "language '$1' not supported", mwUnsupportedField = "field '$1' not supported", @@ -473,12 +474,42 @@ type hasPeers: bool # has headings on the same level of hierarchy? LevelMap = seq[LevelInfo] # Saves for each possible title adornment # style its level in the current document. + SubstitutionKind = enum + rstSubstitution = "substitution", + hyperlinkAlias = "hyperlink alias", + implicitHyperlinkAlias = "implicitly-generated hyperlink alias" Substitution = object + kind*: SubstitutionKind key*: string value*: PRstNode - AnchorSubst = tuple - mainAnchor: string - aliases: seq[string] + info*: TLineInfo # place where the substitution was defined + AnchorRule = enum + arInternalRst, ## For automatically generated RST anchors (from + ## headings, footnotes, inline internal targets): + ## case-insensitive, 1-space-significant (by RST spec) + arNim ## For anchors generated by ``docgen.rst``: Nim-style case + ## sensitivity, etc. (see `proc normalizeNimName`_ for details) + arHyperlink, ## For links with manually set anchors in + ## form `text `_ + RstAnchorKind = enum + manualDirectiveAnchor = "manual directive anchor", + manualInlineAnchor = "manual inline anchor", + footnoteAnchor = "footnote anchor", + headlineAnchor = "implicitly-generated headline anchor" + AnchorSubst = object + mainAnchor: ref string # A reference name that will be inserted directly + # into HTML/Latex. It's declared as `ref` because + # it can be shared between aliases. + info: TLineInfo # where the anchor was defined + priority: int + case kind: range[arInternalRst .. arNim] + of arInternalRst: + anchorType: RstAnchorKind + of arNim: + tooltip: string # displayed tooltip for Nim-generated anchors + langSym: LangSymbol + AnchorSubstTable = Table[string, seq[AnchorSubst]] + # use `seq` to account for duplicate anchors FootnoteType = enum fnManualNumber, # manually numbered footnote like [3] fnAutoNumber, # auto-numbered footnote [#] @@ -505,7 +536,8 @@ type currRoleKind: RstNodeKind # ... and its node kind subs: seq[Substitution] # substitutions refs*: seq[Substitution] # references - anchors*: seq[AnchorSubst] # internal target substitutions + anchors*: AnchorSubstTable + # internal target substitutions lineFootnoteNum: seq[TLineInfo] # footnote line, auto numbers .. [#] lineFootnoteNumRef: seq[TLineInfo] # footnote line, their reference [#]_ currFootnoteNumRef: int # ... their counter for `resolveSubs` @@ -518,7 +550,7 @@ type findFile: FindFileHandler # How to find files. filenames*: RstFileTable # map file name <-> FileIndex (for storing # file names for warnings after 1st stage) - currFileIdx: FileIndex # current index in `filesnames` + currFileIdx*: FileIndex # current index in `filenames` hasToc*: bool PRstSharedState* = ref RstSharedState @@ -532,6 +564,7 @@ type ## in case of error/warning reporting to ## (relative) line/column of the token. curAnchor*: string # variable to track latest anchor in s.anchors + curAnchorName*: string # corresponding name in human-readable format EParseError* = object of ValueError @@ -590,13 +623,16 @@ proc whichRoleAux(sym: string): RstNodeKind = proc len(filenames: RstFileTable): int = filenames.idxToFilename.len -proc setCurrFilename(s: PRstSharedState, file1: string) = +proc addFilename*(s: PRstSharedState, file1: string): FileIndex = + ## Returns index of filename, adding it if it has not been used before let nextIdx = s.filenames.len.FileIndex - let v = getOrDefault(s.filenames.filenameToIdx, file1, default = nextIdx) - if v == nextIdx: - s.filenames.filenameToIdx[file1] = v + result = getOrDefault(s.filenames.filenameToIdx, file1, default = nextIdx) + if result == nextIdx: + s.filenames.filenameToIdx[file1] = result s.filenames.idxToFilename.add file1 - s.currFileIdx = v + +proc setCurrFilename*(s: PRstSharedState, file1: string) = + s.currFileIdx = addFilename(s, file1) proc getFilename(filenames: RstFileTable, fid: FileIndex): string = doAssert(0 <= fid.int and fid.int < filenames.len, @@ -730,6 +766,8 @@ proc initParser(p: var RstParser, sharedState: PRstSharedState) = p.s = sharedState proc addNodesAux(n: PRstNode, result: var string) = + if n == nil: + return if n.kind == rnLeaf: result.add(n.text) else: @@ -738,6 +776,11 @@ proc addNodesAux(n: PRstNode, result: var string) = proc addNodes(n: PRstNode): string = n.addNodesAux(result) +proc linkName(n: PRstNode): string = + ## Returns a normalized reference name, see: + ## https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#reference-names + n.addNodes.toLowerAscii + proc rstnodeToRefnameAux(n: PRstNode, r: var string, b: var bool) = template special(s) = if b: @@ -804,15 +847,26 @@ proc findSub(s: PRstSharedState, n: PRstNode): int = return i result = -1 +proc lineInfo(p: RstParser, iTok: int): TLineInfo = + result.col = int16(p.col + p.tok[iTok].col) + result.line = uint16(p.line + p.tok[iTok].line) + result.fileIndex = p.s.currFileIdx + +proc lineInfo(p: RstParser): TLineInfo = lineInfo(p, p.idx) +# TODO: we need this simplification because we don't preserve exact starting +# token of currently parsed element: +proc prevLineInfo(p: RstParser): TLineInfo = lineInfo(p, p.idx-1) + proc setSub(p: var RstParser, key: string, value: PRstNode) = var length = p.s.subs.len for i in 0 ..< length: if key == p.s.subs[i].key: p.s.subs[i].value = value return - p.s.subs.add(Substitution(key: key, value: value)) + p.s.subs.add(Substitution(key: key, value: value, info: prevLineInfo(p))) -proc setRef(p: var RstParser, key: string, value: PRstNode) = +proc setRef(p: var RstParser, key: string, value: PRstNode, + refType: SubstitutionKind) = var length = p.s.refs.len for i in 0 ..< length: if key == p.s.refs[i].key: @@ -820,37 +874,111 @@ proc setRef(p: var RstParser, key: string, value: PRstNode) = rstMessage(p, mwRedefinitionOfLabel, key) p.s.refs[i].value = value return - p.s.refs.add(Substitution(key: key, value: value)) + p.s.refs.add(Substitution(kind: refType, key: key, value: value, + info: prevLineInfo(p))) -proc findRef(s: PRstSharedState, key: string): PRstNode = +proc findRef(s: PRstSharedState, key: string): seq[Substitution] = for i in countup(0, high(s.refs)): if key == s.refs[i].key: - return s.refs[i].value + result.add s.refs[i] -proc addAnchor(p: var RstParser, refn: string, reset: bool) = - ## add anchor `refn` to anchor aliases and update last anchor ``curAnchor`` - if p.curAnchor == "": - p.s.anchors.add (refn, @[refn]) +# Ambiguity in links: we don't follow procedure of removing implicit targets +# defined in https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#implicit-hyperlink-targets +# Instead we just give explicit links a higher priority than to implicit ones +# and report ambiguities as warnings. Hopefully it is easy to remove +# ambiguities manually. Nim auto-generated links from ``docgen.nim`` +# have lowest priority: 1 (for procs) and below for other symbol types. + +proc refPriority(k: SubstitutionKind): int = + case k + of rstSubstitution: result = 8 + of hyperlinkAlias: result = 7 + of implicitHyperlinkAlias: result = 2 + +proc internalRefPriority(k: RstAnchorKind): int = + case k + of manualDirectiveAnchor: result = 6 + of manualInlineAnchor: result = 5 + of footnoteAnchor: result = 4 + of headlineAnchor: result = 3 + +proc addAnchorRst(p: var RstParser, name: string, refn: string, reset: bool, + anchorType: RstAnchorKind) = + ## Adds anchor `refn` with an alias `name` and + ## updates the corresponding `curAnchor` / `curAnchorName`. + let prio = internalRefPriority(anchorType) + if p.curAnchorName == "": + var anchRef = new string + anchRef[] = refn + p.s.anchors.mgetOrPut(name, newSeq[AnchorSubst]()).add( + AnchorSubst(kind: arInternalRst, mainAnchor: anchRef, priority: prio, + info: prevLineInfo(p), anchorType: anchorType)) else: - p.s.anchors[^1].mainAnchor = refn - p.s.anchors[^1].aliases.add refn + # override previous mainAnchor by `ref` in all aliases + var anchRef = p.s.anchors[p.curAnchorName][0].mainAnchor + anchRef[] = refn + p.s.anchors.mgetOrPut(name, newSeq[AnchorSubst]()).add( + AnchorSubst(kind: arInternalRst, mainAnchor: anchRef, priority: prio, + info: prevLineInfo(p), anchorType: anchorType)) if reset: p.curAnchor = "" + p.curAnchorName = "" else: p.curAnchor = refn + p.curAnchorName = name -proc findMainAnchor(s: PRstSharedState, refn: string): string = - for subst in s.anchors: - if subst.mainAnchor == refn: # no need to rename - result = subst.mainAnchor - break - var toLeave = false - for anchor in subst.aliases: - if anchor == refn: # this anchor will be named as mainAnchor - result = subst.mainAnchor - toLeave = true - if toLeave: - break +proc addAnchorNim*(s: var PRstSharedState, refn: string, tooltip: string, + langSym: LangSymbol, priority: int, + info: TLineInfo) = + ## Adds an anchor `refn` (`mainAnchor`), which follows + ## the rule `arNim` (i.e. a symbol in ``*.nim`` file) + var anchRef = new string + anchRef[] = refn + s.anchors.mgetOrPut(langSym.name, newSeq[AnchorSubst]()).add( + AnchorSubst(kind: arNim, mainAnchor: anchRef, langSym: langSym, + tooltip: tooltip, priority: priority, + info: info)) + +proc findMainAnchorNim(s: PRstSharedState, signature: PRstNode, + info: TLineInfo): + seq[AnchorSubst] = + let langSym = toLangSymbol(signature) + let substitutions = s.anchors.getOrDefault(langSym.name, + newSeq[AnchorSubst]()) + if substitutions.len == 0: + return + # map symKind (like "proc") -> found symbols/groups: + var found: Table[string, seq[AnchorSubst]] + for s in substitutions: + if s.kind == arNim: + if match(s.langSym, langSym): + found.mgetOrPut(s.langSym.symKind, newSeq[AnchorSubst]()).add s + for symKind, sList in found: + if sList.len == 1: + result.add sList[0] + else: # > 1, there are overloads, potential ambiguity in this `symKind` + if langSym.parametersProvided: + # there are non-group signatures, select only them + for s in sList: + if not s.langSym.isGroup: + result.add s + else: # when there are many overloads a link like foo_ points to all + # of them, so selecting the group + var foundGroup = true + for s in sList: + if s.langSym.isGroup: + result.add s + foundGroup = true + break + doAssert foundGroup, "docgen has not generated the group" + +proc findMainAnchorRst(s: PRstSharedState, linkText: string, info: TLineInfo): + seq[AnchorSubst] = + let name = linkText.toLowerAscii + let substitutions = s.anchors.getOrDefault(name, newSeq[AnchorSubst]()) + for s in substitutions: + if s.kind == arInternalRst: + result.add s proc addFootnoteNumManual(p: var RstParser, num: int) = ## add manually-numbered footnote @@ -860,13 +988,6 @@ proc addFootnoteNumManual(p: var RstParser, num: int) = return p.s.footnotes.add((fnManualNumber, num, -1, -1, $num)) -proc lineInfo(p: RstParser, iTok: int): TLineInfo = - result.col = int16(p.col + p.tok[iTok].col) - result.line = uint16(p.line + p.tok[iTok].line) - result.fileIndex = p.s.currFileIdx - -proc lineInfo(p: RstParser): TLineInfo = lineInfo(p, p.idx) - proc addFootnoteNumAuto(p: var RstParser, label: string) = ## add auto-numbered footnote. ## Empty label [#] means it'll be resolved by the occurrence. @@ -989,6 +1110,7 @@ proc newRstNodeA(p: var RstParser, kind: RstNodeKind): PRstNode = if p.curAnchor != "": result.anchor = p.curAnchor p.curAnchor = "" + p.curAnchorName = "" template newLeaf(s: string): PRstNode = newRstLeaf(s) @@ -1255,7 +1377,7 @@ proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode = else: newKind = rnHyperlink newSons = @[a, b] - setRef(p, rstnodeToRefname(a), b) + setRef(p, rstnodeToRefname(a), b, implicitHyperlinkAlias) result = newRstNode(newKind, newSons) else: # some link that will be resolved in `resolveSubs` newKind = rnRef @@ -1562,7 +1684,8 @@ proc parseInline(p: var RstParser, father: PRstNode) = inc p.idx parseUntil(p, n, "`", false) let refn = rstnodeToRefname(n) - p.s.anchors.add (refn, @[refn]) + addAnchorRst(p, name = linkName(n), refn = refn, reset = true, + anchorType=manualInlineAnchor) father.add(n) elif roSupportMarkdown in p.s.options and currentTok(p).symbol == "```": inc p.idx @@ -2084,7 +2207,8 @@ proc parseHeadline(p: var RstParser): PRstNode = result.level = getLevel(p, c, hasOverline=false) checkHeadingHierarchy(p, result.level) p.s.hCurLevel = result.level - addAnchor(p, rstnodeToRefname(result), reset=true) + addAnchorRst(p, linkName(result), rstnodeToRefname(result), reset=true, + anchorType=headlineAnchor) proc parseOverline(p: var RstParser): PRstNode = var c = currentTok(p).symbol[0] @@ -2106,7 +2230,8 @@ proc parseOverline(p: var RstParser): PRstNode = if currentTok(p).kind == tkAdornment: inc p.idx if currentTok(p).kind == tkIndent: inc p.idx - addAnchor(p, rstnodeToRefname(result), reset=true) + addAnchorRst(p, linkName(result), rstnodeToRefname(result), reset=true, + anchorType=headlineAnchor) type IntSeq = seq[int] @@ -2837,7 +2962,7 @@ proc parseFootnote(p: var RstParser): PRstNode = anchor.add $p.s.lineFootnoteSym.len of fnCitation: anchor.add rstnodeToRefname(label) - addAnchor(p, anchor, reset=true) + addAnchorRst(p, anchor, anchor, reset=true, anchorType=footnoteAnchor) result.anchor = anchor if currentTok(p).kind == tkWhite: inc p.idx discard parseBlockContent(p, result, parseSectionWrapper) @@ -2858,13 +2983,23 @@ proc parseDotDot(p: var RstParser): PRstNode = elif match(p, p.idx, " _"): # hyperlink target: inc p.idx, 2 - var a = getReferenceName(p, ":") + var ending = ":" + if currentTok(p).symbol == "`": + inc p.idx + ending = "`" + var a = getReferenceName(p, ending) + if ending == "`": + if currentTok(p).symbol == ":": + inc p.idx + else: + rstMessage(p, meExpected, ":") if currentTok(p).kind == tkWhite: inc p.idx var b = untilEol(p) if len(b) == 0: # set internal anchor - addAnchor(p, rstnodeToRefname(a), reset=false) + addAnchorRst(p, linkName(a), rstnodeToRefname(a), reset=false, + anchorType=manualDirectiveAnchor) else: # external hyperlink - setRef(p, rstnodeToRefname(a), b) + setRef(p, rstnodeToRefname(a), b, refType=hyperlinkAlias) elif match(p, p.idx, " |"): # substitution definitions: inc p.idx, 2 @@ -2892,7 +3027,7 @@ proc rstParsePass1*(fragment: string, sharedState: PRstSharedState): PRstNode = ## Parses an RST `fragment`. ## The result should be further processed by - ## `preparePass2` and `resolveSubs` (which is pass 2). + ## preparePass2_ and resolveSubs_ (which is pass 2). var p: RstParser initParser(p, sharedState) p.line = line @@ -2905,6 +3040,65 @@ proc preparePass2*(s: PRstSharedState, mainNode: PRstNode) = countTitles(s, mainNode) orderFootnotes(s) +proc resolveLink(s: PRstSharedState, n: PRstNode) : PRstNode = + # Associate this link alias with its target and change node kind to + # rnHyperlink or rnInternalRef appropriately. + type LinkDef = object + ar: AnchorRule + priority: int + tooltip: string + target: PRstNode + info: TLineInfo + proc cmp(x, y: LinkDef): int = + result = cmp(x.priority, y.priority) + if result == 0: + result = cmp(x.target, y.target) + var foundLinks: seq[LinkDef] + let text = newRstNode(rnInner, n.sons) + let refn = rstnodeToRefname(n) + var hyperlinks = findRef(s, refn) + for y in hyperlinks: + foundLinks.add LinkDef(ar: arHyperlink, priority: refPriority(y.kind), + target: y.value, info: y.info, + tooltip: "(" & $y.kind & ")") + let substRst = findMainAnchorRst(s, text.addNodes, n.info) + for subst in substRst: + foundLinks.add LinkDef(ar: arInternalRst, priority: subst.priority, + target: newLeaf(subst.mainAnchor[]), + info: subst.info, + tooltip: "(" & $subst.anchorType & ")") + if roNimFile in s.options: + let substNim = findMainAnchorNim(s, signature=text, n.info) + for subst in substNim: + foundLinks.add LinkDef(ar: arNim, priority: subst.priority, + target: newLeaf(subst.mainAnchor[]), + info: subst.info, tooltip: subst.tooltip) + foundLinks.sort(cmp = cmp, order = Descending) + let linkText = addNodes(n) + if foundLinks.len >= 1: + let kind = if foundLinks[0].ar == arHyperlink: rnHyperlink + elif foundLinks[0].ar == arNim: rnNimdocRef + else: rnInternalRef + result = newRstNode(kind) + result.sons = @[text, foundLinks[0].target] + if kind == rnNimdocRef: result.tooltip = foundLinks[0].tooltip + if foundLinks.len > 1: # report ambiguous link + var targets = newSeq[string]() + for l in foundLinks: + var t = " " + if s.filenames.len > 1: + t.add getFilename(s.filenames, l.info.fileIndex) + let n = l.info.line + let c = l.info.col + ColRstOffset + t.add "($1, $2): $3" % [$n, $c, l.tooltip] + targets.add t + rstMessage(s.filenames, s.msgHandler, n.info, mwAmbiguousLink, + "`$1`\n clash:\n$2" % [ + linkText, targets.join("\n")]) + else: # nothing found + result = n + rstMessage(s.filenames, s.msgHandler, n.info, mwBrokenLink, linkText) + proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode = ## Makes pass 2 of RST parsing. ## Resolves substitutions and anchor aliases, groups footnotes. @@ -2933,21 +3127,7 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode = elif s.hTitleCnt == 0: n.level += 1 of rnRef: - let refn = rstnodeToRefname(n) - var y = findRef(s, refn) - if y != nil: - result = newRstNode(rnHyperlink) - let text = newRstNode(rnInner, n.sons) - result.sons = @[text, y] - else: - let anchor = findMainAnchor(s, refn) - if anchor != "": - result = newRstNode(rnInternalRef) - let text = newRstNode(rnInner, n.sons) - result.sons = @[text, # visible text of reference - newLeaf(anchor)] # link itself - else: - rstMessage(s.filenames, s.msgHandler, n.info, mwBrokenLink, refn) + result = resolveLink(s, n) of rnFootnote: var (fnType, num) = getFootnoteType(n.sons[0]) case fnType @@ -2993,9 +3173,10 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode = of fnCitation: result.add n.sons[0] refn.add rstnodeToRefname(n) - let anch = findMainAnchor(s, refn) - if anch != "": - result.add newLeaf(anch) # add link + # TODO: correctly report ambiguities + let anchorInfo = findMainAnchorRst(s, refn, n.info) + if anchorInfo.len != 0: + result.add newLeaf(anchorInfo[0].mainAnchor[]) # add link else: rstMessage(s.filenames, s.msgHandler, n.info, mwBrokenLink, refn) result.add newLeaf(refn) # add link diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim index e1ed7c099..bc7b9a650 100644 --- a/lib/packages/docutils/rstast.nim +++ b/lib/packages/docutils/rstast.nim @@ -43,6 +43,7 @@ type rnFootnoteGroup, # footnote group - exists for a purely stylistic # reason: to display a few footnotes as 1 block rnStandaloneHyperlink, rnHyperlink, rnRef, rnInternalRef, rnFootnoteRef, + rnNimdocRef, # reference to automatically generated Nim symbol rnDirective, # a general directive rnDirArg, # a directive argument (for some directives). # here are directives that are not rnDirective: @@ -104,6 +105,8 @@ type rnInterpretedText, rnField, rnInlineCode, rnCodeBlock, rnFootnoteRef: info*: TLineInfo ## To have line/column info for warnings at ## nodes that are post-processed after parsing + of rnNimdocRef: + tooltip*: string else: discard anchor*: string ## anchor, internal link target diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index 8eac37307..a556aa2e3 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -60,8 +60,8 @@ type MetaEnum* = enum metaNone, metaTitle, metaSubtitle, metaAuthor, metaVersion - EscapeMode = enum # in Latex text inside options [] and URLs is - # escaped slightly differently than in normal text + EscapeMode* = enum # in Latex text inside options [] and URLs is + # escaped slightly differently than in normal text emText, emOption, emUrl # emText is currently used for code also RstGenerator* = object of RootObj @@ -201,7 +201,9 @@ proc addTexChar(dest: var string, c: char, escMode: EscapeMode) = ## All escapes that need to work in text and code blocks (`emText` mode) ## should start from \ (to be compatible with fancyvrb/fvextra). case c - of '_', '$', '&', '#', '%': add(dest, "\\" & c) + of '_', '&', '#', '%': add(dest, "\\" & c) + # commands \label and \pageref don't accept \$ by some reason but OK with $: + of '$': (if escMode == emUrl: add(dest, c) else: add(dest, "\\" & c)) # \~ and \^ have a special meaning unless they are followed by {} of '~', '^': add(dest, "\\" & c & "{}") # Latex loves to substitute ` to opening quote, even in texttt mode! @@ -1180,7 +1182,8 @@ proc renderAdmonition(d: PDoc, n: PRstNode, result: var string) = "$1\n\\end{rstadmonition}\n", result) -proc renderHyperlink(d: PDoc, text, link: PRstNode, result: var string, external: bool) = +proc renderHyperlink(d: PDoc, text, link: PRstNode, result: var string, + external: bool, nimdoc = false, tooltip="") = var linkStr = "" block: let mode = d.escMode @@ -1189,14 +1192,19 @@ proc renderHyperlink(d: PDoc, text, link: PRstNode, result: var string, external d.escMode = mode var textStr = "" renderRstToOut(d, text, textStr) + let nimDocStr = if nimdoc: " nimdoc" else: "" + var tooltipStr = "" + if tooltip != "": + tooltipStr = """ title="$1"""" % [ esc(d.target, tooltip) ] if external: dispA(d.target, result, - "$1", - "\\href{$2}{$1}", [textStr, linkStr]) + "$1", + "\\href{$2}{$1}", [textStr, linkStr, nimDocStr, tooltipStr]) else: dispA(d.target, result, - "$1", - "\\hyperlink{$2}{$1} (p.~\\pageref{$2})", [textStr, linkStr]) + "$1", + "\\hyperlink{$2}{$1} (p.~\\pageref{$2})", + [textStr, linkStr, nimDocStr, tooltipStr]) proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = if n == nil: return @@ -1329,6 +1337,9 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = renderHyperlink(d, text=n.sons[0], link=n.sons[0], result, external=true) of rnInternalRef: renderHyperlink(d, text=n.sons[0], link=n.sons[1], result, external=false) + of rnNimdocRef: + renderHyperlink(d, text=n.sons[0], link=n.sons[1], result, external=false, + nimdoc=true, tooltip=n.tooltip) of rnHyperlink: renderHyperlink(d, text=n.sons[0], link=n.sons[1], result, external=true) of rnFootnoteRef: diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 873949ca6..2234a8625 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -273,6 +273,11 @@ func nimIdentNormalize*(s: string): string = ## ## That means to convert to lower case and remove any '_' on all characters ## except first one. + ## + ## .. Warning:: Backticks (`) are not handled: they remain *as is* and + ## spaces are preserved. See `nimIdentBackticksNormalize + ## `_ for + ## an alternative approach. runnableExamples: doAssert nimIdentNormalize("Foo_bar") == "Foobar" result = newString(s.len) diff --git a/nimdoc/test_out_index_dot_html/expected/index.html b/nimdoc/test_out_index_dot_html/expected/index.html index 3499b5326..69dd2eb2e 100644 --- a/nimdoc/test_out_index_dot_html/expected/index.html +++ b/nimdoc/test_out_index_dot_html/expected/index.html @@ -121,6 +121,8 @@ window.addEventListener('DOMContentLoaded', main);

    Procs

    + +
    proc foo() {....raises: [], tags: [].}
    @@ -130,6 +132,8 @@ I do foo
    +
    +
    diff --git a/nimdoc/testproject/expected/nimdoc.out.css b/nimdoc/testproject/expected/nimdoc.out.css index 3353f1cda..7c819ea30 100644 --- a/nimdoc/testproject/expected/nimdoc.out.css +++ b/nimdoc/testproject/expected/nimdoc.out.css @@ -263,6 +263,10 @@ a.reference-toplevel { font-weight: bold; } +a.nimdoc { + word-spacing: 0.3em; +} + a.toc-backref { text-decoration: none; color: var(--text); } diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html index 47901ce0d..c64039941 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html @@ -106,7 +106,10 @@ window.addEventListener('DOMContentLoaded', main);
  • Types
      -
    • G
      • +
    • SomeType
      • @@ -115,9 +118,54 @@ window.addEventListener('DOMContentLoaded', main);
    • Procs
        -
          fn2 + + + + + + + +
            fn3 @@ -159,6 +207,11 @@ window.addEventListener('DOMContentLoaded', main);
          • fn10(a: int): int
            • +
          +
            someType
            • +
          • + Iterators + +
          • Templates
              @@ -197,7 +261,13 @@ window.addEventListener('DOMContentLoaded', main);
              -

              +

              This is a description of the utils module.

              +

              Links work:

              + +

              This is now a header

              Next header

              And so on

              @@ -209,10 +279,31 @@ window.addEventListener('DOMContentLoaded', main);
              1. Other case value
              2. Second case.
              -

              +

              Ref group fn2 or specific function like fn2() or fn2( int ) or fn2(int, float).

              +

              Ref generics like this: binarySearch or binarySearch(openArray[T], K, proc (T, K)) or proc binarySearch(openArray[T], K, proc (T, K)) or in different style: proc binarysearch(openarray[T], K, proc(T, K)). Can be combined with export symbols and type parameters: binarysearch*[T, K](openArray[T], K, proc (T, K)). With spaces binary search.

              +

              Note that proc can be used in postfix form: binarySearch proc.

              +

              Ref. type like G and type G and G[T] and type G*[T].

              +Ref. [] is the same as proc `[]`(G[T]) because there are no overloads. The full form: proc `[]`*[T](x: G[T]): TRef. []= aka `[]=`(G[T], int, T).Ref. $ aka proc $ or proc `$`.Ref. $(a: ref SomeType).Ref. foo_bar aka iterator foo_bar_.Ref. fn[T; U,V: SomeFloat]().Ref. 'big or func `'big` or `'big`(string).

              Types

              +
              +
              G[T] = object
+  val: T
+
              +
              + + + +
              +
              SomeType = enum
   enumValueA, enumValueB, enumValueC
              @@ -227,6 +318,109 @@ window.addEventListener('DOMContentLoaded', main);

              Procs

              + +
              +
              +
              proc `$`[T](a: G[T]): string
              +
              + + + +
              +
              +
              +
              proc `$`[T](a: ref SomeType): string
              +
              + + + +
              +
              + +
              + +
              +
              +
              func `'big`(a: string): SomeType {....raises: [], tags: [].}
              +
              + + + +
              +
              + +
              + +
              +
              +
              proc `[]`[T](x: G[T]): T
              +
              + + + +
              +
              + +
              + +
              +
              +
              proc `[]=`[T](a: var G[T]; index: int; value: T)
              +
              + + + +
              +
              + +
              + +
              +
              +
              proc binarySearch[T, K](a: openArray[T]; key: K;
+                        cmp: proc (x: T; y: K): int {.closure.}): int
              +
              + + + +
              +
              + +
              + +
              +
              +
              proc f(x: G[int]) {....raises: [], tags: [].}
              +
              + +There is also variant f(G[string]) + +
              +
              +
              +
              proc f(x: G[string]) {....raises: [], tags: [].}
              +
              + +See also f(G[int]). + +
              +
              + +
              + +
              +
              +
              proc fn[T; U, V: SomeFloat]()
              +
              + + + +
              +
              + +
              + +
              proc fn2() {....raises: [], tags: [].}
              @@ -235,6 +429,26 @@ comment
              +
              +
              proc fn2(x: int) {....raises: [], tags: [].}
              +
              + +fn2 comment + +
              +
              +
              +
              proc fn2(x: int; y: float) {....raises: [], tags: [].}
              +
              + + + +
              +
              + +
              + +
              proc fn3(): auto {....raises: [], tags: [].}
              @@ -243,6 +457,10 @@ comment
              + +
              + +
              proc fn4(): auto {....raises: [], tags: [].}
              @@ -251,6 +469,10 @@ comment
              + +
              + +
              proc fn5() {....raises: [], tags: [].}
              @@ -259,6 +481,10 @@ comment
              + +
              + +
              proc fn6() {....raises: [], tags: [].}
              @@ -267,6 +493,10 @@ comment
              + +
              + +
              proc fn7() {....raises: [], tags: [].}
              @@ -275,6 +505,10 @@ comment
              + +
              + +
              proc fn8(): auto {....raises: [], tags: [].}
              @@ -283,6 +517,10 @@ comment
              + +
              + +
              func fn9(a: int): int {....raises: [], tags: [].}
              @@ -291,6 +529,10 @@ comment
              + +
              + +
              func fn10(a: int): int {....raises: [], tags: [].}
              @@ -299,6 +541,22 @@ comment
              + +
              + +
              +
              +
              proc funWithGenerics[T, U: SomeFloat](a: T; b: U)
              +
              + + + +
              +
              + +
              + +
              proc someType(): SomeType {....raises: [], tags: [].}
              @@ -308,10 +566,31 @@ constructor.
              +
              + +
              +
              +

              Iterators

              +
              + +
              +
              +
              iterator fooBar(a: seq[SomeType]): int {....raises: [], tags: [].}
              +
              + + + +
              +
              + +
              +

              Templates

              + +
              template aEnum(): untyped
              @@ -320,6 +599,10 @@ constructor.
              + +
              + +
              template bEnum(): untyped
              @@ -328,6 +611,10 @@ constructor.
              + +
              + +
              template fromUtilsGen(): untyped
              @@ -339,6 +626,8 @@ should be shown in utils.html only
              +
              +
              diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.idx b/nimdoc/testproject/expected/subdir/subdir_b/utils.idx index a87393f09..441bd8675 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.idx +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.idx @@ -1,9 +1,14 @@ +funWithGenerics subdir/subdir_b/utils.html#funWithGenerics,T,U utils: funWithGenerics[T, U: SomeFloat](a: T; b: U) enumValueA subdir/subdir_b/utils.html#enumValueA SomeType.enumValueA enumValueB subdir/subdir_b/utils.html#enumValueB SomeType.enumValueB enumValueC subdir/subdir_b/utils.html#enumValueC SomeType.enumValueC SomeType subdir/subdir_b/utils.html#SomeType utils: SomeType +G subdir/subdir_b/utils.html#G utils: G someType subdir/subdir_b/utils.html#someType_2 utils: someType(): SomeType fn2 subdir/subdir_b/utils.html#fn2 utils: fn2() +fn2 subdir/subdir_b/utils.html#fn2,int utils: fn2(x: int) +fn2 subdir/subdir_b/utils.html#fn2,int,float utils: fn2(x: int; y: float) +binarySearch subdir/subdir_b/utils.html#binarySearch,openArray[T],K,proc(T,K) utils: binarySearch[T, K](a: openArray[T]; key: K;\n cmp: proc (x: T; y: K): int {.closure.}): int fn3 subdir/subdir_b/utils.html#fn3 utils: fn3(): auto fn4 subdir/subdir_b/utils.html#fn4 utils: fn4(): auto fn5 subdir/subdir_b/utils.html#fn5 utils: fn5() @@ -15,6 +20,15 @@ fn10 subdir/subdir_b/utils.html#fn10,int utils: fn10(a: int): int aEnum subdir/subdir_b/utils.html#aEnum.t utils: aEnum(): untyped bEnum subdir/subdir_b/utils.html#bEnum.t utils: bEnum(): untyped fromUtilsGen subdir/subdir_b/utils.html#fromUtilsGen.t utils: fromUtilsGen(): untyped +f subdir/subdir_b/utils.html#f,G[int] utils: f(x: G[int]) +f subdir/subdir_b/utils.html#f,G[string] utils: f(x: G[string]) +`[]` subdir/subdir_b/utils.html#[],G[T] utils: `[]`[T](x: G[T]): T +`[]=` subdir/subdir_b/utils.html#[]=,G[T],int,T utils: `[]=`[T](a: var G[T]; index: int; value: T) +`$` subdir/subdir_b/utils.html#$,G[T] utils: `$`[T](a: G[T]): string +`$` subdir/subdir_b/utils.html#$,ref.SomeType utils: `$`[T](a: ref SomeType): string +fooBar subdir/subdir_b/utils.html#fooBar.i,seq[SomeType] utils: fooBar(a: seq[SomeType]): int +fn subdir/subdir_b/utils.html#fn utils: fn[T; U, V: SomeFloat]() +`'big` subdir/subdir_b/utils.html#'big,string utils: `'big`(a: string): SomeType This is now a header subdir/subdir_b/utils.html#this-is-now-a-header This is now a header Next header subdir/subdir_b/utils.html#this-is-now-a-header-next-header Next header And so on subdir/subdir_b/utils.html#next-header-and-so-on And so on diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html index dbacb57a7..e45492dac 100644 --- a/nimdoc/testproject/expected/testproject.html +++ b/nimdoc/testproject/expected/testproject.html @@ -559,6 +559,8 @@ This should be visible.

              Procs

              + +
              proc addfBug14485() {....raises: [], tags: [].}
              @@ -579,6 +581,10 @@ Some proc
              + +
              + +
              proc anything() {....raises: [], tags: [].}
              @@ -587,6 +593,10 @@ There is no block quote after blank lines at the beginning.
              + +
              + +
              proc asyncFun1(): Future[int] {....raises: [Exception, ValueError],
                                 tags: [RootEffect].}
              @@ -596,6 +606,10 @@ ok1
              + +
              + +
              proc asyncFun2(): owned(Future[void]) {....raises: [Exception], tags: [RootEffect].}
              @@ -604,6 +618,10 @@ ok1
              + +
              + +
              proc asyncFun3(): owned(Future[void]) {....raises: [Exception], tags: [RootEffect].}
              @@ -614,6 +632,10 @@ ok1
              + +
              + +
              proc bar[T](a, b: T): T
              @@ -622,6 +644,10 @@ ok1
              + +
              + +
              proc baz() {....raises: [], tags: [].}
              @@ -641,6 +667,10 @@ This is deprecated without message.
              + +
              + +
              proc buzz[T](a, b: T): T {....deprecated: "since v0.20".}
              @@ -652,6 +682,10 @@ This is deprecated with a message.
              + +
              + +
              proc c_nonexistent(frmt: cstring): cint {.importc: "nonexistent",
     header: "<stdio.h>", varargs, discardable, ...raises: [], tags: [].}
              @@ -661,6 +695,10 @@ This is deprecated with a message.
              + +
              + +
              proc c_printf(frmt: cstring): cint {.importc: "printf", header: "<stdio.h>",
                                      varargs, discardable, ...raises: [], tags: [].}
              @@ -670,6 +708,10 @@ the c printf. etc.
              + +
              + +
              proc fromUtils3() {....raises: [], tags: [].}
              @@ -681,6 +723,10 @@ came form utils but should be shown where
              + +
              + +
              proc isValid[T](x: T): bool
              @@ -689,6 +735,27 @@ came form utils but should be shown where
              + +
              + +
              +
              +
              proc low[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect,
+    ...raises: [], tags: [].}
              +
              + +

              Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.

              +

              See also:

              + +
              low(2) # => -9223372036854775808
              + +
              +
              + +
              + +
              proc low2[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect,
     ...raises: [], tags: [].}
              @@ -704,19 +771,10 @@ came form utils but should be shown where
              -
              -
              proc low[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect,
-    ...raises: [], tags: [].}
              -
              -

              Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.

              -

              See also:

              - -
              low(2) # => -9223372036854775808
              - -
              + +
              proc p1() {....raises: [], tags: [].}
              @@ -741,6 +799,10 @@ this is a nested doc comment
              + +
              + +
              func someFunc() {....raises: [], tags: [].}
              @@ -749,6 +811,10 @@ My someFunc. Stuff in
              + +
              + +
              proc tripleStrLitTest() {....raises: [], tags: [].}
              @@ -793,6 +859,10 @@ at indent 0
              + +
              + +
              proc z1(): Foo {....raises: [], tags: [].}
              @@ -801,6 +871,10 @@ cz1
              + +
              + +
              proc z2() {....raises: [], tags: [].}
              @@ -811,6 +885,10 @@ cz2
              + +
              + +
              proc z3() {....raises: [], tags: [].}
              @@ -819,6 +897,10 @@ cz3
              + +
              + +
              proc z4() {....raises: [], tags: [].}
              @@ -827,6 +909,10 @@ cz4
              + +
              + +
              proc z5(): int {....raises: [], tags: [].}
              @@ -835,6 +921,10 @@ cz5
              + +
              + +
              proc z6(): int {....raises: [], tags: [].}
              @@ -843,6 +933,10 @@ cz6
              + +
              + +
              proc z7(): int {....raises: [], tags: [].}
              @@ -851,6 +945,10 @@ cz7
              + +
              + +
              proc z8(): int {....raises: [], tags: [].}
              @@ -859,6 +957,10 @@ cz8
              + +
              + +
              proc z9() {....raises: [], tags: [].}
              @@ -869,6 +971,10 @@ cz8
              + +
              + +
              proc z10() {....raises: [], tags: [].}
              @@ -879,6 +985,10 @@ cz8
              + +
              + +
              proc z11() {....raises: [], tags: [].}
              @@ -889,6 +999,10 @@ cz8
              + +
              + +
              proc z12(): int {....raises: [], tags: [].}
              @@ -899,6 +1013,10 @@ cz8
              + +
              + +
              proc z13() {....raises: [], tags: [].}
              @@ -909,6 +1027,10 @@ cz13
              + +
              + +
              proc z17() {....raises: [], tags: [].}
              @@ -920,10 +1042,14 @@ cz17 rest
              +
              +

              Methods

              + +
              method method1(self: Moo) {.base, ...raises: [], tags: [].}
              @@ -932,6 +1058,10 @@ foo1
              + +
              + +
              method method2(self: Moo): int {.base, ...raises: [], tags: [].}
              @@ -940,6 +1070,10 @@ foo2
              + +
              + +
              method method3(self: Moo): int {.base, ...raises: [], tags: [].}
              @@ -949,10 +1083,14 @@ foo3
              +
              +

              Iterators

              + +
              iterator fromUtils1(): int {....raises: [], tags: [].}
              @@ -965,6 +1103,10 @@ foo3
              + +
              + +
              iterator iter1(n: int): int {....raises: [], tags: [].}
              @@ -973,6 +1115,10 @@ foo1
              + +
              + +
              iterator iter2(n: int): int {....raises: [], tags: [].}
              @@ -984,10 +1130,14 @@ foo2
              +
              +

              Macros

              + +
              macro bar(): untyped
              @@ -996,6 +1146,10 @@ foo2
              + +
              + +
              macro z16()
              @@ -1008,6 +1162,10 @@ foo2
              + +
              + +
              macro z18(): int
              @@ -1017,10 +1175,14 @@ cz18
              +
              +

              Templates

              + +
              template foo(a, b: SomeType)
              @@ -1029,6 +1191,10 @@ This does nothing
              + +
              + +
              template fromUtils2()
              @@ -1040,6 +1206,10 @@ ok3
              + +
              + +
              template myfn()
              @@ -1063,6 +1233,10 @@ bar
              + +
              + +
              template testNimDocTrailingExample()
              @@ -1073,6 +1247,10 @@ bar
              + +
              + +
              template z6t(): int
              @@ -1081,6 +1259,10 @@ cz6t
              + +
              + +
              template z14()
              @@ -1091,6 +1273,10 @@ cz14
              + +
              + +
              template z15()
              @@ -1110,6 +1296,8 @@ cz15
              +
              +
              diff --git a/nimdoc/testproject/expected/theindex.html b/nimdoc/testproject/expected/theindex.html index f72f6c37e..da76334d4 100644 --- a/nimdoc/testproject/expected/theindex.html +++ b/nimdoc/testproject/expected/theindex.html @@ -71,7 +71,25 @@ window.addEventListener('DOMContentLoaded', main);

              Index

              Modules: subdir/subdir_b/utils, testproject.

              API symbols

              -
              A:
              +
              binarySearch:
              buzz:
              +
              f:
              +
              fn:
              fn10:
              • utils: fn10(a: int): int
                • @@ -178,6 +212,10 @@ window.addEventListener('DOMContentLoaded', main);
                fn2:
                fn3:
                +
                fooBar:
                FooBuzz:
                +
                funWithGenerics:
                +
                G:
                isValid:
                • testproject: isValid[T](x: T): bool
                  • diff --git a/nimdoc/testproject/subdir/subdir_b/utils.nim b/nimdoc/testproject/subdir/subdir_b/utils.nim index 37c91dd3c..5d881e620 100644 --- a/nimdoc/testproject/subdir/subdir_b/utils.nim +++ b/nimdoc/testproject/subdir/subdir_b/utils.nim @@ -1,5 +1,7 @@ ##[ +.. include:: ./utils_overview.rst + # This is now a header ## Next header @@ -19,13 +21,32 @@ More text. 1. Other case value 2. Second case. +Ref group fn2_ or specific function like `fn2()`_ +or `fn2( int )`_ or `fn2(int, +float)`_. + +Ref generics like this: binarySearch_ or `binarySearch(openArray[T], K, +proc (T, K))`_ or `proc binarySearch(openArray[T], K, proc (T, K))`_ or +in different style: `proc binarysearch(openarray[T], K, proc(T, K))`_. +Can be combined with export symbols and type parameters: +`binarysearch*[T, K](openArray[T], K, proc (T, K))`_. +With spaces `binary search`_. + +Note that `proc` can be used in postfix form: `binarySearch proc`_. + +Ref. type like G_ and `type G`_ and `G[T]`_ and `type G*[T]`_. + ]## +include ./utils_helpers + type SomeType* = enum enumValueA, enumValueB, enumValueC + G*[T] = object + val: T proc someType*(): SomeType = ## constructor. @@ -33,6 +54,14 @@ proc someType*(): SomeType = proc fn2*() = discard ## comment +proc fn2*(x: int) = + ## fn2 comment + discard +proc fn2*(x: int, y: float) = + discard +proc binarySearch*[T, K](a: openArray[T]; key: K; + cmp: proc (x: T; y: K): int {.closure.}): int = + discard proc fn3*(): auto = 1 ## comment proc fn4*(): auto = 2 * 3 + 4 ## comment proc fn5*() ## comment @@ -89,3 +118,39 @@ template fromUtilsGen*(): untyped = ## came form utils but should be shown where `fromUtilsGen` is called runnableExamples: discard """should be shown as examples for fromUtils3 in module calling fromUtilsGen""" + +proc f*(x: G[int]) = + ## There is also variant `f(G[string])`_ + discard +proc f*(x: G[string]) = + ## See also `f(G[int])`_. + discard + +## Ref. `[]`_ is the same as `proc \`[]\`(G[T])`_ because there are no +## overloads. The full form: `proc \`[]\`*[T](x: G[T]): T`_ + +proc `[]`*[T](x: G[T]): T = x.val + +## Ref. `[]=`_ aka `\`[]=\`(G[T], int, T)`_. + +proc `[]=`*[T](a: var G[T], index: int, value: T) = discard + +## Ref. `$`_ aka `proc $`_ or `proc \`$\``_. + +proc `$`*[T](a: G[T]): string = "" + +## Ref. `$(a: ref SomeType)`_. + +proc `$`*[T](a: ref SomeType): string = "" + +## Ref. foo_bar_ aka `iterator foo_bar_`_. + +iterator fooBar*(a: seq[SomeType]): int = discard + +## Ref. `fn[T; U,V: SomeFloat]()`_. + +proc fn*[T; U, V: SomeFloat]() = discard + +## Ref. `'big`_ or `func \`'big\``_ or `\`'big\`(string)`_. + +func `'big`*(a: string): SomeType = discard diff --git a/nimdoc/testproject/subdir/subdir_b/utils_helpers.nim b/nimdoc/testproject/subdir/subdir_b/utils_helpers.nim new file mode 100644 index 000000000..2c45ffb83 --- /dev/null +++ b/nimdoc/testproject/subdir/subdir_b/utils_helpers.nim @@ -0,0 +1 @@ +proc funWithGenerics*[T, U: SomeFloat](a: T, b: U) = discard diff --git a/nimdoc/testproject/subdir/subdir_b/utils_overview.rst b/nimdoc/testproject/subdir/subdir_b/utils_overview.rst new file mode 100644 index 000000000..58ce930bf --- /dev/null +++ b/nimdoc/testproject/subdir/subdir_b/utils_overview.rst @@ -0,0 +1,8 @@ +This is a description of the utils module. + +Links work: + +* other module: `iterators `_ (not in this dir, just an example) +* internal: `fn2(x)`_ +* internal included from another module: `funWithGenerics*[T, U: + SomeFloat](a: T, b: U)`_. diff --git a/tests/stdlib/tdochelpers.nim b/tests/stdlib/tdochelpers.nim new file mode 100644 index 000000000..5bcdf32d5 --- /dev/null +++ b/tests/stdlib/tdochelpers.nim @@ -0,0 +1,155 @@ +discard """ + output: ''' + +[Suite] Integration with Nim +''' +""" + +# tests for dochelpers.nim module + +import ../../lib/packages/docutils/[rstast, rst, dochelpers] +import unittest + +proc rstParseTest(text: string): PRstNode = + proc testMsgHandler(filename: string, line, col: int, msgkind: MsgKind, + arg: string) = + doAssert msgkind == mwBrokenLink + let r = rstParse(text, "-input-", LineRstInit, ColRstInit, + {roPreferMarkdown, roSupportMarkdown, roNimFile}, + msgHandler=testMsgHandler) + result = r.node + +suite "Integration with Nim": + test "simple symbol parsing (shortest form)": + let input1 = "g_".rstParseTest + check input1.toLangSymbol == LangSymbol(symKind: "", name: "g") + + test "simple symbol parsing (group of words)": + let input1 = "`Y`_".rstParseTest + check input1.toLangSymbol == LangSymbol(symKind: "", name: "Y") + + # this means not a statement 'type', it's a backticked identifier `type`: + let input2 = "`type`_".rstParseTest + check input2.toLangSymbol == LangSymbol(symKind: "", name: "type") + + let input3 = "`[]`_".rstParseTest + check input3.toLangSymbol == LangSymbol(symKind: "", name: "[]") + + let input4 = "`X Y Z`_".rstParseTest + check input4.toLangSymbol == LangSymbol(symKind: "", name: "Xyz") + + test "simple proc parsing": + let input1 = "proc f".rstParseTest + check input1.toLangSymbol == LangSymbol(symKind: "proc", name: "f") + + test "another backticked name": + let input1 = """`template \`type\``_""".rstParseTest + check input1.toLangSymbol == LangSymbol(symKind: "template", name: "type") + + test "simple proc parsing with parameters": + let input1 = "`proc f*()`_".rstParseTest + let input2 = "`proc f()`_".rstParseTest + let expected = LangSymbol(symKind: "proc", name: "f", + parametersProvided: true) + check input1.toLangSymbol == expected + check input2.toLangSymbol == expected + + test "symbol parsing with 1 parameter": + let input = "`f(G[int])`_".rstParseTest + let expected = LangSymbol(symKind: "", name: "f", + parameters: @[("G[int]", "")], + parametersProvided: true) + check input.toLangSymbol == expected + + test "more proc parsing": + let input1 = "`proc f[T](x:G[T]):M[T]`_".rstParseTest + let input2 = "`proc f[ T ] ( x: G [T] ): M[T]`_".rstParseTest + let input3 = "`proc f*[T](x: G[T]): M[T]`_".rstParseTest + let expected = LangSymbol(symKind: "proc", + name: "f", + generics: "[T]", + parameters: @[("x", "G[T]")], + parametersProvided: true, + outType: "M[T]") + check(input1.toLangSymbol == expected) + check(input2.toLangSymbol == expected) + check(input3.toLangSymbol == expected) + + test "advanced proc parsing with Nim identifier normalization": + let input = """`proc binarySearch*[T, K](a: openArray[T]; key: K; + cmp: proc (x: T; y: K): int)`_""".rstParseTest + let expected = LangSymbol(symKind: "proc", + name: "binarysearch", + generics: "[T,K]", + parameters: @[ + ("a", "openarray[T]"), + ("key", "K"), + ("cmp", "proc(x:T;y:K):int")], + parametersProvided: true, + outType: "") + check(input.toLangSymbol == expected) + + test "the same without proc": + let input = """`binarySearch*[T, K](a: openArray[T]; key: K; + cmp: proc (x: T; y: K): int {.closure.})`_""".rstParseTest + let expected = LangSymbol(symKind: "", + name: "binarysearch", + generics: "[T,K]", + parameters: @[ + ("a", "openarray[T]"), + ("key", "K"), + ("cmp", "proc(x:T;y:K):int")], + parametersProvided: true, + outType: "") + check(input.toLangSymbol == expected) + + test "operator $ with and without backticks": + let input1 = """`func \`$\`*[T](a: \`open Array\`[T]): string`_""". + rstParseTest + let input2 = """`func $*[T](a: \`open Array\`[T]): string`_""". + rstParseTest + let expected = LangSymbol(symKind: "func", + name: "$", + generics: "[T]", + parameters: @[("a", "openarray[T]")], + parametersProvided: true, + outType: "string") + check(input1.toLangSymbol == expected) + check(input2.toLangSymbol == expected) + + test "operator [] with and without backticks": + let input1 = """`func \`[]\`[T](a: \`open Array\`[T], idx: int): T`_""". + rstParseTest + let input2 = """`func [][T](a: \`open Array\`[T], idx: int): T`_""". + rstParseTest + let expected = LangSymbol(symKind: "func", + name: "[]", + generics: "[T]", + parameters: @[("a", "openarray[T]"), + ("idx", "int")], + parametersProvided: true, + outType: "T") + check(input1.toLangSymbol == expected) + check(input2.toLangSymbol == expected) + + test "postfix symbol specifier #1": + let input = """`walkDir iterator`_""". + rstParseTest + let expected = LangSymbol(symKind: "iterator", + name: "walkdir") + check(input.toLangSymbol == expected) + + test "postfix symbol specifier #2": + let input1 = """`\`[]\`[T](a: \`open Array\`[T], idx: int): T func`_""". + rstParseTest + let input2 = """`[][T](a: \`open Array\`[T], idx: int): T func`_""". + rstParseTest + let expected = LangSymbol(symKind: "func", + name: "[]", + generics: "[T]", + parameters: @[("a", "openarray[T]"), + ("idx", "int")], + parametersProvided: true, + outType: "T") + check(input1.toLangSymbol == expected) + check(input2.toLangSymbol == expected) diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index e9f67324c..e4609a671 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -17,9 +17,7 @@ discard """ # tests for rst module -import ../../lib/packages/docutils/rstgen -import ../../lib/packages/docutils/rst -import ../../lib/packages/docutils/rstast +import ../../lib/packages/docutils/[rstgen, rst, rstast] import unittest, strutils import std/private/miscdollars import os @@ -61,6 +59,52 @@ proc toAst(input: string, result = e.msg suite "RST parsing": + test "References are whitespace-neutral and case-insensitive": + # refname is 'lexical-analysis', the same for all the 3 variants: + check(dedent""" + Lexical Analysis + ================ + + Ref. `Lexical Analysis`_ or `Lexical analysis`_ or `lexical analysis`_. + """.toAst == + dedent""" + rnInner + rnHeadline level=1 + rnLeaf 'Lexical' + rnLeaf ' ' + rnLeaf 'Analysis' + rnParagraph + rnLeaf 'Ref' + rnLeaf '.' + rnLeaf ' ' + rnInternalRef + rnInner + rnLeaf 'Lexical' + rnLeaf ' ' + rnLeaf 'Analysis' + rnLeaf 'lexical-analysis' + rnLeaf ' ' + rnLeaf 'or' + rnLeaf ' ' + rnInternalRef + rnInner + rnLeaf 'Lexical' + rnLeaf ' ' + rnLeaf 'analysis' + rnLeaf 'lexical-analysis' + rnLeaf ' ' + rnLeaf 'or' + rnLeaf ' ' + rnInternalRef + rnInner + rnLeaf 'lexical' + rnLeaf ' ' + rnLeaf 'analysis' + rnLeaf 'lexical-analysis' + rnLeaf '.' + rnLeaf ' ' + """) + test "option list has priority over definition list": check(dedent""" --defusages @@ -376,9 +420,9 @@ suite "Warnings": let output = input.toAst(warnings=warnings) check(warnings[] == @[ "input(3, 14) Warning: broken link 'citation-som'", - "input(5, 7) Warning: broken link 'a-broken-link'", + "input(5, 7) Warning: broken link 'a broken Link'", "input(7, 15) Warning: unknown substitution 'undefined subst'", - "input(9, 6) Warning: broken link 'shortdotlink'" + "input(9, 6) Warning: broken link 'short.link'" ]) test "With include directive and blank lines at the beginning": @@ -391,7 +435,7 @@ suite "Warnings": let input = ".. include:: other.rst" var warnings = new seq[string] let output = input.toAst(warnings=warnings) - check warnings[] == @["other.rst(5, 6) Warning: broken link 'brokenlink'"] + check warnings[] == @["other.rst(5, 6) Warning: broken link 'brokenLink'"] check(output == dedent""" rnInner rnParagraph @@ -404,6 +448,59 @@ suite "Warnings": """) removeFile("other.rst") + test "warnings for ambiguous links (references + anchors)": + # Reference like `x`_ generates a link alias x that may clash with others + let input = dedent""" + Manual reference: `foo <#foo,string,string>`_ + + .. _foo: + + Paragraph. + + Ref foo_ + """ + var warnings = new seq[string] + let output = input.toAst(warnings=warnings) + check(warnings[] == @[ + dedent """ + input(7, 5) Warning: ambiguous doc link `foo` + clash: + (3, 8): (manual directive anchor) + (1, 45): (implicitly-generated hyperlink alias)""" + ]) + # reference should be resolved to the manually set anchor: + check(output == + dedent""" + rnInner + rnParagraph + rnLeaf 'Manual' + rnLeaf ' ' + rnLeaf 'reference' + rnLeaf ':' + rnLeaf ' ' + rnHyperlink + rnInner + rnLeaf 'foo' + rnInner + rnLeaf '#' + rnLeaf 'foo' + rnLeaf ',' + rnLeaf 'string' + rnLeaf ',' + rnLeaf 'string' + rnParagraph anchor='foo' + rnLeaf 'Paragraph' + rnLeaf '.' + rnParagraph + rnLeaf 'Ref' + rnLeaf ' ' + rnInternalRef + rnInner + rnLeaf 'foo' + rnLeaf 'foo' + rnLeaf ' ' + """) + suite "RST include directive": test "Include whole": "other.rst".writeFile("**test1**") diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index 867677404..45cf1a68e 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -1054,8 +1054,9 @@ Test1 Paragraph2 ref `internal anchor`_. """ let output9 = input9.toHtml - #doAssert "id=\"internal-anchor\"" in output9 - #doAssert "internal anchor" notin output9 + # _`internal anchor` got erased: + check "href=\"#internal-anchor\"" notin output9 + check "href=\"#citation-another\"" in output9 doAssert output9.count("
                  " & "
                  ") == 1 doAssert output9.count("
                  ") == 3 @@ -1330,12 +1331,12 @@ Test1 """ let output1 = input1.toHtml # "target101" should be erased and changed to "section-xyz": - doAssert "href=\"#target101\"" notin output1 - doAssert "id=\"target101\"" notin output1 - doAssert "href=\"#target102\"" notin output1 - doAssert "id=\"target102\"" notin output1 - doAssert "id=\"section-xyz\"" in output1 - doAssert "href=\"#section-xyz\"" in output1 + check "href=\"#target101\"" notin output1 + check "id=\"target101\"" notin output1 + check "href=\"#target102\"" notin output1 + check "id=\"target102\"" notin output1 + check "id=\"section-xyz\"" in output1 + check "href=\"#section-xyz\"" in output1 let input2 = dedent """ .. _target300: @@ -1405,7 +1406,7 @@ Test1 let output1 = input1.toHtml doAssert "id=\"secdot1\"" in output1 doAssert "id=\"Z2minusothercolonsecplusc-2\"" in output1 - doAssert "id=\"linkdot1-2021\"" in output1 + check "id=\"linkdot1-2021\"" in output1 let ref1 = "sec.1" let ref2 = "2-other:sec+c_2" let ref3 = "link.1_2021" From ee703c5db4ceb8d0228b4429c53d85d5f1ff91ea Mon Sep 17 00:00:00 2001 From: Tomasz Kramkowski Date: Fri, 29 Oct 2021 10:39:41 +0100 Subject: [PATCH 1318/2472] Fix nimIdentNormalize, fixes #19067 (#19068) * Make nimIdentNormalize return "" when passed ""; fixes #19067 Fixes #19067 * Add tests for nimIdentNormalize --- lib/pure/strutils.nim | 5 +++-- tests/stdlib/tstrutils.nim | 7 +++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 2234a8625..e4a4a5ec1 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -281,8 +281,9 @@ func nimIdentNormalize*(s: string): string = runnableExamples: doAssert nimIdentNormalize("Foo_bar") == "Foobar" result = newString(s.len) - if s.len > 0: - result[0] = s[0] + if s.len == 0: + return + result[0] = s[0] var j = 1 for i in 1..len(s) - 1: if s[i] in {'A'..'Z'}: diff --git a/tests/stdlib/tstrutils.nim b/tests/stdlib/tstrutils.nim index 234991bdb..8000b6b07 100644 --- a/tests/stdlib/tstrutils.nim +++ b/tests/stdlib/tstrutils.nim @@ -841,5 +841,12 @@ bar doAssert s.endsWith('a') == false doAssert s.endsWith('\0') == false + block: # nimIdentNormalize + doAssert nimIdentNormalize("") == "" + doAssert nimIdentNormalize("foo") == "foo" + doAssert nimIdentNormalize("foo_bar") == "foobar" + doAssert nimIdentNormalize("Foo_bar") == "Foobar" + doAssert nimIdentNormalize("_Foo_bar") == "_foobar" + static: main() main() From cfdac6666f5772479724e082fb08c2db694d8d40 Mon Sep 17 00:00:00 2001 From: Tail Wag Games Date: Fri, 29 Oct 2021 06:42:44 -0500 Subject: [PATCH 1319/2472] Freeing critical sections via atexit in system/alloc and system/io (#19062) * adding new system module sysexitprocs and including system exit procedures when registering exit handlers defined in userland * fixing failing tests and adding initialization guard to handle cases where the module's global init logic isn't invoked first as is the case with some gc implementaions * js backend shouldn't try to invoke actual system exit procs * fixing formatting in sysexitprocs.nim * 256 was too much - my max number of plugins in my engine is 64 and I require two hooks per runtime it looks like with tls emulation turned off, so for my purposes 128 should be sufficient * so atExit should be enough here, can get rid of all the extra cruft I had added on top since I didn't realize atExit already provided a stack * done being cute - since newruntime prevents correct cpp codegen for object variants apparently and breaks tests if I try to use std/exitprocs, ddSysExitProc is just going into both modules. Since system doesn't include system/io, polluting system with it doesn't make sense either... at least it is only importc'd when it is required in either module and we don't have to have any weird when defined(nimOwnedEnabled) with a comment explaining why --- lib/std/exitprocs.nim | 3 --- lib/system/alloc.nim | 3 +++ lib/system/io.nim | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/std/exitprocs.nim b/lib/std/exitprocs.nim index c76583a8c..c6537f7f8 100644 --- a/lib/std/exitprocs.nim +++ b/lib/std/exitprocs.nim @@ -44,9 +44,6 @@ proc callClosures() {.noconv.} = of kClosure: fun.fun1() of kNoconv: fun.fun2() - when not defined(js) and not defined(nimOwnedEnabled): - deinitLock(gFunsLock) - template fun() = if gFuns.len == 0: addAtExit(callClosures) diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index 6d83a2c17..2c6ab4462 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -1054,9 +1054,12 @@ template instantiateForRegion(allocator: untyped) {.dirty.} = it = it.next when hasThreadSupport: + proc addSysExitProc(quitProc: proc() {.noconv.}) {.importc: "atexit", header: "".} + var sharedHeap: MemRegion var heapLock: SysLock initSysLock(heapLock) + addSysExitProc(proc() {.noconv.} = deinitSys(heapLock)) proc getFreeMem(): int = #sysAssert(result == countFreeMem()) diff --git a/lib/system/io.nim b/lib/system/io.nim index e7369392a..2ad43acdb 100644 --- a/lib/system/io.nim +++ b/lib/system/io.nim @@ -777,15 +777,15 @@ proc setStdIoUnbuffered*() {.tags: [], benign.} = when declared(stdout): when defined(windows) and compileOption("threads"): + proc addSysExitProc(quitProc: proc() {.noconv.}) {.importc: "atexit", header: "".} + const insideRLocksModule = false include "system/syslocks" + var echoLock: SysLock initSysLock echoLock - - when not defined(js) and not defined(nimOwnedEnabled): - import std/exitprocs - addExitProc(proc() {.noconv.} = deinitSys echoLock) + addSysExitProc(proc() {.noconv.} = deinitSys(echoLock)) const stdOutLock = not defined(windows) and not defined(android) and From f755e452d2a2c7656f07793b7f01dc654a7c5253 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Derek=20=E5=91=86?= <116649+derekdai@users.noreply.github.com> Date: Sat, 30 Oct 2021 01:55:48 +0800 Subject: [PATCH 1320/2472] fix #18971 (#19070) [backport:1.6] since the example code return value from global variable, instead of first argument, the `n.len` is 1 which causes compiler crashes. --- compiler/varpartitions.nim | 2 +- tests/arc/t18971.nim | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 tests/arc/t18971.nim diff --git a/compiler/varpartitions.nim b/compiler/varpartitions.nim index 8f422face..709d00fa0 100644 --- a/compiler/varpartitions.nim +++ b/compiler/varpartitions.nim @@ -479,7 +479,7 @@ proc destMightOwn(c: var Partitions; dest: var VarIndex; n: PNode) = # calls do construct, what we construct must be destroyed, # so dest cannot be a cursor: dest.flags.incl ownsData - elif n.typ.kind in {tyLent, tyVar}: + elif n.typ.kind in {tyLent, tyVar} and n.len > 1: # we know the result is derived from the first argument: var roots: seq[(PSym, int)] allRoots(n[1], roots, RootEscapes) diff --git a/tests/arc/t18971.nim b/tests/arc/t18971.nim new file mode 100644 index 000000000..9b587d956 --- /dev/null +++ b/tests/arc/t18971.nim @@ -0,0 +1,10 @@ +discard """ + cmd: "nim c --gc:arc $file" +""" + +type MyObj = ref object + +var o = MyObj() +proc x: var MyObj = o + +var o2 = x() From 2bda4a30a6b12dd0840dc347e454e54fe26721e7 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 30 Oct 2021 10:14:23 +0200 Subject: [PATCH 1321/2472] fixes #19000 (#19032) * fixes #19000 * progress --- lib/system/memalloc.nim | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/system/memalloc.nim b/lib/system/memalloc.nim index f8ebc8c5f..49766e69d 100644 --- a/lib/system/memalloc.nim +++ b/lib/system/memalloc.nim @@ -116,6 +116,9 @@ when hasAlloc and not defined(js): ## ## See also: ## * `create <#create,typedesc>`_ + static: + when sizeof(T) <= 0: + {.fatal: "createU does not support types T where sizeof(T) == 0".} cast[ptr T](alloc(T.sizeof * size)) template alloc0*(size: Natural): pointer = @@ -141,6 +144,9 @@ when hasAlloc and not defined(js): ## ## The allocated memory belongs to its allocating thread! ## Use `createShared <#createShared,typedesc>`_ to allocate from a shared heap. + static: + when sizeof(T) <= 0: + {.fatal: "create does not support types T where sizeof(T) == 0".} cast[ptr T](alloc0(sizeof(T) * size)) template realloc*(p: pointer, newSize: Natural): pointer = From 2f730afe9e7eb981258aea257deba0240f488dd1 Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 31 Oct 2021 13:22:00 +0800 Subject: [PATCH 1322/2472] fix #18410 (Errors initializing an object of RootObj with the C++ backend) [backport] (#18836) * fix #18410 * one line comment * typo * typo * cover cpp --- compiler/cgen.nim | 3 ++- tests/arc/tconst_to_sink.nim | 3 ++- tests/cpp/torc.nim | 15 +++++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 tests/cpp/torc.nim diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 412a428da..b669483cc 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -363,7 +363,8 @@ proc genObjectInit(p: BProc, section: TCProcSection, t: PType, a: var TLoc, else: linefmt(p, section, "$1.m_type = $2;$n", [r, genTypeInfoV1(p.module, t, a.lode.info)]) of frEmbedded: - if optTinyRtti in p.config.globalOptions: + # inheritance in C++ does not allow struct initialization: bug #18410 + if not p.module.compileToCpp and optTinyRtti in p.config.globalOptions: var tmp: TLoc if mode == constructRefObj: let objType = t.skipTypes(abstractInst+{tyRef}) diff --git a/tests/arc/tconst_to_sink.nim b/tests/arc/tconst_to_sink.nim index ddcc46e67..25f659341 100644 --- a/tests/arc/tconst_to_sink.nim +++ b/tests/arc/tconst_to_sink.nim @@ -1,6 +1,7 @@ discard """ output: '''@[(s1: "333", s2: ""), (s1: "abc", s2: "def"), (s1: "3x", s2: ""), (s1: "3x", s2: ""), (s1: "3x", s2: ""), (s1: "3x", s2: ""), (s1: "lastone", s2: "")]''' - cmd: "nim c --gc:arc $file" + matrix: "--gc:arc" + targets: "c cpp" """ # bug #13240 diff --git a/tests/cpp/torc.nim b/tests/cpp/torc.nim new file mode 100644 index 000000000..7fc474f94 --- /dev/null +++ b/tests/cpp/torc.nim @@ -0,0 +1,15 @@ +discard """ + targets: "cpp" + matrix: "--gc:orc" +""" + +import std/options + +# bug #18410 +type + O = object of RootObj + val: pointer + +proc p(): Option[O] = none(O) + +doAssert $p() == "none(O)" \ No newline at end of file From 4c510d5577a0ced6a31b8417fed74acdd604b9a3 Mon Sep 17 00:00:00 2001 From: haxscramper Date: Tue, 2 Nov 2021 10:54:04 +0300 Subject: [PATCH 1323/2472] [FIX] Do not break formatted string line (#19085) [backport] Otherwise, compiler produces broken error message - `$1` is not interpolated `Error: The $1 type doesn't have a default value. The following fields must be initialized: importGraph.` --- compiler/semobjconstr.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/compiler/semobjconstr.nim b/compiler/semobjconstr.nim index 001956ede..30ab76957 100644 --- a/compiler/semobjconstr.nim +++ b/compiler/semobjconstr.nim @@ -368,8 +368,7 @@ proc defaultConstructionError(c: PContext, t: PType, info: TLineInfo) = let initResult = semConstructTypeAux(c, constrCtx, {}) assert constrCtx.missingFields.len > 0 localError(c.config, info, - "The $1 type doesn't have a default value. The following fields must " & - "be initialized: $2." % [typeToString(t), listSymbolNames(constrCtx.missingFields)]) + "The $1 type doesn't have a default value. The following fields must be initialized: $2." % [typeToString(t), listSymbolNames(constrCtx.missingFields)]) elif objType.kind == tyDistinct: localError(c.config, info, "The $1 distinct type doesn't have a default value." % typeToString(t)) From 24a75842998709b29ef50ef4cc10d30304efe3aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Derek=20=E5=91=86?= <116649+derekdai@users.noreply.github.com> Date: Tue, 2 Nov 2021 15:55:50 +0800 Subject: [PATCH 1324/2472] update numbers of lifetime-tracking hooks in doc/destructors.rst (#19088) --- doc/destructors.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/destructors.rst b/doc/destructors.rst index 6e4509666..d6f240dee 100644 --- a/doc/destructors.rst +++ b/doc/destructors.rst @@ -101,7 +101,7 @@ The memory management for Nim's standard `string` and `seq` types as well as other standard collections is performed via so-called "Lifetime-tracking hooks", which are particular `type bound operators `_. -There are 3 different hooks for each (generic or concrete) object type `T` (`T` can also be a +There are 4 different hooks for each (generic or concrete) object type `T` (`T` can also be a `distinct` type) that are called implicitly by the compiler. (Note: The word "hook" here does not imply any kind of dynamic binding From cc984217a98438de5a36447ea6574eb2c5da4ae4 Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Tue, 2 Nov 2021 02:02:53 -0600 Subject: [PATCH 1325/2472] Fix VM's sametype impl to work for generics/typedescs (#19073) * Fix vm's sametype implementation to properly handle generics and typedescs * actually fixed sametype + have test * added comments and removed unsafe code --- compiler/types.nim | 5 ++++ compiler/vm.nim | 3 ++- tests/stdlib/tmacros.nim | 58 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/compiler/types.nim b/compiler/types.nim index 2c7b91d9f..7cf02c240 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -1128,6 +1128,11 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool = a = a.skipTypes({tyDistinct, tyGenericInst}) if a.kind != b.kind: return false + #[ + The following code should not run in the case either side is an generic alias, + but it's not presently possible to distinguish the genericinsts from aliases of + objects ie `type A[T] = SomeObject` + ]# # this is required by tunique_type but makes no sense really: if tyDistinct notin {x.kind, y.kind} and x.kind == tyGenericInst and IgnoreTupleFields notin c.flags: let diff --git a/compiler/vm.nim b/compiler/vm.nim index 1b443aff1..b64545081 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -1044,7 +1044,8 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = strictSymEquality=true)) of opcSameNodeType: decodeBC(rkInt) - regs[ra].intVal = ord(regs[rb].node.typ.sameTypeOrNil regs[rc].node.typ) + regs[ra].intVal = ord(regs[rb].node.typ.sameTypeOrNil(regs[rc].node.typ, {ExactTypeDescValues, ExactGenericParams})) + # The types should exactly match which is why we pass `{ExactTypeDescValues..ExactGcSafety}`. of opcXor: decodeBC(rkInt) regs[ra].intVal = ord(regs[rb].intVal != regs[rc].intVal) diff --git a/tests/stdlib/tmacros.nim b/tests/stdlib/tmacros.nim index 18cfdc75c..64a474743 100644 --- a/tests/stdlib/tmacros.nim +++ b/tests/stdlib/tmacros.nim @@ -66,3 +66,61 @@ block: # unpackVarargs doAssert call1(toString) == "" doAssert call1(toString, 10) == "10" doAssert call1(toString, 10, 11) == "1011" + +block: # SameType + type + A = int + B = distinct int + C = object + Generic[T, Y] = object + macro isSameType(a, b: typed): untyped = + newLit(sameType(a, b)) + + static: + assert Generic[int, int].isSameType(Generic[int, int]) + assert Generic[A, string].isSameType(Generic[int, string]) + assert not Generic[A, string].isSameType(Generic[B, string]) + assert not Generic[int, string].isSameType(Generic[int, int]) + assert isSameType(int, A) + assert isSameType(10, 20) + assert isSameType("Hello", "world") + assert not isSameType("Hello", cstring"world") + assert not isSameType(int, B) + assert not isSameType(int, Generic[int, int]) + assert not isSameType(C, string) + assert not isSameType(C, int) + + + #[ + # compiler sameType fails for the following, read more in `types.nim`'s `sameTypeAux`. + type + D[T] = C + G[T] = T + static: + assert isSameType(D[int], C) + assert isSameType(D[int], D[float]) + assert isSameType(G[float](1.0), float(1.0)) + assert isSameType(float(1.0), G[float](1.0)) + ]# + + type Tensor[T] = object + data: T + + macro testTensorInt(x: typed): untyped = + let + tensorIntType = getTypeInst(Tensor[int])[1] + xTyp = x.getTypeInst + + newLit(xTyp.sameType(tensorIntType)) + + var + x: Tensor[int] + x1 = Tensor[float]() + x2 = Tensor[A]() + x3 = Tensor[B]() + + static: + assert testTensorInt(x) + assert not testTensorInt(x1) + assert testTensorInt(x2) + assert not testTensorInt(x3) From 2e0db988e7efd05b07f5165cf1357efea9391eb3 Mon Sep 17 00:00:00 2001 From: Kaushal Modi Date: Tue, 2 Nov 2021 04:06:46 -0400 Subject: [PATCH 1326/2472] tests: Fix warnings in tstrscans (#19082) --- tests/stdlib/tstrscans.nim | 60 +++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/tests/stdlib/tstrscans.nim b/tests/stdlib/tstrscans.nim index d443c5dda..24a3c02f7 100644 --- a/tests/stdlib/tstrscans.nim +++ b/tests/stdlib/tstrscans.nim @@ -15,7 +15,7 @@ block ParsePasswd: else: break - const etc_passwd = """root:x:0:0:root:/root:/bin/bash + const etcPasswd = """root:x:0:0:root:/root:/bin/bash daemon:x:1:1:daemon:/usr/sbin:/bin/sh bin:x:2:2:bin:/bin:/bin/sh sys:x:3:3:sys:/dev:/bin/sh @@ -23,7 +23,7 @@ nobody:x:65534:65534:nobody:/nonexistent:/bin/sh messagebus:x:103:107::/var/run/dbus:/bin/false """ - const parsed_etc_passwd = @[ + const parsedEtcPasswd = @[ "root:x:0:0:root:/root:/bin/bash", "daemon:x:1:1:daemon:/usr/sbin:/bin/sh", "bin:x:2:2:bin:/bin:/bin/sh", @@ -31,7 +31,7 @@ messagebus:x:103:107::/var/run/dbus:/bin/false "nobody:x:65534:65534:nobody:/nonexistent:/bin/sh", "messagebus:x:103:107::/var/run/dbus:/bin/false", ] - doAssert etc_passwd.parsePasswd == parsed_etc_passwd + doAssert etcPasswd.parsePasswd == parsedEtcPasswd block LastNot: var idx : int @@ -139,27 +139,27 @@ block: break var key, val: string - var intval: int - var floatval: float - doAssert scanf("abc:: xyz 89 33.25", "$w$s::$s$w$s$i $f", key, val, intval, floatVal) + var intVal: int + var floatVal: float + doAssert scanf("abc:: xyz 89 33.25", "$w$s::$s$w$s$i $f", key, val, intVal, floatVal) doAssert key == "abc" doAssert val == "xyz" - doAssert intval == 89 + doAssert intVal == 89 doAssert floatVal == 33.25 - var binval: int - var octval: int - var hexval: int - doAssert scanf("0b0101 0o1234 0xabcd", "$b$s$o$s$h", binval, octval, hexval) - doAssert binval == 0b0101 - doAssert octval == 0o1234 - doAssert hexval == 0xabcd + var binVal: int + var octVal: int + var hexVal: int + doAssert scanf("0b0101 0o1234 0xabcd", "$b$s$o$s$h", binVal, octVal, hexVal) + doAssert binVal == 0b0101 + doAssert octVal == 0o1234 + doAssert hexVal == 0xabcd - let xx = scanf("$abc", "$$$i", intval) + let xx = scanf("$abc", "$$$i", intVal) doAssert xx == false - let xx2 = scanf("$1234", "$$$i", intval) + let xx2 = scanf("$1234", "$$$i", intVal) doAssert xx2 let yy = scanf(";.--Breakpoint00 [output]", @@ -246,24 +246,32 @@ block: result = 0 while start+result < input.len and input[start+result] in seps: inc result + type + ScanRetType = tuple + success: bool + lo: int + hi: int + ch: char + word: string + var res = 0 for line in input.splitLines: - let (success, lo, hi, c ,w) = scanTuple(line, "$i-$i $c: $w") - if success: + let ret: ScanRetType = scanTuple(line, "$i-$i $c: $w") + if ret.success: inc res doAssert res == 4 - let (_, key, val, intval, floatVal) = scanTuple("abc:: xyz 89 33.25", "$w$s::$s$w$s$i $f") + let (_, key, val, intVal, floatVal) = scanTuple("abc:: xyz 89 33.25", "$w$s::$s$w$s$i $f") doAssert key == "abc" doAssert val == "xyz" - doAssert intval == 89 + doAssert intVal == 89 doAssert floatVal == 33.25 - let (_, binVal, octVal, hexVal) = scanTuple("0b0101 0o1234 0xabcd", "$b$s$o$s$h", binval, octval, hexval) - doAssert binval == 0b0101 - doAssert octval == 0o1234 - doAssert hexval == 0xabcd + let (_, binVal, octVal, hexVal) = scanTuple("0b0101 0o1234 0xabcd", "$b$s$o$s$h", binVal, octVal, hexVal) + doAssert binVal == 0b0101 + doAssert octVal == 0o1234 + doAssert hexVal == 0xabcd var (xx,_) = scanTuple("$abc", "$$$i") doAssert xx == false @@ -272,9 +280,9 @@ block: let (xx2, _) = block: scanTuple("$1234", "$$$i") doAssert xx2 - var (yy, intval2, key2) = scanTuple(";.--Breakpoint00 [output]", + var (yy, intVal2, key2) = scanTuple(";.--Breakpoint00 [output]", "$[someSep]Breakpoint${twoDigits}$[someSep({';','.','-'})] [$+]$.", int) doAssert yy doAssert key2 == "output" - doAssert intVal2 == 13 \ No newline at end of file + doAssert intVal2 == 13 From e01e543fce8eab147d4e7eb5b89bba80c25e9cb3 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 2 Nov 2021 16:10:25 +0800 Subject: [PATCH 1327/2472] bootstrapping Nim compiler with `cpp --gc:orc` (#19087) --- koch.nim | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/koch.nim b/koch.nim index 8dd897028..06515aad7 100644 --- a/koch.nim +++ b/koch.nim @@ -602,9 +602,8 @@ proc runCI(cmd: string) = execFold("Run atlas tests", "nim c -r -d:atlasTests tools/atlas/atlas.nim clone https://github.com/disruptek/balls") when not defined(bsd): - if not doUseCpp: - # the BSDs are overwhelmed already, so only run this test on the other machines: - kochExecFold("Boot Nim ORC", "boot -d:release --gc:orc --lib:lib") + # the BSDs are overwhelmed already, so only run this test on the other machines: + kochExecFold("Boot Nim ORC", "boot -d:release --gc:orc --lib:lib") proc testUnixInstall(cmdLineRest: string) = csource("-d:danger" & cmdLineRest) From 12d2901e89d394cb1eb1242acc0107a2d20e3b3d Mon Sep 17 00:00:00 2001 From: Kaushal Modi Date: Tue, 2 Nov 2021 04:12:54 -0400 Subject: [PATCH 1328/2472] libs/impore/re: Add note about the requirement of `matches` to be pre-allocated (#19081) Add few runnableExamples for `findBounds` for clarity. Fixes https://github.com/nim-lang/Nim/issues/18775 --- lib/impure/re.nim | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/lib/impure/re.nim b/lib/impure/re.nim index d6922f4f1..8e5efbb28 100644 --- a/lib/impure/re.nim +++ b/lib/impure/re.nim @@ -152,6 +152,9 @@ proc findBounds*(buf: cstring, pattern: Regex, matches: var openArray[string], ## and the captured ## substrings in the array `matches`. If it does not match, nothing ## is written into `matches` and `(-1,0)` is returned. + ## + ## Note: The memory for `matches` needs to be allocated before this function is + ## called, otherwise it will just remain empty. var rtarray = initRtArray[cint]((matches.len+1)*3) rawMatches = rtarray.getRawData @@ -171,6 +174,14 @@ proc findBounds*(s: string, pattern: Regex, matches: var openArray[string], ## and the captured substrings in the array `matches`. ## If it does not match, nothing ## is written into `matches` and `(-1,0)` is returned. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. + runnableExamples: + var matches = newSeq[string](1) + let (first, last) = findBounds("Hello World", re"(W\w+)", matches) + doAssert first == 6 + doAssert last == 10 + doAssert matches[0] == "World" result = findBounds(cstring(s), pattern, matches, min(start, MaxReBufSize), min(s.len, MaxReBufSize)) @@ -182,6 +193,8 @@ proc findBounds*(buf: cstring, pattern: Regex, ## and the captured substrings in the array `matches`. ## If it does not match, nothing is written into `matches` and ## `(-1,0)` is returned. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. var rtarray = initRtArray[cint]((matches.len+1)*3) rawMatches = rtarray.getRawData @@ -202,6 +215,14 @@ proc findBounds*(s: string, pattern: Regex, ## and the captured substrings in the array `matches`. ## If it does not match, nothing is written into `matches` and ## `(-1,0)` is returned. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. + runnableExamples: + var matches = newSeq[tuple[first, last: int]](1) + let (first, last) = findBounds("Hello World", re"(\w+)", matches) + doAssert first == 0 + doAssert last == 4 + doAssert matches[0] == (0, 4) result = findBounds(cstring(s), pattern, matches, min(start, MaxReBufSize), min(s.len, MaxReBufSize)) @@ -255,6 +276,8 @@ proc matchLen*(s: string, pattern: Regex, matches: var openArray[string], ## the same as `match`, but it returns the length of the match, ## if there is no match, `-1` is returned. Note that a match length ## of zero can happen. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. result = matchOrFind(cstring(s), pattern, matches, start.cint, s.len.cint, pcre.ANCHORED) proc matchLen*(buf: cstring, pattern: Regex, matches: var openArray[string], @@ -262,6 +285,8 @@ proc matchLen*(buf: cstring, pattern: Regex, matches: var openArray[string], ## the same as `match`, but it returns the length of the match, ## if there is no match, `-1` is returned. Note that a match length ## of zero can happen. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. return matchOrFind(buf, pattern, matches, start.cint, bufSize.cint, pcre.ANCHORED) proc matchLen*(s: string, pattern: Regex, start = 0): int {.inline.} = @@ -292,6 +317,7 @@ proc match*(s: string, pattern: Regex, matches: var openArray[string], ## match, nothing is written into `matches` and `false` is ## returned. ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. runnableExamples: import std/sequtils var matches: array[2, string] @@ -306,6 +332,8 @@ proc match*(buf: cstring, pattern: Regex, matches: var openArray[string], ## match, nothing is written into `matches` and `false` is ## returned. ## `buf` has length `bufSize` (not necessarily `'\0'` terminated). + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. result = matchLen(buf, pattern, matches, start, bufSize) != -1 proc find*(buf: cstring, pattern: Regex, matches: var openArray[string], @@ -314,6 +342,8 @@ proc find*(buf: cstring, pattern: Regex, matches: var openArray[string], ## substrings in the array `matches`. If it does not match, nothing ## is written into `matches` and `-1` is returned. ## `buf` has length `bufSize` (not necessarily `'\0'` terminated). + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. var rtarray = initRtArray[cint]((matches.len+1)*3) rawMatches = rtarray.getRawData @@ -332,6 +362,8 @@ proc find*(s: string, pattern: Regex, matches: var openArray[string], ## returns the starting position of `pattern` in `s` and the captured ## substrings in the array `matches`. If it does not match, nothing ## is written into `matches` and `-1` is returned. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. result = find(cstring(s), pattern, matches, start, s.len) proc find*(buf: cstring, pattern: Regex, start = 0, bufSize: int): int = @@ -433,6 +465,8 @@ proc contains*(s: string, pattern: Regex, start = 0): bool {.inline.} = proc contains*(s: string, pattern: Regex, matches: var openArray[string], start = 0): bool {.inline.} = ## same as `find(s, pattern, matches, start) >= 0` + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. return find(s, pattern, matches, start) >= 0 proc startsWith*(s: string, prefix: Regex): bool {.inline.} = From efaaa38eefa38d842ef9225b6ac3f60c4c842209 Mon Sep 17 00:00:00 2001 From: Kaushal Modi Date: Tue, 2 Nov 2021 05:34:11 -0400 Subject: [PATCH 1329/2472] docstyle.rst: Do not export it to HTML as a standalone doc (#19083) .. This doc is included in contributing.rst. Fixes https://github.com/nim-lang/Nim/issues/14593. --- tools/kochdocs.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 0c5c8f4a3..42e648fb4 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -111,7 +111,10 @@ proc getRst2html(): seq[string] = for a in walkDirRecFilter("doc"): let path = a.path if a.kind == pcFile and path.splitFile.ext == ".rst" and path.lastPathPart notin - ["docs.rst", "nimfix.rst"]: + ["docs.rst", "nimfix.rst", + "docstyle.rst" # docstyle.rst shouldn't be converted to html separately; + # it's included in contributing.rst. + ]: # maybe we should still show nimfix, could help reviving it # `docs` is redundant with `overview`, might as well remove that file? result.add path From a55b90827ed87b61972147e8120c9afa949d17f9 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 2 Nov 2021 23:22:11 +0800 Subject: [PATCH 1330/2472] remove the mention of travis from readme (#19093) --- readme.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/readme.md b/readme.md index 7f333d5d4..af07c2a7b 100644 --- a/readme.md +++ b/readme.md @@ -149,7 +149,7 @@ read [this guide][pull-request-instructions]. Ideally, you should make sure that all tests pass before submitting a pull request. However, if you are short on time, you can just run the tests specific to your -changes by only running the corresponding categories of tests. Travis CI verifies +changes by only running the corresponding categories of tests. CI verifies that all tests pass before allowing the pull request to be accepted, so only running specific tests should be harmless. Integration tests should go in ``tests/untestable``. @@ -220,7 +220,6 @@ Copyright © 2006-2021 Andreas Rumpf, all rights reserved. [nimsuggest-repo]: https://github.com/nim-lang/nimsuggest [csources-repo-deprecated]: https://github.com/nim-lang/csources [csources-v1-repo]: https://github.com/nim-lang/csources_v1 -[badge-nim-travisci]: https://img.shields.io/travis/nim-lang/Nim/devel.svg?style=flat-square [badge-nim-irc]: https://img.shields.io/badge/chat-on_irc-blue.svg?style=flat-square [badge-nim-discord]: https://img.shields.io/discord/371759389889003530?color=blue&label=discord&logo=discord&logoColor=gold&style=flat-square [badge-nim-gitter]: https://img.shields.io/badge/chat-on_gitter-blue.svg?style=flat-square From 9d51197aa4eb404c3cce8c5989146dea8d3de024 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 2 Nov 2021 16:30:59 +0100 Subject: [PATCH 1331/2472] fixes #19078 [backport] (#19090) --- compiler/condsyms.nim | 2 ++ compiler/pragmas.nim | 4 +++- compiler/wordrecg.nim | 2 +- lib/std/private/digitsutils.nim | 9 ++++++--- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index df355bc26..5a7e78d5f 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -138,3 +138,5 @@ proc initDefines*(symbols: StringTableRef) = defineSymbol("nimHasHintAll") defineSymbol("nimHasTrace") defineSymbol("nimHasEffectsOf") + + defineSymbol("nimHasEnforceNoRaises") diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index a6e6131d7..702a9e082 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -31,7 +31,7 @@ const wAsmNoStackFrame, wDiscardable, wNoInit, wCodegenDecl, wGensym, wInject, wRaises, wEffectsOf, wTags, wLocks, wDelegator, wGcSafe, wConstructor, wLiftLocals, wStackTrace, wLineTrace, wNoDestroy, - wRequires, wEnsures} + wRequires, wEnsures, wEnforceNoRaises} converterPragmas* = procPragmas methodPragmas* = procPragmas+{wBase}-{wImportCpp} templatePragmas* = {wDeprecated, wError, wGensym, wInject, wDirty, @@ -1237,6 +1237,8 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, pragmaProposition(c, it) of wEnsures: pragmaEnsures(c, it) + of wEnforceNoRaises: + sym.flags.incl sfNeverRaises else: invalidPragma(c, it) elif comesFromPush and whichKeyword(ident) != wInvalid: discard "ignore the .push pragma; it doesn't apply" diff --git a/compiler/wordrecg.nim b/compiler/wordrecg.nim index ce06effab..22f6cc71d 100644 --- a/compiler/wordrecg.nim +++ b/compiler/wordrecg.nim @@ -86,7 +86,7 @@ type wAsmNoStackFrame = "asmNoStackFrame", wImplicitStatic = "implicitStatic", wGlobal = "global", wCodegenDecl = "codegenDecl", wUnchecked = "unchecked", wGuard = "guard", wLocks = "locks", wPartial = "partial", wExplain = "explain", - wLiftLocals = "liftlocals", + wLiftLocals = "liftlocals", wEnforceNoRaises = "enforceNoRaises", wAuto = "auto", wBool = "bool", wCatch = "catch", wChar = "char", wClass = "class", wCompl = "compl", wConst_cast = "const_cast", wDefault = "default", diff --git a/lib/std/private/digitsutils.nim b/lib/std/private/digitsutils.nim index 9be2ab3ef..588bcaec0 100644 --- a/lib/std/private/digitsutils.nim +++ b/lib/std/private/digitsutils.nim @@ -78,14 +78,17 @@ func addIntImpl(result: var string, x: uint64) {.inline.} = dec next addChars(result, tmp, next, tmp.len - next) -func addInt*(result: var string, x: uint64) = +when not defined(nimHasEnforceNoRaises): + {.pragma: enforceNoRaises.} + +func addInt*(result: var string, x: uint64) {.enforceNoRaises.} = when nimvm: addIntImpl(result, x) else: when not defined(js): addIntImpl(result, x) else: addChars(result, numToString(x)) -proc addInt*(result: var string; x: int64) = +proc addInt*(result: var string; x: int64) {.enforceNoRaises.} = ## Converts integer to its string representation and appends it to `result`. runnableExamples: var s = "foo" @@ -110,5 +113,5 @@ proc addInt*(result: var string; x: int64) = addChars(result, numToString(x)) else: impl() -proc addInt*(result: var string; x: int) {.inline.} = +proc addInt*(result: var string; x: int) {.inline, enforceNoRaises.} = addInt(result, int64(x)) From b155864967ba08861c39403003095d4822b03b4e Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 2 Nov 2021 23:31:30 +0800 Subject: [PATCH 1332/2472] build testament in package CI (#19092) * build testament in package CI * Update testament/important_packages.nim --- koch.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/koch.nim b/koch.nim index 06515aad7..f87828cdc 100644 --- a/koch.nim +++ b/koch.nim @@ -559,7 +559,8 @@ proc runCI(cmd: string) = let batchParam = "--batch:$1" % "NIM_TESTAMENT_BATCH".getEnv("_") if getEnv("NIM_TEST_PACKAGES", "0") == "1": - execFold("Test selected Nimble packages", "nim r testament/testament $# pcat nimble-packages" % batchParam) + nimCompileFold("Compile testament", "testament/testament.nim", options = "-d:release") + execFold("Test selected Nimble packages", "testament $# pcat nimble-packages" % batchParam) else: buildTools() From 539bced70df035ddbdbf4c3d9c7fd6074f0b1073 Mon Sep 17 00:00:00 2001 From: Kaushal Modi Date: Wed, 3 Nov 2021 01:46:34 -0400 Subject: [PATCH 1333/2472] Add test for issue 15435 (#19079) * Add test for issue 15435 Closes https://github.com/nim-lang/Nim/issues/15435. * Specify bug # in comment Addresses https://github.com/nim-lang/Nim/pull/19079#discussion_r740279547 --- tests/sets/t15435.nim | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 tests/sets/t15435.nim diff --git a/tests/sets/t15435.nim b/tests/sets/t15435.nim new file mode 100644 index 000000000..5ead7e641 --- /dev/null +++ b/tests/sets/t15435.nim @@ -0,0 +1,29 @@ +# bug #15435 +discard """ +errormsg: "type mismatch: got " +nimout: '''t15435.nim(28, 13) Error: type mismatch: got +but expected one of: +proc `<`[T](x, y: set[T]): bool + first type mismatch at position: 2 + required type for y: set[T] + but expression 'x' is of type: set[range 1..5(uint8)] +20 other mismatching symbols have been suppressed; compile with --showAllMismatches:on to see them + +expression: {1'u8, 5} < x''' +""" + + + + + + +## line 20 +var + x: set[range[1u8..5u8]] + +x.incl(1) +x.incl(3) +x.incl(5) + +if {1u8, 5} < x: + echo "successful" From 5fed1c05ce7ea9cbeb2b70bb8583e836c981abd4 Mon Sep 17 00:00:00 2001 From: Kaushal Modi Date: Wed, 3 Nov 2021 01:48:30 -0400 Subject: [PATCH 1334/2472] manual: Document the use of `static` as a proc call (#19084) * manual: Document the use of `static` as a proc call Also adds tests. Fixes https://github.com/nim-lang/Nim/issues/16987 . * Update doc/manual.rst Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Use the "bug #NNNN" comment syntax for consistency Ref: https://nim-lang.github.io/Nim/contributing.html#writing-tests-stdlib > Always refer to a GitHub issue using the following exact syntax: bug for tooling. * manual: Undocument usage of foo.static foo.static and foo.static() are not expected to work. Ref: https://github.com/nim-lang/Nim/pull/19084/files#r741203578 Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- doc/manual.rst | 14 ++++++++++++++ tests/system/tstatic_callable.nim | 12 ++++++++++++ tests/system/tstatic_callable_error.nim | 14 ++++++++++++++ 3 files changed, 40 insertions(+) create mode 100644 tests/system/tstatic_callable.nim create mode 100644 tests/system/tstatic_callable_error.nim diff --git a/doc/manual.rst b/doc/manual.rst index 4b212fe19..1371dba17 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -2975,6 +2975,20 @@ Even some code that has side effects is permitted in a static block: static: echo "echo at compile time" +`static` can also be used like a routine. + +.. code-block:: nim + + proc getNum(a: int): int = a + + # Below calls "echo getNum(123)" at compile time. + static: + echo getNum(123) + + # Below call evaluates the "getNum(123)" at compile time, but its + # result gets used at run time. + echo static(getNum(123)) + There are limitations on what Nim code can be executed at compile time; see `Restrictions on Compile-Time Execution <#restrictions-on-compileminustime-execution>`_ for details. diff --git a/tests/system/tstatic_callable.nim b/tests/system/tstatic_callable.nim new file mode 100644 index 000000000..92d1fca8d --- /dev/null +++ b/tests/system/tstatic_callable.nim @@ -0,0 +1,12 @@ +# bug #16987 + +proc getNum(a: int): int = a + +# Below calls "doAssert getNum(123) == 123" at compile time. +static: + doAssert getNum(123) == 123 + +# Below calls evaluate the "getNum(123)" at compile time, but the +# results of those calls get used at run time. +doAssert (static getNum(123)) == 123 +doAssert (static(getNum(123))) == 123 diff --git a/tests/system/tstatic_callable_error.nim b/tests/system/tstatic_callable_error.nim new file mode 100644 index 000000000..c6f1e3d07 --- /dev/null +++ b/tests/system/tstatic_callable_error.nim @@ -0,0 +1,14 @@ +# bug #16987 + +discard """ +errormsg: "cannot evaluate at compile time: inp" +nimout: ''' +tstatic_callable_error.nim(14, 21) Error: cannot evaluate at compile time: inp''' +""" + + +# line 10 +proc getNum(a: int): int = a + +let inp = 123 +echo (static getNum(inp)) From dbbafd320ce1edb0e51167c8e9f6ba3d865672c3 Mon Sep 17 00:00:00 2001 From: Kaushal Modi Date: Wed, 3 Nov 2021 03:08:50 -0400 Subject: [PATCH 1335/2472] manual: Document that comma propagates the default values of parameters (#19080) * manual: Document that comma propagates the default values of parameters Fixes https://github.com/nim-lang/Nim/issues/15949. * Use the "bug #NNNN" comment syntax for consistency Ref: https://nim-lang.github.io/Nim/contributing.html#writing-tests-stdlib > Always refer to a GitHub issue using the following exact syntax: bug for tooling. --- doc/manual.rst | 10 +++++++++- tests/proc/t15949.nim | 20 ++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 tests/proc/t15949.nim diff --git a/doc/manual.rst b/doc/manual.rst index 1371dba17..001402036 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -3603,9 +3603,17 @@ does not provide a value for the argument. The value will be reevaluated every time the function is called. .. code-block:: nim - # b is optional with 47 as its default value + # b is optional with 47 as its default value. proc foo(a: int, b: int = 47): int +Just as the comma propagates the types from right to left until the +first parameter or until a semicolon is hit, it also propagates the +default value starting from the parameter declared with it. + +.. code-block:: nim + # Both a and b are optional with 47 as their default values. + proc foo(a, b: int = 47): int + Parameters can be declared mutable and so allow the proc to modify those arguments, by using the type modifier `var`. diff --git a/tests/proc/t15949.nim b/tests/proc/t15949.nim new file mode 100644 index 000000000..bc3fddc84 --- /dev/null +++ b/tests/proc/t15949.nim @@ -0,0 +1,20 @@ +# bug #15949 + +discard """ +errormsg: "parameter 'a' requires a type" +nimout: ''' +t15949.nim(20, 14) Error: parameter 'a' requires a type''' +""" + + +# line 10 +proc procGood(a, b = 1): (int, int) = (a, b) + +doAssert procGood() == (1, 1) +doAssert procGood(b = 3) == (1, 3) +doAssert procGood(a = 2) == (2, 1) +doAssert procGood(a = 5, b = 6) == (5, 6) + +# The type (and default value propagation breaks in the below example +# as semicolon is used instead of comma. +proc procBad(a; b = 1): (int, int) = (a, b) From f7cfc46ae17b81d9ccdc84573da00561f24ea99f Mon Sep 17 00:00:00 2001 From: David Krause Date: Wed, 3 Nov 2021 09:14:59 +0100 Subject: [PATCH 1336/2472] Added support for sending "ehlo" and receiving multiline "ehlo" response. (#19077) * Added support for sending "ehlo" and receiving multiline "ehlo" response. Signed-off-by: David Krause * fix typo Signed-off-by: David Krause * send "EHLO" first, if not supported, send "HELO" to smtp server. Signed-off-by: David Krause * fix english Signed-off-by: David Krause * add changelog entry for smtp `ehlo` Signed-off-by: David Krause * recvEhlo must not be exported Signed-off-by: David Krause --- changelog.md | 2 ++ lib/pure/smtp.nim | 33 ++++++++++++++++++++++++--------- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/changelog.md b/changelog.md index fbda70582..1636d125f 100644 --- a/changelog.md +++ b/changelog.md @@ -7,7 +7,9 @@ ## Standard library additions and changes +## `std/smtp` +- Sends `ehlo` first. If the mail server does not understand, it sends `helo` as a fallback. ## Language changes diff --git a/lib/pure/smtp.nim b/lib/pure/smtp.nim index 5ba048608..f5196ce1e 100644 --- a/lib/pure/smtp.nim +++ b/lib/pure/smtp.nim @@ -100,7 +100,6 @@ proc debugRecv*(smtp: Smtp | AsyncSmtp): Future[string] {.multisync.} = ## `SMTP extensions`_. ## ## See `checkReply(reply)<#checkReply,AsyncSmtp,string>`_. - result = await smtp.sock.recvLine() if smtp.debug: echo("S:" & result) @@ -171,8 +170,7 @@ proc `$`*(msg: Message): string = result.add("\c\L") result.add(msg.msgBody) -proc newSmtp*(useSsl = false, debug = false, - sslContext: SslContext = nil): Smtp = +proc newSmtp*(useSsl = false, debug = false, sslContext: SslContext = nil): Smtp = ## Creates a new `Smtp` instance. new result result.debug = debug @@ -186,12 +184,10 @@ proc newSmtp*(useSsl = false, debug = false, else: {.error: "SMTP module compiled without SSL support".} -proc newAsyncSmtp*(useSsl = false, debug = false, - sslContext: SslContext = nil): AsyncSmtp = +proc newAsyncSmtp*(useSsl = false, debug = false, sslContext: SslContext = nil): AsyncSmtp = ## Creates a new `AsyncSmtp` instance. new result result.debug = debug - result.sock = newAsyncSocket() if useSsl: when compiledWithSsl: @@ -220,7 +216,6 @@ proc checkReply*(smtp: Smtp | AsyncSmtp, reply: string) {.multisync.} = ## would need to call when using this module. One exception to ## this is if you are implementing any ## `SMTP extensions`_. - var line = await smtp.debugRecv() if not line.startsWith(reply): await quitExcpt(smtp, "Expected " & reply & " reply, got: " & line) @@ -230,6 +225,22 @@ proc helo*(smtp: Smtp | AsyncSmtp) {.multisync.} = await smtp.debugSend("HELO " & smtp.address & "\c\L") await smtp.checkReply("250") +proc recvEhlo(smtp: Smtp | AsyncSmtp): Future[bool] {.multisync.} = + ## Skips "250-" lines, read until "250 " found. + ## Return `true` if server supports `EHLO`, false otherwise. + while true: + var line = await smtp.sock.recvLine() + if smtp.debug: + echo("S:" & line) + if line.startsWith("250-"): continue + elif line.startsWith("250 "): return true # last line + else: return false + +proc ehlo*(smtp: Smtp | AsyncSmtp): Future[bool] {.multisync.} = + ## Sends EHLO request. + await smtp.debugSend("EHLO " & smtp.address & "\c\L") + return await smtp.recvEhlo() + proc connect*(smtp: Smtp | AsyncSmtp, address: string, port: Port) {.multisync.} = ## Establishes a connection with a SMTP server. @@ -237,7 +248,9 @@ proc connect*(smtp: Smtp | AsyncSmtp, smtp.address = address await smtp.sock.connect(address, port) await smtp.checkReply("220") - await smtp.helo() + let speaksEsmtp = await smtp.ehlo() + if not speaksEsmtp: + await smtp.helo() proc startTls*(smtp: Smtp | AsyncSmtp, sslContext: SslContext = nil) {.multisync.} = ## Put the SMTP connection in TLS (Transport Layer Security) mode. @@ -249,7 +262,9 @@ proc startTls*(smtp: Smtp | AsyncSmtp, sslContext: SslContext = nil) {.multisync getSSLContext().wrapConnectedSocket(smtp.sock, handshakeAsClient) else: sslContext.wrapConnectedSocket(smtp.sock, handshakeAsClient) - await smtp.helo() + let speaksEsmtp = await smtp.ehlo() + if not speaksEsmtp: + await smtp.helo() else: {.error: "SMTP module compiled without SSL support".} From b2edc3468cf0857b337bff09cb68649d49803ccf Mon Sep 17 00:00:00 2001 From: Constantine Molchanov Date: Wed, 3 Nov 2021 11:16:24 +0300 Subject: [PATCH 1337/2472] Use testament to check Norm test pass (#19018) * Use testament to check Norm test pass This is what I actually use to test Norm, so it's better to use it. This should not currently pass. This is expected because this is exactly the problem I want to highlight with this PR. My tests do indeed not pass at the moment. * Remove clearNimblePath from testament command. Co-authored-by: flywind --- testament/important_packages.nim | 2 +- testament/specs.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index e037cc609..fc5c808ab 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -115,7 +115,7 @@ pkg "nimterop", "nimble minitest" pkg "nimwc", "nim c nimwc.nim" pkg "nimx", "nim c --threads:on test/main.nim", allowFailure = true pkg "nitter", "nim c src/nitter.nim", "https://github.com/zedeus/nitter" -pkg "norm", "nim c -r tests/sqlite/trows.nim" +pkg "norm", "testament r tests/sqlite/trows.nim" pkg "npeg", "nimble testarc" pkg "numericalnim", "nim c -r tests/test_integrate.nim" pkg "optionsutils" diff --git a/testament/specs.nim b/testament/specs.nim index 6fd0ab22d..c7e332262 100644 --- a/testament/specs.nim +++ b/testament/specs.nim @@ -103,7 +103,7 @@ type proc getCmd*(s: TSpec): string = if s.cmd.len == 0: - result = compilerPrefix & " $target --hints:on -d:testing --clearNimblePath --nimblePath:build/deps/pkgs $options $file" + result = compilerPrefix & " $target --hints:on -d:testing --nimblePath:build/deps/pkgs $options $file" else: result = s.cmd From f2f15e972645496f901fcfde197048de368453d6 Mon Sep 17 00:00:00 2001 From: Don-Duong Quach Date: Wed, 3 Nov 2021 08:47:31 -0700 Subject: [PATCH 1338/2472] fix for #19020, credit to @ElegantBeef (#19021) --- lib/core/macros.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 806ea7e31..6dbb92875 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -1611,7 +1611,7 @@ macro getCustomPragmaVal*(n: typed, cp: typed{nkSym}): untyped = let pragmaNode = customPragmaNode(n) for p in pragmaNode: if p.kind in nnkPragmaCallKinds and p.len > 0 and p[0].kind == nnkSym and p[0] == cp: - if p.len == 2: + if p.len == 2 or (p.len == 3 and p[1].kind == nnkSym and p[1].symKind == nskType): result = p[1] else: let def = p[0].getImpl[3] From fce89cb60a3428c61da3c7e6c2d3e6cdcb14bb3b Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 7 Nov 2021 16:38:02 +0100 Subject: [PATCH 1339/2472] fixes another effect inference bug [backport:1.6] (#19100) * fixes another effect inference bug [backport:1.6] --- compiler/sempass2.nim | 3 ++ lib/pure/asyncdispatch.nim | 4 +- lib/pure/json.nim | 10 +++-- lib/system/excpt.nim | 13 ++++--- lib/system/seqs_v2.nim | 2 +- tests/effects/tnestedprocs.nim | 63 +++++++++++++++++++++++++++++++ tests/pragmas/tcompile_pragma.nim | 1 + 7 files changed, 85 insertions(+), 11 deletions(-) create mode 100644 tests/effects/tnestedprocs.nim diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index 9a27d14fa..5624d7adb 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -851,6 +851,9 @@ proc trackCall(tracked: PEffects; n: PNode) = elif isIndirectCall(tracked, a): assumeTheWorst(tracked, n, op) gcsafeAndSideeffectCheck() + else: + if strictEffects in tracked.c.features and a.kind == nkSym and a.sym.kind in routineKinds: + propagateEffects(tracked, n, a.sym) else: mergeRaises(tracked, effectList[exceptionEffects], n) mergeTags(tracked, effectList[tagEffects], n) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index e15fb0851..c924d0a3d 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -733,7 +733,7 @@ when defined(windows) or defined(nimdoc): proc acceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}, inheritable = defined(nimInheritHandles)): - owned(Future[tuple[address: string, client: AsyncFD]]) = + owned(Future[tuple[address: string, client: AsyncFD]]) {.gcsafe.} = ## Accepts a new connection. Returns a future containing the client socket ## corresponding to that connection and the remote address of the client. ## The future will complete when the connection is successfully accepted. @@ -800,7 +800,7 @@ when defined(windows) or defined(nimdoc): var ol = newCustom() ol.data = CompletionData(fd: socket, cb: - proc (fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) = + proc (fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) {.gcsafe.} = if not retFuture.finished: if errcode == OSErrorCode(-1): completeAccept() diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 85c3393b2..c831bf85d 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -437,7 +437,7 @@ macro `%*`*(x: untyped): untyped = ## `%` for every element. result = toJsonImpl(x) -proc `==`*(a, b: JsonNode): bool = +proc `==`*(a, b: JsonNode): bool {.noSideEffect.} = ## Check two nodes for equality if a.isNil: if b.isNil: return true @@ -464,12 +464,16 @@ proc `==`*(a, b: JsonNode): bool = if a.fields.len != b.fields.len: return false for key, val in a.fields: if not b.fields.hasKey(key): return false - if b.fields[key] != val: return false + when defined(nimHasEffectsOf): + {.noSideEffect.}: + if b.fields[key] != val: return false + else: + if b.fields[key] != val: return false result = true proc hash*(n: OrderedTable[string, JsonNode]): Hash {.noSideEffect.} -proc hash*(n: JsonNode): Hash = +proc hash*(n: JsonNode): Hash {.noSideEffect.} = ## Compute the hash for a JSON node case n.kind of JArray: diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index 3e520570f..a71328c14 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -354,7 +354,7 @@ var onUnhandledException*: (proc (errorMsg: string) {. ## The default is to write a stacktrace to `stderr` and then call `quit(1)`. ## Unstable API. -proc reportUnhandledErrorAux(e: ref Exception) {.nodestroy.} = +proc reportUnhandledErrorAux(e: ref Exception) {.nodestroy, gcsafe.} = when hasSomeStackTrace: var buf = newStringOfCap(2000) if e.trace.len == 0: @@ -362,7 +362,8 @@ proc reportUnhandledErrorAux(e: ref Exception) {.nodestroy.} = else: var trace = $e.trace add(buf, trace) - `=destroy`(trace) + {.gcsafe.}: + `=destroy`(trace) add(buf, "Error: unhandled exception: ") add(buf, e.msg) add(buf, " [") @@ -373,7 +374,8 @@ proc reportUnhandledErrorAux(e: ref Exception) {.nodestroy.} = onUnhandledException(buf) else: showErrorMessage2(buf) - `=destroy`(buf) + {.gcsafe.}: + `=destroy`(buf) else: # ugly, but avoids heap allocations :-) template xadd(buf, s, slen) = @@ -387,7 +389,8 @@ proc reportUnhandledErrorAux(e: ref Exception) {.nodestroy.} = if e.trace.len != 0: var trace = $e.trace add(buf, trace) - `=destroy`(trace) + {.gcsafe.}: + `=destroy`(trace) add(buf, "Error: unhandled exception: ") add(buf, e.msg) add(buf, " [") @@ -398,7 +401,7 @@ proc reportUnhandledErrorAux(e: ref Exception) {.nodestroy.} = else: showErrorMessage(buf.addr, L) -proc reportUnhandledError(e: ref Exception) {.nodestroy.} = +proc reportUnhandledError(e: ref Exception) {.nodestroy, gcsafe.} = if unhandledExceptionHook != nil: unhandledExceptionHook(e) when hostOS != "any": diff --git a/lib/system/seqs_v2.nim b/lib/system/seqs_v2.nim index 375eef340..0ac51109e 100644 --- a/lib/system/seqs_v2.nim +++ b/lib/system/seqs_v2.nim @@ -73,7 +73,7 @@ proc prepareSeqAdd(len: int; p: pointer; addlen, elemSize, elemAlign: int): poin q.cap = newCap result = q -proc shrink*[T](x: var seq[T]; newLen: Natural) = +proc shrink*[T](x: var seq[T]; newLen: Natural) {.tags: [].} = when nimvm: setLen(x, newLen) else: diff --git a/tests/effects/tnestedprocs.nim b/tests/effects/tnestedprocs.nim new file mode 100644 index 000000000..125896d44 --- /dev/null +++ b/tests/effects/tnestedprocs.nim @@ -0,0 +1,63 @@ +discard """ + cmd: "nim check --hints:off $file" + nimout: '''tnestedprocs.nim(27, 8) Error: 'inner' can have side effects +> tnestedprocs.nim(29, 13) Hint: 'inner' calls `.sideEffect` 'outer2' +>> tnestedprocs.nim(26, 6) Hint: 'outer2' called by 'inner' + +tnestedprocs.nim(45, 8) Error: 'inner' can have side effects +> tnestedprocs.nim(47, 13) Hint: 'inner' calls `.sideEffect` 'outer6' +>> tnestedprocs.nim(44, 6) Hint: 'outer6' called by 'inner' + +tnestedprocs.nim(58, 41) Error: type mismatch: got but expected 'proc (){.closure, noSideEffect.}' + Pragma mismatch: got '{..}', but expected '{.noSideEffect.}'. +''' + errormsg: "type mismatch: got but expected 'proc (){.closure, noSideEffect.}'" +""" +{.experimental: "strictEffects".} +proc outer {.noSideEffect.} = + proc inner(p: int) = + if p == 0: + outer() + + inner(4) + +outer() + +proc outer2 = + proc inner(p: int) {.noSideEffect.} = + if p == 0: + outer2() + + inner(4) + +outer2() + +proc outer3(p: int) {.noSideEffect.} = + proc inner(p: int) {.noSideEffect.} = + if p == 0: + p.outer3() + + inner(4) + +outer3(5) + +proc outer6 = + proc inner(p: int) {.noSideEffect.} = + if p == 0: + outer6() + + inner(4) + echo "bad" + +outer6() + + +proc outer4 = + proc inner(p: int) {.noSideEffect.} = + if p == 0: + let x: proc () {.noSideEffect.} = outer4 + x() + + inner(4) + +outer4() diff --git a/tests/pragmas/tcompile_pragma.nim b/tests/pragmas/tcompile_pragma.nim index 4e09a7501..5b99352dd 100644 --- a/tests/pragmas/tcompile_pragma.nim +++ b/tests/pragmas/tcompile_pragma.nim @@ -1,5 +1,6 @@ discard """ output: '''34''' + joinable: false """ {.compile("cfunction.c", "-DNUMBER_HERE=34").} From b423ab138f41c24db3b168fd30f1ecce7850a170 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Mon, 8 Nov 2021 07:21:15 +0000 Subject: [PATCH 1340/2472] Remove invalid statements about try in async docs. (#19108) --- lib/pure/asyncdispatch.nim | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index c924d0a3d..20ad60427 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -101,16 +101,9 @@ ## Handling Exceptions ## ------------------- ## -## The most reliable way to handle exceptions is to use `yield` on a future -## then check the future's `failed` property. For example: +## You can handle exceptions in the same way as in ordinary Nim code; +## by using the try statement: ## -## .. code-block:: Nim -## var future = sock.recv(100) -## yield future -## if future.failed: -## # Handle exception -## -## The `async` procedures also offer limited support for the try statement. ## ## .. code-block:: Nim ## try: @@ -119,9 +112,16 @@ ## except: ## # Handle exception ## -## Unfortunately the semantics of the try statement may not always be correct, -## and occasionally the compilation may fail altogether. -## As such it is better to use the former style when possible. +## +## +## An alternative approach to handling exceptions is to use `yield` on a future +## then check the future's `failed` property. For example: +## +## .. code-block:: Nim +## var future = sock.recv(100) +## yield future +## if future.failed: +## # Handle exception ## ## ## Discarding futures @@ -1987,4 +1987,4 @@ when defined(linux) or defined(windows) or defined(macosx) or defined(bsd) or var fdLim: RLimit if getrlimit(RLIMIT_NOFILE, fdLim) < 0: raiseOSError(osLastError()) - result = int(fdLim.rlim_cur) - 1 \ No newline at end of file + result = int(fdLim.rlim_cur) - 1 From b21eb1ed36aa068cc9b0a304742e4c81b5112304 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Mon, 8 Nov 2021 15:10:01 +0300 Subject: [PATCH 1341/2472] change os.nim doc links to new style (#19102) --- compiler/docgen.nim | 7 + doc/docgen.rst | 7 + lib/packages/docutils/dochelpers.nim | 19 +- lib/pure/includes/osenv.nim | 42 +- lib/pure/includes/oserr.nim | 21 +- lib/pure/includes/osseps.nim | 6 +- lib/pure/os.nim | 558 +++++++++++++-------------- tests/stdlib/tdochelpers.nim | 6 + 8 files changed, 345 insertions(+), 321 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 2639840d1..7ba952083 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -954,6 +954,13 @@ proc toLangSymbol(k: TSymKind, n: PNode, baseName: string): LangSymbol = if kind != tkSpaces: result.generics.add(literal.nimIdentNormalize) + if k == skType: + case n[2].kind + of nkEnumTy: result.symTypeKind = "enum" + of nkObjectTy: result.symTypeKind = "object" + of nkTupleTy: result.symTypeKind = "tuple" + else: discard + proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) = if (docFlags != kForceExport) and not isVisible(d, nameNode): return let diff --git a/doc/docgen.rst b/doc/docgen.rst index 51e7102eb..48166b2c5 100644 --- a/doc/docgen.rst +++ b/doc/docgen.rst @@ -399,6 +399,13 @@ recognized fine:: no backticks: `func [][T](x: openArray[T]): T`_ escaped: `func \`[]\`[T](x: openArray[T]): T`_ +.. Note:: Types that defined as `enum`, or `object`, or `tuple` can also be + referenced with those names directly (instead of `type`):: + + type CopyFlag = enum + ... + ## Ref. `CopyFlag enum`_ + Related Options =============== diff --git a/lib/packages/docutils/dochelpers.nim b/lib/packages/docutils/dochelpers.nim index c488c4d99..8f97a7119 100644 --- a/lib/packages/docutils/dochelpers.nim +++ b/lib/packages/docutils/dochelpers.nim @@ -17,7 +17,9 @@ import rstast type LangSymbol* = object ## symbol signature in Nim - symKind*: string ## "proc", "const", etc + symKind*: string ## "proc", "const", "type", etc + symTypeKind*: string ## ""|enum|object|tuple - + ## valid only when `symKind == "type"` name*: string ## plain symbol name without any parameters generics*: string ## generic parameters (without brackets) isGroup*: bool ## is LangSymbol a group with overloads? @@ -79,7 +81,14 @@ proc toLangSymbol*(linkText: PRstNode): LangSymbol = assert linkText.kind in {rnRef, rnInner} const NimDefs = ["proc", "func", "macro", "method", "iterator", - "template", "converter", "const", "type", "var"] + "template", "converter", "const", "type", "var", + "enum", "object", "tuple"] + template resolveSymKind(x: string) = + if x in ["enum", "object", "tuple"]: + result.symKind = "type" + result.symTypeKind = x + else: + result.symKind = x type State = enum inBeginning @@ -97,7 +106,7 @@ proc toLangSymbol*(linkText: PRstNode): LangSymbol = if curIdent != "": case state of inBeginning: doAssert false, "incorrect state inBeginning" - of afterSymKind: result.symKind = curIdent + of afterSymKind: resolveSymKind curIdent of beforeSymbolName: doAssert false, "incorrect state beforeSymbolName" of atSymbolName: result.name = curIdent.nimIdentBackticksNormalize of afterSymbolName: doAssert false, "incorrect state afterSymbolName" @@ -195,7 +204,7 @@ proc toLangSymbol*(linkText: PRstNode): LangSymbol = let isPostfixSymKind = i > 0 and i == L - 1 and result.symKind == "" and s(i) in NimDefs if isPostfixSymKind: # for links like `foo proc`_ - result.symKind = s(i) + resolveSymKind s(i) else: case state of inBeginning: @@ -235,6 +244,8 @@ proc match*(generated: LangSymbol, docLink: LangSymbol): bool = result = docLink.symKind in ["proc", "func"] else: result = generated.symKind == docLink.symKind + if result and docLink.symKind == "type" and docLink.symTypeKind != "": + result = generated.symTypeKind == docLink.symTypeKind if not result: return result = generated.name == docLink.name if not result: return diff --git a/lib/pure/includes/osenv.nim b/lib/pure/includes/osenv.nim index 00a82327c..1a01ab9bc 100644 --- a/lib/pure/includes/osenv.nim +++ b/lib/pure/includes/osenv.nim @@ -56,13 +56,13 @@ when not defined(nimscript): ## ## If the variable does not exist, `""` is returned. To distinguish ## whether a variable exists or it's value is just `""`, call - ## `existsEnv(key) proc <#existsEnv,string>`_. + ## `existsEnv(key) proc`_. ## ## See also: - ## * `existsEnv proc <#existsEnv,string>`_ - ## * `putEnv proc <#putEnv,string,string>`_ - ## * `delEnv proc <#delEnv,string>`_ - ## * `envPairs iterator <#envPairs.i>`_ + ## * `existsEnv proc`_ + ## * `putEnv proc`_ + ## * `delEnv proc`_ + ## * `envPairs iterator`_ runnableExamples: assert getEnv("unknownEnv") == "" assert getEnv("unknownEnv", "doesn't exist") == "doesn't exist" @@ -76,10 +76,10 @@ when not defined(nimscript): ## Returns true if it exists, false otherwise. ## ## See also: - ## * `getEnv proc <#getEnv,string,string>`_ - ## * `putEnv proc <#putEnv,string,string>`_ - ## * `delEnv proc <#delEnv,string>`_ - ## * `envPairs iterator <#envPairs.i>`_ + ## * `getEnv proc`_ + ## * `putEnv proc`_ + ## * `delEnv proc`_ + ## * `envPairs iterator`_ runnableExamples: assert not existsEnv("unknownEnv") @@ -90,10 +90,10 @@ when not defined(nimscript): ## If an error occurs, `OSError` is raised. ## ## See also: - ## * `getEnv proc <#getEnv,string,string>`_ - ## * `existsEnv proc <#existsEnv,string>`_ - ## * `delEnv proc <#delEnv,string>`_ - ## * `envPairs iterator <#envPairs.i>`_ + ## * `getEnv proc`_ + ## * `existsEnv proc`_ + ## * `delEnv proc`_ + ## * `envPairs iterator`_ when defined(windows): if key.len == 0 or '=' in key: raise newException(OSError, "invalid key, got: " & $(key, val)) @@ -108,10 +108,10 @@ when not defined(nimscript): ## If an error occurs, `OSError` is raised. ## ## See also:ven - ## * `getEnv proc <#getEnv,string,string>`_ - ## * `existsEnv proc <#existsEnv,string>`_ - ## * `putEnv proc <#putEnv,string,string>`_ - ## * `envPairs iterator <#envPairs.i>`_ + ## * `getEnv proc`_ + ## * `existsEnv proc`_ + ## * `putEnv proc`_ + ## * `envPairs iterator`_ template bail = raiseOSError(osLastError(), key) when defined(windows): #[ @@ -190,10 +190,10 @@ iterator envPairs*(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = ## in the second its value. ## ## Works in native backends, nodejs and vm, like the following APIs: - ## * `getEnv proc <#getEnv,string,string>`_ - ## * `existsEnv proc <#existsEnv,string>`_ - ## * `putEnv proc <#putEnv,string,string>`_ - ## * `delEnv proc <#delEnv,string>`_ + ## * `getEnv proc`_ + ## * `existsEnv proc`_ + ## * `putEnv proc`_ + ## * `delEnv proc`_ when nimvm: for ai in envPairsImplSeq(): yield ai else: diff --git a/lib/pure/includes/oserr.nim b/lib/pure/includes/oserr.nim index e42771d52..01403fa70 100644 --- a/lib/pure/includes/oserr.nim +++ b/lib/pure/includes/oserr.nim @@ -18,7 +18,7 @@ proc `$`*(err: OSErrorCode): string {.borrow.} proc osErrorMsg*(errorCode: OSErrorCode): string = ## Converts an OS error code into a human readable string. ## - ## The error code can be retrieved using the `osLastError proc <#osLastError>`_. + ## The error code can be retrieved using the `osLastError proc`_. ## ## If conversion fails, or `errorCode` is `0` then `""` will be ## returned. @@ -28,8 +28,8 @@ proc osErrorMsg*(errorCode: OSErrorCode): string = ## message. ## ## See also: - ## * `raiseOSError proc <#raiseOSError,OSErrorCode,string>`_ - ## * `osLastError proc <#osLastError>`_ + ## * `raiseOSError proc`_ + ## * `osLastError proc`_ runnableExamples: when defined(linux): assert osErrorMsg(OSErrorCode(0)) == "" @@ -63,18 +63,17 @@ proc newOSError*( ## Creates a new `OSError exception `_. ## ## The `errorCode` will determine the - ## message, `osErrorMsg proc <#osErrorMsg,OSErrorCode>`_ will be used + ## message, `osErrorMsg proc`_ will be used ## to get this message. ## - ## The error code can be retrieved using the `osLastError proc - ## <#osLastError>`_. + ## The error code can be retrieved using the `osLastError proc`_. ## ## If the error code is `0` or an error message could not be retrieved, ## the message `unknown OS error` will be used. ## ## See also: - ## * `osErrorMsg proc <#osErrorMsg,OSErrorCode>`_ - ## * `osLastError proc <#osLastError>`_ + ## * `osErrorMsg proc`_ + ## * `osLastError proc`_ var e: owned(ref OSError); new(e) e.errorCode = errorCode.int32 e.msg = osErrorMsg(errorCode) @@ -90,7 +89,7 @@ proc newOSError*( proc raiseOSError*(errorCode: OSErrorCode, additionalInfo = "") {.noinline.} = ## Raises an `OSError exception `_. ## - ## Read the description of the `newOSError proc <#newOSError,OSErrorCode,string>`_ to learn + ## Read the description of the `newOSError proc`_ to learn ## how the exception object is created. raise newOSError(errorCode, additionalInfo) @@ -109,8 +108,8 @@ proc osLastError*(): OSErrorCode {.sideEffect.} = ## immediately after an OS call fails. On POSIX systems this is not a problem. ## ## See also: - ## * `osErrorMsg proc <#osErrorMsg,OSErrorCode>`_ - ## * `raiseOSError proc <#raiseOSError,OSErrorCode,string>`_ + ## * `osErrorMsg proc`_ + ## * `raiseOSError proc`_ when defined(nimscript): discard elif defined(windows): diff --git a/lib/pure/includes/osseps.nim b/lib/pure/includes/osseps.nim index 10c85047b..1ea587e3c 100644 --- a/lib/pure/includes/osseps.nim +++ b/lib/pure/includes/osseps.nim @@ -37,9 +37,9 @@ const when doslikeFileSystem: '/' else: DirSep ## An alternative character used by the operating system to separate - ## pathname components, or the same as `DirSep <#DirSep>`_ if only one separator + ## pathname components, or the same as DirSep_ if only one separator ## character exists. This is set to `'/'` on Windows systems - ## where `DirSep <#DirSep>`_ is a backslash (`'\\'`). + ## where DirSep_ is a backslash (`'\\'`). PathSep* = when defined(macos) or defined(RISCOS): ',' @@ -55,7 +55,7 @@ const defined(PalmOS) or defined(MorphOS): false else: true ## True if the file system is case sensitive, false otherwise. Used by - ## `cmpPaths proc <#cmpPaths,string,string>`_ to compare filenames properly. + ## `cmpPaths proc`_ to compare filenames properly. ExeExt* = when doslikeFileSystem: "exe" diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 5320aa87e..513e09b48 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -21,9 +21,9 @@ runnableExamples: ## **See also:** ## * `osproc module `_ for process communication beyond -## `execShellCmd proc <#execShellCmd,string>`_ +## `execShellCmd proc`_ ## * `parseopt module `_ for command-line parser beyond -## `parseCmdLine proc <#parseCmdLine,string>`_ +## `parseCmdLine proc`_ ## * `uri module `_ ## * `distros module `_ ## * `dynlib module `_ @@ -152,9 +152,9 @@ proc joinPath*(head, tail: string): string {. ## head has one). ## ## See also: - ## * `joinPath(varargs) proc <#joinPath,varargs[string]>`_ - ## * `/ proc <#/,string,string>`_ - ## * `splitPath proc <#splitPath,string>`_ + ## * `joinPath(parts: varargs[string]) proc`_ + ## * `/ proc`_ + ## * `splitPath proc`_ ## * `uri.combine proc `_ ## * `uri./ proc `_ runnableExamples: @@ -189,17 +189,17 @@ proc joinPath*(head, tail: string): string {. proc joinPath*(parts: varargs[string]): string {.noSideEffect, rtl, extern: "nos$1OpenArray".} = - ## The same as `joinPath(head, tail) proc <#joinPath,string,string>`_, + ## The same as `joinPath(head, tail) proc`_, ## but works with any number of directory parts. ## ## You need to pass at least one element or the proc ## will assert in debug builds and crash on release builds. ## ## See also: - ## * `joinPath(head, tail) proc <#joinPath,string,string>`_ - ## * `/ proc <#/,string,string>`_ - ## * `/../ proc <#/../,string,string>`_ - ## * `splitPath proc <#splitPath,string>`_ + ## * `joinPath(head, tail) proc`_ + ## * `/ proc`_ + ## * `/../ proc`_ + ## * `splitPath proc`_ runnableExamples: when defined(posix): assert joinPath("a") == "a" @@ -214,13 +214,13 @@ proc joinPath*(parts: varargs[string]): string {.noSideEffect, joinPathImpl(result, state, parts[i]) proc `/`*(head, tail: string): string {.noSideEffect, inline.} = - ## The same as `joinPath(head, tail) proc <#joinPath,string,string>`_. + ## The same as `joinPath(head, tail) proc`_. ## ## See also: - ## * `/../ proc <#/../,string,string>`_ - ## * `joinPath(head, tail) proc <#joinPath,string,string>`_ - ## * `joinPath(varargs) proc <#joinPath,varargs[string]>`_ - ## * `splitPath proc <#splitPath,string>`_ + ## * `/../ proc`_ + ## * `joinPath(head, tail) proc`_ + ## * `joinPath(parts: varargs[string]) proc`_ + ## * `splitPath proc`_ ## * `uri.combine proc `_ ## * `uri./ proc `_ runnableExamples: @@ -239,11 +239,11 @@ proc splitPath*(path: string): tuple[head, tail: string] {. ## ``head / tail == path`` (except for edge cases like "/usr"). ## ## See also: - ## * `joinPath(head, tail) proc <#joinPath,string,string>`_ - ## * `joinPath(varargs) proc <#joinPath,varargs[string]>`_ - ## * `/ proc <#/,string,string>`_ - ## * `/../ proc <#/../,string,string>`_ - ## * `relativePath proc <#relativePath,string,string>`_ + ## * `joinPath(head, tail) proc`_ + ## * `joinPath(parts: varargs[string]) proc`_ + ## * `/ proc`_ + ## * `/../ proc`_ + ## * `relativePath proc`_ runnableExamples: assert splitPath("usr/local/bin") == ("usr/local", "bin") assert splitPath("usr/local/bin/") == ("usr/local/bin", "") @@ -366,7 +366,7 @@ proc relativePath*(path, base: string, sep = DirSep): string {. rtl, extern: "nos$1".} = ## Converts `path` to a path relative to `base`. ## - ## The `sep` (default: `DirSep <#DirSep>`_) is used for the path normalizations, + ## The `sep` (default: DirSep_) is used for the path normalizations, ## this can be useful to ensure the relative path only contains `'/'` ## so that it can be used for URL constructions. ## @@ -375,9 +375,9 @@ proc relativePath*(path, base: string, sep = DirSep): string {. ## That means an absolute path can be returned. ## ## See also: - ## * `splitPath proc <#splitPath,string>`_ - ## * `parentDir proc <#parentDir,string>`_ - ## * `tailDir proc <#tailDir,string>`_ + ## * `splitPath proc`_ + ## * `parentDir proc`_ + ## * `tailDir proc`_ runnableExamples: assert relativePath("/Users/me/bar/z.nim", "/Users/other/bad", '/') == "../../me/bar/z.nim" assert relativePath("/Users/me/bar/z.nim", "/Users/other", '/') == "../me/bar/z.nim" @@ -476,14 +476,13 @@ proc parentDir*(path: string): string {. ## ## This is similar to ``splitPath(path).head`` when ``path`` doesn't end ## in a dir separator, but also takes care of path normalizations. - ## The remainder can be obtained with `lastPathPart(path) proc - ## <#lastPathPart,string>`_. + ## The remainder can be obtained with `lastPathPart(path) proc`_. ## ## See also: - ## * `relativePath proc <#relativePath,string,string>`_ - ## * `splitPath proc <#splitPath,string>`_ - ## * `tailDir proc <#tailDir,string>`_ - ## * `parentDirs iterator <#parentDirs.i,string>`_ + ## * `relativePath proc`_ + ## * `splitPath proc`_ + ## * `tailDir proc`_ + ## * `parentDirs iterator`_ runnableExamples: assert parentDir("") == "" when defined(posix): @@ -511,9 +510,9 @@ proc tailDir*(path: string): string {. ## Returns the tail part of `path`. ## ## See also: - ## * `relativePath proc <#relativePath,string,string>`_ - ## * `splitPath proc <#splitPath,string>`_ - ## * `parentDir proc <#parentDir,string>`_ + ## * `relativePath proc`_ + ## * `splitPath proc`_ + ## * `parentDir proc`_ runnableExamples: assert tailDir("/bin") == "bin" assert tailDir("bin") == "" @@ -556,7 +555,7 @@ iterator parentDirs*(path: string, fromRoot=false, inclusive=true): string = ## only the directories appearing in the relative path. ## ## See also: - ## * `parentDir proc <#parentDir,string>`_ + ## * `parentDir proc`_ ## runnableExamples: let g = "a/b/c" @@ -599,8 +598,8 @@ proc `/../`*(head, tail: string): string {.noSideEffect.} = ## directory. Then ``head / tail`` is performed instead. ## ## See also: - ## * `/ proc <#/,string,string>`_ - ## * `parentDir proc <#parentDir,string>`_ + ## * `/ proc`_ + ## * `parentDir proc`_ runnableExamples: when defined(posix): assert "a/b/c" /../ "d/e" == "a/b/d/e" @@ -621,11 +620,11 @@ proc searchExtPos*(path: string): int = ## of extension. Returns -1 otherwise. ## ## See also: - ## * `splitFile proc <#splitFile,string>`_ - ## * `extractFilename proc <#extractFilename,string>`_ - ## * `lastPathPart proc <#lastPathPart,string>`_ - ## * `changeFileExt proc <#changeFileExt,string,string>`_ - ## * `addFileExt proc <#addFileExt,string,string>`_ + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ runnableExamples: assert searchExtPos("a/b/c") == -1 assert searchExtPos("c.nim") == 1 @@ -645,7 +644,7 @@ proc splitFile*(path: string): tuple[dir, name, ext: string] {. noSideEffect, rtl, extern: "nos$1".} = ## Splits a filename into `(dir, name, extension)` tuple. ## - ## `dir` does not end in `DirSep <#DirSep>`_ unless it's `/`. + ## `dir` does not end in DirSep_ unless it's `/`. ## `extension` includes the leading dot. ## ## If `path` has no extension, `ext` is the empty string. @@ -653,11 +652,11 @@ proc splitFile*(path: string): tuple[dir, name, ext: string] {. ## If `path` has no filename component, `name` and `ext` are empty strings. ## ## See also: - ## * `searchExtPos proc <#searchExtPos,string>`_ - ## * `extractFilename proc <#extractFilename,string>`_ - ## * `lastPathPart proc <#lastPathPart,string>`_ - ## * `changeFileExt proc <#changeFileExt,string,string>`_ - ## * `addFileExt proc <#addFileExt,string,string>`_ + ## * `searchExtPos proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ runnableExamples: var (dir, name, ext) = splitFile("usr/local/nimc.html") assert dir == "usr/local" @@ -698,15 +697,14 @@ proc extractFilename*(path: string): string {. noSideEffect, rtl, extern: "nos$1".} = ## Extracts the filename of a given `path`. ## - ## This is the same as ``name & ext`` from `splitFile(path) proc - ## <#splitFile,string>`_. + ## This is the same as ``name & ext`` from `splitFile(path) proc`_. ## ## See also: - ## * `searchExtPos proc <#searchExtPos,string>`_ - ## * `splitFile proc <#splitFile,string>`_ - ## * `lastPathPart proc <#lastPathPart,string>`_ - ## * `changeFileExt proc <#changeFileExt,string,string>`_ - ## * `addFileExt proc <#addFileExt,string,string>`_ + ## * `searchExtPos proc`_ + ## * `splitFile proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ runnableExamples: assert extractFilename("foo/bar/") == "" assert extractFilename("foo/bar") == "bar" @@ -719,15 +717,15 @@ proc extractFilename*(path: string): string {. proc lastPathPart*(path: string): string {. noSideEffect, rtl, extern: "nos$1".} = - ## Like `extractFilename proc <#extractFilename,string>`_, but ignores + ## Like `extractFilename proc`_, but ignores ## trailing dir separator; aka: `baseName`:idx: in some other languages. ## ## See also: - ## * `searchExtPos proc <#searchExtPos,string>`_ - ## * `splitFile proc <#splitFile,string>`_ - ## * `extractFilename proc <#extractFilename,string>`_ - ## * `changeFileExt proc <#changeFileExt,string,string>`_ - ## * `addFileExt proc <#addFileExt,string,string>`_ + ## * `searchExtPos proc`_ + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ runnableExamples: assert lastPathPart("foo/bar/") == "bar" assert lastPathPart("foo/bar") == "bar" @@ -747,11 +745,11 @@ proc changeFileExt*(filename, ext: string): string {. ## of none such beast.) ## ## See also: - ## * `searchExtPos proc <#searchExtPos,string>`_ - ## * `splitFile proc <#splitFile,string>`_ - ## * `extractFilename proc <#extractFilename,string>`_ - ## * `lastPathPart proc <#lastPathPart,string>`_ - ## * `addFileExt proc <#addFileExt,string,string>`_ + ## * `searchExtPos proc`_ + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `addFileExt proc`_ runnableExamples: assert changeFileExt("foo.bar", "baz") == "foo.baz" assert changeFileExt("foo.bar", "") == "foo" @@ -771,11 +769,11 @@ proc addFileExt*(filename, ext: string): string {. ## (Although I know of none such beast.) ## ## See also: - ## * `searchExtPos proc <#searchExtPos,string>`_ - ## * `splitFile proc <#splitFile,string>`_ - ## * `extractFilename proc <#extractFilename,string>`_ - ## * `lastPathPart proc <#lastPathPart,string>`_ - ## * `changeFileExt proc <#changeFileExt,string,string>`_ + ## * `searchExtPos proc`_ + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ runnableExamples: assert addFileExt("foo.bar", "baz") == "foo.bar" assert addFileExt("foo.bar", "") == "foo.bar" @@ -875,15 +873,15 @@ proc getHomeDir*(): string {.rtl, extern: "nos$1", tags: [ReadEnvEffect, ReadIOEffect].} = ## Returns the home directory of the current user. ## - ## This proc is wrapped by the `expandTilde proc <#expandTilde,string>`_ + ## This proc is wrapped by the `expandTilde proc`_ ## for the convenience of processing paths coming from user configuration files. ## ## See also: - ## * `getConfigDir proc <#getConfigDir>`_ - ## * `getTempDir proc <#getTempDir>`_ - ## * `expandTilde proc <#expandTilde,string>`_ - ## * `getCurrentDir proc <#getCurrentDir>`_ - ## * `setCurrentDir proc <#setCurrentDir,string>`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `expandTilde proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ runnableExamples: assert getHomeDir() == expandTilde("~") @@ -903,11 +901,11 @@ proc getConfigDir*(): string {.rtl, extern: "nos$1", ## returned string: `\\` on Windows and `/` on all other OSs. ## ## See also: - ## * `getHomeDir proc <#getHomeDir>`_ - ## * `getTempDir proc <#getTempDir>`_ - ## * `expandTilde proc <#expandTilde,string>`_ - ## * `getCurrentDir proc <#getCurrentDir>`_ - ## * `setCurrentDir proc <#setCurrentDir,string>`_ + ## * `getHomeDir proc`_ + ## * `getTempDir proc`_ + ## * `expandTilde proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ when defined(windows): result = getEnv("APPDATA") else: @@ -927,9 +925,9 @@ proc getCacheDir*(): string = ## * On other platforms: `getEnv("XDG_CACHE_HOME", getEnv("HOME") / ".cache")` ## ## **See also:** - ## * `getHomeDir proc <#getHomeDir>`_ - ## * `getTempDir proc <#getTempDir>`_ - ## * `getConfigDir proc <#getConfigDir>`_ + ## * `getHomeDir proc`_ + ## * `getTempDir proc`_ + ## * `getConfigDir proc`_ # follows https://crates.io/crates/platform-dirs when defined(windows): result = getEnv("LOCALAPPDATA") @@ -985,11 +983,11 @@ proc getTempDir*(): string {.rtl, extern: "nos$1", ## **Note:** This proc does not check whether the returned path exists. ## ## See also: - ## * `getHomeDir proc <#getHomeDir>`_ - ## * `getConfigDir proc <#getConfigDir>`_ - ## * `expandTilde proc <#expandTilde,string>`_ - ## * `getCurrentDir proc <#getCurrentDir>`_ - ## * `setCurrentDir proc <#setCurrentDir,string>`_ + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `expandTilde proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ const tempDirDefault = "/tmp" when defined(tempDir): const tempDir {.strdefine.}: string = tempDirDefault @@ -1015,17 +1013,17 @@ proc getTempDir*(): string {.rtl, extern: "nos$1", proc expandTilde*(path: string): string {. tags: [ReadEnvEffect, ReadIOEffect].} = ## Expands ``~`` or a path starting with ``~/`` to a full path, replacing - ## ``~`` with `getHomeDir() <#getHomeDir>`_ (otherwise returns ``path`` unmodified). + ## ``~`` with `getHomeDir()`_ (otherwise returns ``path`` unmodified). ## ## Windows: this is still supported despite the Windows platform not having this ## convention; also, both ``~/`` and ``~\`` are handled. ## ## See also: - ## * `getHomeDir proc <#getHomeDir>`_ - ## * `getConfigDir proc <#getConfigDir>`_ - ## * `getTempDir proc <#getTempDir>`_ - ## * `getCurrentDir proc <#getCurrentDir>`_ - ## * `setCurrentDir proc <#setCurrentDir,string>`_ + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ runnableExamples: assert expandTilde("~" / "appname.cfg") == getHomeDir() / "appname.cfg" assert expandTilde("~/foo/bar") == getHomeDir() / "foo/bar" @@ -1089,9 +1087,8 @@ when defined(windows) or defined(posix) or defined(nintendoswitch): proc quoteShell*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = ## Quote ``s``, so it can be safely passed to shell. ## - ## When on Windows, it calls `quoteShellWindows proc - ## <#quoteShellWindows,string>`_. Otherwise, calls `quoteShellPosix proc - ## <#quoteShellPosix,string>`_. + ## When on Windows, it calls `quoteShellWindows proc`_. + ## Otherwise, calls `quoteShellPosix proc`_. when defined(windows): result = quoteShellWindows(s) else: @@ -1158,8 +1155,8 @@ proc fileExists*(filename: string): bool {.rtl, extern: "nos$1", ## Directories, device files, named pipes and sockets return false. ## ## See also: - ## * `dirExists proc <#dirExists,string>`_ - ## * `symlinkExists proc <#symlinkExists,string>`_ + ## * `dirExists proc`_ + ## * `symlinkExists proc`_ when defined(windows): when useWinUnicode: wrapUnary(a, getFileAttributesW, filename) @@ -1177,8 +1174,8 @@ proc dirExists*(dir: string): bool {.rtl, extern: "nos$1", tags: [ReadDirEffect] ## is returned. Follows symlinks. ## ## See also: - ## * `fileExists proc <#fileExists,string>`_ - ## * `symlinkExists proc <#symlinkExists,string>`_ + ## * `fileExists proc`_ + ## * `symlinkExists proc`_ when defined(windows): when useWinUnicode: wrapUnary(a, getFileAttributesW, dir) @@ -1197,8 +1194,8 @@ proc symlinkExists*(link: string): bool {.rtl, extern: "nos$1", ## regardless of whether the link points to a directory or file. ## ## See also: - ## * `fileExists proc <#fileExists,string>`_ - ## * `dirExists proc <#dirExists,string>`_ + ## * `fileExists proc`_ + ## * `dirExists proc`_ when defined(windows): when useWinUnicode: wrapUnary(a, getFileAttributesW, link) @@ -1228,7 +1225,7 @@ proc findExe*(exe: string, followSymlinks: bool = true; ## in directories listed in the ``PATH`` environment variable. ## ## Returns `""` if the `exe` cannot be found. `exe` - ## is added the `ExeExts <#ExeExts>`_ file extensions if it has none. + ## is added the `ExeExts`_ file extensions if it has none. ## ## If the system supports symlinks it also resolves them until it ## meets the actual file. This behavior can be disabled if desired @@ -1283,9 +1280,9 @@ proc getLastModificationTime*(file: string): times.Time {.rtl, extern: "nos$1", ## Returns the `file`'s last modification time. ## ## See also: - ## * `getLastAccessTime proc <#getLastAccessTime,string>`_ - ## * `getCreationTime proc <#getCreationTime,string>`_ - ## * `fileNewer proc <#fileNewer,string,string>`_ + ## * `getLastAccessTime proc`_ + ## * `getCreationTime proc`_ + ## * `fileNewer proc`_ when defined(posix): var res: Stat if stat(file, res) < 0'i32: raiseOSError(osLastError(), file) @@ -1301,9 +1298,9 @@ proc getLastAccessTime*(file: string): times.Time {.rtl, extern: "nos$1", noWeir ## Returns the `file`'s last read or write access time. ## ## See also: - ## * `getLastModificationTime proc <#getLastModificationTime,string>`_ - ## * `getCreationTime proc <#getCreationTime,string>`_ - ## * `fileNewer proc <#fileNewer,string,string>`_ + ## * `getLastModificationTime proc`_ + ## * `getCreationTime proc`_ + ## * `fileNewer proc`_ when defined(posix): var res: Stat if stat(file, res) < 0'i32: raiseOSError(osLastError(), file) @@ -1323,9 +1320,9 @@ proc getCreationTime*(file: string): times.Time {.rtl, extern: "nos$1", noWeirdT ## `here `_ for details. ## ## See also: - ## * `getLastModificationTime proc <#getLastModificationTime,string>`_ - ## * `getLastAccessTime proc <#getLastAccessTime,string>`_ - ## * `fileNewer proc <#fileNewer,string,string>`_ + ## * `getLastModificationTime proc`_ + ## * `getLastAccessTime proc`_ + ## * `fileNewer proc`_ when defined(posix): var res: Stat if stat(file, res) < 0'i32: raiseOSError(osLastError(), file) @@ -1342,9 +1339,9 @@ proc fileNewer*(a, b: string): bool {.rtl, extern: "nos$1", noWeirdTarget.} = ## modification time is later than `b`'s. ## ## See also: - ## * `getLastModificationTime proc <#getLastModificationTime,string>`_ - ## * `getLastAccessTime proc <#getLastAccessTime,string>`_ - ## * `getCreationTime proc <#getCreationTime,string>`_ + ## * `getLastModificationTime proc`_ + ## * `getLastAccessTime proc`_ + ## * `getCreationTime proc`_ when defined(posix): # If we don't have access to nanosecond resolution, use '>=' when not StatHasNanoseconds: @@ -1362,10 +1359,10 @@ when not defined(nimscript): ## So the path returned by this proc is determined at run time. ## ## See also: - ## * `getHomeDir proc <#getHomeDir>`_ - ## * `getConfigDir proc <#getConfigDir>`_ - ## * `getTempDir proc <#getTempDir>`_ - ## * `setCurrentDir proc <#setCurrentDir,string>`_ + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `setCurrentDir proc`_ ## * `currentSourcePath template `_ ## * `getProjectPath proc `_ when defined(nodejs): @@ -1421,10 +1418,10 @@ proc setCurrentDir*(newDir: string) {.inline, tags: [], noWeirdTarget.} = ## is raised if `newDir` cannot been set. ## ## See also: - ## * `getHomeDir proc <#getHomeDir>`_ - ## * `getConfigDir proc <#getConfigDir>`_ - ## * `getTempDir proc <#getTempDir>`_ - ## * `getCurrentDir proc <#getCurrentDir>`_ + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `getCurrentDir proc`_ when defined(windows): when useWinUnicode: if setCurrentDirectoryW(newWideCString(newDir)) == 0'i32: @@ -1441,8 +1438,8 @@ proc absolutePath*(path: string, root = getCurrentDir()): string = ## If `path` is absolute, return it, ignoring `root`. ## ## See also: - ## * `normalizedPath proc <#normalizedPath,string>`_ - ## * `normalizePath proc <#normalizePath,string>`_ + ## * `normalizedPath proc`_ + ## * `normalizePath proc`_ runnableExamples: assert absolutePath("a") == getCurrentDir() / "a" @@ -1479,9 +1476,9 @@ proc normalizePath*(path: var string) {.rtl, extern: "nos$1", tags: [].} = ## Triple dot is not handled. ## ## See also: - ## * `absolutePath proc <#absolutePath,string>`_ - ## * `normalizedPath proc <#normalizedPath,string>`_ for outplace version - ## * `normalizeExe proc <#normalizeExe,string>`_ + ## * `absolutePath proc`_ + ## * `normalizedPath proc`_ for outplace version + ## * `normalizeExe proc`_ runnableExamples: when defined(posix): var a = "a///b//..//c///d" @@ -1522,8 +1519,8 @@ proc normalizedPath*(path: string): string {.rtl, extern: "nos$1", tags: [].} = ## Returns a normalized path for the current OS. ## ## See also: - ## * `absolutePath proc <#absolutePath,string>`_ - ## * `normalizePath proc <#normalizePath,string>`_ for the in-place version + ## * `absolutePath proc`_ + ## * `normalizePath proc`_ for the in-place version runnableExamples: when defined(posix): assert normalizedPath("a///b//..//c///d") == "a/c/d" @@ -1561,7 +1558,7 @@ proc sameFile*(path1, path2: string): bool {.rtl, extern: "nos$1", ## sym-linked paths to the same file or directory. ## ## See also: - ## * `sameFileContent proc <#sameFileContent,string,string>`_ + ## * `sameFileContent proc`_ when defined(windows): var success = true var f1 = openHandle(path1) @@ -1598,9 +1595,9 @@ type FilePermission* = enum ## File access permission, modelled after UNIX. ## ## See also: - ## * `getFilePermissions <#getFilePermissions,string>`_ - ## * `setFilePermissions <#setFilePermissions,string,set[FilePermission]>`_ - ## * `FileInfo object <#FileInfo>`_ + ## * `getFilePermissions`_ + ## * `setFilePermissions`_ + ## * `FileInfo object`_ fpUserExec, ## execute access for the file owner fpUserWrite, ## write access for the file owner fpUserRead, ## read access for the file owner @@ -1620,8 +1617,8 @@ proc getFilePermissions*(filename: string): set[FilePermission] {. ## permission is available in any case. ## ## See also: - ## * `setFilePermissions proc <#setFilePermissions,string,set[FilePermission]>`_ - ## * `FilePermission enum <#FilePermission>`_ + ## * `setFilePermissions proc`_ + ## * `FilePermission enum`_ when defined(posix): var a: Stat if stat(filename, a) < 0'i32: raiseOSError(osLastError(), filename) @@ -1666,8 +1663,8 @@ proc setFilePermissions*(filename: string, permissions: set[FilePermission], ## ``fpUserWrite`` permission. ## ## See also: - ## * `getFilePermissions <#getFilePermissions,string>`_ - ## * `FilePermission enum <#FilePermission>`_ + ## * `getFilePermissions proc`_ + ## * `FilePermission enum`_ when defined(posix): var p = 0.Mode if fpUserRead in permissions: p = p or S_IRUSR.Mode @@ -1744,8 +1741,8 @@ proc createSymlink*(src, dest: string) {.noWeirdTarget.} = ## of symlinks to root users (administrators) or users with developper mode enabled. ## ## See also: - ## * `createHardlink proc <#createHardlink,string,string>`_ - ## * `expandSymlink proc <#expandSymlink,string>`_ + ## * `createHardlink proc`_ + ## * `expandSymlink proc`_ when defined(windows): const SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE = 2 @@ -1769,7 +1766,7 @@ proc expandSymlink*(symlinkPath: string): string {.noWeirdTarget.} = ## On Windows this is a noop, `symlinkPath` is simply returned. ## ## See also: - ## * `createSymlink proc <#createSymlink,string,string>`_ + ## * `createSymlink proc`_ when defined(windows): result = symlinkPath else: @@ -1823,11 +1820,11 @@ proc copyFile*(source, dest: string, options = {cfSymlinkFollow}) {.rtl, ## copy the source file's attributes into dest. ## ## On other platforms you need - ## to use `getFilePermissions <#getFilePermissions,string>`_ and - ## `setFilePermissions <#setFilePermissions,string,set[FilePermission]>`_ + ## to use `getFilePermissions`_ and + ## `setFilePermissions`_ ## procs ## to copy them by hand (or use the convenience `copyFileWithPermissions - ## proc <#copyFileWithPermissions,string,string>`_), + ## proc`_), ## otherwise `dest` will inherit the default permissions of a newly ## created file for the user. ## @@ -1838,12 +1835,12 @@ proc copyFile*(source, dest: string, options = {cfSymlinkFollow}) {.rtl, ## `-d:nimLegacyCopyFile` is used. ## ## See also: - ## * `CopyFlag enum <#CopyFlag>`_ - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ - ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ - ## * `removeFile proc <#removeFile,string>`_ - ## * `moveFile proc <#moveFile,string,string>`_ + ## * `CopyFlag enum`_ + ## * `copyDir proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `tryRemoveFile proc`_ + ## * `removeFile proc`_ + ## * `moveFile proc`_ doAssert card(copyFlagSymlink * options) == 1, "There should be exactly " & "one cfSymlink* in options" @@ -1908,8 +1905,8 @@ proc copyFileToDir*(source, dir: string, options = {cfSymlinkFollow}) ## ignored on Windows: symlinks are skipped. ## ## See also: - ## * `CopyFlag enum <#CopyFlag>`_ - ## * `copyFile proc <#copyDir,string,string>`_ + ## * `CopyFlag enum`_ + ## * `copyFile proc`_ if dir.len == 0: # treating "" as "." is error prone raise newException(ValueError, "dest is empty") copyFile(source, dir / source.lastPathPart, options) @@ -1942,10 +1939,10 @@ proc tryRemoveFile*(file: string): bool {.rtl, extern: "nos$1", tags: [WriteDirE ## On Windows, ignores the read-only attribute. ## ## See also: - ## * `copyFile proc <#copyFile,string,string>`_ - ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ - ## * `removeFile proc <#removeFile,string>`_ - ## * `moveFile proc <#moveFile,string,string>`_ + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `removeFile proc`_ + ## * `moveFile proc`_ result = true when defined(windows): when useWinUnicode: @@ -1974,11 +1971,11 @@ proc removeFile*(file: string) {.rtl, extern: "nos$1", tags: [WriteDirEffect], n ## On Windows, ignores the read-only attribute. ## ## See also: - ## * `removeDir proc <#removeDir,string>`_ - ## * `copyFile proc <#copyFile,string,string>`_ - ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ - ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ - ## * `moveFile proc <#moveFile,string,string>`_ + ## * `removeDir proc`_ + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `tryRemoveFile proc`_ + ## * `moveFile proc`_ if not tryRemoveFile(file): raiseOSError(osLastError(), file) @@ -2022,11 +2019,11 @@ proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", ## Can be used to `rename files`:idx:. ## ## See also: - ## * `moveDir proc <#moveDir,string,string>`_ - ## * `copyFile proc <#copyFile,string,string>`_ - ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ - ## * `removeFile proc <#removeFile,string>`_ - ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ + ## * `moveDir proc`_ + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `removeFile proc`_ + ## * `tryRemoveFile proc`_ if not tryMoveFSObject(source, dest, isDir = false): when defined(windows): @@ -2135,10 +2132,10 @@ iterator walkPattern*(pattern: string): string {.tags: [ReadDirEffect], noWeirdT ## notation is supported. ## ## See also: - ## * `walkFiles iterator <#walkFiles.i,string>`_ - ## * `walkDirs iterator <#walkDirs.i,string>`_ - ## * `walkDir iterator <#walkDir.i,string>`_ - ## * `walkDirRec iterator <#walkDirRec.i,string>`_ + ## * `walkFiles iterator`_ + ## * `walkDirs iterator`_ + ## * `walkDir iterator`_ + ## * `walkDirRec iterator`_ runnableExamples: import std/sequtils let paths = toSeq(walkPattern("lib/pure/*")) # works on Windows too @@ -2154,10 +2151,10 @@ iterator walkFiles*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTar ## notation is supported. ## ## See also: - ## * `walkPattern iterator <#walkPattern.i,string>`_ - ## * `walkDirs iterator <#walkDirs.i,string>`_ - ## * `walkDir iterator <#walkDir.i,string>`_ - ## * `walkDirRec iterator <#walkDirRec.i,string>`_ + ## * `walkPattern iterator`_ + ## * `walkDirs iterator`_ + ## * `walkDir iterator`_ + ## * `walkDirRec iterator`_ runnableExamples: import std/sequtils assert "lib/pure/os.nim".unixToNativePath in toSeq(walkFiles("lib/pure/*.nim")) # works on Windows too @@ -2171,10 +2168,10 @@ iterator walkDirs*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTarg ## notation is supported. ## ## See also: - ## * `walkPattern iterator <#walkPattern.i,string>`_ - ## * `walkFiles iterator <#walkFiles.i,string>`_ - ## * `walkDir iterator <#walkDir.i,string>`_ - ## * `walkDirRec iterator <#walkDirRec.i,string>`_ + ## * `walkPattern iterator`_ + ## * `walkFiles iterator`_ + ## * `walkDir iterator`_ + ## * `walkDirRec iterator`_ runnableExamples: import std/sequtils let paths = toSeq(walkDirs("lib/pure/*")) # works on Windows too @@ -2235,15 +2232,15 @@ type PathComponent* = enum ## Enumeration specifying a path component. ## ## See also: - ## * `walkDirRec iterator <#walkDirRec.i,string>`_ - ## * `FileInfo object <#FileInfo>`_ + ## * `walkDirRec iterator`_ + ## * `FileInfo object`_ pcFile, ## path refers to a file pcLinkToFile, ## path refers to a symbolic link to a file pcDir, ## path refers to a directory pcLinkToDir ## path refers to a symbolic link to a directory proc getCurrentCompilerExe*(): string {.compileTime.} = discard - ## This is `getAppFilename() <#getAppFilename>`_ at compile time. + ## This is `getAppFilename()`_ at compile time. ## ## Can be used to retrieve the currently executing ## Nim compiler from a Nim or nimscript program, or the nimble binary @@ -2291,10 +2288,10 @@ iterator walkDir*(dir: string; relative = false, checkDir = false): assert collect(for k in walkDir("dirA"): k.path).join(" ") == "dirA/dirB dirA/dirC dirA/fileA2.txt dirA/fileA1.txt" ## See also: - ## * `walkPattern iterator <#walkPattern.i,string>`_ - ## * `walkFiles iterator <#walkFiles.i,string>`_ - ## * `walkDirs iterator <#walkDirs.i,string>`_ - ## * `walkDirRec iterator <#walkDirRec.i,string>`_ + ## * `walkPattern iterator`_ + ## * `walkFiles iterator`_ + ## * `walkDirs iterator`_ + ## * `walkDirRec iterator`_ when nimvm: for k, v in items(staticWalkDir(dir, relative)): @@ -2401,10 +2398,10 @@ iterator walkDirRec*(dir: string, ## ## ## See also: - ## * `walkPattern iterator <#walkPattern.i,string>`_ - ## * `walkFiles iterator <#walkFiles.i,string>`_ - ## * `walkDirs iterator <#walkDirs.i,string>`_ - ## * `walkDir iterator <#walkDir.i,string>`_ + ## * `walkPattern iterator`_ + ## * `walkFiles iterator`_ + ## * `walkDirs iterator`_ + ## * `walkDir iterator`_ var stack = @[""] var checkDir = checkDir @@ -2444,13 +2441,13 @@ proc removeDir*(dir: string, checkDir = false) {.rtl, extern: "nos$1", tags: [ ## existed in the first place, unless `checkDir` = true. ## ## See also: - ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ - ## * `removeFile proc <#removeFile,string>`_ - ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ - ## * `createDir proc <#createDir,string>`_ - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - ## * `moveDir proc <#moveDir,string,string>`_ + ## * `tryRemoveFile proc`_ + ## * `removeFile proc`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + ## * `copyDir proc`_ + ## * `copyDirWithPermissions proc`_ + ## * `moveDir proc`_ for kind, path in walkDir(dir, checkDir = checkDir): case kind of pcFile, pcLinkToFile, pcLinkToDir: removeFile(path) @@ -2512,11 +2509,11 @@ proc existsOrCreateDir*(dir: string): bool {.rtl, extern: "nos$1", ## Returns `true` if the directory already exists, and `false` otherwise. ## ## See also: - ## * `removeDir proc <#removeDir,string>`_ - ## * `createDir proc <#createDir,string>`_ - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - ## * `moveDir proc <#moveDir,string,string>`_ + ## * `removeDir proc`_ + ## * `createDir proc`_ + ## * `copyDir proc`_ + ## * `copyDirWithPermissions proc`_ + ## * `moveDir proc`_ result = not rawCreateDir(dir) if result: # path already exists - need to check that it is indeed a directory @@ -2534,11 +2531,11 @@ proc createDir*(dir: string) {.rtl, extern: "nos$1", ## most usages this does not indicate an error. ## ## See also: - ## * `removeDir proc <#removeDir,string>`_ - ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - ## * `moveDir proc <#moveDir,string,string>`_ + ## * `removeDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `copyDir proc`_ + ## * `copyDirWithPermissions proc`_ + ## * `moveDir proc`_ var omitNext = false when doslikeFileSystem: omitNext = isAbsolute(dir) @@ -2568,17 +2565,17 @@ proc copyDir*(source, dest: string) {.rtl, extern: "nos$1", ## ## On other platforms created files and directories will inherit the ## default permissions of a newly created file/directory for the user. - ## Use `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ + ## Use `copyDirWithPermissions proc`_ ## to preserve attributes recursively on these platforms. ## ## See also: - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - ## * `copyFile proc <#copyFile,string,string>`_ - ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ - ## * `removeDir proc <#removeDir,string>`_ - ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ - ## * `createDir proc <#createDir,string>`_ - ## * `moveDir proc <#moveDir,string,string>`_ + ## * `copyDirWithPermissions proc`_ + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `removeDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + ## * `moveDir proc`_ createDir(dest) for kind, path in walkDir(source): var noSource = splitPath(path).tail @@ -2596,12 +2593,12 @@ proc moveDir*(source, dest: string) {.tags: [ReadIOEffect, WriteIOEffect], noWei ## If this fails, `OSError` is raised. ## ## See also: - ## * `moveFile proc <#moveFile,string,string>`_ - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - ## * `removeDir proc <#removeDir,string>`_ - ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ - ## * `createDir proc <#createDir,string>`_ + ## * `moveFile proc`_ + ## * `copyDir proc`_ + ## * `copyDirWithPermissions proc`_ + ## * `removeDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ if not tryMoveFSObject(source, dest, isDir = true): # Fallback to copy & del copyDir(source, dest) @@ -2615,7 +2612,7 @@ proc createHardlink*(src, dest: string) {.noWeirdTarget.} = ## root users (administrators). ## ## See also: - ## * `createSymlink proc <#createSymlink,string,string>`_ + ## * `createSymlink proc`_ when defined(windows): when useWinUnicode: var wSrc = newWideCString(src) @@ -2638,13 +2635,12 @@ proc copyFileWithPermissions*(source, dest: string, ## if `source` is a symlink, copies the file symlink points to. `options` is ## ignored on Windows: symlinks are skipped. ## - ## This is a wrapper proc around `copyFile <#copyFile,string,string>`_, - ## `getFilePermissions <#getFilePermissions,string>`_ and - ## `setFilePermissions<#setFilePermissions,string,set[FilePermission]>`_ + ## This is a wrapper proc around `copyFile`_, + ## `getFilePermissions`_ and `setFilePermissions`_ ## procs on non-Windows platforms. ## - ## On Windows this proc is just a wrapper for `copyFile proc - ## <#copyFile,string,string>`_ since that proc already copies attributes. + ## On Windows this proc is just a wrapper for `copyFile proc`_ since + ## that proc already copies attributes. ## ## On non-Windows systems permissions are copied after the file itself has ## been copied, which won't happen atomically and could lead to a race @@ -2653,13 +2649,13 @@ proc copyFileWithPermissions*(source, dest: string, ## `OSError`. ## ## See also: - ## * `CopyFlag enum <#CopyFlag>`_ - ## * `copyFile proc <#copyFile,string,string>`_ - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ - ## * `removeFile proc <#removeFile,string>`_ - ## * `moveFile proc <#moveFile,string,string>`_ - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ + ## * `CopyFlag enum`_ + ## * `copyFile proc`_ + ## * `copyDir proc`_ + ## * `tryRemoveFile proc`_ + ## * `removeFile proc`_ + ## * `moveFile proc`_ + ## * `copyDirWithPermissions proc`_ copyFile(source, dest, options) when not defined(windows): try: @@ -2678,13 +2674,12 @@ proc copyDirWithPermissions*(source, dest: string, ## On non-Windows OSes, symlinks are copied as symlinks. On Windows, symlinks ## are skipped. ## - ## If this fails, `OSError` is raised. This is a wrapper proc around `copyDir - ## <#copyDir,string,string>`_ and `copyFileWithPermissions - ## <#copyFileWithPermissions,string,string>`_ procs + ## If this fails, `OSError` is raised. This is a wrapper proc around + ## `copyDir`_ and `copyFileWithPermissions`_ procs ## on non-Windows platforms. ## - ## On Windows this proc is just a wrapper for `copyDir proc - ## <#copyDir,string,string>`_ since that proc already copies attributes. + ## On Windows this proc is just a wrapper for `copyDir proc`_ since + ## that proc already copies attributes. ## ## On non-Windows systems permissions are copied after the file or directory ## itself has been copied, which won't happen atomically and could lead to a @@ -2693,13 +2688,13 @@ proc copyDirWithPermissions*(source, dest: string, ## `OSError`. ## ## See also: - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyFile proc <#copyFile,string,string>`_ - ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ - ## * `removeDir proc <#removeDir,string>`_ - ## * `moveDir proc <#moveDir,string,string>`_ - ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ - ## * `createDir proc <#createDir,string>`_ + ## * `copyDir proc`_ + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `removeDir proc`_ + ## * `moveDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ createDir(dest) when not defined(windows): try: @@ -2769,9 +2764,9 @@ proc parseCmdLine*(c: string): seq[string] {. ## ## See also: ## * `parseopt module `_ - ## * `paramCount proc <#paramCount>`_ - ## * `paramStr proc <#paramStr,int>`_ - ## * `commandLineParams proc <#commandLineParams>`_ + ## * `paramCount proc`_ + ## * `paramStr proc`_ + ## * `commandLineParams proc`_ result = @[] var i = 0 @@ -2838,9 +2833,8 @@ when defined(nimdoc): ## ## Unlike `argc`:idx: in C, if your binary was called without parameters this ## will return zero. - ## You can query each individual parameter with `paramStr proc <#paramStr,int>`_ - ## or retrieve all of them in one go with `commandLineParams proc - ## <#commandLineParams>`_. + ## You can query each individual parameter with `paramStr proc`_ + ## or retrieve all of them in one go with `commandLineParams proc`_. ## ## **Availability**: When generating a dynamic library (see `--app:lib`) on ## Posix this proc is not defined. @@ -2848,9 +2842,9 @@ when defined(nimdoc): ## ## See also: ## * `parseopt module `_ - ## * `parseCmdLine proc <#parseCmdLine,string>`_ - ## * `paramStr proc <#paramStr,int>`_ - ## * `commandLineParams proc <#commandLineParams>`_ + ## * `parseCmdLine proc`_ + ## * `paramStr proc`_ + ## * `commandLineParams proc`_ ## ## **Examples:** ## @@ -2865,13 +2859,13 @@ when defined(nimdoc): ## ## `i` should be in the range `1..paramCount()`, the `IndexDefect` ## exception will be raised for invalid values. Instead of iterating - ## over `paramCount() <#paramCount>`_ with this proc you can - ## call the convenience `commandLineParams() <#commandLineParams>`_. + ## over `paramCount()`_ with this proc you can + ## call the convenience `commandLineParams()`_. ## ## Similarly to `argv`:idx: in C, ## it is possible to call `paramStr(0)` but this will return OS specific ## contents (usually the name of the invoked executable). You should avoid - ## this and call `getAppFilename() <#getAppFilename>`_ instead. + ## this and call `getAppFilename()`_ instead. ## ## **Availability**: When generating a dynamic library (see `--app:lib`) on ## Posix this proc is not defined. @@ -2879,10 +2873,10 @@ when defined(nimdoc): ## ## See also: ## * `parseopt module `_ - ## * `parseCmdLine proc <#parseCmdLine,string>`_ - ## * `paramCount proc <#paramCount>`_ - ## * `commandLineParams proc <#commandLineParams>`_ - ## * `getAppFilename proc <#getAppFilename>`_ + ## * `parseCmdLine proc`_ + ## * `paramCount proc`_ + ## * `commandLineParams proc`_ + ## * `getAppFilename proc`_ ## ## **Examples:** ## @@ -2971,7 +2965,7 @@ when declared(paramCount) or defined(nimdoc): ## Convenience proc which returns the command line parameters. ## ## This returns **only** the parameters. If you want to get the application - ## executable filename, call `getAppFilename() <#getAppFilename>`_. + ## executable filename, call `getAppFilename()`_. ## ## **Availability**: On Posix there is no portable way to get the command ## line from a DLL and thus the proc isn't defined in this environment. You @@ -2980,10 +2974,10 @@ when declared(paramCount) or defined(nimdoc): ## ## See also: ## * `parseopt module `_ - ## * `parseCmdLine proc <#parseCmdLine,string>`_ - ## * `paramCount proc <#paramCount>`_ - ## * `paramStr proc <#paramStr,int>`_ - ## * `getAppFilename proc <#getAppFilename>`_ + ## * `parseCmdLine proc`_ + ## * `paramCount proc`_ + ## * `paramStr proc`_ + ## * `getAppFilename proc`_ ## ## **Examples:** ## @@ -3133,8 +3127,8 @@ proc getAppFilename*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noW ## This proc will resolve symlinks. ## ## See also: - ## * `getAppDir proc <#getAppDir>`_ - ## * `getCurrentCompilerExe proc <#getCurrentCompilerExe>`_ + ## * `getAppDir proc`_ + ## * `getCurrentCompilerExe proc`_ # Linux: /proc//exe # Solaris: @@ -3198,7 +3192,7 @@ proc getAppDir*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noWeirdT ## Returns the directory of the application's executable. ## ## See also: - ## * `getAppFilename proc <#getAppFilename>`_ + ## * `getAppFilename proc`_ result = splitFile(getAppFilename()).dir proc sleep*(milsecs: int) {.rtl, extern: "nos$1", tags: [TimeEffect], noWeirdTarget.} = @@ -3242,9 +3236,9 @@ type ## Contains information associated with a file object. ## ## See also: - ## * `getFileInfo(handle) proc <#getFileInfo,FileHandle>`_ - ## * `getFileInfo(file) proc <#getFileInfo,File>`_ - ## * `getFileInfo(path) proc <#getFileInfo,string>`_ + ## * `getFileInfo(handle) proc`_ + ## * `getFileInfo(file) proc`_ + ## * `getFileInfo(path, followSymlink) proc`_ id*: tuple[device: DeviceId, file: FileId] ## Device and file id. kind*: PathComponent ## Kind of file object - directory, symlink, etc. size*: BiggestInt ## Size of file. @@ -3334,8 +3328,8 @@ proc getFileInfo*(handle: FileHandle): FileInfo {.noWeirdTarget.} = ## is invalid, `OSError` is raised. ## ## See also: - ## * `getFileInfo(file) proc <#getFileInfo,File>`_ - ## * `getFileInfo(path) proc <#getFileInfo,string>`_ + ## * `getFileInfo(file) proc`_ + ## * `getFileInfo(path, followSymlink) proc`_ # Done: ID, Kind, Size, Permissions, Link Count when defined(windows): @@ -3356,8 +3350,8 @@ proc getFileInfo*(file: File): FileInfo {.noWeirdTarget.} = ## Retrieves file information for the file object. ## ## See also: - ## * `getFileInfo(handle) proc <#getFileInfo,FileHandle>`_ - ## * `getFileInfo(path) proc <#getFileInfo,string>`_ + ## * `getFileInfo(handle) proc`_ + ## * `getFileInfo(path, followSymlink) proc`_ if file.isNil: raise newException(IOError, "File is nil") result = getFileInfo(file.getFileHandle()) @@ -3366,7 +3360,7 @@ proc getFileInfo*(path: string, followSymlink = true): FileInfo {.noWeirdTarget. ## Retrieves file information for the file object pointed to by `path`. ## ## Due to intrinsic differences between operating systems, the information - ## contained by the returned `FileInfo object <#FileInfo>`_ will be slightly + ## contained by the returned `FileInfo object`_ will be slightly ## different across platforms, and in some cases, incomplete or inaccurate. ## ## When `followSymlink` is true (default), symlinks are followed and the @@ -3378,8 +3372,8 @@ proc getFileInfo*(path: string, followSymlink = true): FileInfo {.noWeirdTarget. ## file information, `OSError` is raised. ## ## See also: - ## * `getFileInfo(handle) proc <#getFileInfo,FileHandle>`_ - ## * `getFileInfo(file) proc <#getFileInfo,File>`_ + ## * `getFileInfo(handle) proc`_ + ## * `getFileInfo(file) proc`_ when defined(windows): var handle = openHandle(path, followSymlink) @@ -3406,7 +3400,7 @@ proc sameFileContent*(path1, path2: string): bool {.rtl, extern: "nos$1", ## binary content. ## ## See also: - ## * `sameFile proc <#sameFile,string,string>`_ + ## * `sameFile proc`_ var a, b: File if not open(a, path1): return false diff --git a/tests/stdlib/tdochelpers.nim b/tests/stdlib/tdochelpers.nim index 5bcdf32d5..9d440a1d6 100644 --- a/tests/stdlib/tdochelpers.nim +++ b/tests/stdlib/tdochelpers.nim @@ -153,3 +153,9 @@ suite "Integration with Nim": outType: "T") check(input1.toLangSymbol == expected) check(input2.toLangSymbol == expected) + + test "type of type": + check ("`CopyFlag enum`_".rstParseTest.toLangSymbol == + LangSymbol(symKind: "type", + symTypeKind: "enum", + name: "Copyflag")) From 83a9c3ba31d180cd5e31026d8b7603bf7adea18c Mon Sep 17 00:00:00 2001 From: Zoom Date: Mon, 8 Nov 2021 18:47:20 +0000 Subject: [PATCH 1342/2472] Update readme.md: matrix links (#19109) Added links to the main #nim-lang room on Matrix and to the Nim matrix space. Added Matrix badge. --- readme.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/readme.md b/readme.md index af07c2a7b..7d45556db 100644 --- a/readme.md +++ b/readme.md @@ -11,6 +11,7 @@ the latest release, check out [Nim's website][nim-site] or [bleeding edge docs]( [![Join the IRC chat][badge-nim-irc]][nim-irc] [![Join the Discord server][badge-nim-discord]][nim-discord] [![Join the Gitter chat][badge-nim-gitter]][nim-gitter] +[![Join the Matrix room](https://img.shields.io/matrix/nim-lang:matrix.org?color=blue&style=flat&logo=matrix)](https://matrix.to/#/#nim-lang:matrix.org) [![Get help][badge-nim-forum-gethelp]][nim-forum] [![View Nim posts on Stack Overflow][badge-nim-stackoverflow]][nim-stackoverflow-newest] [![Follow @nim_lang on Twitter][badge-nim-twitter]][nim-twitter] @@ -22,6 +23,7 @@ the latest release, check out [Nim's website][nim-site] or [bleeding edge docs]( channels there are bridged to IRC. * [Gitter][nim-gitter] - an additional place to discuss Nim in real-time. There is a bridge between Gitter and the IRC channel. +* [Matrix][nim-matrix] - the main room to discuss Nim in real-time. [Matrix space][nim-matrix-space] contains a list of rooms, most of them are bridged to IRC. * [Telegram][nim-telegram] - an additional place to discuss Nim in real-time. There is the official Telegram channel. Not bridged to IRC. * [Stack Overflow][nim-stackoverflow] - a popular Q/A site for programming related @@ -213,6 +215,8 @@ Copyright © 2006-2021 Andreas Rumpf, all rights reserved. [nim-stackoverflow-newest]: https://stackoverflow.com/questions/tagged/nim-lang?sort=newest&pageSize=15 [nim-discord]: https://discord.gg/nim [nim-gitter]: https://gitter.im/nim-lang/Nim +[nim-matrix]: https://matrix.to/#/#nim-lang:matrix.org +[nim-matrix-space]: https://matrix.to/#/#nim:envs.net [nim-telegram]: https://t.me/nim_lang [nim-bountysource]: https://www.bountysource.com/teams/nim [nim-bitcoin]: https://blockchain.info/address/1BXfuKM2uvoD6mbx4g5xM3eQhLzkCK77tJ From b7c66ce860cfbadfa8ed60784fc387a0818e2e5d Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 9 Nov 2021 12:43:16 +0100 Subject: [PATCH 1343/2472] fixes #19013 [backport:1.6] (#19111) * fixes #19013 [backport:1.6] * added test case --- compiler/ast.nim | 8 ++++++++ compiler/isolation_check.nim | 17 ++++++++++++++++- compiler/varpartitions.nim | 7 ------- tests/isolate/tisolate2.nim | 22 ++++++++++++++++++++++ 4 files changed, 46 insertions(+), 8 deletions(-) create mode 100644 tests/isolate/tisolate2.nim diff --git a/compiler/ast.nim b/compiler/ast.nim index abd2ff01e..9086860b9 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -2101,3 +2101,11 @@ proc skipAddr*(n: PNode): PNode {.inline.} = proc isNewStyleConcept*(n: PNode): bool {.inline.} = assert n.kind == nkTypeClassTy result = n[0].kind == nkEmpty + +const + nodesToIgnoreSet* = {nkNone..pred(nkSym), succ(nkSym)..nkNilLit, + nkTypeSection, nkProcDef, nkConverterDef, + nkMethodDef, nkIteratorDef, nkMacroDef, nkTemplateDef, nkLambda, nkDo, + nkFuncDef, nkConstSection, nkConstDef, nkIncludeStmt, nkImportStmt, + nkExportStmt, nkPragma, nkCommentStmt, nkBreakState, + nkTypeOfExpr, nkMixinStmt, nkBindStmt} diff --git a/compiler/isolation_check.nim b/compiler/isolation_check.nim index 777e7f6ce..a8c5a3651 100644 --- a/compiler/isolation_check.nim +++ b/compiler/isolation_check.nim @@ -77,6 +77,17 @@ proc canAlias*(arg, ret: PType): bool = var marker = initIntSet() result = canAlias(arg, ret, marker) +proc containsVariable(n: PNode): bool = + case n.kind + of nodesToIgnoreSet: + result = false + of nkSym: + result = n.sym.kind in {skForVar, skParam, skVar, skLet, skConst, skResult, skTemp} + else: + for ch in n: + if containsVariable(ch): return true + result = false + proc checkIsolate*(n: PNode): bool = if types.containsTyRef(n.typ): # XXX Maybe require that 'n.typ' is acyclic. This is not much @@ -96,7 +107,11 @@ proc checkIsolate*(n: PNode): bool = else: let argType = n[i].typ if argType != nil and not isCompileTimeOnly(argType) and containsTyRef(argType): - if argType.canAlias(n.typ): + if argType.canAlias(n.typ) or containsVariable(n[i]): + # bug #19013: Alias information is not enough, we need to check for potential + # "overlaps". I claim the problem can only happen by reading again from a location + # that materialized which is only possible if a variable that contains a `ref` + # is involved. return false result = true of nkIfStmt, nkIfExpr: diff --git a/compiler/varpartitions.nim b/compiler/varpartitions.nim index 709d00fa0..721de900e 100644 --- a/compiler/varpartitions.nim +++ b/compiler/varpartitions.nim @@ -647,13 +647,6 @@ proc deps(c: var Partitions; dest, src: PNode) = when explainCursors: echo "D not a cursor ", d.sym, " reassignedTo ", c.s[srcid].reassignedTo c.s[vid].flags.incl preventCursor -const - nodesToIgnoreSet = {nkNone..pred(nkSym), succ(nkSym)..nkNilLit, - nkTypeSection, nkProcDef, nkConverterDef, - nkMethodDef, nkIteratorDef, nkMacroDef, nkTemplateDef, nkLambda, nkDo, - nkFuncDef, nkConstSection, nkConstDef, nkIncludeStmt, nkImportStmt, - nkExportStmt, nkPragma, nkCommentStmt, nkBreakState, - nkTypeOfExpr, nkMixinStmt, nkBindStmt} proc potentialMutationViaArg(c: var Partitions; n: PNode; callee: PType) = if constParameters in c.goals and tfNoSideEffect in callee.flags: diff --git a/tests/isolate/tisolate2.nim b/tests/isolate/tisolate2.nim new file mode 100644 index 000000000..9bf92d82e --- /dev/null +++ b/tests/isolate/tisolate2.nim @@ -0,0 +1,22 @@ +discard """ + errormsg: "expression cannot be isolated: a_to_b(a)" + line: 22 +""" + +# bug #19013 +import std/isolation + +type Z = ref object + i: int + +type A = object + z: Z + +type B = object + z: Z + +func a_to_b(a: A): B = + result = B(z: a.z) + +let a = A(z: Z(i: 3)) +let b = isolate(a_to_b(a)) From 6ff61766daec10d4cf78488b38bbe05d034e3a1c Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 9 Nov 2021 18:59:59 +0100 Subject: [PATCH 1344/2472] fixes #19011 [backport:1.6] (#19114) --- compiler/parser.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/parser.nim b/compiler/parser.nim index 5e9a7424a..aedf62d54 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -1877,7 +1877,7 @@ proc parseEnum(p: var Parser): PNode = var symPragma = a var pragma: PNode - if p.tok.tokType == tkCurlyDotLe: + if (p.tok.indent < 0 or p.tok.indent >= p.currInd) and p.tok.tokType == tkCurlyDotLe: pragma = optPragmas(p) symPragma = newNodeP(nkPragmaExpr, p) symPragma.add(a) From b0f4a9b845e4d483547f1a3818e3caf53cd90964 Mon Sep 17 00:00:00 2001 From: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Date: Tue, 9 Nov 2021 19:00:33 +0100 Subject: [PATCH 1345/2472] Add deprecation pragmas in lib/deprecated/pure (#19113) --- lib/deprecated/pure/LockFreeHash.nim | 6 ++++-- lib/deprecated/pure/events.nim | 2 ++ lib/deprecated/pure/ospaths.nim | 5 +++-- lib/deprecated/pure/securehash.nim | 5 ++--- lib/deprecated/pure/sharedstrings.nim | 2 ++ 5 files changed, 13 insertions(+), 7 deletions(-) diff --git a/lib/deprecated/pure/LockFreeHash.nim b/lib/deprecated/pure/LockFreeHash.nim index 97e1ef6ad..69e12226f 100644 --- a/lib/deprecated/pure/LockFreeHash.nim +++ b/lib/deprecated/pure/LockFreeHash.nim @@ -1,10 +1,12 @@ #nim c -t:-march=i686 --cpu:amd64 --threads:on -d:release lockfreehash.nim -import math, hashes - #------------------------------------------------------------------------------ ## Memory Utility Functions +{.deprecated.} + +import math, hashes + proc newHeap*[T](): ptr T = result = cast[ptr T](alloc0(sizeof(T))) diff --git a/lib/deprecated/pure/events.nim b/lib/deprecated/pure/events.nim index 218cb45d5..85fc349c2 100644 --- a/lib/deprecated/pure/events.nim +++ b/lib/deprecated/pure/events.nim @@ -36,6 +36,8 @@ ## myobj.SomeEvent.addHandler(handleevent) ## ee.emit(myobj.SomeEvent, genericargs) +{.deprecated.} + type EventArgs* = object of RootObj ## Base object for event arguments that are passed to callback functions. EventHandler* = tuple[name: string, handlers: seq[proc(e: EventArgs) {.closure.}]] ## An eventhandler for an event. diff --git a/lib/deprecated/pure/ospaths.nim b/lib/deprecated/pure/ospaths.nim index 6c7fe4fb3..e253a3e5a 100644 --- a/lib/deprecated/pure/ospaths.nim +++ b/lib/deprecated/pure/ospaths.nim @@ -7,8 +7,9 @@ # distribution, for details about the copyright. # -## This module is deprecated, `import os` instead. -{.deprecated: "import os.nim instead".} +## This module is deprecated, `import std/os` instead. + +{.deprecated: "use `std/os` instead".} import os export ReadEnvEffect, WriteEnvEffect, ReadDirEffect, WriteDirEffect, OSErrorCode, diff --git a/lib/deprecated/pure/securehash.nim b/lib/deprecated/pure/securehash.nim index 2f4530d88..ea7930f81 100644 --- a/lib/deprecated/pure/securehash.nim +++ b/lib/deprecated/pure/securehash.nim @@ -1,6 +1,5 @@ - - ## This module is a deprecated alias for the `sha1` module. -{.deprecated.} + +{.deprecated: "use `std/sha1` instead".} include "../std/sha1" diff --git a/lib/deprecated/pure/sharedstrings.nim b/lib/deprecated/pure/sharedstrings.nim index 4d4ae151b..99963e1f2 100644 --- a/lib/deprecated/pure/sharedstrings.nim +++ b/lib/deprecated/pure/sharedstrings.nim @@ -9,6 +9,8 @@ ## Shared string support for Nim. +{.deprecated.} + type UncheckedCharArray = UncheckedArray[char] From 46fb85542518c8594c24d168b025cc8a64b8fb3d Mon Sep 17 00:00:00 2001 From: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Date: Tue, 9 Nov 2021 19:00:43 +0100 Subject: [PATCH 1346/2472] Deprecate `std/sharedlist` and `std/sharedtables` (#19112) --- changelogs/changelog.md | 2 ++ changelogs/changelog_1_4_0.md | 2 +- doc/lib.rst | 6 ------ lib/pure/collections/lists.nim | 1 - lib/pure/collections/sharedlist.nim | 2 ++ lib/pure/collections/sharedtables.nim | 2 ++ lib/pure/collections/tables.nim | 1 - 7 files changed, 7 insertions(+), 9 deletions(-) diff --git a/changelogs/changelog.md b/changelogs/changelog.md index 475f568ec..d793c4313 100644 --- a/changelogs/changelog.md +++ b/changelogs/changelog.md @@ -4,6 +4,8 @@ ## Standard library additions and changes +- `std/sharedlist` and `std/sharedtables` are now deprecated, see RFC [#433](https://github.com/nim-lang/RFCs/issues/433). + ### New compile flag (`-d:nimNoGetRandom`) when building `std/sysrand` to remove dependency on linux `getrandom` syscall This compile flag only affects linux builds and is necessary if either compiling on a linux kernel version < 3.17, or if code built will be executing on kernel < 3.17. diff --git a/changelogs/changelog_1_4_0.md b/changelogs/changelog_1_4_0.md index 091048ad1..77f66ffde 100644 --- a/changelogs/changelog_1_4_0.md +++ b/changelogs/changelog_1_4_0.md @@ -282,7 +282,7 @@ - Removed the deprecated `asyncdispatch.newAsyncNativeSocket`. - Removed the deprecated `dom.releaseEvents` and `dom.captureEvents`. -- Removed `sharedlists.initSharedList`, was deprecated and produces undefined behaviour. +- Removed `sharedlist.initSharedList`, was deprecated and produces undefined behaviour. - There is a new experimental feature called "strictFuncs" which makes the definition of `.noSideEffect` stricter. [See here](manual_experimental.html#stricts-funcs) diff --git a/doc/lib.rst b/doc/lib.rst index 54728d681..881d857d3 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -134,12 +134,6 @@ Collections * `sets `_ Nim hash set support. -* `sharedlist `_ - Nim shared linked list support. Contains a shared singly-linked list. - -* `sharedtables `_ - Nim shared hash table support. Contains shared tables. - * `tables `_ Nim hash table support. Contains tables, ordered tables, and count tables. diff --git a/lib/pure/collections/lists.nim b/lib/pure/collections/lists.nim index af53bb68b..8b255fb60 100644 --- a/lib/pure/collections/lists.nim +++ b/lib/pure/collections/lists.nim @@ -56,7 +56,6 @@ runnableExamples: ## # See also ## * `deques module `_ for double-ended queues -## * `sharedlist module `_ for shared singly-linked lists import std/private/since diff --git a/lib/pure/collections/sharedlist.nim b/lib/pure/collections/sharedlist.nim index 73e147e05..e61883220 100644 --- a/lib/pure/collections/sharedlist.nim +++ b/lib/pure/collections/sharedlist.nim @@ -11,6 +11,8 @@ ## ## Unstable API. +{.deprecated.} + {.push stackTrace: off.} import diff --git a/lib/pure/collections/sharedtables.nim b/lib/pure/collections/sharedtables.nim index a85674df9..816ab49ab 100644 --- a/lib/pure/collections/sharedtables.nim +++ b/lib/pure/collections/sharedtables.nim @@ -14,6 +14,8 @@ ## ## Unstable API. +{.deprecated.} + import hashes, math, locks diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 2695f3693..00f71ef1d 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -191,7 +191,6 @@ runnableExamples: ## ## * `json module`_ for table-like structure which allows ## heterogeneous members -## * `sharedtables module`_ for shared hash table support ## * `strtabs module`_ for efficient hash tables ## mapping from strings to strings ## * `hashes module`_ for helper functions for hashing From 997ccc588955dab5cdbfd4c0fc067d91ccc1a7c8 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Tue, 9 Nov 2021 21:01:47 +0300 Subject: [PATCH 1347/2472] fix nimindexterm in rst2tex/doc2tex [backport] (#19106) * fix nimindexterm (rst2tex/doc2tex) [backport] * Add support for indexing in rst --- config/nimdoc.tex.cfg | 12 +++++++++++- lib/packages/docutils/rstgen.nim | 2 +- tools/kochdocs.nim | 6 ++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/config/nimdoc.tex.cfg b/config/nimdoc.tex.cfg index d7ab652eb..457f452dd 100644 --- a/config/nimdoc.tex.cfg +++ b/config/nimdoc.tex.cfg @@ -63,6 +63,11 @@ doc.file = """ % % Compile it by: xelatex (up to 3 times to get labels generated) % ------- +% For example: +% xelatex file.tex +% xelatex file.tex +% makeindex file +% xelatex file.tex % \documentclass[a4paper,11pt]{article} \usepackage[a4paper,xetex,left=3cm,right=3cm,top=1.5cm,bottom=2cm]{geometry} @@ -105,7 +110,9 @@ doc.file = """ \usepackage{parskip} % paragraphs delimited by vertical space, no indent \usepackage{graphicx} -\newcommand{\nimindexterm}[2]{#2\label{#1}} +\usepackage{makeidx} +\newcommand{\nimindexterm}[2]{#2\index{#2}\label{#1}} +\makeindex \usepackage{dingbat} % for \carriagereturn, etc \usepackage{fvextra} % for code blocks (works better than original fancyvrb) @@ -249,5 +256,8 @@ doc.file = """ \maketitle $content + +\printindex + \end{document} """ diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index a556aa2e3..4cbd05379 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -408,7 +408,7 @@ proc renderIndexTerm*(d: PDoc, n: PRstNode, result: var string) = var term = "" renderAux(d, n, term) setIndexTerm(d, changeFileExt(extractFilename(d.filename), HtmlExt), id, term, d.currentSection) - dispA(d.target, result, "$2", "\\nimindexterm{$2}{$1}", + dispA(d.target, result, "$2", "\\nimindexterm{$1}{$2}", [id, term]) type diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 42e648fb4..70ee5ad07 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -301,6 +301,12 @@ proc nim2pdf(src: string, dst: string, nimArgs: string) = # `>` should work on windows, if not, we can use `execCmdEx` let cmd = "xelatex -interaction=nonstopmode -output-directory=$# $# > $#" % [outDir.quoteShell, texFile.quoteShell, xelatexLog.quoteShell] exec(cmd) # on error, user can inspect `xelatexLog` + if i == 1: # build .ind file + var texFileBase = texFile + texFileBase.removeSuffix(".tex") + let cmd = "makeindex $# > $#" % [ + texFileBase.quoteShell, xelatexLog.quoteShell] + exec(cmd) moveFile(texFile.changeFileExt("pdf"), dst) proc buildPdfDoc*(nimArgs, destPath: string) = From 15157d06c3e0bba0c372a094917a13b916e15b54 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 9 Nov 2021 22:48:04 +0100 Subject: [PATCH 1348/2472] fixes #14470 [backport:1.2] (#19115) --- compiler/ccgtypes.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 23e1a5867..e7e099659 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -220,7 +220,8 @@ proc isInvalidReturnType(conf: ConfigRef; rettype: PType): bool = # such a poor programming language. # We exclude records with refs too. This enhances efficiency and # is necessary for proper code generation of assignments. - if rettype == nil: result = true + if rettype == nil or getSize(conf, rettype) > conf.target.floatSize*3: + result = true else: case mapType(conf, rettype, skResult) of ctArray: From a78ee8ae8453aabde04fa0103e02378efe870827 Mon Sep 17 00:00:00 2001 From: treeform Date: Tue, 9 Nov 2021 23:52:34 -0800 Subject: [PATCH 1349/2472] Call {.cursor.} a pragma. (#19116) * Call {.cursor.} a pragma. Its hard to find .curser annotation while googling because all other things like it are called pragmas. See https://nim-lang.org/docs/manual.html#pragmas Also the . in front of the name makes it hard to find and search for. Can we just call it cursor pragma? * Small fix for comment. --- doc/destructors.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/destructors.rst b/doc/destructors.rst index d6f240dee..83a50230b 100644 --- a/doc/destructors.rst +++ b/doc/destructors.rst @@ -555,14 +555,14 @@ for expressions of type `lent T` or of type `var T`. echo t[0] # accessor does not copy the element! -The .cursor annotation -====================== +The cursor pragma +================= Under the `--gc:arc|orc`:option: modes Nim's `ref` type is implemented via the same runtime "hooks" and thus via reference counting. This means that cyclic structures cannot be freed immediately (`--gc:orc`:option: ships with a cycle collector). -With the `.cursor` annotation one can break up cycles declaratively: +With the `cursor` pragma one can break up cycles declaratively: .. code-block:: nim @@ -575,7 +575,7 @@ But please notice that this is not C++'s weak_ptr, it means the right field is n involved in the reference counting, it is a raw pointer without runtime checks. Automatic reference counting also has the disadvantage that it introduces overhead -when iterating over linked structures. The `.cursor` annotation can also be used +when iterating over linked structures. The `cursor` pragma can also be used to avoid this overhead: .. code-block:: nim @@ -586,10 +586,10 @@ to avoid this overhead: it = it.next -In fact, `.cursor` more generally prevents object construction/destruction pairs +In fact, `cursor` more generally prevents object construction/destruction pairs and so can also be useful in other contexts. The alternative solution would be to use raw pointers (`ptr`) instead which is more cumbersome and also more dangerous -for Nim's evolution: Later on, the compiler can try to prove `.cursor` annotations +for Nim's evolution: Later on, the compiler can try to prove `cursor` pragmas to be safe, but for `ptr` the compiler has to remain silent about possible problems. @@ -597,7 +597,7 @@ problems. Cursor inference / copy elision =============================== -The current implementation also performs `.cursor` inference. Cursor inference is +The current implementation also performs `cursor` inference. Cursor inference is a form of copy elision. To see how and when we can do that, think about this question: In `dest = src` when @@ -612,7 +612,7 @@ indirections: .. code-block:: nim proc main(tab: Table[string, string]) = - let v = tab["key"] # inferred as .cursor because 'tab' is not mutated. + let v = tab["key"] # inferred as cursor because 'tab' is not mutated. # no copy into 'v', no destruction of 'v'. use(v) useItAgain(v) From 77b696c2c92b5f478526290c5e184a4c41060f7b Mon Sep 17 00:00:00 2001 From: Ryan Oldenburg Date: Wed, 10 Nov 2021 23:48:22 -0600 Subject: [PATCH 1350/2472] Remove tlsEmulation enabled from Windows + GCC config (#19119) [backport:1.6] This flag has a very significant performance impact on programs compiled with --threads:on. It is also apparently not needed anymore for standard circumstances. Can we remove the config? See https://github.com/nim-lang/Nim/issues/18146#issuecomment-876802676 for discussion and perf impact. [backport:1.6] --- config/nim.cfg | 3 --- 1 file changed, 3 deletions(-) diff --git a/config/nim.cfg b/config/nim.cfg index 3b964d124..86c6e2141 100644 --- a/config/nim.cfg +++ b/config/nim.cfg @@ -154,9 +154,6 @@ nimblepath="$home/.nimble/pkgs/" # Configuration for the GNU C/C++ compiler: @if windows: #gcc.path = r"$nim\dist\mingw\bin" - @if gcc or tcc: - tlsEmulation:on - @end @end gcc.maxerrorsimpl = "-fmax-errors=3" From 036d894e6a13cf9792c3250e34252a225c73e468 Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Thu, 11 Nov 2021 07:41:21 +0000 Subject: [PATCH 1351/2472] Add security tip for setCookie (#19117) * Add security tip for setCookie * Update lib/pure/cookies.nim Co-authored-by: Dominik Picheta * Update lib/pure/cookies.nim Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Co-authored-by: Andreas Rumpf Co-authored-by: Dominik Picheta Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- lib/pure/cookies.nim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/pure/cookies.nim b/lib/pure/cookies.nim index 25d701eb4..132f64637 100644 --- a/lib/pure/cookies.nim +++ b/lib/pure/cookies.nim @@ -50,6 +50,9 @@ proc setCookie*(key, value: string, domain = "", path = "", maxAge = none(int), sameSite = SameSite.Default): string = ## Creates a command in the format of ## `Set-Cookie: key=value; Domain=...; ...` + ## + + ## .. tip: Cookies can be vulnerable. Consider setting `secure=true`, `httpOnly=true` and `sameSite=Strict`. result = "" if not noName: result.add("Set-Cookie: ") result.add key & "=" & value From 7313b70a6af3fec6f367dd72bdd4f7651bf60d21 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 11 Nov 2021 15:53:27 +0800 Subject: [PATCH 1352/2472] correct cookie docs (#19122) --- lib/pure/cookies.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/pure/cookies.nim b/lib/pure/cookies.nim index 132f64637..7bf34f75b 100644 --- a/lib/pure/cookies.nim +++ b/lib/pure/cookies.nim @@ -51,8 +51,7 @@ proc setCookie*(key, value: string, domain = "", path = "", ## Creates a command in the format of ## `Set-Cookie: key=value; Domain=...; ...` ## - - ## .. tip: Cookies can be vulnerable. Consider setting `secure=true`, `httpOnly=true` and `sameSite=Strict`. + ## .. tip:: Cookies can be vulnerable. Consider setting `secure=true`, `httpOnly=true` and `sameSite=Strict`. result = "" if not noName: result.add("Set-Cookie: ") result.add key & "=" & value From cde83d90aa888ccc83bd78e6b6fb54d0ab0b2c26 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 11 Nov 2021 19:07:08 +0100 Subject: [PATCH 1353/2472] refactoring: orc can use getThreadId() (#19123) * refactoring: orc can use getThreadId() * progress --- lib/system/orc.nim | 7 +- lib/system/threadids.nim | 103 ++++++++++++++++++++++++++++++ lib/system/threadlocalstorage.nim | 3 - lib/system/threads.nim | 93 +-------------------------- 4 files changed, 110 insertions(+), 96 deletions(-) create mode 100644 lib/system/threadids.nim diff --git a/lib/system/orc.nim b/lib/system/orc.nim index 6273e1a21..f9f5cd81f 100644 --- a/lib/system/orc.nim +++ b/lib/system/orc.nim @@ -79,10 +79,12 @@ proc trace(s: Cell; desc: PNimTypeV2; j: var GcEnv) {.inline.} = var p = s +! sizeof(RefHeader) cast[TraceProc](desc.traceImpl)(p, addr(j)) +include threadids + when logOrc: proc writeCell(msg: cstring; s: Cell; desc: PNimTypeV2) = - cfprintf(cstderr, "%s %s %ld root index: %ld; RC: %ld; color: %ld\n", - msg, desc.name, s.refId, s.rootIdx, s.rc shr rcShift, s.color) + cfprintf(cstderr, "%s %s %ld root index: %ld; RC: %ld; color: %ld; thread: %ld\n", + msg, desc.name, s.refId, s.rootIdx, s.rc shr rcShift, s.color, getThreadId()) proc free(s: Cell; desc: PNimTypeV2) {.inline.} = when traceCollector: @@ -419,6 +421,7 @@ proc registerCycle(s: Cell; desc: PNimTypeV2) = proc GC_runOrc* = ## Forces a cycle collection pass. collectCycles() + orcAssert roots.len == 0, "roots not empty!" proc GC_enableOrc*() = ## Enables the cycle collector subsystem of `--gc:orc`. This is a `--gc:orc` diff --git a/lib/system/threadids.nim b/lib/system/threadids.nim new file mode 100644 index 000000000..3a6eadcbb --- /dev/null +++ b/lib/system/threadids.nim @@ -0,0 +1,103 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# we need to cache current threadId to not perform syscall all the time +var threadId {.threadvar.}: int + +when defined(windows): + proc getCurrentThreadId(): int32 {. + stdcall, dynlib: "kernel32", importc: "GetCurrentThreadId".} + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(getCurrentThreadId()) + result = threadId + +elif defined(linux): + proc syscall(arg: clong): clong {.varargs, importc: "syscall", header: "".} + when defined(amd64): + const NR_gettid = clong(186) + else: + var NR_gettid {.importc: "__NR_gettid", header: "".}: clong + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(syscall(NR_gettid)) + result = threadId + +elif defined(dragonfly): + proc lwp_gettid(): int32 {.importc, header: "unistd.h".} + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(lwp_gettid()) + result = threadId + +elif defined(openbsd): + proc getthrid(): int32 {.importc: "getthrid", header: "".} + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(getthrid()) + result = threadId + +elif defined(netbsd): + proc lwp_self(): int32 {.importc: "_lwp_self", header: "".} + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(lwp_self()) + result = threadId + +elif defined(freebsd): + proc syscall(arg: cint, arg0: ptr cint): cint {.varargs, importc: "syscall", header: "".} + var SYS_thr_self {.importc:"SYS_thr_self", header:"".}: cint + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + var tid = 0.cint + if threadId == 0: + discard syscall(SYS_thr_self, addr tid) + threadId = tid + result = threadId + +elif defined(macosx): + proc syscall(arg: cint): cint {.varargs, importc: "syscall", header: "".} + var SYS_thread_selfid {.importc:"SYS_thread_selfid", header:"".}: cint + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(syscall(SYS_thread_selfid)) + result = threadId + +elif defined(solaris): + type thread_t {.importc: "thread_t", header: "".} = distinct int + proc thr_self(): thread_t {.importc, header: "".} + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(thr_self()) + result = threadId + +elif defined(haiku): + type thr_id {.importc: "thread_id", header: "".} = distinct int32 + proc find_thread(name: cstring): thr_id {.importc, header: "".} + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(find_thread(nil)) + result = threadId diff --git a/lib/system/threadlocalstorage.nim b/lib/system/threadlocalstorage.nim index 30962ef77..e772f270c 100644 --- a/lib/system/threadlocalstorage.nim +++ b/lib/system/threadlocalstorage.nim @@ -29,9 +29,6 @@ when defined(windows): proc terminateThread(hThread: SysThread, dwExitCode: int32): int32 {. stdcall, dynlib: "kernel32", importc: "TerminateThread".} - proc getCurrentThreadId(): int32 {. - stdcall, dynlib: "kernel32", importc: "GetCurrentThreadId".} - type ThreadVarSlot = distinct int32 diff --git a/lib/system/threads.nim b/lib/system/threads.nim index a9006b0c5..a8a413bcb 100644 --- a/lib/system/threads.nim +++ b/lib/system/threads.nim @@ -343,94 +343,5 @@ else: proc createThread*(t: var Thread[void], tp: proc () {.thread, nimcall.}) = createThread[void](t, tp) -# we need to cache current threadId to not perform syscall all the time -var threadId {.threadvar.}: int - -when defined(windows): - proc getThreadId*(): int = - ## Gets the ID of the currently running thread. - if threadId == 0: - threadId = int(getCurrentThreadId()) - result = threadId - -elif defined(linux): - proc syscall(arg: clong): clong {.varargs, importc: "syscall", header: "".} - when defined(amd64): - const NR_gettid = clong(186) - else: - var NR_gettid {.importc: "__NR_gettid", header: "".}: clong - - proc getThreadId*(): int = - ## Gets the ID of the currently running thread. - if threadId == 0: - threadId = int(syscall(NR_gettid)) - result = threadId - -elif defined(dragonfly): - proc lwp_gettid(): int32 {.importc, header: "unistd.h".} - - proc getThreadId*(): int = - ## Gets the ID of the currently running thread. - if threadId == 0: - threadId = int(lwp_gettid()) - result = threadId - -elif defined(openbsd): - proc getthrid(): int32 {.importc: "getthrid", header: "".} - - proc getThreadId*(): int = - ## get the ID of the currently running thread. - if threadId == 0: - threadId = int(getthrid()) - result = threadId - -elif defined(netbsd): - proc lwp_self(): int32 {.importc: "_lwp_self", header: "".} - - proc getThreadId*(): int = - ## Gets the ID of the currently running thread. - if threadId == 0: - threadId = int(lwp_self()) - result = threadId - -elif defined(freebsd): - proc syscall(arg: cint, arg0: ptr cint): cint {.varargs, importc: "syscall", header: "".} - var SYS_thr_self {.importc:"SYS_thr_self", header:"".}: cint - - proc getThreadId*(): int = - ## Gets the ID of the currently running thread. - var tid = 0.cint - if threadId == 0: - discard syscall(SYS_thr_self, addr tid) - threadId = tid - result = threadId - -elif defined(macosx): - proc syscall(arg: cint): cint {.varargs, importc: "syscall", header: "".} - var SYS_thread_selfid {.importc:"SYS_thread_selfid", header:"".}: cint - - proc getThreadId*(): int = - ## Gets the ID of the currently running thread. - if threadId == 0: - threadId = int(syscall(SYS_thread_selfid)) - result = threadId - -elif defined(solaris): - type thread_t {.importc: "thread_t", header: "".} = distinct int - proc thr_self(): thread_t {.importc, header: "".} - - proc getThreadId*(): int = - ## Gets the ID of the currently running thread. - if threadId == 0: - threadId = int(thr_self()) - result = threadId - -elif defined(haiku): - type thr_id {.importc: "thread_id", header: "".} = distinct int32 - proc find_thread(name: cstring): thr_id {.importc, header: "".} - - proc getThreadId*(): int = - ## Gets the ID of the currently running thread. - if threadId == 0: - threadId = int(find_thread(nil)) - result = threadId +when not defined(gcOrc): + include threadids From 528ef6c218d90653f0f814c3dd256caa2cc389dc Mon Sep 17 00:00:00 2001 From: orthoplex <39635932+orthoplex@users.noreply.github.com> Date: Thu, 11 Nov 2021 20:01:51 +0100 Subject: [PATCH 1354/2472] fixed colorNames sorting mistake (#19125) [backport] --- lib/pure/colors.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/colors.nim b/lib/pure/colors.nim index 4a3f47e92..2193f69a6 100644 --- a/lib/pure/colors.nim +++ b/lib/pure/colors.nim @@ -353,8 +353,8 @@ const ("lightcoral", colLightCoral), ("lightcyan", colLightCyan), ("lightgoldenrodyellow", colLightGoldenRodYellow), - ("lightgrey", colLightGrey), ("lightgreen", colLightGreen), + ("lightgrey", colLightGrey), ("lightpink", colLightPink), ("lightsalmon", colLightSalmon), ("lightseagreen", colLightSeaGreen), From 3aaa12dbe5796cc55c6161e4d30a70a7f75cb3b5 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 12 Nov 2021 14:59:48 +0800 Subject: [PATCH 1355/2472] update manual (#19130) [backport] --- doc/manual.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual.rst b/doc/manual.rst index 001402036..328035b6d 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -6721,11 +6721,11 @@ statement, as seen in stack backtraces: if not cond: # change run-time line information of the 'raise' statement: {.line: instantiationInfo().}: - raise newException(EAssertionFailed, msg) + raise newException(AssertionDefect, msg) If the `line` pragma is used with a parameter, the parameter needs be a `tuple[filename: string, line: int]`. If it is used without a parameter, -`system.InstantiationInfo()` is used. +`system.instantiationInfo()` is used. linearScanEnd pragma From f8b71f7075b68aefc3955388ad5ec3178aee7658 Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Fri, 12 Nov 2021 18:00:19 +1100 Subject: [PATCH 1356/2472] Fix type mismatch with future logging (#19131) --- lib/pure/asyncfutures.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index 6906d03fd..4782f0a73 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -225,7 +225,7 @@ proc complete*[T](future: FutureVar[T], val: T) = fut.finished = true fut.value = val fut.callbacks.call() - when isFutureLoggingEnabled: logFutureFinish(future) + when isFutureLoggingEnabled: logFutureFinish(fut) proc fail*[T](future: Future[T], error: ref Exception) = ## Completes `future` with `error`. From c6fc3b2eae5aac68480a787b7fa3b81e3748bc2f Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 12 Nov 2021 11:19:24 +0100 Subject: [PATCH 1357/2472] fixes #19051 [backport:1.6] (#19133) --- compiler/sizealignoffsetimpl.nim | 19 +++++++++++++++++-- tests/ccgbugs/t5296.nim | 9 ++++++++- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/compiler/sizealignoffsetimpl.nim b/compiler/sizealignoffsetimpl.nim index 44ba32579..c5cd3ba64 100644 --- a/compiler/sizealignoffsetimpl.nim +++ b/compiler/sizealignoffsetimpl.nim @@ -196,6 +196,11 @@ proc computeUnionObjectOffsetsFoldFunction(conf: ConfigRef; n: PNode; packed: bo accum.offset = szUnknownSize proc computeSizeAlign(conf: ConfigRef; typ: PType) = + template setSize(typ, s) = + typ.size = s + typ.align = s + typ.paddingAtEnd = 0 + ## computes and sets ``size`` and ``align`` members of ``typ`` assert typ != nil let hasSize = typ.size != szUncomputedSize @@ -258,14 +263,14 @@ proc computeSizeAlign(conf: ConfigRef; typ: PType) = of tyArray: computeSizeAlign(conf, typ[1]) - let elemSize = typ[1].size + let elemSize = typ[1].size let len = lengthOrd(conf, typ[0]) if elemSize < 0: typ.size = elemSize typ.align = int16(elemSize) elif len < 0: typ.size = szUnknownSize - typ.align = szUnknownSize + typ.align = szUnknownSize else: typ.size = toInt64Checked(len * int32(elemSize), szTooBigSize) typ.align = typ[1].align @@ -445,6 +450,16 @@ proc computeSizeAlign(conf: ConfigRef; typ: PType) = typ.size = szUnknownSize typ.align = szUnknownSize typ.paddingAtEnd = szUnknownSize + of tyInt, tyUInt: + setSize typ, conf.target.intSize.int16 + of tyBool, tyChar, tyUInt8, tyInt8: + setSize typ, 1 + of tyInt16, tyUInt16: + setSize typ, 2 + of tyInt32, tyUInt32: + setSize typ, 4 + of tyInt64, tyUInt64: + setSize typ, 8 else: typ.size = szUnknownSize typ.align = szUnknownSize diff --git a/tests/ccgbugs/t5296.nim b/tests/ccgbugs/t5296.nim index 990b4bee2..8fbed35c4 100644 --- a/tests/ccgbugs/t5296.nim +++ b/tests/ccgbugs/t5296.nim @@ -1,6 +1,7 @@ discard """ cmd: "nim c -d:release $file" -output: 1 +output: '''1 +-1''' """ proc bug() : void = @@ -12,3 +13,9 @@ proc bug() : void = echo x bug() + +# bug #19051 +type GInt[T] = int + +var a = 1 +echo -a From 0a1049881eaa08cb7d10194d9b446d8a9ca52122 Mon Sep 17 00:00:00 2001 From: Clay Sweetser Date: Fri, 12 Nov 2021 22:51:43 -0500 Subject: [PATCH 1358/2472] Merge file size fields correctly on Windows (#19141) * Merge file size fields correctly on Windows Merge file size fields correctly on Windows - Merge the two 32-bit file size fields from `BY_HANDLE_FILE_INFORMATION` correctly in `rawToFormalFileInfo`. - Fixes #19135 * Update os.nim --- lib/pure/os.nim | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 513e09b48..9b08fe2e1 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -3255,7 +3255,11 @@ template rawToFormalFileInfo(rawInfo, path, formalInfo): untyped = ## 'rawInfo' is either a 'BY_HANDLE_FILE_INFORMATION' structure on Windows, ## or a 'Stat' structure on posix when defined(windows): - template merge(a, b): untyped = a or (b shl 32) + template merge(a, b): untyped = + int64( + (uint64(cast[uint32](a))) or + (uint64(cast[uint32](b)) shl 32) + ) formalInfo.id.device = rawInfo.dwVolumeSerialNumber formalInfo.id.file = merge(rawInfo.nFileIndexLow, rawInfo.nFileIndexHigh) formalInfo.size = merge(rawInfo.nFileSizeLow, rawInfo.nFileSizeHigh) From ade85ee91fee849f4b036ed8e42e31b978144522 Mon Sep 17 00:00:00 2001 From: Christoph Krybus Date: Sat, 13 Nov 2021 04:51:58 +0100 Subject: [PATCH 1359/2472] Fix punycode.decode function (#19136) * Refactor: rename proc to func * Fix punycode.decode function This function could only properly decode punycodes containing a single encoded unicode character. As soon as there was more than one punycode character group to decode it produced invalid output - the number of characters was correct, but their position was not. * Update tpunycode.nim Co-authored-by: Clay Sweetser --- lib/pure/punycode.nim | 31 +++--- tests/stdlib/tpunycode.nim | 202 ++++++++++++++++++++++++++++++++++++- 2 files changed, 217 insertions(+), 16 deletions(-) diff --git a/lib/pure/punycode.nim b/lib/pure/punycode.nim index dfeeea35e..49e46496e 100644 --- a/lib/pure/punycode.nim +++ b/lib/pure/punycode.nim @@ -28,7 +28,7 @@ const type PunyError* = object of ValueError -proc decodeDigit(x: char): int {.raises: [PunyError].} = +func decodeDigit(x: char): int {.raises: [PunyError].} = if '0' <= x and x <= '9': result = ord(x) - (ord('0') - 26) elif 'A' <= x and x <= 'Z': @@ -38,7 +38,7 @@ proc decodeDigit(x: char): int {.raises: [PunyError].} = else: raise newException(PunyError, "Bad input") -proc encodeDigit(digit: int): Rune {.raises: [PunyError].} = +func encodeDigit(digit: int): Rune {.raises: [PunyError].} = if 0 <= digit and digit < 26: result = Rune(digit + ord('a')) elif 26 <= digit and digit < 36: @@ -46,10 +46,10 @@ proc encodeDigit(digit: int): Rune {.raises: [PunyError].} = else: raise newException(PunyError, "internal error in punycode encoding") -proc isBasic(c: char): bool = ord(c) < 0x80 -proc isBasic(r: Rune): bool = int(r) < 0x80 +func isBasic(c: char): bool = ord(c) < 0x80 +func isBasic(r: Rune): bool = int(r) < 0x80 -proc adapt(delta, numPoints: int, first: bool): int = +func adapt(delta, numPoints: int, first: bool): int = var d = if first: delta div Damp else: delta div 2 d += d div numPoints var k = 0 @@ -58,7 +58,7 @@ proc adapt(delta, numPoints: int, first: bool): int = k += Base result = k + (Base - TMin + 1) * d div (d + Skew) -proc encode*(prefix, s: string): string {.raises: [PunyError].} = +func encode*(prefix, s: string): string {.raises: [PunyError].} = ## Encode a string that may contain Unicode. ## Prepend `prefix` to the result result = prefix @@ -114,18 +114,18 @@ proc encode*(prefix, s: string): string {.raises: [PunyError].} = inc d inc n -proc encode*(s: string): string {.raises: [PunyError].} = +func encode*(s: string): string {.raises: [PunyError].} = ## Encode a string that may contain Unicode. Prefix is empty. result = encode("", s) -proc decode*(encoded: string): string {.raises: [PunyError].} = +func decode*(encoded: string): string {.raises: [PunyError].} = ## Decode a Punycode-encoded string var n = InitialN i = 0 bias = InitialBias var d = rfind(encoded, Delimiter) - result = "" + var output: seq[Rune] if d > 0: # found Delimiter @@ -133,7 +133,7 @@ proc decode*(encoded: string): string {.raises: [PunyError].} = var c = encoded[j] # char if not c.isBasic: raise newException(PunyError, "Encoded contains a non-basic char") - result.add(c) # add the character + output.add(Rune(c)) # add the character inc d else: d = 0 # set to first index @@ -161,16 +161,17 @@ proc decode*(encoded: string): string {.raises: [PunyError].} = break w *= Base - t k += Base - bias = adapt(i - oldi, runeLen(result) + 1, oldi == 0) + bias = adapt(i - oldi, len(output) + 1, oldi == 0) - if i div (runeLen(result) + 1) > high(int32) - n: + if i div (len(output) + 1) > high(int32) - n: raise newException(PunyError, "Value too large") - n += i div (runeLen(result) + 1) - i = i mod (runeLen(result) + 1) - insert(result, $Rune(n), i) + n += i div (len(output) + 1) + i = i mod (len(output) + 1) + insert(output, Rune(n), i) inc i + result = $output runnableExamples: static: diff --git a/tests/stdlib/tpunycode.nim b/tests/stdlib/tpunycode.nim index 596c5ef63..bb2b4beb4 100644 --- a/tests/stdlib/tpunycode.nim +++ b/tests/stdlib/tpunycode.nim @@ -1,5 +1,205 @@ -import punycode +import punycode, std/unicode doAssert(decode(encode("", "bücher")) == "bücher") doAssert(decode(encode("münchen")) == "münchen") doAssert encode("xn--", "münchen") == "xn--mnchen-3ya" + +# source: https://datatracker.ietf.org/doc/html/rfc3492.html#section-7 +let punycode_testcases = [ + # (A) Arabic (Egyptian): + ( + input: + "\u0644\u064A\u0647\u0645\u0627\u0628\u062A\u0643\u0644" & + "\u0645\u0648\u0634\u0639\u0631\u0628\u064A\u061F", + output: + "egbpdaj6bu4bxfgehfvwxn" + ), + + # (B) Chinese (simplified): + ( + input: + "\u4ED6\u4EEC\u4E3A\u4EC0\u4E48\u4E0D\u8BF4\u4E2D\u6587", + output: + "ihqwcrb4cv8a8dqg056pqjye" + ), + + # (C) Chinese (traditional): + ( + input: + "\u4ED6\u5011\u7232\u4EC0\u9EBD\u4E0D\u8AAA\u4E2D\u6587", + output: + "ihqwctvzc91f659drss3x8bo0yb" + ), + + # (D) Czech: Proprostnemluvesky + ( + input: + "\u0050\u0072\u006F\u010D\u0070\u0072\u006F\u0073\u0074" & + "\u011B\u006E\u0065\u006D\u006C\u0075\u0076\u00ED\u010D" & + "\u0065\u0073\u006B\u0079", + output: + "Proprostnemluvesky-uyb24dma41a" + ), + + # (E) Hebrew: + ( + input: + "\u05DC\u05DE\u05D4\u05D4\u05DD\u05E4\u05E9\u05D5\u05D8" & + "\u05DC\u05D0\u05DE\u05D3\u05D1\u05E8\u05D9\u05DD\u05E2" & + "\u05D1\u05E8\u05D9\u05EA", + output: + "4dbcagdahymbxekheh6e0a7fei0b" + ), + + # (F) Hindi (Devanagari): + ( + input: + "\u092F\u0939\u0932\u094B\u0917\u0939\u093F\u0928\u094D" & + "\u0926\u0940\u0915\u094D\u092F\u094B\u0902\u0928\u0939" & + "\u0940\u0902\u092C\u094B\u0932\u0938\u0915\u0924\u0947" & + "\u0939\u0948\u0902", + output: + "i1baa7eci9glrd9b2ae1bj0hfcgg6iyaf8o0a1dig0cd" + ), + + # (G) Japanese (kanji and hiragana): + ( + input: + "\u306A\u305C\u307F\u3093\u306A\u65E5\u672C\u8A9E\u3092" & + "\u8A71\u3057\u3066\u304F\u308C\u306A\u3044\u306E\u304B", + output: + "n8jok5ay5dzabd5bym9f0cm5685rrjetr6pdxa" + ), + + # (H) Korean (Hangul syllables): + ( + input: + "\uC138\uACC4\uC758\uBAA8\uB4E0\uC0AC\uB78C\uB4E4\uC774" & + "\uD55C\uAD6D\uC5B4\uB97C\uC774\uD574\uD55C\uB2E4\uBA74" & + "\uC5BC\uB9C8\uB098\uC88B\uC744\uAE4C", + output: + "989aomsvi5e83db1d2a355cv1e0vak1dwrv93d5xbh15a0dt30a5jpsd879ccm6fea98c" + ), + + # (I) Russian (Cyrillic): + ( + input: + "\u043F\u043E\u0447\u0435\u043C\u0443\u0436\u0435\u043E" & + "\u043D\u0438\u043D\u0435\u0433\u043E\u0432\u043E\u0440" & + "\u044F\u0442\u043F\u043E\u0440\u0443\u0441\u0441\u043A" & + "\u0438", + output: + "b1abfaaepdrnnbgefbaDotcwatmq2g4l" + ), + + # (J) Spanish: PorqunopuedensimplementehablarenEspaol + ( + input: + "\u0050\u006F\u0072\u0071\u0075\u00E9\u006E\u006F\u0070" & + "\u0075\u0065\u0064\u0065\u006E\u0073\u0069\u006D\u0070" & + "\u006C\u0065\u006D\u0065\u006E\u0074\u0065\u0068\u0061" & + "\u0062\u006C\u0061\u0072\u0065\u006E\u0045\u0073\u0070" & + "\u0061\u00F1\u006F\u006C", + output: + "PorqunopuedensimplementehablarenEspaol-fmd56a" + ), + + # (K) Vietnamese: + # Tisaohkhngthch\ + # nitingVit + ( + input: + "\u0054\u1EA1\u0069\u0073\u0061\u006F\u0068\u1ECD\u006B" & + "\u0068\u00F4\u006E\u0067\u0074\u0068\u1EC3\u0063\u0068" & + "\u1EC9\u006E\u00F3\u0069\u0074\u0069\u1EBF\u006E\u0067" & + "\u0056\u0069\u1EC7\u0074", + output: + "TisaohkhngthchnitingVit-kjcr8268qyxafd2f1b9g" + ), + + # (L) 3B + ( + input: + "\u0033\u5E74\u0042\u7D44\u91D1\u516B\u5148\u751F", + output: + "3B-ww4c5e180e575a65lsy2b" + ), + + # (M) -with-SUPER-MONKEYS + ( + input: + "\u5B89\u5BA4\u5948\u7F8E\u6075\u002D\u0077\u0069\u0074" & + "\u0068\u002D\u0053\u0055\u0050\u0045\u0052\u002D\u004D" & + "\u004F\u004E\u004B\u0045\u0059\u0053", + output: + "-with-SUPER-MONKEYS-pc58ag80a8qai00g7n9n" + ), + + # (N) Hello-Another-Way- + ( + input: + "\u0048\u0065\u006C\u006C\u006F\u002D\u0041\u006E\u006F" & + "\u0074\u0068\u0065\u0072\u002D\u0057\u0061\u0079\u002D" & + "\u305D\u308C\u305E\u308C\u306E\u5834\u6240", + output: + "Hello-Another-Way--fc4qua05auwb3674vfr0b" + ), + + # (O) 2 + ( + input: + "\u3072\u3068\u3064\u5C4B\u6839\u306E\u4E0B\u0032", + output: + "2-u9tlzr9756bt3uc0v" + ), + + # (P) MajiKoi5 + ( + input: + "\u004D\u0061\u006A\u0069\u3067\u004B\u006F\u0069\u3059" & + "\u308B\u0035\u79D2\u524D", + output: + "MajiKoi5-783gue6qz075azm5e" + ), + + # (Q) de + ( + input: + "\u30D1\u30D5\u30A3\u30FC\u0064\u0065\u30EB\u30F3\u30D0", + output: + "de-jg4avhby1noc0d" + ), + + # (R) + ( + input: + "\u305D\u306E\u30B9\u30D4\u30FC\u30C9\u3067", + output: + "d9juau41awczczp" + ), + + # (S) -> $1.00 <- + ( + input: + "\u002D\u003E\u0020\u0024\u0031\u002E\u0030\u0030\u0020" & + "\u003C\u002D", + output: + "-> $1.00 <--" + ) +] + +block encodeTests: + for (uni, puny) in punycode_testcases: + # Need to convert both strings to lower case, since + # some of the extended encodings use upper case, but our + # code produces only lower case. Converting just puny to + # lower is also insufficient, since some of the input characters + # are upper case. + doAssert encode(uni).toLower() == puny.toLower() + +block decodeTests: + for (uni, puny) in punycode_testcases: + doAssert decode(puny) == uni + +block decodeInvalidTest: + doAssertRaises(PunyError): discard decode("xn--w&") From 270a5a372ddcb8fa97eacdf926dd90bfe189298f Mon Sep 17 00:00:00 2001 From: Anuken Date: Sat, 13 Nov 2021 19:09:15 -0500 Subject: [PATCH 1360/2472] Fix undeclared 'SYS_getrandom' on emscripten (#19144) --- lib/std/sysrand.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/sysrand.nim b/lib/std/sysrand.nim index b35f24a72..6b354e5a3 100644 --- a/lib/std/sysrand.nim +++ b/lib/std/sysrand.nim @@ -164,7 +164,7 @@ elif defined(windows): result = randomBytes(addr dest[0], size) -elif defined(linux) and not defined(nimNoGetRandom): +elif defined(linux) and not defined(nimNoGetRandom) and not defined(emscripten): # TODO using let, pending bootstrap >= 1.4.0 var SYS_getrandom {.importc: "SYS_getrandom", header: "".}: clong const syscallHeader = """#include From bec490016548e73f01317736ff8863f9ce6aab31 Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 14 Nov 2021 19:46:21 +0800 Subject: [PATCH 1361/2472] wrong spaces (3 => 2) (#19145) --- lib/system/threads.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/system/threads.nim b/lib/system/threads.nim index a8a413bcb..0e5198035 100644 --- a/lib/system/threads.nim +++ b/lib/system/threads.nim @@ -190,7 +190,7 @@ when defined(windows): threadProcWrapperBody(closure) # implicitly return 0 elif defined(genode): - proc threadProcWrapper[TArg](closure: pointer) {.noconv.} = + proc threadProcWrapper[TArg](closure: pointer) {.noconv.} = threadProcWrapperBody(closure) else: proc threadProcWrapper[TArg](closure: pointer): pointer {.noconv.} = From 6976d18519d102c9cc857a84a4b10b009c1e1a38 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Sun, 14 Nov 2021 03:49:30 -0800 Subject: [PATCH 1362/2472] Implement zephyr urandom and monotime (#19142) * implement urandom for Zephyr * add monotime on zephyr Co-authored-by: Jaremy Creechley --- lib/std/monotimes.nim | 7 +++++++ lib/std/sysrand.nim | 12 +++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/lib/std/monotimes.nim b/lib/std/monotimes.nim index bd078d7c0..5c67a5d4c 100644 --- a/lib/std/monotimes.nim +++ b/lib/std/monotimes.nim @@ -76,6 +76,10 @@ when defined(js): elif defined(posix) and not defined(osx): import posix +when defined(zephyr): + proc k_uptime_ticks(): int64 {.importc: "k_uptime_ticks", header: "".} + proc k_ticks_to_ns_floor64(ticks: int64): int64 {.importc: "k_ticks_to_ns_floor64", header: "".} + elif defined(windows): proc QueryPerformanceCounter(res: var uint64) {. importc: "QueryPerformanceCounter", stdcall, dynlib: "kernel32".} @@ -98,6 +102,9 @@ proc getMonoTime*(): MonoTime {.tags: [TimeEffect].} = mach_timebase_info(machAbsoluteTimeFreq) result = MonoTime(ticks: ticks * machAbsoluteTimeFreq.numer div machAbsoluteTimeFreq.denom) + elif defined(zephyr): + let ticks = k_ticks_to_ns_floor64(k_uptime_ticks()) + result = MonoTime(ticks: ticks) elif defined(posix): var ts: Timespec discard clock_gettime(CLOCK_MONOTONIC, ts) diff --git a/lib/std/sysrand.nim b/lib/std/sysrand.nim index 6b354e5a3..1b7b2c024 100644 --- a/lib/std/sysrand.nim +++ b/lib/std/sysrand.nim @@ -63,7 +63,7 @@ when defined(posix): import posix const - batchImplOS = defined(freebsd) or defined(openbsd) or (defined(macosx) and not defined(ios)) + batchImplOS = defined(freebsd) or defined(openbsd) or defined(zephyr) or (defined(macosx) and not defined(ios)) batchSize {.used.} = 256 when batchImplOS: @@ -207,6 +207,16 @@ elif defined(openbsd): proc getRandomImpl(p: pointer, size: int): int {.inline.} = result = getentropy(p, cint(size)).int +elif defined(zephyr): + proc sys_csrand_get(dst: pointer, length: csize_t): cint {.importc: "sys_csrand_get", header: "".} + # Fill the destination buffer with cryptographically secure + # random data values + # + + proc getRandomImpl(p: pointer, size: int): int {.inline.} = + # 0 if success, -EIO if entropy reseed error + result = sys_csrand_get(p, csize_t(size)).int + elif defined(freebsd): type cssize_t {.importc: "ssize_t", header: "".} = int From 309ec7167e6ea17112901533539be0a1884c8b65 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 16 Nov 2021 15:49:04 +0100 Subject: [PATCH 1363/2472] fixes .raises inference for newSeq builtin under --gc:orc [backport] (#19158) --- lib/system/seqs_v2.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/system/seqs_v2.nim b/lib/system/seqs_v2.nim index 0ac51109e..0c487b31d 100644 --- a/lib/system/seqs_v2.nim +++ b/lib/system/seqs_v2.nim @@ -73,7 +73,7 @@ proc prepareSeqAdd(len: int; p: pointer; addlen, elemSize, elemAlign: int): poin q.cap = newCap result = q -proc shrink*[T](x: var seq[T]; newLen: Natural) {.tags: [].} = +proc shrink*[T](x: var seq[T]; newLen: Natural) {.tags: [], raises: [].} = when nimvm: setLen(x, newLen) else: From 92d6fb86c66ceda154972314ba695cea4aac9301 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 16 Nov 2021 11:30:07 -0800 Subject: [PATCH 1364/2472] Enable customizing PageShift to set PageSize for embedded targets (#19129) * Enable customizing PageSize (via PageShift). This enables adjusting PageSize for embedded targets without abusing cpu16. * copy nimPageXYZ settings for mmpaptest * add docs for Nim manual * add docs for Nim manual * docs tweaks Co-authored-by: Jaremy Creechley --- doc/nimc.rst | 24 ++++++++++++++++++++++++ lib/system/bitmasks.nim | 10 +++++++--- tests/mmaptest.nim | 7 +++++-- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/doc/nimc.rst b/doc/nimc.rst index 98b754729..d35dad6c2 100644 --- a/doc/nimc.rst +++ b/doc/nimc.rst @@ -656,6 +656,30 @@ is not available but C's `malloc` is. You can use the `nimAllocPagesViaMalloc` define to use `malloc` instead of `mmap`. `nimAllocPagesViaMalloc` is currently only supported with `--gc:arc` or `--gc:orc`. (Since version 1.6) +nimPage256 / nimPage512 / nimPage1k +=================================== + +Adjust the page size for Nim's GC allocator. This enables using +`nimAllocPagesViaMalloc` on devices with less RAM. The default +page size requires too much RAM to work. + +Recommended settings: + +- < 32 kB of RAM use `nimPage256` + +- < 512 kB of RAM use `nimPage512` + +- < 2 MB of RAM use `nimPage1k` + +Initial testing hasn't shown much difference between 512B or 1kB page sizes +in terms of performance or latency. Using `nimPages256` will limit the +total amount of allocatable RAM. + +nimMemAlignTiny +=============== + +Sets `MemAlign` to `4` bytes which reduces the memory alignment +to better match some embedded devices. Nim for realtime systems ======================== diff --git a/lib/system/bitmasks.nim b/lib/system/bitmasks.nim index caf86568a..9cb57bc45 100644 --- a/lib/system/bitmasks.nim +++ b/lib/system/bitmasks.nim @@ -10,14 +10,18 @@ # Page size of the system; in most cases 4096 bytes. For exotic OS or # CPU this needs to be changed: const - PageShift = when defined(cpu16): 8 else: 12 # \ - # my tests showed no improvements for using larger page sizes. + PageShift = when defined(nimPage256) or defined(cpu16): 8 + elif defined(nimPage512): 9 + elif defined(nimPage1k): 10 + else: 12 # \ # my tests showed no improvements for using larger page sizes. + PageSize = 1 shl PageShift PageMask = PageSize-1 MemAlign = # also minimal allocatable memory block - when defined(useMalloc): + when defined(nimMemAlignTiny): 4 + elif defined(useMalloc): when defined(amd64): 16 else: 8 else: 16 diff --git a/tests/mmaptest.nim b/tests/mmaptest.nim index 7a93cdd45..33010606f 100644 --- a/tests/mmaptest.nim +++ b/tests/mmaptest.nim @@ -19,8 +19,11 @@ proc `+!!`(p: pointer, size: int): pointer {.inline.} = result = cast[pointer](cast[int](p) + size) const - PageShift = when defined(cpu16): 8 else: 12 # \ - # my tests showed no improvements for using larger page sizes. + PageShift = when defined(nimPage256) or defined(cpu16): 8 + elif defined(nimPage512): 9 + elif defined(nimPage1k): 10 + else: 12 # \ # my tests showed no improvements for using larger page sizes. + PageSize = 1 shl PageShift var p = osAllocPages(3 * PageSize) From fe46c8b5f13f854e4109183d2199b922f939548c Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 17 Nov 2021 17:15:54 +0800 Subject: [PATCH 1365/2472] fix marshal bugs in VM (#19161) [backport:1.6] --- compiler/vmmarshal.nim | 4 ++-- tests/ccgbugs/t8967.nim | 12 ++++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/compiler/vmmarshal.nim b/compiler/vmmarshal.nim index d48d2408c..d28f0325b 100644 --- a/compiler/vmmarshal.nim +++ b/compiler/vmmarshal.nim @@ -92,7 +92,8 @@ proc storeAny(s: var string; t: PType; a: PNode; stored: var IntSet; if a[i].kind == nkRange: var x = copyNode(a[i][0]) storeAny(s, t.lastSon, x, stored, conf) - while x.intVal+1 <= a[i][1].intVal: + inc x.intVal + while x.intVal <= a[i][1].intVal: s.add(", ") storeAny(s, t.lastSon, x, stored, conf) inc x.intVal @@ -231,7 +232,6 @@ proc loadAny(p: var JsonParser, t: PType, result = newNode(nkCurly) while p.kind != jsonArrayEnd and p.kind != jsonEof: result.add loadAny(p, t.lastSon, tab, cache, conf, idgen) - next(p) if p.kind == jsonArrayEnd: next(p) else: raiseParseErr(p, "']' end of array expected") of tyPtr, tyRef: diff --git a/tests/ccgbugs/t8967.nim b/tests/ccgbugs/t8967.nim index e342b7eae..0301a2e4f 100644 --- a/tests/ccgbugs/t8967.nim +++ b/tests/ccgbugs/t8967.nim @@ -4,7 +4,11 @@ discard """ import marshal -let orig: set[char] = {'A'..'Z'} -let m = $$orig -let old = to[set[char]](m) -doAssert orig - old == {} +template main() = + let orig: set[char] = {'A'..'Z'} + let m = $$orig + let old = to[set[char]](m) + doAssert orig - old == {} + +static: main() +main() From a8611c665f26adbf7f6dd2438359b4c5d930bf86 Mon Sep 17 00:00:00 2001 From: OxFred <88140942+OxFred@users.noreply.github.com> Date: Wed, 17 Nov 2021 13:16:12 +0100 Subject: [PATCH 1366/2472] Update dependency (#19151) --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index fc5c808ab..e11799e73 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -55,7 +55,7 @@ pkg "chronos", "nim c -r -d:release tests/testall", allowFailure = true # pendin pkg "cligen", "nim c --path:. -r cligen.nim" pkg "combparser", "nimble test --gc:orc" pkg "compactdict" -pkg "comprehension", "nimble test", "https://github.com/alehander42/comprehension" +pkg "comprehension", "nimble test", "https://github.com/alehander92/comprehension" pkg "criterion", allowFailure = true # pending https://github.com/disruptek/criterion/issues/3 (wrongly closed) pkg "datamancer" pkg "dashing", "nim c tests/functional.nim" From eb5358dcdb1b650db658def14cc8c60b4bb7c203 Mon Sep 17 00:00:00 2001 From: quantimnot <54247259+quantimnot@users.noreply.github.com> Date: Fri, 19 Nov 2021 02:19:01 -0500 Subject: [PATCH 1367/2472] Testament now checks OS, CPU, endianess, bitsizes 8-64 in discard statements (#19137) * Testament checks host/cpu in compiler/platform for discard spec statement * Remove duplicated value checks, add comments, add all cpu bit sizes and endianesses Co-authored-by: quantimnot --- testament/specs.nim | 70 ++++++++++++++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 20 deletions(-) diff --git a/testament/specs.nim b/testament/specs.nim index c7e332262..42400f22d 100644 --- a/testament/specs.nim +++ b/testament/specs.nim @@ -9,6 +9,7 @@ import sequtils, parseutils, strutils, os, streams, parsecfg, tables, hashes, sets +import compiler/platform type TestamentData* = ref object # better to group globals under 1 object; could group the other ones here too @@ -329,42 +330,71 @@ proc parseSpec*(filename: string): TSpec = # Valgrind only supports OSX <= 17.x result.useValgrind = disabled of "disabled": - case e.value.normalize + let value = e.value.normalize + case value of "y", "yes", "true", "1", "on": result.err = reDisabled of "n", "no", "false", "0", "off": discard - of "win", "windows": + # These values are defined in `compiler/options.isDefined` + of "win": when defined(windows): result.err = reDisabled of "linux": when defined(linux): result.err = reDisabled of "bsd": when defined(bsd): result.err = reDisabled - of "osx", "macosx": # xxx remove `macosx` alias? + of "osx": when defined(osx): result.err = reDisabled - of "unix": - when defined(unix): result.err = reDisabled - of "posix": + of "unix", "posix": when defined(posix): result.err = reDisabled + of "freebsd": + when defined(freebsd): result.err = reDisabled + of "littleendian": + when defined(littleendian): result.err = reDisabled + of "bigendian": + when defined(bigendian): result.err = reDisabled + of "cpu8", "8bit": + when defined(cpu8): result.err = reDisabled + of "cpu16", "16bit": + when defined(cpu16): result.err = reDisabled + of "cpu32", "32bit": + when defined(cpu32): result.err = reDisabled + of "cpu64", "64bit": + when defined(cpu64): result.err = reDisabled + # These values are for CI environments of "travis": # deprecated if isTravis: result.err = reDisabled of "appveyor": # deprecated if isAppVeyor: result.err = reDisabled of "azure": if isAzure: result.err = reDisabled - of "32bit": - if sizeof(int) == 4: - result.err = reDisabled - of "freebsd": - when defined(freebsd): result.err = reDisabled - of "arm64": - when defined(arm64): result.err = reDisabled - of "i386": - when defined(i386): result.err = reDisabled - of "openbsd": - when defined(openbsd): result.err = reDisabled - of "netbsd": - when defined(netbsd): result.err = reDisabled else: - result.parseErrors.addLine "cannot interpret as a bool: ", e.value + # Check whether the value exists as an OS or CPU that is + # defined in `compiler/platform`. + block checkHost: + for os in platform.OS: + # Check if the value exists as OS. + if value == os.name.normalize: + # The value exists; is it the same as the current host? + if value == hostOS.normalize: + # The value exists and is the same as the current host, + # so disable the test. + result.err = reDisabled + # The value was defined, so there is no need to check further + # values or raise an error. + break checkHost + for cpu in platform.CPU: + # Check if the value exists as CPU. + if value == cpu.name.normalize: + # The value exists; is it the same as the current host? + if value == hostCPU.normalize: + # The value exists and is the same as the current host, + # so disable the test. + result.err = reDisabled + # The value was defined, so there is no need to check further + # values or raise an error. + break checkHost + # The value doesn't exist as an OS, CPU, or any previous value + # defined in this case statement, so raise an error. + result.parseErrors.addLine "cannot interpret as a bool: ", e.value of "cmd": if e.value.startsWith("nim "): result.cmd = compilerPrefix & e.value[3..^1] From 1b143f5e79c940ba7f70e0512f36b5c61a6bc24d Mon Sep 17 00:00:00 2001 From: hlaaftana <10591326+hlaaftana@users.noreply.github.com> Date: Mon, 22 Nov 2021 12:41:35 +0300 Subject: [PATCH 1368/2472] Accept nnkTypeSection from typedef macro pragmas (#19168) --- changelog.md | 29 ++++++++++++++ compiler/semstmts.nim | 18 ++++++--- doc/manual.rst | 1 + tests/pragmas/ttypedef_macro.nim | 66 ++++++++++++++++++++++++++++++++ 4 files changed, 109 insertions(+), 5 deletions(-) create mode 100644 tests/pragmas/ttypedef_macro.nim diff --git a/changelog.md b/changelog.md index 1636d125f..8e70513e0 100644 --- a/changelog.md +++ b/changelog.md @@ -13,7 +13,36 @@ ## Language changes +- Pragma macros on type definitions can now return `nnkTypeSection` nodes as well as `nnkTypeDef`, + allowing multiple type definitions to be injected in place of the original type definition. + ```nim + import macros + + macro multiply(amount: static int, s: untyped): untyped = + let name = $s[0].basename + result = newNimNode(nnkTypeSection) + for i in 1 .. amount: + result.add(newTree(nnkTypeDef, ident(name & $i), s[1], s[2])) + + type + Foo = object + Bar {.multiply: 3.} = object + x, y, z: int + Baz = object + + # becomes + + type + Foo = object + Bar1 = object + x, y, z: int + Bar2 = object + x, y, z: int + Bar3 = object + x, y, z: int + Baz = object + ``` ## Compiler changes diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 4607e857a..36d76608e 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -1112,7 +1112,12 @@ proc typeDefLeftSidePass(c: PContext, typeSection: PNode, i: int) = if name.kind == nkPragmaExpr: let rewritten = applyTypeSectionPragmas(c, name[1], typeDef) if rewritten != nil: - typeSection[i] = rewritten + case rewritten.kind + of nkTypeDef: + typeSection[i] = rewritten + of nkTypeSection: + typeSection.sons[i .. i] = rewritten.sons + else: illFormedAst(rewritten, c.config) typeDefLeftSidePass(c, typeSection, i) return pragma(c, s, name[1], typePragmas) @@ -1143,16 +1148,19 @@ proc typeDefLeftSidePass(c: PContext, typeSection: PNode, i: int) = proc typeSectionLeftSidePass(c: PContext, n: PNode) = # process the symbols on the left side for the whole type section, before # we even look at the type definitions on the right - for i in 0.. Date: Tue, 23 Nov 2021 00:13:03 -0800 Subject: [PATCH 1369/2472] Implement threads on Zephyr (#19156) * pthreads setup for zephyr - enable tweak stack size - update lib/system/threads.nim - Fix int/uint in casting pointer. * add documentation and tweak flag names * add documentation and tweak flag names * fix configuration flag names * fix configuration flag names * cleanup Co-authored-by: Jaremy Creechley --- doc/nimc.rst | 9 ++++++++ lib/system/threadlocalstorage.nim | 2 ++ lib/system/threads.nim | 34 +++++++++++++++++++++++-------- 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/doc/nimc.rst b/doc/nimc.rst index d35dad6c2..6be4c204e 100644 --- a/doc/nimc.rst +++ b/doc/nimc.rst @@ -681,6 +681,15 @@ nimMemAlignTiny Sets `MemAlign` to `4` bytes which reduces the memory alignment to better match some embedded devices. +Thread stack size +================= + +Nim's thread API provides a simple wrapper around more advanced +RTOS task features. Customizing the stack size and stack guard size can +be done by setting `-d:nimThreadStackSize=16384` or `-d:nimThreadStackGuard=32`. + +Currently only Zephyr and FreeRTOS support these configurations. + Nim for realtime systems ======================== diff --git a/lib/system/threadlocalstorage.nim b/lib/system/threadlocalstorage.nim index e772f270c..cea9abb42 100644 --- a/lib/system/threadlocalstorage.nim +++ b/lib/system/threadlocalstorage.nim @@ -153,6 +153,8 @@ else: proc pthread_attr_init(a1: var Pthread_attr): cint {. importc, header: pthreadh.} + proc pthread_attr_setstack*(a1: ptr Pthread_attr, a2: pointer, a3: int): cint {. + importc, header: pthreadh.} proc pthread_attr_setstacksize(a1: var Pthread_attr, a2: int): cint {. importc, header: pthreadh.} proc pthread_attr_destroy(a1: var Pthread_attr): cint {. diff --git a/lib/system/threads.nim b/lib/system/threads.nim index 0e5198035..ac727be0a 100644 --- a/lib/system/threads.nim +++ b/lib/system/threads.nim @@ -47,14 +47,23 @@ when not declared(ThisIsSystem): {.error: "You must not import this module explicitly".} -const - StackGuardSize = 4096 - ThreadStackMask = - when defined(genode): - 1024*64*sizeof(int)-1 - else: - 1024*256*sizeof(int)-1 - ThreadStackSize = ThreadStackMask+1 - StackGuardSize +when defined(zephyr) or defined(freertos): + const + nimThreadStackSize {.intdefine.} = 8192 + nimThreadStackGuard {.intdefine.} = 128 + + StackGuardSize = nimThreadStackGuard + ThreadStackSize = nimThreadStackSize - nimThreadStackGuard +else: + const + StackGuardSize = 4096 + ThreadStackMask = + when defined(genode): + 1024*64*sizeof(int)-1 + else: + 1024*256*sizeof(int)-1 + + ThreadStackSize = ThreadStackMask+1 - StackGuardSize #const globalsSlot = ThreadVarSlot(0) #sysAssert checkSlot.int == globalsSlot.int @@ -321,7 +330,14 @@ else: when hasSharedHeap: t.core.stackSize = ThreadStackSize var a {.noinit.}: Pthread_attr doAssert pthread_attr_init(a) == 0 - let setstacksizeResult = pthread_attr_setstacksize(a, ThreadStackSize) + when defined(zephyr): + var + rawstk = allocShared0(ThreadStackSize + StackGuardSize) + stk = cast[pointer](cast[uint](rawstk) + StackGuardSize) + let setstacksizeResult = pthread_attr_setstack(addr a, stk, ThreadStackSize) + else: + let setstacksizeResult = pthread_attr_setstacksize(a, ThreadStackSize) + when not defined(ios): # This fails on iOS doAssert(setstacksizeResult == 0) From 040d23e799e94aa3e0c6b254f1dc708b2eada96b Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Tue, 23 Nov 2021 15:02:03 +0300 Subject: [PATCH 1370/2472] implement RST & Markdown quote blocks (#19147) * implement RST & Markdown quote blocks * compile with nim 1.0 * Fix indentation --- config/nimdoc.tex.cfg | 10 +- doc/nimdoc.css | 5 + lib/packages/docutils/rst.nim | 210 +++++++++++- lib/packages/docutils/rstast.nim | 9 +- lib/packages/docutils/rstgen.nim | 29 +- nimdoc/testproject/expected/nimdoc.out.css | 5 + tests/stdlib/trst.nim | 361 +++++++++++++++++++++ 7 files changed, 617 insertions(+), 12 deletions(-) diff --git a/config/nimdoc.tex.cfg b/config/nimdoc.tex.cfg index 457f452dd..3912d1279 100644 --- a/config/nimdoc.tex.cfg +++ b/config/nimdoc.tex.cfg @@ -139,9 +139,17 @@ doc.file = """ \usepackage[most]{tcolorbox} % boxes around admonitions, code blocks, doc.item \newtcolorbox{rstadmonition}[1][]{blanker, breakable, - left=3mm, right=3mm, top=1mm, bottom=1mm, + left=3mm, right=0mm, top=1mm, bottom=1mm, before upper=\indent, parbox=false, #1} +\newtcolorbox{rstquote}[1][]{blanker, breakable, + left=3mm, right=3mm, top=1mm, bottom=1mm, + parbox=false, + borderline west={0.3em}{0pt}{lightgray}, + borderline north={0.05em}{0pt}{lightgray}, + borderline east={0.05em}{0pt}{lightgray}, + borderline south={0.05em}{0pt}{lightgray}} + \definecolor{rstframecolor}{rgb}{0.85, 0.8, 0.6} \newtcolorbox{rstprebox}[1][]{blanker, breakable, diff --git a/doc/nimdoc.css b/doc/nimdoc.css index 7c819ea30..0014cf196 100644 --- a/doc/nimdoc.css +++ b/doc/nimdoc.css @@ -567,6 +567,11 @@ blockquote { border-left: 5px solid #bbc; } +blockquote.markdown-quote { + font-size: 0.9rem; /* use rem to avoid recursion */ + font-style: normal; +} + .pre, span.tok { font-family: "Source Code Pro", Monaco, Menlo, Consolas, "Courier New", monospace; font-weight: 500; diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 2e908f4e5..4c28894fb 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -53,6 +53,8 @@ ## + field lists ## + option lists ## + indented literal blocks +## + quoted literal blocks +## + line blocks ## + simple tables ## + directives (see official documentation in `RST directives list`_): ## - ``image``, ``figure`` for including images and videos @@ -121,6 +123,7 @@ ## * Markdown code blocks ## * Markdown links ## * Markdown headlines +## * Markdown block quotes ## * using ``1`` as auto-enumerator in enumerated lists like RST ``#`` ## (auto-enumerator ``1`` can not be used with ``#`` in the same list) ## @@ -145,7 +148,7 @@ ## 2) Compatibility mode which is RST rules. ## ## .. Note:: in both modes the parser interpretes text between single -## backticks (code) identically: +## backticks (code) identically: ## backslash does not escape; the only exception: ``\`` folowed by ` ## does escape so that we can always input a single backtick ` in ## inline code. However that makes impossible to input code with @@ -156,13 +159,35 @@ ## ``\`` -- GOOD ## So single backticks can always be input: `\`` will turn to ` code ## +## .. Attention:: +## We don't support some obviously poor design choices of Markdown (or RST). +## +## - no support for the rule of 2 spaces causing a line break in Markdown +## (use RST "line blocks" syntax for making line breaks) +## +## - interpretation of Markdown block quotes is also slightly different, +## e.g. case +## +## :: +## +## >>> foo +## > bar +## >>baz +## +## is a single 3rd-level quote `foo bar baz` in original Markdown, while +## in Nim we naturally see it as 3rd-level quote `foo` + 1st level `bar` + +## 2nd level `baz`: +## +## >>> foo +## > bar +## >>baz +## ## Limitations ## ----------- ## ## * no Unicode support in character width calculations ## * body elements ## - no roman numerals in enumerated lists -## - no quoted literal blocks ## - no doctest blocks ## - no grid tables ## - some directives are missing (check official `RST directives list`_): @@ -472,6 +497,10 @@ type line: int # the last line of this style occurrence # (for error message) hasPeers: bool # has headings on the same level of hierarchy? + LiteralBlockKind = enum # RST-style literal blocks after `::` + lbNone, + lbIndentedLiteralBlock, + lbQuotedLiteralBlock LevelMap = seq[LevelInfo] # Saves for each possible title adornment # style its level in the current document. SubstitutionKind = enum @@ -1953,6 +1982,44 @@ proc parseLiteralBlock(p: var RstParser): PRstNode = inc p.idx result.add(n) +proc parseQuotedLiteralBlock(p: var RstParser): PRstNode = + result = newRstNodeA(p, rnLiteralBlock) + var n = newLeaf("") + if currentTok(p).kind == tkIndent: + var indent = currInd(p) + while currentTok(p).kind == tkIndent: inc p.idx # skip blank lines + var quoteSym = currentTok(p).symbol[0] + while true: + case currentTok(p).kind + of tkEof: + break + of tkIndent: + if currentTok(p).ival < indent: + break + elif currentTok(p).ival == indent: + if nextTok(p).kind == tkPunct and nextTok(p).symbol[0] == quoteSym: + n.text.add("\n") + inc p.idx + elif nextTok(p).kind == tkIndent: + break + else: + rstMessage(p, mwRstStyle, "no newline after quoted literal block") + break + else: + rstMessage(p, mwRstStyle, + "unexpected indentation in quoted literal block") + break + else: + n.text.add(currentTok(p).symbol) + inc p.idx + result.add(n) + +proc parseRstLiteralBlock(p: var RstParser, kind: LiteralBlockKind): PRstNode = + if kind == lbIndentedLiteralBlock: + result = parseLiteralBlock(p) + else: + result = parseQuotedLiteralBlock(p) + proc getLevel(p: var RstParser, c: char, hasOverline: bool): int = ## Returns (preliminary) heading level corresponding to `c` and ## `hasOverline`. If level does not exist, add it first. @@ -2023,6 +2090,33 @@ proc isLineBlock(p: RstParser): bool = p.tok[j].col > currentTok(p).col or p.tok[j].symbol == "\n" +proc isMarkdownBlockQuote(p: RstParser): bool = + result = currentTok(p).symbol[0] == '>' + +proc whichRstLiteralBlock(p: RstParser): LiteralBlockKind = + ## Checks that the following tokens are either Indented Literal Block or + ## Quoted Literal Block (which is not quite the same as Markdown quote block). + ## https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#quoted-literal-blocks + if currentTok(p).symbol == "::" and nextTok(p).kind == tkIndent: + if currInd(p) > nextTok(p).ival: + result = lbNone + if currInd(p) < nextTok(p).ival: + result = lbIndentedLiteralBlock + elif currInd(p) == nextTok(p).ival: + var i = p.idx + 1 + while p.tok[i].kind == tkIndent: inc i + const validQuotingCharacters = { + '!', '"', '#', '$', '%', '&', '\'', '(', ')', '*', '+', ',', '-', + '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', + '_', '`', '{', '|', '}', '~'} + if p.tok[i].kind in {tkPunct, tkAdornment} and + p.tok[i].symbol[0] in validQuotingCharacters: + result = lbQuotedLiteralBlock + else: + result = lbNone + else: + result = lbNone + proc predNL(p: RstParser): bool = result = true if p.idx > 0: @@ -2078,6 +2172,8 @@ proc whichSection(p: RstParser): RstNodeKind = elif match(p, p.idx + 1, " a"): result = rnTable elif currentTok(p).symbol == "|" and isLineBlock(p): result = rnLineBlock + elif roSupportMarkdown in p.s.options and isMarkdownBlockQuote(p): + result = rnMarkdownBlockQuote elif match(p, p.idx + 1, "i") and isAdornmentHeadline(p, p.idx): result = rnOverline else: @@ -2090,6 +2186,8 @@ proc whichSection(p: RstParser): RstNodeKind = result = rnMarkdownTable elif currentTok(p).symbol == "|" and isLineBlock(p): result = rnLineBlock + elif roSupportMarkdown in p.s.options and isMarkdownBlockQuote(p): + result = rnMarkdownBlockQuote elif match(p, tokenAfterNewline(p), "aI") and isAdornmentHeadline(p, tokenAfterNewline(p)): result = rnHeadline @@ -2143,6 +2241,102 @@ proc parseLineBlock(p: var RstParser): PRstNode = else: break +proc parseDoc(p: var RstParser): PRstNode {.gcsafe.} + +proc getQuoteSymbol(p: RstParser, idx: int): tuple[sym: string, depth: int, tokens: int] = + result = ("", 0, 0) + var i = idx + result.sym &= p.tok[i].symbol + result.depth += p.tok[i].symbol.len + inc result.tokens + inc i + while p.tok[i].kind == tkWhite and i+1 < p.tok.len and + p.tok[i+1].kind == tkPunct and p.tok[i+1].symbol[0] == '>': + result.sym &= p.tok[i].symbol + result.sym &= p.tok[i+1].symbol + result.depth += p.tok[i+1].symbol.len + inc result.tokens, 2 + inc i, 2 + +proc parseMarkdownQuoteSegment(p: var RstParser, curSym: string, col: int): + PRstNode = + ## We define *segment* as a group of lines that starts with exactly the + ## same quote symbol. If the following lines don't contain any `>` (*lazy* + ## continuation) they considered as continuation of the current segment. + var q: RstParser # to delete `>` at a start of line and then parse normally + initParser(q, p.s) + q.col = p.col + q.line = p.line + var minCol = int.high # minimum colum num in the segment + while true: # move tokens of segment from `p` to `q` skipping `curSym` + case currentTok(p).kind + of tkEof: + break + of tkIndent: + if nextTok(p).kind in {tkIndent, tkEof}: + break + else: + if nextTok(p).symbol[0] == '>': + var (quoteSym, _, quoteTokens) = getQuoteSymbol(p, p.idx + 1) + if quoteSym == curSym: # the segment continues + var iTok = tokenAfterNewline(p, p.idx+1) + if p.tok[iTok].kind notin {tkEof, tkIndent} and + p.tok[iTok].symbol[0] != '>': + rstMessage(p, mwRstStyle, + "two or more quoted lines are followed by unquoted line " & + $(curLine(p) + 1)) + break + q.tok.add currentTok(p) + var ival = currentTok(p).ival + quoteSym.len + inc p.idx, (1 + quoteTokens) # skip newline and > > > + if currentTok(p).kind == tkWhite: + ival += currentTok(p).symbol.len + inc p.idx + # fix up previous `tkIndent`s to ival (as if >>> were not there) + var j = q.tok.len - 1 + while j >= 0 and q.tok[j].kind == tkIndent: + q.tok[j].ival = ival + dec j + else: # next segment started + break + elif currentTok(p).ival < col: + break + else: # the segment continues, a case like: + # > beginning + # continuation + q.tok.add currentTok(p) + inc p.idx + else: + if currentTok(p).col < minCol: minCol = currentTok(p).col + q.tok.add currentTok(p) + inc p.idx + q.indentStack = @[minCol] + # if initial indentation `minCol` is > 0 then final newlines + # should be omitted so that parseDoc could advance to the end of tokens: + var j = q.tok.len - 1 + while q.tok[j].kind == tkIndent: dec j + q.tok.setLen (j+1) + q.tok.add Token(kind: tkEof, line: currentTok(p).line) + result = parseDoc(q) + +proc parseMarkdownBlockQuote(p: var RstParser): PRstNode = + var (curSym, quotationDepth, quoteTokens) = getQuoteSymbol(p, p.idx) + let col = currentTok(p).col + result = newRstNodeA(p, rnMarkdownBlockQuote) + inc p.idx, quoteTokens # skip first > + while true: + var item = newRstNode(rnMarkdownBlockQuoteItem) + item.quotationDepth = quotationDepth + if currentTok(p).kind == tkWhite: inc p.idx + item.add parseMarkdownQuoteSegment(p, curSym, col) + result.add(item) + if currentTok(p).kind == tkIndent and currentTok(p).ival == col and + nextTok(p).kind != tkEof and nextTok(p).symbol[0] == '>': + (curSym, quotationDepth, quoteTokens) = getQuoteSymbol(p, p.idx + 1) + inc p.idx, (1 + quoteTokens) # skip newline and > > > + else: + break + proc parseParagraph(p: var RstParser, result: PRstNode) = while true: case currentTok(p).kind @@ -2158,16 +2352,17 @@ proc parseParagraph(p: var RstParser, result: PRstNode) = result.add newLeaf(" ") of rnLineBlock: result.addIfNotNil(parseLineBlock(p)) + of rnMarkdownBlockQuote: + result.addIfNotNil(parseMarkdownBlockQuote(p)) else: break else: break of tkPunct: - if currentTok(p).symbol == "::" and - nextTok(p).kind == tkIndent and - currInd(p) < nextTok(p).ival: + if (let literalBlockKind = whichRstLiteralBlock(p); + literalBlockKind != lbNone): result.add newLeaf(":") inc p.idx # skip '::' - result.add(parseLiteralBlock(p)) + result.add(parseRstLiteralBlock(p, literalBlockKind)) break else: parseInline(p, result) @@ -2257,8 +2452,6 @@ proc getColumns(p: var RstParser, cols: var IntSeq) = # last column has no limit: cols[L - 1] = 32000 -proc parseDoc(p: var RstParser): PRstNode {.gcsafe.} - proc parseSimpleTable(p: var RstParser): PRstNode = var cols: IntSeq @@ -2585,6 +2778,7 @@ proc parseSection(p: var RstParser, result: PRstNode) = a = parseLiteralBlock(p) of rnBulletList: a = parseBulletList(p) of rnLineBlock: a = parseLineBlock(p) + of rnMarkdownBlockQuote: a = parseMarkdownBlockQuote(p) of rnDirective: a = parseDotDot(p) of rnEnumList: a = parseEnumList(p) of rnLeaf: rstMessage(p, meNewSectionExpected, "(syntax error)") diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim index bc7b9a650..1d5da5e1c 100644 --- a/lib/packages/docutils/rstast.nim +++ b/lib/packages/docutils/rstast.nim @@ -32,7 +32,10 @@ type rnFieldName, # consisting of a field name ... rnFieldBody, # ... and a field body rnOptionList, rnOptionListItem, rnOptionGroup, rnOption, rnOptionString, - rnOptionArgument, rnDescription, rnLiteralBlock, rnQuotedLiteralBlock, + rnOptionArgument, rnDescription, rnLiteralBlock, + rnMarkdownBlockQuote, # a quote starting from punctuation like >>> + rnMarkdownBlockQuoteItem, # a quotation block, quote lines starting with + # the same number of chars rnLineBlock, # the | thingie rnLineBlockItem, # a son of rnLineBlock - one line inside it. # When `RstNode` lineIndent="\n" the line's empty @@ -101,6 +104,8 @@ type of rnFootnote, rnCitation, rnOptionListItem: order*: int ## footnote order (for auto-symbol footnotes and ## auto-numbered ones without a label) + of rnMarkdownBlockQuoteItem: + quotationDepth*: int ## number of characters in line prefix of rnRef, rnSubstitutionReferences, rnInterpretedText, rnField, rnInlineCode, rnCodeBlock, rnFootnoteRef: info*: TLineInfo ## To have line/column info for warnings at @@ -409,6 +414,8 @@ proc treeRepr*(node: PRstNode, indent=0): string = result.add " level=" & $node.level of rnFootnote, rnCitation, rnOptionListItem: result.add (if node.order == 0: "" else: " order=" & $node.order) + of rnMarkdownBlockQuoteItem: + result.add " quotationDepth=" & $node.quotationDepth else: discard result.add (if node.anchor == "": "" else: " anchor='" & node.anchor & "'") diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index 4cbd05379..dc6ca25c1 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -89,6 +89,7 @@ type onTestSnippet*: proc (d: var RstGenerator; filename, cmd: string; status: int; content: string) escMode*: EscapeMode + curQuotationDepth: int PDoc = var RstGenerator ## Alias to type less. @@ -167,6 +168,7 @@ proc initRstGenerator*(g: var RstGenerator, target: OutputTarget, g.currentSection = "" g.id = 0 g.escMode = emText + g.curQuotationDepth = 0 let fileParts = filename.splitFile if fileParts.ext == ".nim": g.currentSection = "Module " & fileParts.name @@ -1268,8 +1270,31 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = of rnLiteralBlock: renderAux(d, n, "$1\n", "\n\n$2\\begin{rstpre}\n$1\n\\end{rstpre}\n\n", result) - of rnQuotedLiteralBlock: - doAssert false, "renderRstToOut" + of rnMarkdownBlockQuote: + d.curQuotationDepth = 1 + var tmp = "" + renderAux(d, n, "$1", "$1", tmp) + let itemEnding = + if d.target == outHtml: "" else: "\\end{rstquote}" + tmp.add itemEnding.repeat(d.curQuotationDepth - 1) + dispA(d.target, result, + "$1\n", + "\n\\begin{rstquote}\n$2\n$1\\end{rstquote}\n", [tmp, n.anchor.idS]) + of rnMarkdownBlockQuoteItem: + let addQuotationDepth = n.quotationDepth - d.curQuotationDepth + var itemPrefix: string # start or ending (quotation grey bar on the left) + if addQuotationDepth >= 0: + let s = + if d.target == outHtml: "
                  " + else: "\\begin{rstquote}" + itemPrefix = s.repeat(addQuotationDepth) + else: + let s = + if d.target == outHtml: "
                  " + else: "\\end{rstquote}" + itemPrefix = s.repeat(-addQuotationDepth) + renderAux(d, n, itemPrefix & "

                  $1

                  ", itemPrefix & "\n$1", result) + d.curQuotationDepth = n.quotationDepth of rnLineBlock: if n.sons.len == 1 and n.sons[0].lineIndent == "\n": # whole line block is one empty line, no need to add extra spacing diff --git a/nimdoc/testproject/expected/nimdoc.out.css b/nimdoc/testproject/expected/nimdoc.out.css index 7c819ea30..0014cf196 100644 --- a/nimdoc/testproject/expected/nimdoc.out.css +++ b/nimdoc/testproject/expected/nimdoc.out.css @@ -567,6 +567,11 @@ blockquote { border-left: 5px solid #bbc; } +blockquote.markdown-quote { + font-size: 0.9rem; /* use rem to avoid recursion */ + font-style: normal; +} + .pre, span.tok { font-family: "Source Code Pro", Monaco, Menlo, Consolas, "Courier New", monospace; font-weight: 500; diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index e4609a671..0a9eb8038 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -105,6 +105,367 @@ suite "RST parsing": rnLeaf ' ' """) + test "RST quoted literal blocks": + let expected = + dedent""" + rnInner + rnLeaf 'Paragraph' + rnLeaf ':' + rnLiteralBlock + rnLeaf '>x' + """ + + check(dedent""" + Paragraph:: + + >x""".toAst == expected) + + check(dedent""" + Paragraph:: + + >x""".toAst == expected) + + test "RST quoted literal blocks, :: at a separate line": + let expected = + dedent""" + rnInner + rnInner + rnLeaf 'Paragraph' + rnLiteralBlock + rnLeaf '>x + >>y' + """ + + check(dedent""" + Paragraph + + :: + + >x + >>y""".toAst == expected) + + check(dedent""" + Paragraph + + :: + + >x + >>y""".toAst == expected) + + test "Markdown quoted blocks": + check(dedent""" + Paragraph. + >x""".toAst == + dedent""" + rnInner + rnLeaf 'Paragraph' + rnLeaf '.' + rnMarkdownBlockQuote + rnMarkdownBlockQuoteItem quotationDepth=1 + rnLeaf 'x' + """) + + # bug #17987 + check(dedent""" + foo https://github.com/nim-lang/Nim/issues/8258 + + > bar""".toAst == + dedent""" + rnInner + rnInner + rnLeaf 'foo' + rnLeaf ' ' + rnStandaloneHyperlink + rnLeaf 'https://github.com/nim-lang/Nim/issues/8258' + rnMarkdownBlockQuote + rnMarkdownBlockQuoteItem quotationDepth=1 + rnLeaf 'bar' + """) + + let expected = dedent""" + rnInner + rnLeaf 'Paragraph' + rnLeaf '.' + rnMarkdownBlockQuote + rnMarkdownBlockQuoteItem quotationDepth=1 + rnInner + rnLeaf 'x1' + rnLeaf ' ' + rnLeaf 'x2' + rnMarkdownBlockQuoteItem quotationDepth=2 + rnInner + rnLeaf 'y1' + rnLeaf ' ' + rnLeaf 'y2' + rnMarkdownBlockQuoteItem quotationDepth=1 + rnLeaf 'z' + """ + + check(dedent""" + Paragraph. + >x1 x2 + >>y1 y2 + >z""".toAst == expected) + + check(dedent""" + Paragraph. + > x1 x2 + >> y1 y2 + > z""".toAst == expected) + + check(dedent""" + >x + >y + >z""".toAst == + dedent""" + rnMarkdownBlockQuote + rnMarkdownBlockQuoteItem quotationDepth=1 + rnInner + rnLeaf 'x' + rnLeaf ' ' + rnLeaf 'y' + rnLeaf ' ' + rnLeaf 'z' + """) + + check(dedent""" + > z + > > >y + """.toAst == + dedent""" + rnMarkdownBlockQuote + rnMarkdownBlockQuoteItem quotationDepth=1 + rnLeaf 'z' + rnMarkdownBlockQuoteItem quotationDepth=3 + rnLeaf 'y' + """) + + test "Markdown quoted blocks: lazy": + let expected = dedent""" + rnInner + rnMarkdownBlockQuote + rnMarkdownBlockQuoteItem quotationDepth=2 + rnInner + rnLeaf 'x' + rnLeaf ' ' + rnLeaf 'continuation1' + rnLeaf ' ' + rnLeaf 'continuation2' + rnParagraph + rnLeaf 'newParagraph' + """ + check(dedent""" + >>x + continuation1 + continuation2 + + newParagraph""".toAst == expected) + + check(dedent""" + >> x + continuation1 + continuation2 + + newParagraph""".toAst == expected) + + # however mixing more than 1 non-lazy line and lazy one(s) splits quote + # in our parser, which appeared the easiest way to handle such cases: + var warnings = new seq[string] + check(dedent""" + >> x + >> continuation1 + continuation2 + + newParagraph""".toAst(warnings=warnings) == + dedent""" + rnInner + rnMarkdownBlockQuote + rnMarkdownBlockQuoteItem quotationDepth=2 + rnLeaf 'x' + rnMarkdownBlockQuoteItem quotationDepth=2 + rnInner + rnLeaf 'continuation1' + rnLeaf ' ' + rnLeaf 'continuation2' + rnParagraph + rnLeaf 'newParagraph' + """) + check(warnings[] == @[ + "input(2, 1) Warning: RST style: two or more quoted lines " & + "are followed by unquoted line 3"]) + + test "Markdown quoted blocks: not lazy": + # here is where we deviate from CommonMark specification: 'bar' below is + # not considered as continuation of 2-level '>> foo' quote. + check(dedent""" + >>> foo + > bar + >> baz + """.toAst() == + dedent""" + rnMarkdownBlockQuote + rnMarkdownBlockQuoteItem quotationDepth=3 + rnLeaf 'foo' + rnMarkdownBlockQuoteItem quotationDepth=1 + rnLeaf 'bar' + rnMarkdownBlockQuoteItem quotationDepth=2 + rnLeaf 'baz' + """) + + + test "Markdown quoted blocks: inline markup works": + check(dedent""" + > hi **bold** text + """.toAst == dedent""" + rnMarkdownBlockQuote + rnMarkdownBlockQuoteItem quotationDepth=1 + rnInner + rnLeaf 'hi' + rnLeaf ' ' + rnStrongEmphasis + rnLeaf 'bold' + rnLeaf ' ' + rnLeaf 'text' + """) + + test "Markdown quoted blocks: blank line separator": + let expected = dedent""" + rnInner + rnMarkdownBlockQuote + rnMarkdownBlockQuoteItem quotationDepth=1 + rnInner + rnLeaf 'x' + rnLeaf ' ' + rnLeaf 'y' + rnMarkdownBlockQuote + rnMarkdownBlockQuoteItem quotationDepth=1 + rnInner + rnLeaf 'z' + rnLeaf ' ' + rnLeaf 't' + """ + check(dedent""" + >x + >y + + > z + > t""".toAst == expected) + + check(dedent""" + >x + y + + > z + t""".toAst == expected) + + test "Markdown quoted blocks: nested body blocks/elements work #1": + let expected = dedent""" + rnMarkdownBlockQuote + rnMarkdownBlockQuoteItem quotationDepth=1 + rnBulletList + rnBulletItem + rnInner + rnLeaf 'x' + rnBulletItem + rnInner + rnLeaf 'y' + """ + + check(dedent""" + > - x + - y + """.toAst == expected) + + # TODO: if bug #17340 point 28 is resolved then this may work: + # check(dedent""" + # > - x + # - y + # """.toAst == expected) + + check(dedent""" + > - x + > - y + """.toAst == expected) + + check(dedent""" + > + > - x + > + > - y + > + """.toAst == expected) + + test "Markdown quoted blocks: nested body blocks/elements work #2": + let expected = dedent""" + rnAdmonition adType=note + [nil] + [nil] + rnDefList + rnDefItem + rnDefName + rnLeaf 'deflist' + rnLeaf ':' + rnDefBody + rnMarkdownBlockQuote + rnMarkdownBlockQuoteItem quotationDepth=2 + rnInner + rnLeaf 'quote' + rnLeaf ' ' + rnLeaf 'continuation' + """ + + check(dedent""" + .. Note:: deflist: + >> quote + continuation + """.toAst == expected) + + check(dedent""" + .. Note:: + deflist: + >> quote + continuation + """.toAst == expected) + + check(dedent""" + .. Note:: + deflist: + >> quote + >> continuation + """.toAst == expected) + + # spaces are not significant between `>`: + check(dedent""" + .. Note:: + deflist: + > > quote + > > continuation + """.toAst == expected) + + test "Markdown quoted blocks: de-indent handled well": + check(dedent""" + > + > - x + > - y + > + > Paragraph. + """.toAst == dedent""" + rnMarkdownBlockQuote + rnMarkdownBlockQuoteItem quotationDepth=1 + rnInner + rnBlockQuote + rnBulletList + rnBulletItem + rnInner + rnLeaf 'x' + rnBulletItem + rnInner + rnLeaf 'y' + rnParagraph + rnLeaf 'Paragraph' + rnLeaf '.' + """) + test "option list has priority over definition list": check(dedent""" --defusages From 5933aece9b5f7320e8feb2ef610b59d347c74d27 Mon Sep 17 00:00:00 2001 From: hlaaftana <10591326+hlaaftana@users.noreply.github.com> Date: Tue, 23 Nov 2021 18:30:17 +0300 Subject: [PATCH 1371/2472] `caseStmtMacros` no longer experimental, experimental manual refactor (#19173) * `caseStmtMacros` no longer experimental, experimental manual refactor * Update doc/manual.rst * apply review suggestions * apply review Co-authored-by: Andreas Rumpf --- changelog.md | 2 + compiler/options.nim | 2 +- compiler/semstmts.nim | 8 +- doc/manual.rst | 409 ++++++++--- doc/manual_experimental.rst | 1180 +++++++++++++------------------ lib/core/macros.nim | 7 +- tests/macros/tcasestmtmacro.nim | 2 - 7 files changed, 814 insertions(+), 796 deletions(-) diff --git a/changelog.md b/changelog.md index 8e70513e0..b645bc5c7 100644 --- a/changelog.md +++ b/changelog.md @@ -43,6 +43,8 @@ x, y, z: int Baz = object ``` +- [Case statement macros](manual.html#macros-case-statement-macros) are no longer experimental, + meaning you no longer need to enable the experimental switch `caseStmtMacros` to use them. ## Compiler changes diff --git a/compiler/options.nim b/compiler/options.nim index 2feaa2d43..d8e5057f9 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -195,7 +195,7 @@ type notnil, dynamicBindSym, forLoopMacros, # not experimental anymore; remains here for backwards compatibility - caseStmtMacros, + caseStmtMacros, # ditto codeReordering, compiletimeFFI, ## This requires building nim with `-d:nimHasLibFFI` diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 36d76608e..f302dd4c3 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -987,10 +987,10 @@ proc semCase(c: PContext, n: PNode; flags: TExprFlags): PNode = else: popCaseContext(c) closeScope(c) - if caseStmtMacros in c.features: - result = handleCaseStmtMacro(c, n, flags) - if result != nil: - return result + #if caseStmtMacros in c.features: + result = handleCaseStmtMacro(c, n, flags) + if result != nil: + return result localError(c.config, n[0].info, errSelectorMustBeOfCertainTypes) return for i in 1..`_. +See also `custom numeric literals <#custom-numeric-literals>`_. -Numeric Literals +Numeric literals ---------------- Numeric literals have the form:: @@ -625,7 +625,7 @@ Hence: 0b10000000'u8 == 0x80'u8 == 128, but, 0b10000000'i8 == 0x80'i8 == -1 instead of causing an overflow error. -Custom Numeric Literals +Custom numeric literals ~~~~~~~~~~~~~~~~~~~~~~~ If the suffix is not predefined, then the suffix is assumed to be a call @@ -700,26 +700,6 @@ contain a dot: `{..}` are the three tokens `{`:tok:, `..`:tok:, `}`:tok: and not the two tokens `{.`:tok:, `.}`:tok:. -Unicode Operators ------------------ - -Under the `--experimental:unicodeOperators` switch these Unicode operators are -also parsed as operators:: - - ∙ ∘ × ★ ⊗ ⊘ ⊙ ⊛ ⊠ ⊡ ∩ ∧ ⊓ # same priority as * (multiplication) - ± ⊕ ⊖ ⊞ ⊟ ∪ ∨ ⊔ # same priority as + (addition) - - -If enabled, Unicode operators can be combined with non-Unicode operator -symbols. The usual precedence extensions then apply, for example, `⊠=` is an -assignment like operator just like `*=` is. - -No Unicode normalization step is performed. - -**Note**: Due to parser limitations one **cannot** enable this feature via a -pragma `{.experimental: "unicodeOperators".}` reliably. - - Syntax ====== @@ -1323,46 +1303,6 @@ as `MyEnum.value`: To implement bit fields with enums see `Bit fields <#set-type-bit-fields>`_ -Overloadable enum field names ------------------------------ - -To be enabled via `{.experimental: "overloadableEnums".}`. - -Enum field names are overloadable much like routines. When an overloaded -enum field is used, it produces a closed sym choice construct, here -written as `(E|E)`. -During overload resolution the right `E` is picked, if possible. -For (array/object...) constructors the right `E` is picked, comparable to -how `[byte(1), 2, 3]` works, one needs to use `[T.E, E2, E3]`. Ambiguous -enum fields produce a static error: - -.. code-block:: nim - :test: "nim c $1" - - {.experimental: "overloadableEnums".} - - type - E1 = enum - value1, - value2 - E2 = enum - value1, - value2 = 4 - - const - Lookuptable = [ - E1.value1: "1", - value2: "2" - ] - - proc p(e: E1) = - # disambiguation in 'case' statements: - case e - of value1: echo "A" - of value2: echo "B" - - p value2 - String type ----------- @@ -1684,11 +1624,6 @@ must match the order of the tuple's definition. Different tuple-types are *equivalent* if they specify the same fields of the same type in the same order. The *names* of the fields also have to be the same. -The assignment operator for tuples copies each component. -The default assignment operator for objects copies each component. Overloading -of the assignment operator is described `here -`_. - .. code-block:: nim type @@ -1765,6 +1700,10 @@ introduce new object roots apart from `system.RootObj`. Student = ref object of Person # Error: inheritance only works with non-final objects id: int +The assignment operator for tuples and objects copies each component. +The methods to override this copying behavior are described `here +`_. + Object construction ------------------- @@ -2685,6 +2624,7 @@ Varargs matching See `Varargs <#types-varargs>`_. + iterable -------- @@ -2725,6 +2665,24 @@ available. Let `p` be an overloaded symbol. These contexts are: As usual, ambiguous matches produce a compile-time error. +Named argument overloading +-------------------------- + +Routines with the same type signature can be called individually if +a parameter has different names between them. + +.. code-block:: Nim + proc foo(x: int) = + echo "Using x: ", x + proc foo(y: int) = + echo "Using y: ", y + + foo(x = 2) # Using x: 2 + foo(y = 2) # Using y: 2 + +Not supplying the parameter name in such cases results in an +ambiguity error. + Statements and expressions ========================== @@ -3200,7 +3158,7 @@ Return statement Example: .. code-block:: nim - return 40+2 + return 40 + 2 The `return` statement ends the execution of the current procedure. It is only allowed in procedures. If there is an `expr`, this is syntactic @@ -3836,8 +3794,8 @@ behavior inside loop bodies. See `closureScope `_ and `capture `_ for details on how to change this behavior. -Anonymous Procs ---------------- +Anonymous procedures +-------------------- Unnamed procedures can be used as lambda expressions to pass into other procedures: @@ -3845,8 +3803,8 @@ procedures: .. code-block:: nim var cities = @["Frankfurt", "Tokyo", "New York", "Kyiv"] - cities.sort(proc (x,y: string): int = - cmp(x.len, y.len)) + cities.sort(proc (x, y: string): int = + cmp(x.len, y.len)) Procs as expressions can appear both as nested procs and inside top-level @@ -3854,6 +3812,42 @@ executable code. The `sugar `_ module contains the `=>` macro which enables a more succinct syntax for anonymous procedures resembling lambdas as they are in languages like JavaScript, C#, etc. +Do notation +----------- + +As a special convenience notation that keeps most elements of a +regular proc expression, the `do` keyword can be used to pass +anonymous procedures to routines: + +.. code-block:: nim + var cities = @["Frankfurt", "Tokyo", "New York", "Kyiv"] + + sort(cities) do (x, y: string) -> int: + cmp(x.len, y.len) + + # Less parentheses using the method plus command syntax: + cities = cities.map do (x: string) -> string: + "City of " & x + +`do` is written after the parentheses enclosing the regular proc params. +The proc expression represented by the `do` block is appended to the routine +call as the last argument. In calls using the command syntax, the `do` block +will bind to the immediately preceding expression rather than the command call. + +`do` with a parameter list corresponds to an anonymous `proc`, however +`do` without parameters is treated as a normal statement list. This allows +macros to receive both indented statement lists as an argument in inline +calls, as well as a direct mirror of Nim's routine syntax. + +.. code-block:: nim + # Passing a statement list to an inline macro: + macroResults.add quote do: + if not `ex`: + echo `info`, ": Check failed: ", `expString` + + # Processing a routine definition in a macro: + rpc(router, "add") do (a, b: int) -> int: + result = a + b Func ---- @@ -5133,7 +5127,7 @@ code: deletedKeys: seq[bool] -Type Classes +Type classes ------------ A type class is a special pseudo-type that can be used to match against @@ -5863,7 +5857,15 @@ twice: While macros enable advanced compile-time code transformations, they cannot change Nim's syntax. -Debug Example +**Style note:** For code readability, it is best to use the least powerful +programming construct that remains expressive. So the "check list" is: + +(1) Use an ordinary proc/iterator, if possible. +(2) Else: Use a generic proc/iterator, if possible. +(3) Else: Use a template, if possible. +(4) Else: Use a macro. + +Debug example ------------- The following example implements a powerful `debug` command that accepts a @@ -5921,7 +5923,7 @@ constructor expression. This is why `debug` iterates over all of `args`'s children. -BindSym +bindSym ------- The above `debug` macro relies on the fact that `write`, `writeLine` and @@ -5970,43 +5972,38 @@ However, the symbols `write`, `writeLine` and `stdout` are already bound and are not looked up again. As the example shows, `bindSym` does work with overloaded symbols implicitly. -Case-Of Macro -------------- +Note that the symbol names passed to `bindSym` have to be constant. The +experimental feature `dynamicBindSym` (`experimental manual +`_) +allows this value to be computed dynamically. -In Nim, it is possible to have a macro with the syntax of a *case-of* -expression just with the difference that all *of-branches* are passed to -and processed by the macro implementation. It is then up the macro -implementation to transform the *of-branches* into a valid Nim -statement. The following example should show how this feature could be -used for a lexical analyzer. +Post-statement blocks +--------------------- + +Macros can receive `of`, `elif`, `else`, `except`, `finally` and `do` +blocks (including their different forms such as `do` with routine parameters) +as arguments if called in statement form. .. code-block:: nim - import std/macros + macro performWithUndo(task, undo: untyped) = ... - macro case_token(args: varargs[untyped]): untyped = - echo args.treeRepr - # creates a lexical analyzer from regular expressions - # ... (implementation is an exercise for the reader ;-) - discard - - case_token: # this colon tells the parser it is a macro statement - of r"[A-Za-z_]+[A-Za-z_0-9]*": - return tkIdentifier - of r"0-9+": - return tkInteger - of r"[\+\-\*\?]+": - return tkOperator + performWithUndo do: + # multiple-line block of code + # to perform the task + do: + # code to undo it + + let num = 12 + # a single colon may be used if there is no initial block + match (num mod 3, num mod 5): + of (0, 0): + echo "FizzBuzz" + of (0, _): + echo "Fizz" + of (_, 0): + echo "Buzz" else: - return tkUnknown - - -**Style note**: For code readability, it is best to use the least powerful -programming construct that still suffices. So the "check list" is: - -(1) Use an ordinary proc/iterator, if possible. -(2) Else: Use a generic proc/iterator, if possible. -(3) Else: Use a template, if possible. -(4) Else: Use a macro. + echo num For loop macro @@ -6072,6 +6069,48 @@ Another example: echo a, " ", b +Case statement macros +--------------------- + +Macros named `` `case` `` can provide implementations of `case` statements +for certain types. The following is an example of such an implementation +for tuples, leveraging the existing equality operator for tuples +(as provided in `system.==`): + +.. code-block:: nim + :test: "nim c $1" + import std/macros + + macro `case`(n: tuple): untyped = + result = newTree(nnkIfStmt) + let selector = n[0] + for i in 1 ..< n.len: + let it = n[i] + case it.kind + of nnkElse, nnkElifBranch, nnkElifExpr, nnkElseExpr: + result.add it + of nnkOfBranch: + for j in 0..it.len-2: + let cond = newCall("==", selector, it[j]) + result.add newTree(nnkElifBranch, cond, it[^1]) + else: + error "custom 'case' for tuple cannot handle this node", it + + case ("foo", 78) + of ("foo", 78): echo "yes" + of ("bar", 88): echo "no" + else: discard + +`case` macros are subject to overload resolution. The type of the +`case` statement's selector expression is matched against the type +of the first argument of the `case` macro. Then the complete `case` +statement is passed in place of the argument and the macro is evaluated. + +In other words, the macro needs to transform the full `case` statement +but only the statement's selector expression is used to determine which +macro to call. + + Special Types ============= @@ -6959,7 +6998,8 @@ experimental pragma The `experimental` pragma enables experimental language features. Depending on the concrete feature, this means that the feature is either considered too unstable for an otherwise stable release or that the future of the feature -is uncertain (it may be removed at any time). +is uncertain (it may be removed at any time). See the +`experimental manual `_ for more details. Example: @@ -7060,6 +7100,21 @@ alignment requirement of the type are ignored. This pragma has no effect on the JS backend. +Noalias pragma +============== + +Since version 1.4 of the Nim compiler, there is a `.noalias` annotation for variables +and parameters. It is mapped directly to C/C++'s `restrict`:c: keyword and means that +the underlying pointer is pointing to a unique location in memory, no other aliases to +this location exist. It is *unchecked* that this alias restriction is followed. If the +restriction is violated, the backend optimizer is free to miscompile the code. +This is an **unsafe** language feature. + +Ideally in later versions of the language, the restriction will be enforced at +compile time. (This is also why the name `noalias` was choosen instead of a more +verbose name like `unsafeAssumeNoAlias`.) + + Volatile pragma --------------- The `volatile` pragma is for variables only. It declares the variable as @@ -7641,7 +7696,7 @@ Example: {.pragma: rtl, importc, dynlib: "client.dll", cdecl.} proc p*(a, b: int): int {.rtl.} = - result = a+b + result = a + b In the example, a new pragma named `rtl` is introduced that either imports a symbol from a dynamic library or exports the symbol for dynamic library @@ -8032,3 +8087,137 @@ Threads and exceptions The interaction between threads and exceptions is simple: A *handled* exception in one thread cannot affect any other thread. However, an *unhandled* exception in one thread terminates the whole *process*. + + +Guards and locks +================ + +Nim provides common low level concurrency mechanisms like locks, atomic +intrinsics or condition variables. + +Nim significantly improves on the safety of these features via additional +pragmas: + +1) A `guard`:idx: annotation is introduced to prevent data races. +2) Every access of a guarded memory location needs to happen in an + appropriate `locks`:idx: statement. + + +Guards and locks sections +------------------------- + +Protecting global variables +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Object fields and global variables can be annotated via a `guard` pragma: + +.. code-block:: nim + + var glock: TLock + var gdata {.guard: glock.}: int + +The compiler then ensures that every access of `gdata` is within a `locks` +section: + +.. code-block:: nim + + proc invalid = + # invalid: unguarded access: + echo gdata + + proc valid = + # valid access: + {.locks: [glock].}: + echo gdata + +Top level accesses to `gdata` are always allowed so that it can be initialized +conveniently. It is *assumed* (but not enforced) that every top level statement +is executed before any concurrent action happens. + +The `locks` section deliberately looks ugly because it has no runtime +semantics and should not be used directly! It should only be used in templates +that also implement some form of locking at runtime: + +.. code-block:: nim + + template lock(a: TLock; body: untyped) = + pthread_mutex_lock(a) + {.locks: [a].}: + try: + body + finally: + pthread_mutex_unlock(a) + + +The guard does not need to be of any particular type. It is flexible enough to +model low level lockfree mechanisms: + +.. code-block:: nim + + var dummyLock {.compileTime.}: int + var atomicCounter {.guard: dummyLock.}: int + + template atomicRead(x): untyped = + {.locks: [dummyLock].}: + memoryReadBarrier() + x + + echo atomicRead(atomicCounter) + + +The `locks` pragma takes a list of lock expressions `locks: [a, b, ...]` +in order to support *multi lock* statements. Why these are essential is +explained in the `lock levels <#guards-and-locks-lock-levels>`_ section. + + +Protecting general locations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The `guard` annotation can also be used to protect fields within an object. +The guard then needs to be another field within the same object or a +global variable. + +Since objects can reside on the heap or on the stack, this greatly enhances +the expressivity of the language: + +.. code-block:: nim + + type + ProtectedCounter = object + v {.guard: L.}: int + L: TLock + + proc incCounters(counters: var openArray[ProtectedCounter]) = + for i in 0..counters.high: + lock counters[i].L: + inc counters[i].v + +The access to field `x.v` is allowed since its guard `x.L` is active. +After template expansion, this amounts to: + +.. code-block:: nim + + proc incCounters(counters: var openArray[ProtectedCounter]) = + for i in 0..counters.high: + pthread_mutex_lock(counters[i].L) + {.locks: [counters[i].L].}: + try: + inc counters[i].v + finally: + pthread_mutex_unlock(counters[i].L) + +There is an analysis that checks that `counters[i].L` is the lock that +corresponds to the protected location `counters[i].v`. This analysis is called +`path analysis`:idx: because it deals with paths to locations +like `obj.field[i].fieldB[j]`. + +The path analysis is **currently unsound**, but that doesn't make it useless. +Two paths are considered equivalent if they are syntactically the same. + +This means the following compiles (for now) even though it really should not: + +.. code-block:: nim + + {.locks: [a[i].L].}: + inc i + access a[i].v diff --git a/doc/manual_experimental.rst b/doc/manual_experimental.rst index 458512292..4ed8439df 100644 --- a/doc/manual_experimental.rst +++ b/doc/manual_experimental.rst @@ -19,47 +19,8 @@ Some of these are not covered by the `.experimental` pragma or one may want to use Nim libraries using these features without using them oneself. -**Note**: Unless otherwise indicated, these features are not to be removed, -but refined and overhauled. - - -Package level objects -===================== - -Every Nim module resides in a (nimble) package. An object type can be attached -to the package it resides in. If that is done, the type can be referenced from -other modules as an `incomplete`:idx: object type. This feature allows to -break up recursive type dependencies across module boundaries. Incomplete -object types are always passed `byref` and can only be used in pointer like -contexts (`var/ref/ptr IncompleteObject`) in general since the compiler does -not yet know the size of the object. To complete an incomplete object -the `package` pragma has to be used. `package` implies `byref`. - -As long as a type `T` is incomplete, neither `sizeof(T)` nor runtime -type information for `T` is available. - - -Example: - -.. code-block:: nim - - # module A (in an arbitrary package) - type - Pack.SomeObject = object ## declare as incomplete object of package 'Pack' - Triple = object - a, b, c: ref SomeObject ## pointers to incomplete objects are allowed - - ## Incomplete objects can be used as parameters: - proc myproc(x: SomeObject) = discard - - -.. code-block:: nim - - # module B (in package "Pack") - type - SomeObject* {.package.} = object ## Use 'package' to complete the object - s, t: string - x, y: int +.. note:: Unless otherwise indicated, these features are not to be removed, + but refined and overhauled. Void type @@ -105,23 +66,106 @@ The `void` type is only valid for parameters and return types; other symbols cannot have the type `void`. -Automatic dereferencing -======================= +Unicode Operators +================= -Automatic dereferencing is performed for the first argument of a routine call. -This feature has to be enabled via `{.experimental: "implicitDeref".}`: +Under the `--experimental:unicodeOperators`:option: switch, +these Unicode operators are also parsed as operators:: + + ∙ ∘ × ★ ⊗ ⊘ ⊙ ⊛ ⊠ ⊡ ∩ ∧ ⊓ # same priority as * (multiplication) + ± ⊕ ⊖ ⊞ ⊟ ∪ ∨ ⊔ # same priority as + (addition) + + +If enabled, Unicode operators can be combined with non-Unicode operator +symbols. The usual precedence extensions then apply, for example, `⊠=` is an +assignment like operator just like `*=` is. + +No Unicode normalization step is performed. + +.. note:: Due to parser limitations one **cannot** enable this feature via a + pragma `{.experimental: "unicodeOperators".}` reliably. + + +Overloadable enum value names +============================= + +Enabled via `{.experimental: "overloadableEnums".}`. + +Enum value names are overloadable, much like routines. If both of the enums +`T` and `U` have a member named `foo`, then the identifier `foo` corresponds +to a choice between `T.foo` and `U.foo`. During overload resolution, +the correct type of `foo` is decided from the context. If the type of `foo` is +ambiguous, a static error will be produced. + +.. code-block:: nim + :test: "nim c $1" + + {.experimental: "overloadableEnums".} + + type + E1 = enum + value1, + value2 + E2 = enum + value1, + value2 = 4 + + const + Lookuptable = [ + E1.value1: "1", + # no need to qualify value2, known to be E1.value2 + value2: "2" + ] + + proc p(e: E1) = + # disambiguation in 'case' statements: + case e + of value1: echo "A" + of value2: echo "B" + + p value2 + + +Package level objects +===================== + +Every Nim module resides in a (nimble) package. An object type can be attached +to the package it resides in. If that is done, the type can be referenced from +other modules as an `incomplete`:idx: object type. This feature allows to +break up recursive type dependencies across module boundaries. Incomplete +object types are always passed `byref` and can only be used in pointer like +contexts (`var/ref/ptr IncompleteObject`) in general, since the compiler does +not yet know the size of the object. To complete an incomplete object, +the `package` pragma has to be used. `package` implies `byref`. + +As long as a type `T` is incomplete, no runtime type information for `T` is +available. + + +Example: .. code-block:: nim - {.experimental: "implicitDeref".} + # module A (in an arbitrary package) + type + Pack.SomeObject = object # declare as incomplete object of package 'Pack' + Triple = object + a, b, c: ref SomeObject # pointers to incomplete objects are allowed - proc depth(x: NodeObj): int = ... + # Incomplete objects can be used as parameters: + proc myproc(x: SomeObject) = discard - var - n: Node - new(n) - echo n.depth - # no need to write n[].depth either + +.. code-block:: nim + + # module B (in package "Pack") + type + SomeObject* {.package.} = object # Use 'package' to complete the object + s, t: string + x, y: int + +This feature will likely be superseded in the future by support for +recursive module dependencies. Code reordering @@ -230,65 +274,30 @@ scope. Therefore, the following will *fail to compile:* a() - -Named argument overloading -========================== - -Routines with the same type signature can be called differently if a parameter -has different names. This does not need an `experimental` switch, but is an -unstable feature. - -.. code-block:: Nim - - proc foo(x: int) = - echo "Using x: ", x - proc foo(y: int) = - echo "Using y: ", y - - foo(x = 2) - # Using x: 2 - foo(y = 2) - # Using y: 2 +This feature will likely be replaced with a better solution to remove +the need for forward declarations. -Do notation -=========== +Automatic dereferencing +======================= -As a special more convenient notation, proc expressions involved in procedure -calls can use the `do` keyword: +Automatic dereferencing is performed for the first argument of a routine call. +This feature has to be enabled via `{.experimental: "implicitDeref".}`: .. code-block:: nim - sort(cities) do (x,y: string) -> int: - cmp(x.len, y.len) + {.experimental: "implicitDeref".} - # Less parenthesis using the method plus command syntax: - cities = cities.map do (x:string) -> string: - "City of " & x + type + NodeObj = object + # ... + Node = ref NodeObj - # In macros, the do notation is often used for quasi-quoting - macroResults.add quote do: - if not `ex`: - echo `info`, ": Check failed: ", `expString` + proc depth(x: NodeObj): int = ... -`do` is written after the parentheses enclosing the regular proc params. -The proc expression represented by the do block is appended to them. -In calls using the command syntax, the do block will bind to the immediately -preceding expression, transforming it in a call. - -`do` with parentheses is an anonymous `proc`; however a `do` without -parentheses is just a block of code. The `do` notation can be used to -pass multiple blocks to a macro: - -.. code-block:: nim - - macro performWithUndo(task, undo: untyped) = ... - - performWithUndo do: - # multiple-line block of code - # to perform the task - do: - # code to undo it + let n = Node() + echo n.depth + # no need to write n[].depth Special Operators @@ -297,8 +306,8 @@ Special Operators dot operators ------------- -**Note**: Dot operators are still experimental and so need to be enabled -via `{.experimental: "dotOperators".}`. +.. note:: Dot operators are still experimental and so need to be enabled + via `{.experimental: "dotOperators".}`. Nim offers a special family of dot operators that can be used to intercept and rewrite proc call and field access attempts, referring @@ -418,6 +427,314 @@ here. .. include:: manual_experimental_strictnotnil.rst +Aliasing restrictions in parameter passing +========================================== + +.. note:: The aliasing restrictions are currently not enforced by the + implementation and need to be fleshed out further. + +"Aliasing" here means that the underlying storage locations overlap in memory +at runtime. An "output parameter" is a parameter of type `var T`, +an input parameter is any parameter that is not of type `var`. + +1. Two output parameters should never be aliased. +2. An input and an output parameter should not be aliased. +3. An output parameter should never be aliased with a global or thread local + variable referenced by the called proc. +4. An input parameter should not be aliased with a global or thread local + variable updated by the called proc. + +One problem with rules 3 and 4 is that they affect specific global or thread +local variables, but Nim's effect tracking only tracks "uses no global variable" +via `.noSideEffect`. The rules 3 and 4 can also be approximated by a different rule: + +5. A global or thread local variable (or a location derived from such a location) + can only passed to a parameter of a `.noSideEffect` proc. + + +Strict funcs +============ + +Since version 1.4, a stricter definition of "side effect" is available. +In addition to the existing rule that a side effect is calling a function +with side effects, the following rule is also enforced: + +Any mutation to an object does count as a side effect if that object is reachable +via a parameter that is not declared as a `var` parameter. + +For example: + +.. code-block:: nim + + {.experimental: "strictFuncs".} + + type + Node = ref object + le, ri: Node + data: string + + func len(n: Node): int = + # valid: len does not have side effects + var it = n + while it != nil: + inc result + it = it.ri + + func mut(n: Node) = + let m = n # is the statement that connected the mutation to the parameter + m.data = "yeah" # the mutation is here + # Error: 'mut' can have side effects + # an object reachable from 'n' is potentially mutated + + +The algorithm behind this analysis is described in +the `view types section <#view-types-algorithm>`_. + + +View types +========== + +.. tip:: `--experimental:views`:option: is more effective + with `--experimental:strictFuncs`:option:. + +A view type is a type that is or contains one of the following types: + +- `lent T` (view into `T`) +- `openArray[T]` (pair of (pointer to array of `T`, size)) + +For example: + +.. code-block:: nim + + type + View1 = openArray[byte] + View2 = lent string + View3 = Table[openArray[char], int] + + +Exceptions to this rule are types constructed via `ptr` or `proc`. +For example, the following types are **not** view types: + +.. code-block:: nim + + type + NotView1 = proc (x: openArray[int]) + NotView2 = ptr openArray[char] + NotView3 = ptr array[4, lent int] + + +The mutability aspect of a view type is not part of the type but part +of the locations it's derived from. More on this later. + +A *view* is a symbol (a let, var, const, etc.) that has a view type. + +Since version 1.4, Nim allows view types to be used as local variables. +This feature needs to be enabled via `{.experimental: "views".}`. + +A local variable of a view type *borrows* from the locations and +it is statically enforced that the view does not outlive the location +it was borrowed from. + +For example: + +.. code-block:: nim + + {.experimental: "views".} + + proc take(a: openArray[int]) = + echo a.len + + proc main(s: seq[int]) = + var x: openArray[int] = s # 'x' is a view into 's' + # it is checked that 'x' does not outlive 's' and + # that 's' is not mutated. + for i in 0 .. high(x): + echo x[i] + take(x) + + take(x.toOpenArray(0, 1)) # slicing remains possible + let y = x # create a view from a view + take y + # it is checked that 'y' does not outlive 'x' and + # that 'x' is not mutated as long as 'y' lives. + + + main(@[11, 22, 33]) + + +A local variable of a view type can borrow from a location +derived from a parameter, another local variable, a global `const` or `let` +symbol or a thread-local `var` or `let`. + +Let `p` the proc that is analysed for the correctness of the borrow operation. + +Let `source` be one of: + +- A formal parameter of `p`. Note that this does not cover parameters of + inner procs. +- The `result` symbol of `p`. +- A local `var` or `let` or `const` of `p`. Note that this does + not cover locals of inner procs. +- A thread-local `var` or `let`. +- A global `let` or `const`. +- A constant array/seq/object/tuple constructor. + + +Path expressions +---------------- + +A location derived from `source` is then defined as a path expression that +has `source` as the owner. A path expression `e` is defined recursively: + +- `source` itself is a path expression. +- Container access like `e[i]` is a path expression. +- Tuple access `e[0]` is a path expression. +- Object field access `e.field` is a path expression. +- `system.toOpenArray(e, ...)` is a path expression. +- Pointer dereference `e[]` is a path expression. +- An address `addr e`, `unsafeAddr e` is a path expression. +- A type conversion `T(e)` is a path expression. +- A cast expression `cast[T](e)` is a path expression. +- `f(e, ...)` is a path expression if `f`'s return type is a view type. + Because the view can only have been borrowed from `e`, we then know + that the owner of `f(e, ...)` is `e`. + + +If a view type is used as a return type, the location must borrow from a location +that is derived from the first parameter that is passed to the proc. +See `the manual `_ +for details about how this is done for `var T`. + +A mutable view can borrow from a mutable location, an immutable view can borrow +from both a mutable or an immutable location. + +If a view borrows from a mutable location, the view can be used to update the +location. Otherwise it cannot be used for mutations. + +The *duration* of a borrow is the span of commands beginning from the assignment +to the view and ending with the last usage of the view. + +For the duration of the borrow operation, no mutations to the borrowed locations +may be performed except via the view that borrowed from the +location. The borrowed location is said to be *sealed* during the borrow. + +.. code-block:: nim + + {.experimental: "views".} + + type + Obj = object + field: string + + proc dangerous(s: var seq[Obj]) = + let v: lent Obj = s[0] # seal 's' + s.setLen 0 # prevented at compile-time because 's' is sealed. + echo v.field + + +The scope of the view does not matter: + +.. code-block:: nim + + proc valid(s: var seq[Obj]) = + let v: lent Obj = s[0] # begin of borrow + echo v.field # end of borrow + s.setLen 0 # valid because 'v' isn't used afterwards + + +The analysis requires as much precision about mutations as is reasonably obtainable, +so it is more effective with the experimental `strict funcs <#strict-funcs>`_ +feature. In other words `--experimental:views`:option: works better +with `--experimental:strictFuncs`:option:. + +The analysis is currently control flow insensitive: + +.. code-block:: nim + + proc invalid(s: var seq[Obj]) = + let v: lent Obj = s[0] + if false: + s.setLen 0 + echo v.field + +In this example, the compiler assumes that `s.setLen 0` invalidates the +borrow operation of `v` even though a human being can easily see that it +will never do that at runtime. + + +Start of a borrow +----------------- + +A borrow starts with one of the following: + +- The assignment of a non-view-type to a view-type. +- The assignment of a location that is derived from a local parameter + to a view-type. + + +End of a borrow +--------------- + +A borrow operation ends with the last usage of the view variable. + + +Reborrows +--------- + +A view `v` can borrow from multiple different locations. However, the borrow +is always the full span of `v`'s lifetime and every location that is borrowed +from is sealed during `v`'s lifetime. + + +Algorithm +--------- + +The following section is an outline of the algorithm that the current implementation +uses. The algorithm performs two traversals over the AST of the procedure or global +section of code that uses a view variable. No fixpoint iterations are performed, the +complexity of the analysis is O(N) where N is the number of nodes of the AST. + +The first pass over the AST computes the lifetime of each local variable based on +a notion of an "abstract time", in the implementation it's a simple integer that is +incremented for every visited node. + +In the second pass, information about the underlying object "graphs" is computed. +Let `v` be a parameter or a local variable. Let `G(v)` be the graph +that `v` belongs to. A graph is defined by the set of variables that belong +to the graph. Initially for all `v`: `G(v) = {v}`. Every variable can only +be part of a single graph. + +Assignments like `a = b` "connect" two variables, both variables end up in the +same graph `{a, b} = G(a) = G(b)`. Unfortunately, the pattern to look for is +much more complex than that and can involve multiple assignment targets +and sources:: + + f(x, y) = g(a, b) + +connects `x` and `y` to `a` and `b`: `G(x) = G(y) = G(a) = G(b) = {x, y, a, b}`. +A type based alias analysis rules out some of these combinations, for example +a `string` value cannot possibly be connected to a `seq[int]`. + +A pattern like `v[] = value` or `v.field = value` marks `G(v)` as mutated. +After the second pass a set of disjoint graphs was computed. + +For strict functions it is then enforced that there is no graph that is both mutated +and has an element that is an immutable parameter (that is a parameter that is not +of type `var T`). + +For borrow checking, a different set of checks is performed. Let `v` be the view +and `b` the location that is borrowed from. + +- The lifetime of `v` must not exceed `b`'s lifetime. Note: The lifetime of + a parameter is the complete proc body. +- If `v` is used for a mutation, `b` must be a mutable location too. +- During `v`'s lifetime, `G(b)` can only be modified by `v` (and only if + `v` is a mutable view). +- If `v` is `result` then `b` has to be a location derived from the first + formal parameter or from a constant location. +- A view cannot be used for a read or a write access before it was assigned to. + + Concepts ======== @@ -441,10 +758,10 @@ Concepts are written in the following form: for value in s: value is T -The concept is a match if: +The concept matches if: -a) all of the expressions within the body can be compiled for the tested type -b) all statically evaluable boolean expressions in the body must be true +a) all expressions within the body can be compiled for the tested type +b) all statically evaluable boolean expressions in the body are true The identifiers following the `concept` keyword represent instances of the currently matched type. You can apply any of the standard type modifiers such @@ -874,15 +1191,17 @@ There are 4 operations that are bound to a type: 3. Destruction 4. Deep copying for communication between threads -These operations can be *overridden* instead of *overloaded*. This means the -implementation is automatically lifted to structured types. For instance if type -`T` has an overridden assignment operator `=` this operator is also used -for assignments of the type `seq[T]`. Since these operations are bound to a -type they have to be bound to a nominal type for reasons of simplicity of -implementation: This means an overridden `deepCopy` for `ref T` is really -bound to `T` and not to `ref T`. This also means that one cannot override -`deepCopy` for both `ptr T` and `ref T` at the same time; instead a -helper distinct or object type has to be used for one pointer type. +These operations can be *overridden* instead of *overloaded*. This means that +the implementation is automatically lifted to structured types. For instance, +if the type `T` has an overridden assignment operator `=`, this operator is +also used for assignments of the type `seq[T]`. + +Since these operations are bound to a type, they have to be bound to a +nominal type for reasons of simplicity of implementation; this means an +overridden `deepCopy` for `ref T` is really bound to `T` and not to `ref T`. +This also means that one cannot override `deepCopy` for both `ptr T` and +`ref T` at the same time, instead a distinct or object helper type has to be +used for one pointer type. Assignments, moves and destruction are specified in the `destructors `_ document. @@ -902,60 +1221,29 @@ The signature has to be: proc `=deepCopy`(x: T): T -This mechanism will be used by most data structures that support shared memory -like channels to implement thread safe automatic memory management. +This mechanism will be used by most data structures that support shared memory, +like channels, to implement thread safe automatic memory management. The builtin `deepCopy` can even clone closures and their environments. See the documentation of `spawn <#parallel-amp-spawn-spawn-statement>`_ for details. -Case statement macros -===================== +Dynamic arguments for bindSym +============================= -Macros named `case` can rewrite `case` statements for certain types in order to -implement `pattern matching`:idx:. The following example implements a -simplistic form of pattern matching for tuples, leveraging the existing -equality operator for tuples (as provided in `system.==`): +This experimental feature allows the symbol name argument of `macros.bindSym` +to be computed dynamically. .. code-block:: nim - :test: "nim c $1" + {.experimental: "dynamicBindSym".} - {.experimental: "caseStmtMacros".} + import macros - import std/macros + macro callOp(opName, arg1, arg2): untyped = + result = newCall(bindSym($opName), arg1, arg2) - macro `case`(n: tuple): untyped = - result = newTree(nnkIfStmt) - let selector = n[0] - for i in 1 ..< n.len: - let it = n[i] - case it.kind - of nnkElse, nnkElifBranch, nnkElifExpr, nnkElseExpr: - result.add it - of nnkOfBranch: - for j in 0..it.len-2: - let cond = newCall("==", selector, it[j]) - result.add newTree(nnkElifBranch, cond, it[^1]) - else: - error "custom 'case' for tuple cannot handle this node", it - - case ("foo", 78) - of ("foo", 78): echo "yes" - of ("bar", 88): echo "no" - else: discard - - -Currently case statement macros must be enabled explicitly -via `{.experimental: "caseStmtMacros".}`. - -`case` macros are subject to overload resolution. The type of the -`case` statement's selector expression is matched against the type -of the first argument of the `case` macro. Then the complete `case` -statement is passed in place of the argument and the macro is evaluated. - -In other words, the macro needs to transform the full `case` statement -but only the statement's selector expression is used to determine which -macro to call. + echo callOp("+", 1, 2) + echo callOp("-", 5, 4) Term rewriting macros @@ -968,30 +1256,30 @@ compilation pipeline with user defined optimizations: .. code-block:: nim - template optMul{`*`(a, 2)}(a: int): int = a+a + template optMul{`*`(a, 2)}(a: int): int = a + a let x = 3 echo x * 2 The compiler now rewrites `x * 2` as `x + x`. The code inside the -curlies is the pattern to match against. The operators `*`, `**`, +curly brackets is the pattern to match against. The operators `*`, `**`, `|`, `~` have a special meaning in patterns if they are written in infix notation, so to match verbatim against `*` the ordinary function call syntax needs to be used. -Term rewriting macro are applied recursively, up to a limit. This means that +Term rewriting macros are applied recursively, up to a limit. This means that if the result of a term rewriting macro is eligible for another rewriting, the compiler will try to perform it, and so on, until no more optimizations are applicable. To avoid putting the compiler into an infinite loop, there is a hard limit on how many times a single term rewriting macro can be applied. Once this limit has been passed, the term rewriting macro will be ignored. -Unfortunately optimizations are hard to get right and even the tiny example +Unfortunately optimizations are hard to get right and even this tiny example is **wrong**: .. code-block:: nim - template optMul{`*`(a, 2)}(a: int): int = a+a + template optMul{`*`(a, 2)}(a: int): int = a + a proc f(): int = echo "side effect!" @@ -1004,7 +1292,7 @@ Fortunately Nim supports side effect analysis: .. code-block:: nim - template optMul{`*`(a, 2)}(a: int{noSideEffect}): int = a+a + template optMul{`*`(a, 2)}(a: int{noSideEffect}): int = a + a proc f(): int = echo "side effect!" @@ -1022,13 +1310,13 @@ blindly: .. code-block:: nim - template mulIsCommutative{`*`(a, b)}(a, b: int): int = b*a + template mulIsCommutative{`*`(a, b)}(a, b: int): int = b * a What optimizers really need to do is a *canonicalization*: .. code-block:: nim - template canonMul{`*`(a, b)}(a: int{lit}, b: int): int = b*a + template canonMul{`*`(a, b)}(a: int{lit}, b: int): int = b * a The `int{lit}` parameter pattern matches against an expression of type `int`, but only if it's a literal. @@ -1143,7 +1431,7 @@ constant folding, so the following does not work: The reason is that the compiler already transformed the 1 into "1" for the `echo` statement. However, a term rewriting macro should not change the -semantics anyway. In fact they can be deactivated with the `--patterns:off`:option: +semantics anyway. In fact, they can be deactivated with the `--patterns:off`:option: command line option or temporarily with the `patterns` pragma. @@ -1155,7 +1443,7 @@ notation: .. code-block:: nim - template t{(0|1|2){x}}(x: untyped): untyped = x+1 + template t{(0|1|2){x}}(x: untyped): untyped = x + 1 let a = 1 # outputs 2: echo a @@ -1164,7 +1452,7 @@ notation: The `~` operator ~~~~~~~~~~~~~~~~~~ -The `~` operator is the **not** operator in patterns: +The `~` operator is the 'not' operator in patterns: .. code-block:: nim @@ -1257,14 +1545,14 @@ an `nnkArgList` node containing:: Sym "x" Sym "-" -(Which is the reverse polish notation of `x + y * z - x`.) +(This is the reverse polish notation of `x + y * z - x`.) Parameters ---------- Parameters in a pattern are type checked in the matching process. If a -parameter is of the type `varargs` it is treated specially and it can match +parameter is of the type `varargs`, it is treated specially and can match 0 or more arguments in the AST to be matched against: .. code-block:: nim @@ -1276,6 +1564,27 @@ parameter is of the type `varargs` it is treated specially and it can match w(f, x, y) +noRewrite pragma +---------------- + +Term rewriting macros and templates are currently greedy and +they will rewrite as long as there is a match. +There was no way to ensure some rewrite happens only once, +e.g. when rewriting term to same term plus extra content. + +`noRewrite` pragma can actually prevent further rewriting on marked code, +e.g. with given example `echo("ab")` will be rewritten just once: + +.. code-block:: nim + + template pwnEcho{echo(x)}(x: untyped) = + {.noRewrite.}: echo("pwned!") + + echo "ab" + +`noRewrite` pragma can be useful to control term-rewriting macros recursion. + + Example: Partial evaluation --------------------------- @@ -1319,7 +1628,7 @@ AST based overloading ===================== Parameter constraints can also be used for ordinary routine parameters; these -constraints affect ordinary overloading resolution then: +constraints then affect ordinary overloading resolution: .. code-block:: nim @@ -1356,22 +1665,25 @@ module to work. Somewhat confusingly, `spawn` is also used in the `parallel` statement with slightly different semantics. `spawn` always takes a call expression of -the form `f(a, ...)`. Let `T` be `f`'s return type. If `T` is `void` -then `spawn`'s return type is also `void` otherwise it is `FlowVar[T]`. +the form `f(a, ...)`. Let `T` be `f`'s return type. If `T` is `void`, +then `spawn`'s return type is also `void`, otherwise it is `FlowVar[T]`. -Within a `parallel` section sometimes the `FlowVar[T]` is eliminated +Within a `parallel` section, the `FlowVar[T]` is sometimes eliminated to `T`. This happens when `T` does not contain any GC'ed memory. The compiler can ensure the location in `location = spawn f(...)` is not read prematurely within a `parallel` section and so there is no need for the overhead of an indirection via `FlowVar[T]` to ensure correctness. -**Note**: Currently exceptions are not propagated between `spawn`'ed tasks! +.. note:: Currently exceptions are not propagated between `spawn`'ed tasks! + +This feature is likely to be removed in the future as external packages +can have better solutions. Spawn statement --------------- -`spawn`:idx: can be used to pass a task to the thread pool: +The `spawn`:idx: statement can be used to pass a task to the thread pool: .. code-block:: nim @@ -1393,10 +1705,10 @@ that `spawn` takes is restricted: * `f`'s parameters may not be of type `var`. This means one has to use raw `ptr`'s for data passing reminding the programmer to be careful. -* `ref` parameters are deeply copied which is a subtle semantic change and - can cause performance problems but ensures memory safety. This deep copy - is performed via `system.deepCopy` and so can be overridden. -* For *safe* data exchange between `f` and the caller a global `TChannel` +* `ref` parameters are deeply copied, which is a subtle semantic change and + can cause performance problems, but ensures memory safety. This deep copy + is performed via `system.deepCopy`, so it can be overridden. +* For *safe* data exchange between `f` and the caller, a global `Channel` needs to be used. However, since spawn can return a result, often no further communication is required. @@ -1420,10 +1732,10 @@ wait on multiple flow variables at the same time: responses.del(index) discard blockUntilAny(responses) -Data flow variables ensure that no data races -are possible. Due to technical limitations not every type `T` is possible in -a data flow variable: `T` has to be of the type `ref`, `string`, `seq` -or of a type that doesn't contain a type that is garbage collected. This +Data flow variables ensure that no data races are possible. Due to +technical limitations, not every type `T` can be used in +a data flow variable: `T` has to be a `ref`, `string`, `seq` +or of a type that doesn't contain any GC'd type. This restriction is not hard to work-around in practice. @@ -1436,14 +1748,14 @@ Example: .. code-block:: nim :test: "nim c --threads:on $1" - # Compute PI in an inefficient way + # Compute pi in an inefficient way import std/[strutils, math, threadpool] {.experimental: "parallel".} proc term(k: float): float = 4 * math.pow(-1, k) / (2*k + 1) proc pi(n: int): float = - var ch = newSeq[float](n+1) + var ch = newSeq[float](n + 1) parallel: for k in 0..ch.high: ch[k] = spawn term(float(k)) @@ -1454,16 +1766,16 @@ Example: The parallel statement is the preferred mechanism to introduce parallelism in a -Nim program. A subset of the Nim language is valid within a `parallel` +Nim program. Only a subset of the Nim language is valid within a `parallel` section. This subset is checked during semantic analysis to be free of data races. A sophisticated `disjoint checker`:idx: ensures that no data races are -possible even though shared memory is extensively supported! +possible, even though shared memory is extensively supported! The subset is in fact the full language with the following restrictions / changes: * `spawn` within a `parallel` section has special semantics. -* Every location of the form `a[i]` and `a[i..j]` and `dest` where +* Every location of the form `a[i]`, `a[i..j]` and `dest` where `dest` is part of the pattern `dest = spawn f(...)` has to be provably disjoint. This is called the *disjoint check*. * Every other complex location `loc` that is used in a spawned @@ -1477,152 +1789,16 @@ restrictions / changes: yet performed for ordinary slices outside of a `parallel` section. -Guards and locks -================ - -Apart from `spawn` and `parallel` Nim also provides all the common low level -concurrency mechanisms like locks, atomic intrinsics or condition variables. - -Nim significantly improves on the safety of these features via additional -pragmas: - -1) A `guard`:idx: annotation is introduced to prevent data races. -2) Every access of a guarded memory location needs to happen in an - appropriate `locks`:idx: statement. -3) Locks and routines can be annotated with `lock levels`:idx: to allow - potential deadlocks to be detected during semantic analysis. - - -Guards and the locks section ----------------------------- - -Protecting global variables -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Object fields and global variables can be annotated via a `guard` pragma: - -.. code-block:: nim - - var glock: TLock - var gdata {.guard: glock.}: int - -The compiler then ensures that every access of `gdata` is within a `locks` -section: - -.. code-block:: nim - - proc invalid = - # invalid: unguarded access: - echo gdata - - proc valid = - # valid access: - {.locks: [glock].}: - echo gdata - -Top level accesses to `gdata` are always allowed so that it can be initialized -conveniently. It is *assumed* (but not enforced) that every top level statement -is executed before any concurrent action happens. - -The `locks` section deliberately looks ugly because it has no runtime -semantics and should not be used directly! It should only be used in templates -that also implement some form of locking at runtime: - -.. code-block:: nim - - template lock(a: TLock; body: untyped) = - pthread_mutex_lock(a) - {.locks: [a].}: - try: - body - finally: - pthread_mutex_unlock(a) - - -The guard does not need to be of any particular type. It is flexible enough to -model low level lockfree mechanisms: - -.. code-block:: nim - - var dummyLock {.compileTime.}: int - var atomicCounter {.guard: dummyLock.}: int - - template atomicRead(x): untyped = - {.locks: [dummyLock].}: - memoryReadBarrier() - x - - echo atomicRead(atomicCounter) - - -The `locks` pragma takes a list of lock expressions `locks: [a, b, ...]` -in order to support *multi lock* statements. Why these are essential is -explained in the `lock levels <#guards-and-locks-lock-levels>`_ section. - - -Protecting general locations -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The `guard` annotation can also be used to protect fields within an object. -The guard then needs to be another field within the same object or a -global variable. - -Since objects can reside on the heap or on the stack this greatly enhances the -expressivity of the language: - -.. code-block:: nim - - type - ProtectedCounter = object - v {.guard: L.}: int - L: TLock - - proc incCounters(counters: var openArray[ProtectedCounter]) = - for i in 0..counters.high: - lock counters[i].L: - inc counters[i].v - -The access to field `x.v` is allowed since its guard `x.L` is active. -After template expansion, this amounts to: - -.. code-block:: nim - - proc incCounters(counters: var openArray[ProtectedCounter]) = - for i in 0..counters.high: - pthread_mutex_lock(counters[i].L) - {.locks: [counters[i].L].}: - try: - inc counters[i].v - finally: - pthread_mutex_unlock(counters[i].L) - -There is an analysis that checks that `counters[i].L` is the lock that -corresponds to the protected location `counters[i].v`. This analysis is called -`path analysis`:idx: because it deals with paths to locations -like `obj.field[i].fieldB[j]`. - -The path analysis is **currently unsound**, but that doesn't make it useless. -Two paths are considered equivalent if they are syntactically the same. - -This means the following compiles (for now) even though it really should not: - -.. code-block:: nim - - {.locks: [a[i].L].}: - inc i - access a[i].v - - Lock levels ------------ +=========== Lock levels are used to enforce a global locking order in order to detect potential deadlocks during semantic analysis. A lock level is an constant integer in the range 0..1_000. Lock level 0 means that no lock is acquired at all. -If a section of code holds a lock of level `M` than it can also acquire any +If a section of code holds a lock of level `M`, it can also acquire any lock of level `N < M`. Another lock of level `M` cannot be acquired. Locks of the same level can only be acquired *at the same time* within a single `locks` section: @@ -1685,7 +1861,7 @@ This is essential so that procs can be called within a `locks` section: p() -As usual `locks` is an inferred effect and there is a subtype +As usual, `locks` is an inferred effect and there is a subtype relation: `proc () {.locks: N.}` is a subtype of `proc () {.locks: M.}` iff (M <= N). @@ -1709,346 +1885,4 @@ having unknown lock level as well: if g.memberProc != nil: g.memberProc() - -noRewrite pragma ----------------- - -Term rewriting macros and templates are currently greedy and -they will rewrite as long as there is a match. -There was no way to ensure some rewrite happens only once, -e.g. when rewriting term to same term plus extra content. - -`noRewrite` pragma can actually prevent further rewriting on marked code, -e.g. with given example `echo("ab")` will be rewritten just once: - -.. code-block:: nim - - template pwnEcho{echo(x)}(x: untyped) = - {.noRewrite.}: echo("pwned!") - - echo "ab" - -`noRewrite` pragma can be useful to control term-rewriting macros recursion. - - -Aliasing restrictions in parameter passing -========================================== - -**Note**: The aliasing restrictions are currently not enforced by the -implementation and need to be fleshed out further. - -"Aliasing" here means that the underlying storage locations overlap in memory -at runtime. An "output parameter" is a parameter of type `var T`, -an input parameter is any parameter that is not of type `var`. - -1. Two output parameters should never be aliased. -2. An input and an output parameter should not be aliased. -3. An output parameter should never be aliased with a global or thread local - variable referenced by the called proc. -4. An input parameter should not be aliased with a global or thread local - variable updated by the called proc. - -One problem with rules 3 and 4 is that they affect specific global or thread -local variables, but Nim's effect tracking only tracks "uses no global variable" -via `.noSideEffect`. The rules 3 and 4 can also be approximated by a different rule: - -5. A global or thread local variable (or a location derived from such a location) - can only passed to a parameter of a `.noSideEffect` proc. - - -Noalias annotation -================== - -Since version 1.4 of the Nim compiler, there is a `.noalias` annotation for variables -and parameters. It is mapped directly to C/C++'s `restrict`:c: keyword and means that -the underlying pointer is pointing to a unique location in memory, no other aliases to -this location exist. It is *unchecked* that this alias restriction is followed, if the -restriction is violated, the backend optimizer is free to miscompile the code. -This is an **unsafe** language feature. - -Ideally in later versions of the language, the restriction will be enforced at -compile time. (Which is also why the name `noalias` was choosen instead of a more -verbose name like `unsafeAssumeNoAlias`.) - - -Strict funcs -============ - -Since version 1.4 a stricter definition of "side effect" is available. In addition -to the existing rule that a side effect is calling a function with side effects -the following rule is also enforced: - -Any mutation to an object does count as a side effect if that object is reachable -via a parameter that is not declared as a `var` parameter. - -For example: - -.. code-block:: nim - - {.experimental: "strictFuncs".} - - type - Node = ref object - le, ri: Node - data: string - - func len(n: Node): int = - # valid: len does not have side effects - var it = n - while it != nil: - inc result - it = it.ri - - func mut(n: Node) = - let m = n # is the statement that connected the mutation to the parameter - m.data = "yeah" # the mutation is here - # Error: 'mut' can have side effects - # an object reachable from 'n' is potentially mutated - - -The algorithm behind this analysis is described in -the `view types section <#view-types-algorithm>`_. - - -View types -========== - -**Note**: `--experimental:views`:option: is more effective -with `--experimental:strictFuncs`:option:. - -A view type is a type that is or contains one of the following types: - -- `lent T` (view into `T`) -- `openArray[T]` (pair of (pointer to array of `T`, size)) - -For example: - -.. code-block:: nim - - type - View1 = openArray[byte] - View2 = lent string - View3 = Table[openArray[char], int] - - -Exceptions to this rule are types constructed via `ptr` or `proc`. -For example, the following types are **not** view types: - -.. code-block:: nim - - type - NotView1 = proc (x: openArray[int]) - NotView2 = ptr openArray[char] - NotView3 = ptr array[4, lent int] - - -The mutability aspect of a view type is not part of the type but part -of the locations it's derived from. More on this later. - -A *view* is a symbol (a let, var, const, etc.) that has a view type. - -Since version 1.4 Nim allows view types to be used as local variables. -This feature needs to be enabled via `{.experimental: "views".}`. - -A local variable of a view type *borrows* from the locations and -it is statically enforced that the view does not outlive the location -it was borrowed from. - -For example: - -.. code-block:: nim - - {.experimental: "views".} - - proc take(a: openArray[int]) = - echo a.len - - proc main(s: seq[int]) = - var x: openArray[int] = s # 'x' is a view into 's' - # it is checked that 'x' does not outlive 's' and - # that 's' is not mutated. - for i in 0 .. high(x): - echo x[i] - take(x) - - take(x.toOpenArray(0, 1)) # slicing remains possible - let y = x # create a view from a view - take y - # it is checked that 'y' does not outlive 'x' and - # that 'x' is not mutated as long as 'y' lives. - - - main(@[11, 22, 33]) - - -A local variable of a view type can borrow from a location -derived from a parameter, another local variable, a global `const` or `let` -symbol or a thread-local `var` or `let`. - -Let `p` the proc that is analysed for the correctness of the borrow operation. - -Let `source` be one of: - -- A formal parameter of `p`. Note that this does not cover parameters of - inner procs. -- The `result` symbol of `p`. -- A local `var` or `let` or `const` of `p`. Note that this does - not cover locals of inner procs. -- A thread-local `var` or `let`. -- A global `let` or `const`. -- A constant array/seq/object/tuple constructor. - - -Path expressions ----------------- - -A location derived from `source` is then defined as a path expression that -has `source` as the owner. A path expression `e` is defined recursively: - -- `source` itself is a path expression. -- Container access like `e[i]` is a path expression. -- Tuple access `e[0]` is a path expression. -- Object field access `e.field` is a path expression. -- `system.toOpenArray(e, ...)` is a path expression. -- Pointer dereference `e[]` is a path expression. -- An address `addr e`, `unsafeAddr e` is a path expression. -- A type conversion `T(e)` is a path expression. -- A cast expression `cast[T](e)` is a path expression. -- `f(e, ...)` is a path expression if `f`'s return type is a view type. - Because the view can only have been borrowed from `e`, we then know - that owner of `f(e, ...)` is `e`. - - -If a view type is used as a return type, the location must borrow from a location -that is derived from the first parameter that is passed to the proc. -See https://nim-lang.org/docs/manual.html#procedures-var-return-type for -details about how this is done for `var T`. - -A mutable view can borrow from a mutable location, an immutable view can borrow -from both a mutable or an immutable location. - -If a view borrows from a mutable location, the view can be used to update the -location. Otherwise it cannot be used for mutations. - -The *duration* of a borrow is the span of commands beginning from the assignment -to the view and ending with the last usage of the view. - -For the duration of the borrow operation, no mutations to the borrowed locations -may be performed except via the view that borrowed from the -location. The borrowed location is said to be *sealed* during the borrow. - -.. code-block:: nim - - {.experimental: "views".} - - type - Obj = object - field: string - - proc dangerous(s: var seq[Obj]) = - let v: lent Obj = s[0] # seal 's' - s.setLen 0 # prevented at compile-time because 's' is sealed. - echo v.field - - -The scope of the view does not matter: - -.. code-block:: nim - - proc valid(s: var seq[Obj]) = - let v: lent Obj = s[0] # begin of borrow - echo v.field # end of borrow - s.setLen 0 # valid because 'v' isn't used afterwards - - -The analysis requires as much precision about mutations as is reasonably obtainable, -so it is more effective with the experimental `strict funcs <#strict-funcs>`_ -feature. In other words `--experimental:views`:option: works better -with `--experimental:strictFuncs`:option:. - -The analysis is currently control flow insensitive: - -.. code-block:: nim - - proc invalid(s: var seq[Obj]) = - let v: lent Obj = s[0] - if false: - s.setLen 0 - echo v.field - -In this example, the compiler assumes that `s.setLen 0` invalidates the -borrow operation of `v` even though a human being can easily see that it -will never do that at runtime. - - -Start of a borrow ------------------ - -A borrow starts with one of the following: - -- The assignment of a non-view-type to a view-type. -- The assignment of a location that is derived from a local parameter - to a view-type. - - -End of a borrow ---------------- - -A borrow operation ends with the last usage of the view variable. - - -Reborrows ---------- - -A view `v` can borrow from multiple different locations. However, the borrow -is always the full span of `v`'s lifetime and every location that is borrowed -from is sealed during `v`'s lifetime. - - -Algorithm ---------- - -The following section is an outline of the algorithm that the current implementation -uses. The algorithm performs two traversals over the AST of the procedure or global -section of code that uses a view variable. No fixpoint iterations are performed, the -complexity of the analysis is O(N) where N is the number of nodes of the AST. - -The first pass over the AST computes the lifetime of each local variable based on -a notion of an "abstract time", in the implementation it's a simple integer that is -incremented for every visited node. - -In the second pass information about the underlying object "graphs" is computed. -Let `v` be a parameter or a local variable. Let `G(v)` be the graph -that `v` belongs to. A graph is defined by the set of variables that belong -to the graph. Initially for all `v`: `G(v) = {v}`. Every variable can only -be part of a single graph. - -Assignments like `a = b` "connect" two variables, both variables end up in the -same graph `{a, b} = G(a) = G(b)`. Unfortunately, the pattern to look for is -much more complex than that and can involve multiple assignment targets -and sources:: - - f(x, y) = g(a, b) - -connects `x` and `y` to `a` and `b`: `G(x) = G(y) = G(a) = G(b) = {x, y, a, b}`. -A type based alias analysis rules out some of these combinations, for example -a `string` value cannot possibly be connected to a `seq[int]`. - -A pattern like `v[] = value` or `v.field = value` marks `G(v)` as mutated. -After the second pass a set of disjoint graphs was computed. - -For strict functions it is then enforced that there is no graph that is both mutated -and has an element that is an immutable parameter (that is a parameter that is not -of type `var T`). - -For borrow checking a different set of checks is performed. Let `v` be the view -and `b` the location that is borrowed from. - -- The lifetime of `v` must not exceed `b`'s lifetime. Note: The lifetime of - a parameter is the complete proc body. -- If `v` is used for a mutation, `b` must be a mutable location too. -- During `v`'s lifetime, `G(b)` can only be modified by `v` (and only if - `v` is a mutable view). -- If `v` is `result` then `b` has to be a location derived from the first - formal parameter or from a constant location. -- A view cannot be used for a read or a write access before it was assigned to. +This feature may be removed in the future due to its practical difficulties. diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 6dbb92875..c7d56c0ed 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -465,12 +465,7 @@ proc bindSym*(ident: string | NimNode, rule: BindSymRule = brClosed): NimNode {. ## If `rule == brForceOpen` always an `nnkOpenSymChoice` tree is ## returned even if the symbol is not ambiguous. ## - ## Experimental feature: - ## use {.experimental: "dynamicBindSym".} to activate it. - ## If called from template / regular code, `ident` and `rule` must be - ## constant expression / literal value. - ## If called from macros / compile time procs / static blocks, - ## `ident` and `rule` can be VM computed value. + ## See the `manual `_ for more details. proc genSym*(kind: NimSymKind = nskLet; ident = ""): NimNode {. magic: "NGenSym", noSideEffect.} diff --git a/tests/macros/tcasestmtmacro.nim b/tests/macros/tcasestmtmacro.nim index 26519f637..32019a92a 100644 --- a/tests/macros/tcasestmtmacro.nim +++ b/tests/macros/tcasestmtmacro.nim @@ -4,8 +4,6 @@ yes ''' """ -{.experimental: "caseStmtMacros".} - import macros macro `case`(n: tuple): untyped = From 2859069dbe1c943494cb6be299b200afa748cf52 Mon Sep 17 00:00:00 2001 From: hlaaftana <10591326+hlaaftana@users.noreply.github.com> Date: Wed, 24 Nov 2021 09:08:07 +0300 Subject: [PATCH 1372/2472] fix #12274 (#19180) --- compiler/parser.nim | 2 +- tests/parser/t12274.nim | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 tests/parser/t12274.nim diff --git a/compiler/parser.nim b/compiler/parser.nim index aedf62d54..0ef6d1f54 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -1146,7 +1146,7 @@ proc isExprStart(p: Parser): bool = of tkSymbol, tkAccent, tkOpr, tkNot, tkNil, tkCast, tkIf, tkFor, tkProc, tkFunc, tkIterator, tkBind, tkBuiltInMagics, tkParLe, tkBracketLe, tkCurlyLe, tkIntLit..tkCustomLit, tkVar, tkRef, tkPtr, - tkTuple, tkObject, tkWhen, tkCase, tkOut: + tkTuple, tkObject, tkWhen, tkCase, tkOut, tkTry, tkBlock: result = true else: result = false diff --git a/tests/parser/t12274.nim b/tests/parser/t12274.nim new file mode 100644 index 000000000..40c85f158 --- /dev/null +++ b/tests/parser/t12274.nim @@ -0,0 +1,9 @@ +var s: seq[int] +s.add block: + let i = 1 + i +s.add try: + 2 +except: + 3 +doAssert s == @[1, 2] From ff39f6e26062651a56d6ac4c24fe5a799c177c50 Mon Sep 17 00:00:00 2001 From: hlaaftana <10591326+hlaaftana@users.noreply.github.com> Date: Wed, 24 Nov 2021 10:20:15 +0300 Subject: [PATCH 1373/2472] make JS trunc polyfill opt-in, closes #16144 (#19183) --- changelog.md | 8 +++++++- lib/system/jssys.nim | 6 +++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/changelog.md b/changelog.md index b645bc5c7..f7f92ad3c 100644 --- a/changelog.md +++ b/changelog.md @@ -3,7 +3,13 @@ ## Changes affecting backward compatibility - +- The `Math.trunc` polyfill for targeting Internet Explorer was + previously emitted for every JavaScript output file except if + the symbol `nodejs` was defined via `-d:nodejs`. Now, it is only + emitted if the symbol `nimJsMathTruncPolyfill` is defined. If you are + targeting Internet Explorer, you may choose to enable this option + or define your own `Math.trunc` polyfill using the [`emit` pragma](https://nim-lang.org/docs/manual.html#implementation-specific-pragmas-emit-pragma). Nim uses + `Math.trunc` for the division and modulo operators for integers. ## Standard library additions and changes diff --git a/lib/system/jssys.nim b/lib/system/jssys.nim index cd4f378be..250bd069d 100644 --- a/lib/system/jssys.nim +++ b/lib/system/jssys.nim @@ -763,7 +763,8 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat, start: int): int # Workaround for IE, IE up to version 11 lacks 'Math.trunc'. We produce # 'Math.trunc' for Nim's ``div`` and ``mod`` operators: -const jsMathTrunc = """ +when defined(nimJsMathTruncPolyfill): + {.emit: """ if (!Math.trunc) { Math.trunc = function(v) { v = +v; @@ -771,5 +772,4 @@ if (!Math.trunc) { return (v - v % 1) || (v < 0 ? -0 : v === 0 ? v : 0); }; } -""" -when not defined(nodejs): {.emit: jsMathTrunc .} +""".} From c7c6b13a325958c6b55cbec2eb812eb06074101b Mon Sep 17 00:00:00 2001 From: hlaaftana <10591326+hlaaftana@users.noreply.github.com> Date: Wed, 24 Nov 2021 14:22:40 +0300 Subject: [PATCH 1374/2472] parseExpr/parseStmt accept filename, fixes #13540 (#19182) --- changelog.md | 3 +++ compiler/vm.nim | 9 ++++----- compiler/vmgen.nim | 4 ++-- lib/core/macros.nim | 14 ++++++++------ tests/macros/mparsefile.nim | 4 ++++ tests/macros/tparsefile.nim | 11 +++++++++++ 6 files changed, 32 insertions(+), 13 deletions(-) create mode 100644 tests/macros/mparsefile.nim create mode 100644 tests/macros/tparsefile.nim diff --git a/changelog.md b/changelog.md index f7f92ad3c..63c3eff40 100644 --- a/changelog.md +++ b/changelog.md @@ -13,6 +13,9 @@ ## Standard library additions and changes +- `macros.parseExpr` and `macros.parseStmt` now accept an optional + filename argument for more informative errors. + ## `std/smtp` - Sends `ehlo` first. If the mail server does not understand, it sends `helo` as a fallback. diff --git a/compiler/vm.nim b/compiler/vm.nim index b64545081..d58a49f9e 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -1748,11 +1748,10 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = elif instr.opcode == opcNHint: message(c.config, info, hintUser, a.strVal) of opcParseExprToAst: - decodeB(rkNode) - # c.debug[pc].line.int - countLines(regs[rb].strVal) ? + decodeBC(rkNode) var error: string let ast = parseString(regs[rb].node.strVal, c.cache, c.config, - toFullPath(c.config, c.debug[pc]), c.debug[pc].line.int, + regs[rc].node.strVal, 0, proc (conf: ConfigRef; info: TLineInfo; msg: TMsgKind; arg: string) {.nosinks.} = if error.len == 0 and msg <= errMax: error = formatMsg(conf, info, msg, arg)) @@ -1764,10 +1763,10 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = else: regs[ra].node = ast[0] of opcParseStmtToAst: - decodeB(rkNode) + decodeBC(rkNode) var error: string let ast = parseString(regs[rb].node.strVal, c.cache, c.config, - toFullPath(c.config, c.debug[pc]), c.debug[pc].line.int, + regs[rc].node.strVal, 0, proc (conf: ConfigRef; info: TLineInfo; msg: TMsgKind; arg: string) {.nosinks.} = if error.len == 0 and msg <= errMax: error = formatMsg(conf, info, msg, arg)) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index bb095d3ee..9a823dff4 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1225,9 +1225,9 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = unused(c, n, dest) genBinaryStmtVar(c, n, opcAddSeqElem) of mParseExprToAst: - genUnaryABC(c, n, dest, opcParseExprToAst) + genBinaryABC(c, n, dest, opcParseExprToAst) of mParseStmtToAst: - genUnaryABC(c, n, dest, opcParseStmtToAst) + genBinaryABC(c, n, dest, opcParseStmtToAst) of mTypeTrait: let tmp = c.genx(n[1]) if dest < 0: dest = c.getTemp(n.typ) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index c7d56c0ed..444783b5a 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -511,27 +511,29 @@ proc lineInfo*(arg: NimNode): string = ## Return line info in the form `filepath(line, column)`. $arg.lineInfoObj -proc internalParseExpr(s: string): NimNode {. +proc internalParseExpr(s, filename: string): NimNode {. magic: "ParseExprToAst", noSideEffect.} -proc internalParseStmt(s: string): NimNode {. +proc internalParseStmt(s, filename: string): NimNode {. magic: "ParseStmtToAst", noSideEffect.} proc internalErrorFlag*(): string {.magic: "NError", noSideEffect.} ## Some builtins set an error flag. This is then turned into a proper ## exception. **Note**: Ordinary application code should not call this. -proc parseExpr*(s: string): NimNode {.noSideEffect.} = +proc parseExpr*(s: string; filename: string = ""): NimNode {.noSideEffect.} = ## Compiles the passed string to its AST representation. ## Expects a single expression. Raises `ValueError` for parsing errors. - result = internalParseExpr(s) + ## A filename can be given for more informative errors. + result = internalParseExpr(s, filename) let x = internalErrorFlag() if x.len > 0: raise newException(ValueError, x) -proc parseStmt*(s: string): NimNode {.noSideEffect.} = +proc parseStmt*(s: string; filename: string = ""): NimNode {.noSideEffect.} = ## Compiles the passed string to its AST representation. ## Expects one or more statements. Raises `ValueError` for parsing errors. - result = internalParseStmt(s) + ## A filename can be given for more informative errors. + result = internalParseStmt(s, filename) let x = internalErrorFlag() if x.len > 0: raise newException(ValueError, x) diff --git a/tests/macros/mparsefile.nim b/tests/macros/mparsefile.nim new file mode 100644 index 000000000..8ac99d568 --- /dev/null +++ b/tests/macros/mparsefile.nim @@ -0,0 +1,4 @@ +let a = 1 +let b = 2 +let c = +let d = 4 diff --git a/tests/macros/tparsefile.nim b/tests/macros/tparsefile.nim new file mode 100644 index 000000000..a41223f80 --- /dev/null +++ b/tests/macros/tparsefile.nim @@ -0,0 +1,11 @@ +import macros + +static: + let fn = "mparsefile.nim" + var raised = false + try: + discard parseStmt(staticRead(fn), filename = fn) + except ValueError as e: + raised = true + doAssert e.msg == "mparsefile.nim(4, 1) Error: invalid indentation" + doAssert raised From a0073d2d4c18f030eccef98a130f7f1f2ad9d67a Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 24 Nov 2021 15:49:32 +0100 Subject: [PATCH 1375/2472] renamed 'gc' switch to 'mm'; [backport:1.6] (#19187) * renamed 'gc' switch to 'mm'; [backport:1.6] * better docs --- changelog.md | 5 +- compiler/commands.nim | 4 +- doc/advopt.txt | 7 +-- doc/backends.rst | 38 +------------- doc/destructors.rst | 12 ++--- doc/docs.rst | 4 +- doc/mm.rst | 95 ++++++++++++++++++++++++++++++++++ doc/nimc.rst | 22 ++++---- doc/{gc.rst => refc.rst} | 109 +++++++++++---------------------------- koch.nim | 2 +- tools/kochdocs.nim | 2 +- 11 files changed, 157 insertions(+), 143 deletions(-) create mode 100644 doc/mm.rst rename doc/{gc.rst => refc.rst} (62%) diff --git a/changelog.md b/changelog.md index 63c3eff40..258cf73e4 100644 --- a/changelog.md +++ b/changelog.md @@ -53,7 +53,7 @@ Baz = object ``` - [Case statement macros](manual.html#macros-case-statement-macros) are no longer experimental, - meaning you no longer need to enable the experimental switch `caseStmtMacros` to use them. + meaning you no longer need to enable the experimental switch `caseStmtMacros` to use them. ## Compiler changes @@ -63,5 +63,6 @@ ## Tool changes - +- The `gc` switch has been renamed to `mm` ("memory management") in order to reflect the + reality better. (Nim moved away from all techniques based on "tracing".) diff --git a/compiler/commands.nim b/compiler/commands.nim index a8caad916..d5c5f24e4 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -248,7 +248,7 @@ template deprecatedAlias(oldName, newName: string) = proc testCompileOptionArg*(conf: ConfigRef; switch, arg: string, info: TLineInfo): bool = case switch.normalize - of "gc": + of "gc", "mm": case arg.normalize of "boehm": result = conf.selectedGC == gcBoehm of "refc": result = conf.selectedGC == gcRefc @@ -596,7 +596,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; processOnOffSwitchG(conf, {optForceFullMake}, arg, pass, info) of "project": processOnOffSwitchG(conf, {optWholeProject, optGenIndex}, arg, pass, info) - of "gc": + of "gc", "mm": if conf.backend == backendJs: return # for: bug #16033 expectArg(conf, switch, arg, pass, info) if pass in {passCmd2, passPP}: diff --git a/doc/advopt.txt b/doc/advopt.txt index 063d018d1..79e784fde 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -122,8 +122,9 @@ Advanced options: --skipUserCfg:on|off do not read the user's configuration file --skipParentCfg:on|off do not read the parent dirs' configuration files --skipProjCfg:on|off do not read the project's configuration file - --gc:refc|arc|orc|markAndSweep|boehm|go|none|regions - select the GC to use; default is 'refc' + --mm:orc|arc|refc|markAndSweep|boehm|go|none|regions + select which memory management to use; default is 'refc' + recommended is 'orc' --exceptions:setjmp|cpp|goto|quirky select the exception handling implementation --index:on|off turn index file generation on|off @@ -163,4 +164,4 @@ Advanced options: --profileVM:on|off turn compile time VM profiler on|off --sinkInference:on|off turn sink parameter inference on|off (default: on) --panics:on|off turn panics into process terminations (default: off) - --deepcopy:on|off enable 'system.deepCopy' for ``--gc:arc|orc`` + --deepcopy:on|off enable 'system.deepCopy' for ``--mm:arc|orc`` diff --git a/doc/backends.rst b/doc/backends.rst index 377e899b0..d55503d10 100644 --- a/doc/backends.rst +++ b/doc/backends.rst @@ -105,7 +105,7 @@ file. However, you can also run the code with `nodejs`:idx: If you experience errors saying that `globalThis` is not defined, be sure to run a recent version of Node.js (at least 12.0). - + Interfacing =========== @@ -387,14 +387,8 @@ A similar thing happens with C code invoking Nim code which returns a proc gimme(): cstring {.exportc.} = result = "Hey there C code! " & $rand(100) -Since Nim's garbage collector is not aware of the C code, once the +Since Nim's reference counting mechanism is not aware of the C code, once the `gimme` proc has finished it can reclaim the memory of the `cstring`. -However, from a practical standpoint, the C code invoking the `gimme` -function directly will be able to use it since Nim's garbage collector has -not had a chance to run *yet*. This gives you enough time to make a copy for -the C side of the program, as calling any further Nim procs *might* trigger -garbage collection making the previously returned string garbage. Or maybe you -are `yourself triggering the collection `_. Custom data types @@ -414,31 +408,3 @@ you can clean it up. And of course, once cleaned you should avoid accessing it from Nim (or C for that matter). Typically C data structures have their own `malloc_structure`:c: and `free_structure`:c: specific functions, so wrapping these for the Nim side should be enough. - - -Thread coordination -------------------- - -When the `NimMain()` function is called Nim initializes the garbage -collector to the current thread, which is usually the main thread of your -application. If your C code later spawns a different thread and calls Nim -code, the garbage collector will fail to work properly and you will crash. - -As long as you don't use the threadvar emulation Nim uses native thread -variables, of which you get a fresh version whenever you create a thread. You -can then attach a GC to this thread via - -.. code-block:: nim - - system.setupForeignThreadGc() - -It is **not** safe to disable the garbage collector and enable it after the -call from your background thread even if the code you are calling is short -lived. - -Before the thread exits, you should tear down the thread's GC to prevent memory -leaks by calling - -.. code-block:: nim - - system.tearDownForeignThreadGc() diff --git a/doc/destructors.rst b/doc/destructors.rst index 83a50230b..c98e94536 100644 --- a/doc/destructors.rst +++ b/doc/destructors.rst @@ -43,7 +43,7 @@ written as: dealloc(x.data) proc `=trace`[T](x: var myseq[T]; env: pointer) = - # `=trace` allows the cycle collector `--gc:orc` + # `=trace` allows the cycle collector `--mm:orc` # to understand how to trace the object graph. if x.data != nil: for i in 0..`_ | Description of some tools that come with the standard distribution. -- | `GC `_ - | Additional documentation about Nim's multi-paradigm memory management strategies +- | `Memory management `_ + | Additional documentation about Nim's memory management strategies | and how to operate them in a realtime setting. - | `Source code filters `_ diff --git a/doc/mm.rst b/doc/mm.rst new file mode 100644 index 000000000..b6941a901 --- /dev/null +++ b/doc/mm.rst @@ -0,0 +1,95 @@ +======================= +Nim's Memory Management +======================= + +.. default-role:: code +.. include:: rstcommon.rst + +:Author: Andreas Rumpf +:Version: |nimversion| + +.. + + + "The road to hell is paved with good intentions." + + +Multi-paradigm Memory Management Strategies +=========================================== + +.. default-role:: option + +Nim offers multiple different memory management strategies. +To choose the memory management strategy use the `--mm:` switch. + +**The recommended switch for newly written Nim code is `--mm:orc`.** + + +ARC/ORC +------- + +`--mm:orc` is a memory management mode primarily based on reference counting. Cycles +in the object graph are handled by a "cycle collector" which is based on "trial deletion". +Since algorithms based on "tracing" are not used, the runtime behavior is oblivious to +the involved heap sizes. + +The reference counting operations (= "RC ops") do not use atomic instructions and do not have to -- +instead entire subgraphs are *moved* between threads. The Nim compiler also aggressively +optimizes away RC ops and exploits `move semantics `_. + +Nim performs a fair share of optimizations for ARC/ORC; you can inspect what it did +to your time critical function via `--expandArc:functionName`. + +`--mm:arc` uses the same mechanism as `--mm:orc`, but it leaves out the cycle collector. +Both ARC and ORC offer deterministic performance for `hard realtime`:idx: systems, but +ARC can be easier to reason about for people coming from Ada/C++/C -- roughly speaking +the memory for a variable is freed when it goes "out of scope". + +We generally advise you to use the `acyclic` annotation in order to optimize away the +cycle collector's overhead +but `--mm:orc` also produces more machine code than `--mm:arc`, so if you're on a target +where code size matters and you know that your code does not produce cycles, you can +use `--mm:arc`. Notice that the default `async`:idx: implementation produces cycles +and leaks memory with `--mm:arc`, in other words, for `async` you need to use `--mm:orc`. + + + +Other MM modes +-------------- + +.. note:: The default `refc` GC is incremental, thread-local and not "stop-the-world". + +--mm:refc This is the default memory management strategy. It's a + deferred reference counting based garbage collector + with a simple Mark&Sweep backup GC in order to collect cycles. Heaps are thread-local. + `This document `_ contains further information. +--mm:markAndSweep Simple Mark-And-Sweep based garbage collector. + Heaps are thread-local. +--mm:boehm Boehm based garbage collector, it offers a shared heap. +--mm:go Go's garbage collector, useful for interoperability with Go. + Offers a shared heap. + +--mm:none No memory management strategy nor a garbage collector. Allocated memory is + simply never freed. You should use `--mm:arc` instead. + +Here is a comparison of the different memory management modes: + +================== ======== ================= ============== =================== +Memory Management Heap Reference Cycles Stop-The-World Command line switch +================== ======== ================= ============== =================== +ORC Shared Cycle Collector No `--mm:orc` +ARC Shared Leak No `--mm:arc` +RefC Local Cycle Collector No `--mm:refc` +Mark & Sweep Local Cycle Collector No `--mm:markAndSweep` +Boehm Shared Cycle Collector Yes `--mm:boehm` +Go Shared Cycle Collector Yes `--mm:go` +None Manual Manual Manual `--mm:none` +================== ======== ================= ============== =================== + +.. default-role:: code +.. include:: rstcommon.rst + +JavaScript's garbage collector is used for the `JavaScript and NodeJS +`_ compilation targets. +The `NimScript `_ target uses the memory management strategy built into +the Nim compiler. diff --git a/doc/nimc.rst b/doc/nimc.rst index 6be4c204e..eb066e748 100644 --- a/doc/nimc.rst +++ b/doc/nimc.rst @@ -408,13 +408,13 @@ to your usual `nim c`:cmd: or `nim cpp`:cmd: command and set the `passC`:option: and `passL`:option: command line switches to something like: .. code-block:: cmd - nim c ... --d:nimAllocPagesViaMalloc --gc:orc --passC="-I$DEVKITPRO/libnx/include" ... + nim c ... --d:nimAllocPagesViaMalloc --mm:orc --passC="-I$DEVKITPRO/libnx/include" ... --passL="-specs=$DEVKITPRO/libnx/switch.specs -L$DEVKITPRO/libnx/lib -lnx" or setup a ``nim.cfg`` file like so:: #nim.cfg - --gc:orc + --mm:orc --d:nimAllocPagesViaMalloc --passC="-I$DEVKITPRO/libnx/include" --passL="-specs=$DEVKITPRO/libnx/switch.specs -L$DEVKITPRO/libnx/lib -lnx" @@ -485,10 +485,10 @@ Define Effect `useMalloc` Makes Nim use C's `malloc`:idx: instead of Nim's own memory manager, albeit prefixing each allocation with its size to support clearing memory on reallocation. - This only works with `--gc:none`:option:, - `--gc:arc`:option: and `--gc:orc`:option:. + This only works with `--mm:none`:option:, + `--mm:arc`:option: and `--mm:orc`:option:. `useRealtimeGC` Enables support of Nim's GC for *soft* realtime - systems. See the documentation of the `gc `_ + systems. See the documentation of the `mm `_ for further information. `logGC` Enable GC logging to stdout. `nodejs` The JS target is actually ``node.js``. @@ -614,9 +614,9 @@ A good start is to use the `any` operating target together with the .. code:: cmd - nim c --os:any --gc:arc -d:useMalloc [...] x.nim + nim c --os:any --mm:arc -d:useMalloc [...] x.nim -- `--gc:arc`:option: will enable the reference counting memory management instead +- `--mm:arc`:option: will enable the reference counting memory management instead of the default garbage collector. This enables Nim to use heap memory which is required for strings and seqs, for example. @@ -654,7 +654,7 @@ devices. This allocator gets blocks/pages of memory via a currently undocumented `osalloc` API which usually uses POSIX's `mmap` call. On many environments `mmap` is not available but C's `malloc` is. You can use the `nimAllocPagesViaMalloc` define to use `malloc` instead of `mmap`. `nimAllocPagesViaMalloc` is currently -only supported with `--gc:arc` or `--gc:orc`. (Since version 1.6) +only supported with `--mm:arc` or `--mm:orc`. (Since version 1.6) nimPage256 / nimPage512 / nimPage1k =================================== @@ -681,19 +681,19 @@ nimMemAlignTiny Sets `MemAlign` to `4` bytes which reduces the memory alignment to better match some embedded devices. -Thread stack size +Thread stack size ================= Nim's thread API provides a simple wrapper around more advanced RTOS task features. Customizing the stack size and stack guard size can be done by setting `-d:nimThreadStackSize=16384` or `-d:nimThreadStackGuard=32`. -Currently only Zephyr and FreeRTOS support these configurations. +Currently only Zephyr and FreeRTOS support these configurations. Nim for realtime systems ======================== -See the documentation of Nim's soft realtime `GC `_ for further +See the `--mm:arc` or `--mm:orc` memory management settings in `MM `_ for further information. diff --git a/doc/gc.rst b/doc/refc.rst similarity index 62% rename from doc/gc.rst rename to doc/refc.rst index 5de02f73d..766097f23 100644 --- a/doc/gc.rst +++ b/doc/refc.rst @@ -1,81 +1,3 @@ -======================= -Nim's Memory Management -======================= - -.. default-role:: code -.. include:: rstcommon.rst - -:Author: Andreas Rumpf -:Version: |nimversion| - -.. - - - "The road to hell is paved with good intentions." - - -Introduction -============ - -A memory-management algorithm optimal for every use-case cannot exist. -Nim provides multiple paradigms for needs ranging from large multi-threaded -applications, to games, hard-realtime systems and small microcontrollers. - -This document describes how the management strategies work; -How to tune the garbage collectors for your needs, like (soft) `realtime systems`:idx:, -and how the memory management strategies other than garbage collectors work. - -.. note:: the default GC is incremental, thread-local and not "stop-the-world" - -Multi-paradigm Memory Management Strategies -=========================================== - -.. default-role:: option - -To choose the memory management strategy use the `--gc:` switch. - ---gc:refc This is the default GC. It's a - deferred reference counting based garbage collector - with a simple Mark&Sweep backup GC in order to collect cycles. Heaps are thread-local. ---gc:markAndSweep Simple Mark-And-Sweep based garbage collector. - Heaps are thread-local. ---gc:boehm Boehm based garbage collector, it offers a shared heap. ---gc:go Go's garbage collector, useful for interoperability with Go. - Offers a shared heap. ---gc:arc Plain reference counting with - `move semantic optimizations `_, offers a shared heap. - It offers deterministic performance for `hard realtime`:idx: systems. Reference cycles - cause memory leaks, beware. - ---gc:orc Same as `--gc:arc` but adds a cycle collector based on "trial deletion". - Unfortunately, that makes its performance profile hard to reason about so it is less - useful for hard real-time systems. - ---gc:none No memory management strategy nor a garbage collector. Allocated memory is - simply never freed. You should use `--gc:arc` instead. - - -================== ======== ================= ============== =================== -Memory Management Heap Reference Cycles Stop-The-World Command line switch -================== ======== ================= ============== =================== -RefC Local Cycle Collector No `--gc:refc` -Mark & Sweep Local Cycle Collector No `--gc:markAndSweep` -ARC Shared Leak No `--gc:arc` -ORC Shared Cycle Collector No `--gc:orc` -Boehm Shared Cycle Collector Yes `--gc:boehm` -Go Shared Cycle Collector Yes `--gc:go` -None Manual Manual Manual `--gc:none` -================== ======== ================= ============== =================== - -.. default-role:: code -.. include:: rstcommon.rst - -JavaScript's garbage collector is used for the `JavaScript and NodeJS -`_ compilation targets. -The `NimScript `_ target uses the memory management strategy built into -the Nim compiler. - - Tweaking the refc GC ==================== @@ -164,6 +86,35 @@ that up to 100 objects are traversed and freed before it checks again. Thus highly specialized environments or for older hardware. +Thread coordination +------------------- + +When the `NimMain()` function is called Nim initializes the garbage +collector to the current thread, which is usually the main thread of your +application. If your C code later spawns a different thread and calls Nim +code, the garbage collector will fail to work properly and you will crash. + +As long as you don't use the threadvar emulation Nim uses native thread +variables, of which you get a fresh version whenever you create a thread. You +can then attach a GC to this thread via + +.. code-block:: nim + + system.setupForeignThreadGc() + +It is **not** safe to disable the garbage collector and enable it after the +call from your background thread even if the code you are calling is short +lived. + +Before the thread exits, you should tear down the thread's GC to prevent memory +leaks by calling + +.. code-block:: nim + + system.tearDownForeignThreadGc() + + + Keeping track of memory ======================= @@ -178,7 +129,7 @@ Other useful procs from `system `_ you can use to keep track of mem * `GC_getStatistics()` Garbage collector statistics as a human-readable string. These numbers are usually only for the running thread, not for the whole heap, -with the exception of `--gc:boehm`:option: and `--gc:go`:option:. +with the exception of `--mm:boehm`:option: and `--mm:go`:option:. In addition to `GC_ref` and `GC_unref` you can avoid the garbage collector by manually allocating memory with procs like `alloc`, `alloc0`, `allocShared`, `allocShared0` or `allocCStringArray`. diff --git a/koch.nim b/koch.nim index f87828cdc..295b1584b 100644 --- a/koch.nim +++ b/koch.nim @@ -604,7 +604,7 @@ proc runCI(cmd: string) = when not defined(bsd): # the BSDs are overwhelmed already, so only run this test on the other machines: - kochExecFold("Boot Nim ORC", "boot -d:release --gc:orc --lib:lib") + kochExecFold("Boot Nim ORC", "boot -d:release --mm:orc --lib:lib") proc testUnixInstall(cmdLineRest: string) = csource("-d:danger" & cmdLineRest) diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 70ee5ad07..4fbea8c77 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -129,7 +129,7 @@ tut2.rst tut3.rst nimc.rst niminst.rst -gc.rst +mm.rst """.splitWhitespace().mapIt("doc" / it) doc0 = """ From a59ad2006291eafcd6790f3011b5cd355b219a5b Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Wed, 24 Nov 2021 18:16:20 +0300 Subject: [PATCH 1376/2472] fix inline syntax highlighting in system.nim (#19184) --- lib/system_overview.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/system_overview.rst b/lib/system_overview.rst index df22697e0..0e4ffaf17 100644 --- a/lib/system_overview.rst +++ b/lib/system_overview.rst @@ -1,4 +1,5 @@ .. default-role:: code +.. include:: ../doc/rstcommon.rst The System module imports several separate modules, and their documentation is in separate files: From f91867aa31f3ae2c4f28d24547d62603daaa6d0c Mon Sep 17 00:00:00 2001 From: hlaaftana <10591326+hlaaftana@users.noreply.github.com> Date: Wed, 24 Nov 2021 18:34:42 +0300 Subject: [PATCH 1377/2472] accept object type node from macros (#19179) --- compiler/semexprs.nim | 2 +- tests/macros/ttypenodes.nim | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 tests/macros/ttypenodes.nim diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index c11cecfa9..0df332689 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -2856,7 +2856,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = of nkBind: message(c.config, n.info, warnDeprecated, "bind is deprecated") result = semExpr(c, n[0], flags) - of nkTypeOfExpr, nkTupleTy, nkTupleClassTy, nkRefTy..nkEnumTy, nkStaticTy: + of nkTypeOfExpr..nkTupleClassTy, nkStaticTy, nkRefTy..nkEnumTy: if c.matchedConcept != nil and n.len == 1: let modifier = n.modifierTypeKindOfNode if modifier != tyNone: diff --git a/tests/macros/ttypenodes.nim b/tests/macros/ttypenodes.nim new file mode 100644 index 000000000..233ea9780 --- /dev/null +++ b/tests/macros/ttypenodes.nim @@ -0,0 +1,16 @@ +import macros + +macro makeEnum(): untyped = + newTree(nnkEnumTy, newEmptyNode(), ident"a", ident"b", ident"c") + +macro makeObject(): untyped = + newTree(nnkObjectTy, newEmptyNode(), newEmptyNode(), newTree(nnkRecList, + newTree(nnkIdentDefs, ident"x", ident"y", ident"int", newEmptyNode()))) + +type + Foo = makeEnum() + Bar = makeObject() + +doAssert {a, b, c} is set[Foo] +let bar = Bar(x: 3, y: 4) +doAssert (bar.x, bar.y) == (3, 4) From 0d0c249074d6a1041de16108dc247396efef5513 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 24 Nov 2021 08:37:14 -0800 Subject: [PATCH 1378/2472] swap port to correct port order (#19177) Co-authored-by: Jaremy Creechley --- lib/pure/net.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 7b0ff78e7..06024e46d 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -478,14 +478,14 @@ proc toSockAddr*(address: IpAddress, port: Port, sa: var Sockaddr_storage, sl = sizeof(Sockaddr_in).SockLen let s = cast[ptr Sockaddr_in](addr sa) s.sin_family = typeof(s.sin_family)(toInt(AF_INET)) - s.sin_port = port + s.sin_port = htons(port) copyMem(addr s.sin_addr, unsafeAddr address.address_v4[0], sizeof(s.sin_addr)) of IpAddressFamily.IPv6: sl = sizeof(Sockaddr_in6).SockLen let s = cast[ptr Sockaddr_in6](addr sa) s.sin6_family = typeof(s.sin6_family)(toInt(AF_INET6)) - s.sin6_port = port + s.sin6_port = htons(port) copyMem(addr s.sin6_addr, unsafeAddr address.address_v6[0], sizeof(s.sin6_addr)) From 373c909300c9ecef159666ac196e360fbd7a3ebc Mon Sep 17 00:00:00 2001 From: Iced Quinn <70673392+IcedQuinn@users.noreply.github.com> Date: Sat, 4 Dec 2021 00:42:03 -0600 Subject: [PATCH 1379/2472] feat: TLS-ALPN wrappers for OpenSSL (#19202) Co-authored-by: Iced Quinn --- lib/wrappers/openssl.nim | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/wrappers/openssl.nim b/lib/wrappers/openssl.nim index c756fa938..00ec160cb 100644 --- a/lib/wrappers/openssl.nim +++ b/lib/wrappers/openssl.nim @@ -847,3 +847,17 @@ when not defined(nimDisableCertificateValidation) and not defined(windows): let cert = d2i_X509(certbytes) let encoded = cert.i2d_X509() assert encoded == certbytes + +# Application Layer Protocol Negociation extension (TLS-ALPN, RFC7301) +# Available in at least OpenSSL 1.1.1 and later, not sure if earlier +# --Iced Quinn + +proc SSL_CTX_set_alpn_protos*(ctx: SslCtx; protos: cstring; protos_len: cuint): cint {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_set_alpn_protos*(ssl: SslPtr; protos: cstring; protos_len: cuint): cint {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_CTX_set_alpn_select_cb*(ctx: SslCtx; cb: proc(ssl: SslPtr; out_proto: ptr cstring; outlen: cstring; in_proto: cstring; inlen: cuint; arg: pointer): cint {.cdecl.}; arg: pointer): cint {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_get0_alpn_selected*(ssl: SslPtr; data: ptr cstring; len: ptr cuint) {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_CTX_set_next_protos_advertised_cb*(ctx: SslCtx; cb: proc(ssl: SslPtr; out_proto: ptr cstring; outlen: ptr cuint; arg: pointer): cint {.cdecl.}; arg: pointer) {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_CTX_set_next_proto_select_cb*(ctx: SslCtx; cb: proc(s: SslPtr; out_proto: cstring; outlen: cstring; in_proto: cstring; inlen: cuint; arg: pointer): cint {.cdecl.}; arg: pointer) {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_select_next_proto*(out_proto: ptr cstring; outlen: cstring; server: cstring; server_len: cuint; client: cstring; client_len: cuint): cint {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_get0_next_proto_negotiated*(s: SslPtr; data: ptr cstring; len: ptr cuint) {.cdecl, dynlib: DLLSSLName, importc.} + From 23c117a950a3ce0e3c85d4f89409075329648230 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 4 Dec 2021 07:42:34 +0100 Subject: [PATCH 1380/2472] misc bugfixes [backport:1.2] (#19203) --- compiler/semobjconstr.nim | 6 +++--- doc/backends.rst | 23 +++++++++-------------- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/compiler/semobjconstr.nim b/compiler/semobjconstr.nim index 30ab76957..6d11d321e 100644 --- a/compiler/semobjconstr.nim +++ b/compiler/semobjconstr.nim @@ -366,9 +366,9 @@ proc defaultConstructionError(c: PContext, t: PType, info: TLineInfo) = if objType.kind == tyObject: var constrCtx = initConstrContext(objType, newNodeI(nkObjConstr, info)) let initResult = semConstructTypeAux(c, constrCtx, {}) - assert constrCtx.missingFields.len > 0 - localError(c.config, info, - "The $1 type doesn't have a default value. The following fields must be initialized: $2." % [typeToString(t), listSymbolNames(constrCtx.missingFields)]) + if constrCtx.missingFields.len > 0: + localError(c.config, info, + "The $1 type doesn't have a default value. The following fields must be initialized: $2." % [typeToString(t), listSymbolNames(constrCtx.missingFields)]) elif objType.kind == tyDistinct: localError(c.config, info, "The $1 distinct type doesn't have a default value." % typeToString(t)) diff --git a/doc/backends.rst b/doc/backends.rst index d55503d10..3a3359fca 100644 --- a/doc/backends.rst +++ b/doc/backends.rst @@ -52,7 +52,7 @@ The commands to compile to either C, C++ or Objective-C are: The most significant difference between these commands is that if you look into the ``nimcache`` directory you will find ``.c``, ``.cpp`` or ``.m`` files, other than that all of them will produce a native binary for your -project. This allows you to take the generated code and place it directly +project. This allows you to take the generated code and place it directly into a project using any of these languages. Here are some typical command- line invocations: @@ -123,7 +123,7 @@ Nim code can interface with the backend through the `Foreign function interface `_ mainly through the `importc pragma `_. The `importc` pragma is the *generic* way of making backend symbols available -in Nim and is available in all the target backends (JavaScript too). The C++ +in Nim and is available in all the target backends (JavaScript too). The C++ or Objective-C backends have their respective `ImportCpp `_ and `ImportObjC `_ @@ -246,11 +246,6 @@ Also, C code requires you to specify a forward declaration for functions or the compiler will assume certain types for the return value and parameters which will likely make your program crash at runtime. -The Nim compiler can generate a C interface header through the `--header`:option: -command-line switch. The generated header will contain all the exported -symbols and the `NimMain` proc which you need to call before any other -Nim code. - Nim invocation example from C ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -269,9 +264,10 @@ Create a ``maths.c`` file with the following content: .. code-block:: c - #include "fib.h" #include + extern int fib(int a); + int main(void) { NimMain(); @@ -286,13 +282,12 @@ program: .. code:: cmd - nim c --noMain --noLinking --header:fib.h fib.nim + nim c --noMain --noLinking fib.nim gcc -o m -I$HOME/.cache/nim/fib_d -Ipath/to/nim/lib $HOME/.cache/nim/fib_d/*.c maths.c The first command runs the Nim compiler with three special options to avoid -generating a `main()`:c: function in the generated files, avoid linking the -object files into a final binary, and explicitly generate a header file for C -integration. All the generated files are placed into the ``nimcache`` +generating a `main()`:c: function in the generated files and to avoid linking the +object files into a final binary. All the generated files are placed into the ``nimcache`` directory. That's why the next command compiles the ``maths.c`` source plus all the ``.c`` files from ``nimcache``. In addition to this path, you also have to tell the C compiler where to find Nim's ``nimbase.h`` header file. @@ -302,12 +297,12 @@ also ask the Nim compiler to generate a statically linked library: .. code:: cmd - nim c --app:staticLib --noMain --header fib.nim + nim c --app:staticLib --noMain fib.nim gcc -o m -Inimcache -Ipath/to/nim/lib libfib.nim.a maths.c The Nim compiler will handle linking the source files generated in the ``nimcache`` directory into the ``libfib.nim.a`` static library, which you can -then link into your C program. Note that these commands are generic and will +then link into your C program. Note that these commands are generic and will vary for each system. For instance, on Linux systems you will likely need to use `-ldl`:option: too to link in required dlopen functionality. From d584dd5b99f4c1e32477db2dfb791d52b0139df1 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 4 Dec 2021 07:42:58 +0100 Subject: [PATCH 1381/2472] fixes #19015 [backport:1.6] (#19204) --- changelog.md | 3 +++ compiler/options.nim | 3 ++- compiler/sigmatch.nim | 3 ++- tests/misc/trfc405.nim | 3 +++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/changelog.md b/changelog.md index 258cf73e4..a8f1920e9 100644 --- a/changelog.md +++ b/changelog.md @@ -11,6 +11,9 @@ or define your own `Math.trunc` polyfill using the [`emit` pragma](https://nim-lang.org/docs/manual.html#implementation-specific-pragmas-emit-pragma). Nim uses `Math.trunc` for the division and modulo operators for integers. +- Optional parameters in combination with `: body` syntax (RFC #405) are now opt-in via + `experimental:flexibleOptionalParams`. + ## Standard library additions and changes - `macros.parseExpr` and `macros.parseStmt` now accept an optional diff --git a/compiler/options.nim b/compiler/options.nim index d8e5057f9..383b6fa2a 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -207,7 +207,8 @@ type strictNotNil, overloadableEnums, strictEffects, - unicodeOperators + unicodeOperators, + flexibleOptionalParams LegacyFeature* = enum allowSemcheckedAstModification, diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 6bc6eefe0..93be92676 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -2461,7 +2461,8 @@ proc matchesAux(c: PContext, n, nOrig: PNode, m: var TCandidate, marker: var Int if m.callee.n[f].kind != nkSym: internalError(c.config, n[a].info, "matches") noMatch() - if a >= firstArgBlock: f = max(f, m.callee.n.len - (n.len - a)) + if flexibleOptionalParams in c.features and a >= firstArgBlock: + f = max(f, m.callee.n.len - (n.len - a)) formal = m.callee.n[f].sym m.firstMismatch.kind = kTypeMismatch if containsOrIncl(marker, formal.position) and container.isNil: diff --git a/tests/misc/trfc405.nim b/tests/misc/trfc405.nim index adc39b1df..8c967eb77 100644 --- a/tests/misc/trfc405.nim +++ b/tests/misc/trfc405.nim @@ -1,3 +1,6 @@ + +{.experimental: "flexibleOptionalParams".} + # https://github.com/nim-lang/RFCs/issues/405 template main = From f90620fb325d5b5b924774ce84384cb4098ed18e Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 4 Dec 2021 07:43:20 +0100 Subject: [PATCH 1382/2472] fixes #19198 [backport:1.6] (#19209) * fixes #19198 [backport:1.6] * added a test case --- compiler/vm.nim | 6 ++++-- compiler/vmgen.nim | 28 ++++++++++++++++------------ tests/vm/tvmmisc.nim | 23 +++++++++++++++++++++-- 3 files changed, 41 insertions(+), 16 deletions(-) diff --git a/compiler/vm.nim b/compiler/vm.nim index d58a49f9e..3343eb781 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -434,8 +434,10 @@ proc opConv(c: PCtx; dest: var TFullReg, src: TFullReg, desttyp, srctyp: PType): of tyFloat..tyFloat64: dest.intVal = int(src.floatVal) else: - let srcDist = (sizeof(src.intVal) - styp.size) * 8 - let destDist = (sizeof(dest.intVal) - desttyp.size) * 8 + let srcSize = getSize(c.config, styp) + let destSize = getSize(c.config, desttyp) + let srcDist = (sizeof(src.intVal) - srcSize) * 8 + let destDist = (sizeof(dest.intVal) - destSize) * 8 var value = cast[BiggestUInt](src.intVal) value = (value shl srcDist) shr srcDist value = (value shl destDist) shr destDist diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 9a823dff4..c0c4bac19 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -749,18 +749,20 @@ proc genNarrow(c: PCtx; n: PNode; dest: TDest) = let t = skipTypes(n.typ, abstractVar-{tyTypeDesc}) # uint is uint64 in the VM, we we only need to mask the result for # other unsigned types: - if t.kind in {tyUInt8..tyUInt32} or (t.kind == tyUInt and t.size < 8): - c.gABC(n, opcNarrowU, dest, TRegister(t.size*8)) - elif t.kind in {tyInt8..tyInt32} or (t.kind == tyInt and t.size < 8): - c.gABC(n, opcNarrowS, dest, TRegister(t.size*8)) + let size = getSize(c.config, t) + if t.kind in {tyUInt8..tyUInt32} or (t.kind == tyUInt and size < 8): + c.gABC(n, opcNarrowU, dest, TRegister(size*8)) + elif t.kind in {tyInt8..tyInt32} or (t.kind == tyInt and size < 8): + c.gABC(n, opcNarrowS, dest, TRegister(size*8)) proc genNarrowU(c: PCtx; n: PNode; dest: TDest) = let t = skipTypes(n.typ, abstractVar-{tyTypeDesc}) # uint is uint64 in the VM, we we only need to mask the result for # other unsigned types: + let size = getSize(c.config, t) if t.kind in {tyUInt8..tyUInt32, tyInt8..tyInt32} or - (t.kind in {tyUInt, tyInt} and t.size < 8): - c.gABC(n, opcNarrowU, dest, TRegister(t.size*8)) + (t.kind in {tyUInt, tyInt} and size < 8): + c.gABC(n, opcNarrowU, dest, TRegister(size*8)) proc genBinaryABCnarrow(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode) = genBinaryABC(c, n, dest, opc) @@ -1088,10 +1090,11 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = genBinaryABC(c, n, dest, opcShlInt) # genNarrowU modified let t = skipTypes(n.typ, abstractVar-{tyTypeDesc}) - if t.kind in {tyUInt8..tyUInt32} or (t.kind == tyUInt and t.size < 8): - c.gABC(n, opcNarrowU, dest, TRegister(t.size*8)) - elif t.kind in {tyInt8..tyInt32} or (t.kind == tyInt and t.size < 8): - c.gABC(n, opcSignExtend, dest, TRegister(t.size*8)) + let size = getSize(c.config, t) + if t.kind in {tyUInt8..tyUInt32} or (t.kind == tyUInt and size < 8): + c.gABC(n, opcNarrowU, dest, TRegister(size*8)) + elif t.kind in {tyInt8..tyInt32} or (t.kind == tyInt and size < 8): + c.gABC(n, opcSignExtend, dest, TRegister(size*8)) of mAshrI: genBinaryABC(c, n, dest, opcAshrInt) of mBitandI: genBinaryABC(c, n, dest, opcBitandInt) of mBitorI: genBinaryABC(c, n, dest, opcBitorInt) @@ -1125,8 +1128,9 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = genUnaryABC(c, n, dest, opcBitnotInt) #genNarrowU modified, do not narrow signed types let t = skipTypes(n.typ, abstractVar-{tyTypeDesc}) - if t.kind in {tyUInt8..tyUInt32} or (t.kind == tyUInt and t.size < 8): - c.gABC(n, opcNarrowU, dest, TRegister(t.size*8)) + let size = getSize(c.config, t) + if t.kind in {tyUInt8..tyUInt32} or (t.kind == tyUInt and size < 8): + c.gABC(n, opcNarrowU, dest, TRegister(size*8)) of mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mFloatToStr, mCStrToStr, mStrToStr, mEnumToStr: genConv(c, n, n[1], dest) of mEqStr, mEqCString: genBinaryABC(c, n, dest, opcEqStr) diff --git a/tests/vm/tvmmisc.nim b/tests/vm/tvmmisc.nim index f6fbde0f9..dfe086d10 100644 --- a/tests/vm/tvmmisc.nim +++ b/tests/vm/tvmmisc.nim @@ -290,10 +290,10 @@ block: # bug #10815 const a = P() doAssert $a == "" - + when defined osx: # xxx bug https://github.com/nim-lang/Nim/issues/10815#issuecomment-476380734 block: - type CharSet {.union.} = object + type CharSet {.union.} = object cs: set[char] vs: array[4, uint64] const a = Charset(cs: {'a'..'z'}) @@ -553,3 +553,22 @@ block: # bug #8015 doAssert $viaProc.table[0] == "(kind: Fixed, cost: 999)" doAssert viaProc.table[1].handler() == 100 doAssert viaProc.table[2].handler() == 200 + + +# bug #19198 + +block: + type + Foo[n: static int] = int + +block: + static: + let x = int 1 + echo x.type # Foo + +block: + static: + let x = int 1 + let y = x + 1 + # Error: unhandled exception: value out of range: -8 notin 0 .. 65535 [RangeDefect] + echo y From 1cbdc1573af5db8bf555b0b6454424188c9f345c Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 4 Dec 2021 07:44:26 +0100 Subject: [PATCH 1383/2472] fixes #19159 [backport:1.6] (#19210) --- compiler/sempass2.nim | 7 ++++--- tests/effects/tstrict_effects3.nim | 12 ++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index 5624d7adb..d6cc4180b 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -852,13 +852,14 @@ proc trackCall(tracked: PEffects; n: PNode) = assumeTheWorst(tracked, n, op) gcsafeAndSideeffectCheck() else: - if strictEffects in tracked.c.features and a.kind == nkSym and a.sym.kind in routineKinds: + if strictEffects in tracked.c.features and a.kind == nkSym and + a.sym.kind in routineKinds: propagateEffects(tracked, n, a.sym) else: mergeRaises(tracked, effectList[exceptionEffects], n) mergeTags(tracked, effectList[tagEffects], n) gcsafeAndSideeffectCheck() - if a.kind != nkSym or a.sym.magic notin {mNBindSym, mFinished}: + if a.kind != nkSym or a.sym.magic notin {mNBindSym, mFinished, mExpandToAst, mQuoteAst}: for i in 1.. Date: Sat, 4 Dec 2021 07:54:11 +0100 Subject: [PATCH 1384/2472] Improve documentation around func and method (#19207) * Improve documentation around func and method * Update doc/tut1.rst Co-authored-by: Danil Yarantsev * Update doc/tut1.rst Co-authored-by: Danil Yarantsev * Update doc/tut1.rst Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Update doc/tut1.rst Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Update doc/tut1.rst Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Update doc/tut1.rst Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Update doc/tut1.rst Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Update doc/tut1.rst Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Update doc/tut1.rst Co-authored-by: Zoom * Update doc/tut1.rst Co-authored-by: Zoom * Update doc/tut1.rst Co-authored-by: Zoom * Update doc/tut1.rst Co-authored-by: Zoom * Rewrite of Zooms suggestion * Update doc/tut1.rst Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Co-authored-by: Danil Yarantsev Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Co-authored-by: Zoom --- doc/tut1.rst | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/doc/tut1.rst b/doc/tut1.rst index f492f12b0..2856625b7 100644 --- a/doc/tut1.rst +++ b/doc/tut1.rst @@ -595,8 +595,10 @@ Procedures To define new commands like `echo `_ and `readLine `_ in the examples, the concept of a -*procedure* is needed. (Some languages call them *methods* or *functions*.) -In Nim new procedures are defined with the `proc` keyword: +*procedure* is needed. You might be used to them being called *methods* or +*functions* in other languages, but Nim +`differentiates these concepts `_. In +Nim, new procedures are defined with the `proc` keyword: .. code-block:: nim :test: "nim c $1" @@ -874,6 +876,31 @@ The example also shows that a proc's body can consist of a single expression whose value is then returned implicitly. +Funcs and methods +----------------- + +As mentioned in the introduction, Nim differentiates between procedures, +functions, and methods, defined by the `proc`, `func`, and `method` keywords +respectively. In some ways, Nim is a bit more pedantic in its definitions than +other languages. + +Functions are closer to the concept of a pure mathematical +function, which might be familiar to you if you've ever done functional +programming. Essentially they are procedures with additional limitations set on +them: they can't access global state (except `const`) and can't produce +side-effects. The `func` keyword is basically an alias for `proc` tagged +with `{.noSideEffects.}`. Functions can still change their mutable arguments +however, which are those marked as `var`, along with any `ref` objects. + +Unlike procedures, methods are dynamically dispatched. This sounds a bit +complicated, but it is a concept closely related to inheritance and object oriented +programming. If you overload a procedure (two procedures with the same name but +of different types or with different sets of arguments are said to be overloaded), the procedure to use is determined +at compile-time. Methods, on the other hand, depend on objects that inherit from +the `RootObj`. This is something that is covered in much greater depth in +the `second part of the tutorial`_. + + Iterators ========= From 1ef945668d458163cb7116a8c936ff92b63d679d Mon Sep 17 00:00:00 2001 From: hlaaftana <10591326+hlaaftana@users.noreply.github.com> Date: Sun, 5 Dec 2021 08:44:22 +0300 Subject: [PATCH 1385/2472] treat do with pragmas but no parens as proc (#19191) fixes #19188 --- compiler/parser.nim | 12 +++++++++--- doc/manual.rst | 8 ++++---- tests/parser/tdo.nim | 19 +++++++++++++++++-- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/compiler/parser.nim b/compiler/parser.nim index 0ef6d1f54..9ff847ef6 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -1110,11 +1110,14 @@ proc optPragmas(p: var Parser): PNode = proc parseDoBlock(p: var Parser; info: TLineInfo): PNode = #| doBlock = 'do' paramListArrow pragma? colcom stmt - let params = parseParamList(p, retColon=false) + var params = parseParamList(p, retColon=false) let pragmas = optPragmas(p) colcom(p, result) result = parseStmt(p) - if params.kind != nkEmpty: + if params.kind != nkEmpty or pragmas.kind != nkEmpty: + if params.kind == nkEmpty: + params = newNodeP(nkFormalParams, p) + params.add(p.emptyNode) # return type result = newProcNode(nkDo, info, body = result, params = params, name = p.emptyNode, pattern = p.emptyNode, genericParams = p.emptyNode, pragmas = pragmas, exceptions = p.emptyNode) @@ -1381,7 +1384,10 @@ proc postExprBlocks(p: var Parser, x: PNode): PNode = if stmtList[0].kind == nkStmtList: stmtList = stmtList[0] stmtList.flags.incl nfBlockArg - if openingParams.kind != nkEmpty: + if openingParams.kind != nkEmpty or openingPragmas.kind != nkEmpty: + if openingParams.kind == nkEmpty: + openingParams = newNodeP(nkFormalParams, p) + openingParams.add(p.emptyNode) # return type result.add newProcNode(nkDo, stmtList.info, body = stmtList, params = openingParams, name = p.emptyNode, pattern = p.emptyNode, diff --git a/doc/manual.rst b/doc/manual.rst index 4fcc516ec..6705e673e 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -3834,10 +3834,10 @@ The proc expression represented by the `do` block is appended to the routine call as the last argument. In calls using the command syntax, the `do` block will bind to the immediately preceding expression rather than the command call. -`do` with a parameter list corresponds to an anonymous `proc`, however -`do` without parameters is treated as a normal statement list. This allows -macros to receive both indented statement lists as an argument in inline -calls, as well as a direct mirror of Nim's routine syntax. +`do` with a parameter list or pragma list corresponds to an anonymous `proc`, +however `do` without parameters or pragmas is treated as a normal statement +list. This allows macros to receive both indented statement lists as an +argument in inline calls, as well as a direct mirror of Nim's routine syntax. .. code-block:: nim # Passing a statement list to an inline macro: diff --git a/tests/parser/tdo.nim b/tests/parser/tdo.nim index 7bd1f7411..382d03398 100644 --- a/tests/parser/tdo.nim +++ b/tests/parser/tdo.nim @@ -1,8 +1,12 @@ discard """ - output: '''true + output: ''' true true -true inner B''' +true +true inner B +running with pragma +ran with pragma +''' """ template withValue(a, b, c, d, e: untyped) = @@ -77,3 +81,14 @@ proc main2 = echo "true inner B" main2() + +proc withPragma(foo: int, bar: proc() {.raises: [].}) = + echo "running with pragma" + bar() + +withPragma(3) do {.raises: [].}: + echo "ran with pragma" + +doAssert not (compiles do: + withPragma(3) do {.raises: [].}: + raise newException(Exception)) From e5475768ba5b540489fb47bdb6fdd338d75175e1 Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 5 Dec 2021 14:17:52 +0800 Subject: [PATCH 1386/2472] feature_request should be labelled automatically (#19215) --- .github/ISSUE_TEMPLATE/feature_request.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index b9f7caad5..6a9afe071 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -2,7 +2,7 @@ name: Feature request about: Do you want to suggest a new feature? Use this template. title: '' -labels: '' +labels: ["Feature"] assignees: '' --- From faacd63bf69561fe915f3651e6b917bb2c0e5bc2 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 6 Dec 2021 17:04:54 +0800 Subject: [PATCH 1387/2472] the title of PR should contain a brief description (#19211) In addition to an issue number, the title of a PR should contain a brief description; then users can follow the progress more easily from the title. --- doc/contributing.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/contributing.rst b/doc/contributing.rst index 07fa01714..d3b2770f3 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -10,6 +10,9 @@ Contributing Contributing happens via "Pull requests" (PR) on github. Every PR needs to be reviewed before it can be merged and the Continuous Integration should be green. +The title of a PR should contain a brief description. If it fixes an issue, +in addition to the number of the issue, the title should also contain a description +of the issue. The PR has to be approved by two core developers or by Araq. From 7e3da693f5bd8879d91ccadc469a3ec80d77a34f Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 7 Dec 2021 01:58:01 +0800 Subject: [PATCH 1388/2472] [format minor] remove unnecessary spaces (#19216) --- lib/system/atomics.nim | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/system/atomics.nim b/lib/system/atomics.nim index f69cb753d..75731764b 100644 --- a/lib/system/atomics.nim +++ b/lib/system/atomics.nim @@ -73,7 +73,7 @@ when someGcc and hasThreadSupport: proc atomicCompareExchangeN*[T: AtomType](p, expected: ptr T, desired: T, weak: bool, success_memmodel: AtomMemModel, failure_memmodel: AtomMemModel): bool {. - importc: "__atomic_compare_exchange_n ", nodecl.} + importc: "__atomic_compare_exchange_n", nodecl.} ## This proc implements an atomic compare and exchange operation. This compares the ## contents at p with the contents at expected and if equal, writes desired at p. ## If they are not equal, the current contents at p is written into expected. @@ -100,13 +100,13 @@ when someGcc and hasThreadSupport: proc atomicSubFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. importc: "__atomic_sub_fetch", nodecl.} proc atomicOrFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. - importc: "__atomic_or_fetch ", nodecl.} + importc: "__atomic_or_fetch", nodecl.} proc atomicAndFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. importc: "__atomic_and_fetch", nodecl.} proc atomicXorFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. importc: "__atomic_xor_fetch", nodecl.} proc atomicNandFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. - importc: "__atomic_nand_fetch ", nodecl.} + importc: "__atomic_nand_fetch", nodecl.} ## Perform the operation return the old value, all memory models are valid proc atomicFetchAdd*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. @@ -177,8 +177,8 @@ elif someVcc and hasThreadSupport: proc `==`(x1, x2: AtomMemModel): bool {.borrow.} - proc readBarrier() {.importc: "_ReadBarrier", header: "".} - proc writeBarrier() {.importc: "_WriteBarrier", header: "".} + proc readBarrier() {.importc: "_ReadBarrier", header: "".} + proc writeBarrier() {.importc: "_WriteBarrier", header: "".} proc fence*() {.importc: "_ReadWriteBarrier", header: "".} template barrier(mem: AtomMemModel) = From 0213c7313b2edf8e8ec9d81764685744e874ef84 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Mon, 6 Dec 2021 20:38:23 +0100 Subject: [PATCH 1389/2472] allow `HSlice` bounded by constants of distinct types (#19219) [backport:1.2] When creating heterogenous slices of distinct types, the compiler does not initialize the internal type's `size` before accessing it. This then leads to this crash message: ``` compiler/int128.nim(594, 11) `false` masking only implemented for 1, 2, 4 and 8 bytes [AssertionError] ``` This patch initializes the `size` properly, fixing the problem. --- compiler/semfold.nim | 16 ++++++++-------- tests/slice/tdistinct.nim | 2 ++ 2 files changed, 10 insertions(+), 8 deletions(-) create mode 100644 tests/slice/tdistinct.nim diff --git a/compiler/semfold.nim b/compiler/semfold.nim index 5ae91d306..d0473b5ce 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -133,7 +133,7 @@ proc evalOp(m: TMagic, n, a, b, c: PNode; idgen: IdGenerator; g: ModuleGraph): P of mCard: result = newIntNodeT(toInt128(nimsets.cardSet(g.config, a)), n, idgen, g) of mBitnotI: if n.typ.isUnsigned: - result = newIntNodeT(bitnot(getInt(a)).maskBytes(int(n.typ.size)), n, idgen, g) + result = newIntNodeT(bitnot(getInt(a)).maskBytes(int(getSize(g.config, n.typ))), n, idgen, g) else: result = newIntNodeT(bitnot(getInt(a)), n, idgen, g) of mLengthArray: result = newIntNodeT(lengthOrd(g.config, a.typ), n, idgen, g) @@ -248,23 +248,23 @@ proc evalOp(m: TMagic, n, a, b, c: PNode; idgen: IdGenerator; g: ModuleGraph): P of mBitorI, mOr: result = newIntNodeT(bitor(getInt(a), getInt(b)), n, idgen, g) of mBitxorI, mXor: result = newIntNodeT(bitxor(getInt(a), getInt(b)), n, idgen, g) of mAddU: - let val = maskBytes(getInt(a) + getInt(b), int(n.typ.size)) + let val = maskBytes(getInt(a) + getInt(b), int(getSize(g.config, n.typ))) result = newIntNodeT(val, n, idgen, g) of mSubU: - let val = maskBytes(getInt(a) - getInt(b), int(n.typ.size)) + let val = maskBytes(getInt(a) - getInt(b), int(getSize(g.config, n.typ))) result = newIntNodeT(val, n, idgen, g) # echo "subU: ", val, " n: ", n, " result: ", val of mMulU: - let val = maskBytes(getInt(a) * getInt(b), int(n.typ.size)) + let val = maskBytes(getInt(a) * getInt(b), int(getSize(g.config, n.typ))) result = newIntNodeT(val, n, idgen, g) of mModU: - let argA = maskBytes(getInt(a), int(a.typ.size)) - let argB = maskBytes(getInt(b), int(a.typ.size)) + let argA = maskBytes(getInt(a), int(getSize(g.config, a.typ))) + let argB = maskBytes(getInt(b), int(getSize(g.config, a.typ))) if argB != Zero: result = newIntNodeT(argA mod argB, n, idgen, g) of mDivU: - let argA = maskBytes(getInt(a), int(a.typ.size)) - let argB = maskBytes(getInt(b), int(a.typ.size)) + let argA = maskBytes(getInt(a), int(getSize(g.config, a.typ))) + let argB = maskBytes(getInt(b), int(getSize(g.config, a.typ))) if argB != Zero: result = newIntNodeT(argA div argB, n, idgen, g) of mLeSet: result = newIntNodeT(toInt128(ord(containsSets(g.config, a, b))), n, idgen, g) diff --git a/tests/slice/tdistinct.nim b/tests/slice/tdistinct.nim new file mode 100644 index 000000000..d99b529a7 --- /dev/null +++ b/tests/slice/tdistinct.nim @@ -0,0 +1,2 @@ +type Foo = distinct uint64 +const slice = 0 ..< 42.Foo From ebd88725f9d899440408cc635b79041942bc4fd0 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 7 Dec 2021 05:41:42 +0800 Subject: [PATCH 1390/2472] remove `sysspawn` which is dead code (#19218) --- lib/system/sysspawn.nim | 187 ---------------------------------------- 1 file changed, 187 deletions(-) delete mode 100644 lib/system/sysspawn.nim diff --git a/lib/system/sysspawn.nim b/lib/system/sysspawn.nim deleted file mode 100644 index c9b9a68f0..000000000 --- a/lib/system/sysspawn.nim +++ /dev/null @@ -1,187 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Implements Nim's 'spawn'. - -when not declared(NimString): - {.error: "You must not import this module explicitly".} - -{.push stackTrace:off.} - -# We declare our own condition variables here to get rid of the dummy lock -# on Windows: - -type - CondVar = object - c: SysCond - stupidLock: SysLock - counter: int - -proc createCondVar(): CondVar = - initSysCond(result.c) - initSysLock(result.stupidLock) - #acquireSys(result.stupidLock) - -proc destroyCondVar(c: var CondVar) {.inline.} = - deinitSysCond(c.c) - -proc await(cv: var CondVar) = - acquireSys(cv.stupidLock) - while cv.counter <= 0: - waitSysCond(cv.c, cv.stupidLock) - dec cv.counter - releaseSys(cv.stupidLock) - -proc signal(cv: var CondVar) = - acquireSys(cv.stupidLock) - inc cv.counter - releaseSys(cv.stupidLock) - signalSysCond(cv.c) - -type - FastCondVar = object - event, slowPath: bool - slow: CondVar - -proc createFastCondVar(): FastCondVar = - initSysCond(result.slow.c) - initSysLock(result.slow.stupidLock) - #acquireSys(result.slow.stupidLock) - result.event = false - result.slowPath = false - -proc await(cv: var FastCondVar) = - #for i in 0 .. 50: - # if cas(addr cv.event, true, false): - # # this is a HIT: Triggers > 95% in my tests. - # return - # cpuRelax() - #cv.slowPath = true - # XXX For some reason this crashes some test programs - await(cv.slow) - cv.event = false - -proc signal(cv: var FastCondVar) = - cv.event = true - #if cas(addr cv.slowPath, true, false): - signal(cv.slow) - -type - Barrier* {.compilerproc.} = object - counter: int - cv: CondVar - -proc barrierEnter*(b: ptr Barrier) {.compilerproc.} = - atomicInc b.counter - -proc barrierLeave*(b: ptr Barrier) {.compilerproc.} = - atomicDec b.counter - if b.counter <= 0: signal(b.cv) - -proc openBarrier*(b: ptr Barrier) {.compilerproc.} = - b.counter = 0 - b.cv = createCondVar() - -proc closeBarrier*(b: ptr Barrier) {.compilerproc.} = - await(b.cv) - destroyCondVar(b.cv) - -{.pop.} - -# ---------------------------------------------------------------------------- - -type - WorkerProc = proc (thread, args: pointer) {.nimcall, gcsafe.} - Worker = object - taskArrived: CondVar - taskStarted: FastCondVar #\ - # task data: - f: WorkerProc - data: pointer - ready: bool # put it here for correct alignment! - -proc nimArgsPassingDone(p: pointer) {.compilerproc.} = - let w = cast[ptr Worker](p) - signal(w.taskStarted) - -var gSomeReady = createFastCondVar() - -proc slave(w: ptr Worker) {.thread.} = - while true: - w.ready = true # If we instead signal "workerReady" we need the scheduler - # to notice this. The scheduler could then optimize the - # layout of the worker threads (e.g. keep the list sorted) - # so that no search for a "ready" thread is necessary. - # This might be implemented later, but is more tricky than - # it looks because 'spawn' itself can run concurrently. - signal(gSomeReady) - await(w.taskArrived) - assert(not w.ready) - # shield against spurious wakeups: - if w.data != nil: - w.f(w, w.data) - w.data = nil - -const NumThreads = 4 - -var - workers: array[NumThreads, Thread[ptr Worker]] - workersData: array[NumThreads, Worker] - -proc setup() = - for i in 0 ..< NumThreads: - workersData[i].taskArrived = createCondVar() - workersData[i].taskStarted = createFastCondVar() - createThread(workers[i], slave, addr(workersData[i])) - -proc preferSpawn*(): bool = - ## Use this proc to determine quickly if a 'spawn' or a direct call is - ## preferable. If it returns 'true' a 'spawn' may make sense. In general - ## it is not necessary to call this directly; use 'spawnX' instead. - result = gSomeReady.event - -proc spawn*(call: typed) {.magic: "Spawn".} - ## always spawns a new task, so that the 'call' is never executed on - ## the calling thread. 'call' has to be proc call 'p(...)' where 'p' - ## is gcsafe and has 'void' as the return type. - -template spawnX*(call: typed) = - ## spawns a new task if a CPU core is ready, otherwise executes the - ## call in the calling thread. Usually it is advised to - ## use 'spawn' in order to not block the producer for an unknown - ## amount of time. 'call' has to be proc call 'p(...)' where 'p' - ## is gcsafe and has 'void' as the return type. - if preferSpawn(): spawn call - else: call - -proc nimSpawn(fn: WorkerProc; data: pointer) {.compilerproc.} = - # implementation of 'spawn' that is used by the code generator. - while true: - for i in 0.. high(workers): - let w = addr(workersData[i]) - if cas(addr w.ready, true, false): - w.data = data - w.f = fn - signal(w.taskArrived) - await(w.taskStarted) - return - await(gSomeReady) - -proc sync*() = - ## a simple barrier to wait for all spawn'ed tasks. If you need more elaborate - ## waiting, you have to use an explicit barrier. - while true: - var allReady = true - for i in 0 .. high(workers): - if not allReady: break - allReady = allReady and workersData[i].ready - if allReady: break - await(gSomeReady) - -setup() From 93c8427fca0bdc0f32c8f7c011c661808ea1e6dc Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 7 Dec 2021 17:59:43 +0100 Subject: [PATCH 1391/2472] re-enable chronos testing once again [backport:1.2] (#19222) --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index e11799e73..7e15e7c65 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -51,7 +51,7 @@ pkg "cascade" pkg "cello" pkg "chroma" pkg "chronicles", "nim c -o:chr -r chronicles.nim" -pkg "chronos", "nim c -r -d:release tests/testall", allowFailure = true # pending https://github.com/nim-lang/Nim/issues/17130 +pkg "chronos", "nim c -r -d:release tests/testall" pkg "cligen", "nim c --path:. -r cligen.nim" pkg "combparser", "nimble test --gc:orc" pkg "compactdict" From cd592ed85ba3bf440d7b24a4bc5b117d1a5f8725 Mon Sep 17 00:00:00 2001 From: Tanguy Date: Tue, 7 Dec 2021 18:01:22 +0100 Subject: [PATCH 1392/2472] fix #19193 (#19195) [backport:1.2] --- compiler/cgen.nim | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index b669483cc..c862d4d30 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -443,8 +443,14 @@ proc constructLoc(p: BProc, loc: var TLoc, isTemp = false) = if optSeqDestructors in p.config.globalOptions and skipTypes(typ, abstractInst + {tyStatic}).kind in {tyString, tySequence}: linefmt(p, cpsStmts, "$1.len = 0; $1.p = NIM_NIL;$n", [rdLoc(loc)]) elif not isComplexValueType(typ): - linefmt(p, cpsStmts, "$1 = ($2)0;$n", [rdLoc(loc), - getTypeDesc(p.module, typ, mapTypeChooser(loc))]) + if containsGarbageCollectedRef(loc.t): + var nilLoc: TLoc + initLoc(nilLoc, locTemp, loc.lode, OnStack) + nilLoc.r = rope("NIM_NIL") + genRefAssign(p, loc, nilLoc) + else: + linefmt(p, cpsStmts, "$1 = ($2)0;$n", [rdLoc(loc), + getTypeDesc(p.module, typ, mapTypeChooser(loc))]) else: if not isTemp or containsGarbageCollectedRef(loc.t): # don't use nimZeroMem for temporary values for performance if we can From 7806ec525eec83d250141a32cfa197949ea1a42b Mon Sep 17 00:00:00 2001 From: rockcavera Date: Wed, 8 Dec 2021 02:48:34 -0300 Subject: [PATCH 1393/2472] Making TCC work again on Windows --cpu:amd64 - fix #16326 (#19221) * fix #16326 * removing comments --- lib/system/io.nim | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/system/io.nim b/lib/system/io.nim index 2ad43acdb..661d1a9ba 100644 --- a/lib/system/io.nim +++ b/lib/system/io.nim @@ -109,8 +109,22 @@ when defined(windows): else: proc c_fseek(f: File, offset: int64, whence: cint): cint {. importc: "_fseeki64", header: "", tags: [].} - proc c_ftell(f: File): int64 {. - importc: "_ftelli64", header: "", tags: [].} + when defined(tcc): + proc c_fsetpos(f: File, pos: var int64): int32 {. + importc: "fsetpos", header: "", tags: [].} + proc c_fgetpos(f: File, pos: var int64): int32 {. + importc: "fgetpos", header: "", tags: [].} + proc c_telli64(f: cint): int64 {. + importc: "_telli64", header: "", tags: [].} + proc c_ftell(f: File): int64 = + # Taken from https://pt.osdn.net/projects/mingw/scm/git/mingw-org-wsl/blobs/5.4-trunk/mingwrt/mingwex/stdio/ftelli64.c + result = -1'i64 + var pos: int64 + if c_fgetpos(f, pos) == 0 and c_fsetpos(f, pos) == 0: + result = c_telli64(c_fileno(f)) + else: + proc c_ftell(f: File): int64 {. + importc: "_ftelli64", header: "", tags: [].} else: proc c_fseek(f: File, offset: int64, whence: cint): cint {. importc: "fseeko", header: "", tags: [].} From 099285494124dedd7d83a848e213f895f54f9e81 Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 8 Dec 2021 15:40:35 +0800 Subject: [PATCH 1394/2472] Atomic inc/dec should use ATOMIC_SEQ_CST (#19212) --- lib/system/atomics.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/system/atomics.nim b/lib/system/atomics.nim index 75731764b..8d29c287d 100644 --- a/lib/system/atomics.nim +++ b/lib/system/atomics.nim @@ -217,7 +217,7 @@ else: proc atomicInc*(memLoc: var int, x: int = 1): int = when someGcc and hasThreadSupport: - result = atomicAddFetch(memLoc.addr, x, ATOMIC_RELAXED) + result = atomicAddFetch(memLoc.addr, x, ATOMIC_SEQ_CST) elif someVcc and hasThreadSupport: result = addAndFetch(memLoc.addr, x) inc(result, x) @@ -228,9 +228,9 @@ proc atomicInc*(memLoc: var int, x: int = 1): int = proc atomicDec*(memLoc: var int, x: int = 1): int = when someGcc and hasThreadSupport: when declared(atomicSubFetch): - result = atomicSubFetch(memLoc.addr, x, ATOMIC_RELAXED) + result = atomicSubFetch(memLoc.addr, x, ATOMIC_SEQ_CST) else: - result = atomicAddFetch(memLoc.addr, -x, ATOMIC_RELAXED) + result = atomicAddFetch(memLoc.addr, -x, ATOMIC_SEQ_CST) elif someVcc and hasThreadSupport: result = addAndFetch(memLoc.addr, -x) dec(result, x) From 502ac4ed5e539146d31920d50020ae35668f4755 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 9 Dec 2021 13:56:51 +0100 Subject: [PATCH 1395/2472] fixes a converter handling regression that caused private converters to leak into client modules; fixes #19213; [backport:1.6] (#19229) --- compiler/importer.nim | 6 ++++-- tests/converter/mdontleak.nim | 3 +++ tests/converter/tdontleak.nim | 10 ++++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 tests/converter/mdontleak.nim create mode 100644 tests/converter/tdontleak.nim diff --git a/compiler/importer.nim b/compiler/importer.nim index af392f849..acca2c644 100644 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -185,11 +185,13 @@ template addUnnamedIt(c: PContext, fromMod: PSym; filter: untyped) {.dirty.} = for it in mitems c.graph.ifaces[fromMod.position].converters: if filter: loadPackedSym(c.graph, it) - addConverter(c, it) + if sfExported in it.sym.flags: + addConverter(c, it) for it in mitems c.graph.ifaces[fromMod.position].patterns: if filter: loadPackedSym(c.graph, it) - addPattern(c, it) + if sfExported in it.sym.flags: + addPattern(c, it) for it in mitems c.graph.ifaces[fromMod.position].pureEnums: if filter: loadPackedSym(c.graph, it) diff --git a/tests/converter/mdontleak.nim b/tests/converter/mdontleak.nim new file mode 100644 index 000000000..e55c3f87c --- /dev/null +++ b/tests/converter/mdontleak.nim @@ -0,0 +1,3 @@ + +converter toBool(x: uint32): bool = x != 0 +# Note: This convertes is not exported! diff --git a/tests/converter/tdontleak.nim b/tests/converter/tdontleak.nim new file mode 100644 index 000000000..4965fa90a --- /dev/null +++ b/tests/converter/tdontleak.nim @@ -0,0 +1,10 @@ +discard """ + output: '''5''' +joinable: false +""" + +import mdontleak +# bug #19213 + +let a = 5'u32 +echo a From 742e9d65ad6b56387dc6bf9a2be1b95c510fd0c4 Mon Sep 17 00:00:00 2001 From: wenghongquan <93646063+wenghongquan@users.noreply.github.com> Date: Thu, 9 Dec 2021 21:00:33 +0800 Subject: [PATCH 1396/2472] Add support for LoongArch (#19223) * Add support for LoongArch * Update compiler/installer.ini Co-authored-by: Andreas Rumpf --- compiler/platform.nim | 5 +++-- lib/system.nim | 2 +- lib/system/platforms.nim | 4 +++- tools/nim.zsh-completion | 1 + tools/niminst/buildsh.nimf | 2 ++ tools/niminst/makefile.nimf | 3 +++ 6 files changed, 13 insertions(+), 4 deletions(-) diff --git a/compiler/platform.nim b/compiler/platform.nim index eb1398677..4e6054d5c 100644 --- a/compiler/platform.nim +++ b/compiler/platform.nim @@ -202,7 +202,7 @@ type cpuPowerpc64el, cpuSparc, cpuVm, cpuHppa, cpuIa64, cpuAmd64, cpuMips, cpuMipsel, cpuArm, cpuArm64, cpuJS, cpuNimVM, cpuAVR, cpuMSP430, cpuSparc64, cpuMips64, cpuMips64el, cpuRiscV32, cpuRiscV64, cpuEsp, cpuWasm32, - cpuE2k + cpuE2k, cpuLoongArch64 type TInfoCPU* = tuple[name: string, intSize: int, endian: Endianness, @@ -238,7 +238,8 @@ const (name: "riscv64", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64), (name: "esp", intSize: 32, endian: littleEndian, floatSize: 64, bit: 32), (name: "wasm32", intSize: 32, endian: littleEndian, floatSize: 64, bit: 32), - (name: "e2k", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64)] + (name: "e2k", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64), + (name: "loongarch64", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64)] type Target* = object diff --git a/lib/system.nim b/lib/system.nim index 012aefe6a..c424cbc1b 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1126,7 +1126,7 @@ const ## Possible values: ## `"i386"`, `"alpha"`, `"powerpc"`, `"powerpc64"`, `"powerpc64el"`, ## `"sparc"`, `"amd64"`, `"mips"`, `"mipsel"`, `"arm"`, `"arm64"`, - ## `"mips64"`, `"mips64el"`, `"riscv32"`, `"riscv64"`. + ## `"mips64"`, `"mips64el"`, `"riscv32"`, `"riscv64"`, '"loongarch64"'. seqShallowFlag = low(int) strlitFlag = 1 shl (sizeof(int)*8 - 2) # later versions of the codegen \ diff --git a/lib/system/platforms.nim b/lib/system/platforms.nim index 5bca8cb1c..b4c9f1f06 100644 --- a/lib/system/platforms.nim +++ b/lib/system/platforms.nim @@ -36,7 +36,8 @@ type riscv32, ## RISC-V 32-bit processor riscv64, ## RISC-V 64-bit processor wasm32, ## WASM, 32-bit - e2k ## MCST Elbrus 2000 + e2k, ## MCST Elbrus 2000 + loongarch64 ## LoongArch 64-bit processor OsPlatform* {.pure.} = enum ## the OS this program will run on. none, dos, windows, os2, linux, morphos, skyos, solaris, @@ -95,5 +96,6 @@ const elif defined(riscv64): CpuPlatform.riscv64 elif defined(wasm32): CpuPlatform.wasm32 elif defined(e2k): CpuPlatform.e2k + elif defined(loongarch64): CpuPlatform.loongarch64 else: CpuPlatform.none ## the CPU this program will run on. diff --git a/tools/nim.zsh-completion b/tools/nim.zsh-completion index 07a221e9c..45477d860 100644 --- a/tools/nim.zsh-completion +++ b/tools/nim.zsh-completion @@ -95,6 +95,7 @@ _nim() { '*--cpu=sparc64[compile for SPARC64 architecture]' \ '*--cpu=vm[compile for Nim VM]' \ '*--cpu=wasm32[compile to WASM 32]' \ + '*--cpu=loongarch64[compile for LoongArch64 architecture]' \ '*--gc=refc[use reference counting garbage collection]' \ '*--gc=arc[use ARC garbage collection]' \ '*--gc=orc[use ORC garbage collection]' \ diff --git a/tools/niminst/buildsh.nimf b/tools/niminst/buildsh.nimf index b81ac731f..464c70abf 100644 --- a/tools/niminst/buildsh.nimf +++ b/tools/niminst/buildsh.nimf @@ -229,6 +229,8 @@ case $ucpu in mycpu="riscv64" ;; *e2k* ) mycpu="e2k" ;; + *loongarch64* ) + mycpu="loongarch64" ;; *) echo 2>&1 "Error: unknown processor: $ucpu" exit 1 diff --git a/tools/niminst/makefile.nimf b/tools/niminst/makefile.nimf index c4f2f0e79..a1f3fa977 100644 --- a/tools/niminst/makefile.nimf +++ b/tools/niminst/makefile.nimf @@ -169,6 +169,9 @@ endif ifeq ($(ucpu),e2k) mycpu = e2k endif +ifeq ($(ucpu),loongarch64) + mycpu = loongarch64 +endif ifndef mycpu $(error unknown CPU architecture: $(ucpu) See makefile.nimf) endif From 99f8793502e65cd43f6e301557d428758948c3ca Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 9 Dec 2021 22:37:59 +0800 Subject: [PATCH 1397/2472] remove `std/sharedstrings` (#19228) * remove std/sharedstrings it has been broken since 0.18.0 * rephrase the changelog entry --- changelog.md | 2 + lib/deprecated/pure/sharedstrings.nim | 154 -------------------------- testament/testament.nim | 1 - 3 files changed, 2 insertions(+), 155 deletions(-) delete mode 100644 lib/deprecated/pure/sharedstrings.nim diff --git a/changelog.md b/changelog.md index a8f1920e9..ebad54fe4 100644 --- a/changelog.md +++ b/changelog.md @@ -14,6 +14,8 @@ - Optional parameters in combination with `: body` syntax (RFC #405) are now opt-in via `experimental:flexibleOptionalParams`. +- `std/sharedstrings` module is removed. + ## Standard library additions and changes - `macros.parseExpr` and `macros.parseStmt` now accept an optional diff --git a/lib/deprecated/pure/sharedstrings.nim b/lib/deprecated/pure/sharedstrings.nim deleted file mode 100644 index 99963e1f2..000000000 --- a/lib/deprecated/pure/sharedstrings.nim +++ /dev/null @@ -1,154 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Shared string support for Nim. - -{.deprecated.} - -type - UncheckedCharArray = UncheckedArray[char] - -type - Buffer = ptr object - refcount: int - capacity, realLen: int - data: UncheckedCharArray - - SharedString* = object ## A string that can be shared. Slicing is O(1). - buffer: Buffer - first, len: int - -proc decRef(b: Buffer) {.inline.} = - if atomicDec(b.refcount) <= 0: - deallocShared(b) - -proc incRef(b: Buffer) {.inline.} = - atomicInc(b.refcount) - -{.experimental.} - -proc `=destroy`*(s: SharedString) = - #echo "destroyed" - if not s.buffer.isNil: - decRef(s.buffer) - -when false: - proc `=copy`*(dest: var SharedString; src: SharedString) = - incRef(src.buffer) - if not dest.buffer.isNil: - decRef(dest.buffer) - dest.buffer = src.buffer - dest.first = src.first - dest.len = src.len - -proc len*(s: SharedString): int = s.len - -proc `[]`*(s: SharedString; i: Natural): char = - if i < s.len: result = s.buffer.data[i+s.first] - else: raise newException(IndexDefect, formatErrorIndexBound(i, s.len-1)) - -proc `[]=`*(s: var SharedString; i: Natural; value: char) = - if i < s.len: s.buffer.data[i+s.first] = value - else: raise newException(IndexDefect, formatErrorIndexBound(i, s.len-1)) - -proc `[]`*(s: SharedString; ab: HSlice[int, int]): SharedString = - #incRef(src.buffer) - if ab.a < s.len: - result.buffer = s.buffer - result.first = ab.a - result.len = min(s.len, ab.b - ab.a + 1) - # else: produce empty string ;-) - -proc newBuffer(cap, len: int): Buffer = - assert cap >= len - result = cast[Buffer](allocShared0(sizeof(int)*3 + cap)) - result.refcount = 0 - result.capacity = cap - result.realLen = len - -proc newSharedString*(len: Natural): SharedString = - if len != 0: - # optimization: Don't have an underlying buffer when 'len == 0' - result.buffer = newBuffer(len, len) - result.first = 0 - result.len = len - -proc newSharedString*(s: string): SharedString = - let len = s.len - if len != 0: - # optimization: Don't have an underlying buffer when 'len == 0' - result.buffer = newBuffer(len, len) - copyMem(addr result.buffer.data[0], cstring(s), s.len) - result.first = 0 - result.len = len - -when declared(atomicLoadN): - template load(x): untyped = atomicLoadN(addr x, ATOMIC_SEQ_CST) -else: - # XXX Fixme - template load(x): untyped = x - -proc add*(s: var SharedString; t: cstring; len: Natural) = - if len == 0: return - let newLen = s.len + len - if s.buffer.isNil: - s.buffer = newBuffer(len, len) - copyMem(addr s.buffer.data[0], t, len) - s.len = len - elif newLen >= s.buffer.capacity or s.first != 0 or - s.len != s.buffer.realLen or load(s.buffer.refcount) > 1: - let oldBuf = s.buffer - s.buffer = newBuffer(max(s.buffer.capacity * 3 div 2, newLen), newLen) - copyMem(addr s.buffer.data[0], addr oldBuf.data[s.first], s.len) - copyMem(addr s.buffer.data[s.len], t, len) - decRef(oldBuf) - else: - copyMem(addr s.buffer.data[s.len], t, len) - s.buffer.realLen += len - s.len += len - -proc add*(s: var SharedString; t: string) = - s.add(t.cstring, t.len) - -proc rawData*(s: var SharedString): pointer = - if s.buffer.isNil: result = nil - else: result = addr s.buffer.data[s.first] - -proc add*(s: var SharedString; t: SharedString) = - if t.buffer.isNil: return - s.add(cast[cstring](addr s.buffer.data[s.first]), t.len) - -proc `$`*(s: SharedString): string = - result = newString(s.len) - if s.len > 0: - copyMem(addr result[0], addr s.buffer.data[s.first], s.len) - -proc `==`*(s: SharedString; t: string): bool = - if s.buffer.isNil: result = t.len == 0 - else: result = t.len == s.len and equalMem(addr s.buffer.data[s.first], - cstring(t), t.len) - -proc `==`*(s, t: SharedString): bool = - if s.buffer.isNil: result = t.len == 0 - else: result = t.len == s.len and equalMem(addr s.buffer.data[s.first], - addr t.buffer.data[t.first], t.len) - -iterator items*(s: SharedString): char = - let buf = s.buffer.data - let x = s.first - if buf != nil: - for i in 0.. Date: Fri, 10 Dec 2021 01:49:31 +0800 Subject: [PATCH 1398/2472] add comments to spawn and pinnedSpawn (#19230) `spawn` uses `nimSpawn3` internally and `pinnedSpawn` uses `nimSpawn4` internally. I comment it in order to help contributors get the gist of its functionality. --- lib/pure/concurrency/threadpool.nim | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/pure/concurrency/threadpool.nim b/lib/pure/concurrency/threadpool.nim index f8a5f9ba8..9334b4833 100644 --- a/lib/pure/concurrency/threadpool.nim +++ b/lib/pure/concurrency/threadpool.nim @@ -448,19 +448,21 @@ proc preferSpawn*(): bool = ## <#spawnX.t>`_ instead. result = gSomeReady.counter > 0 -proc spawn*(call: sink typed) {.magic: "Spawn".} +proc spawn*(call: sink typed) {.magic: "Spawn".} = ## Always spawns a new task, so that the `call` is never executed on ## the calling thread. ## ## `call` has to be a proc call `p(...)` where `p` is gcsafe and has a ## return type that is either `void` or compatible with `FlowVar[T]`. + discard "It uses `nimSpawn3` internally" -proc pinnedSpawn*(id: ThreadId; call: sink typed) {.magic: "Spawn".} +proc pinnedSpawn*(id: ThreadId; call: sink typed) {.magic: "Spawn".} = ## Always spawns a new task on the worker thread with `id`, so that ## the `call` is **always** executed on the thread. ## ## `call` has to be a proc call `p(...)` where `p` is gcsafe and has a ## return type that is either `void` or compatible with `FlowVar[T]`. + discard "It uses `nimSpawn4` internally" template spawnX*(call) = ## Spawns a new task if a CPU core is ready, otherwise executes the From 32d4bf352585a4fc4a6fa1bd24b270af087b0372 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 9 Dec 2021 22:23:16 +0100 Subject: [PATCH 1399/2472] fixes an old ARC bug: the produced copy/sink operations don't copy the hidden type field for objects with enabled inheritance; fixes #19205 [backport:1.6] (#19232) --- compiler/ast.nim | 2 +- compiler/ccgexprs.nim | 8 ++++++++ compiler/ccgtypes.nim | 2 +- compiler/liftdestructors.nim | 10 ++++++++++ compiler/types.nim | 3 +++ tests/arc/tarcmisc.nim | 15 +++++++++++++++ 6 files changed, 38 insertions(+), 2 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 9086860b9..4286f940f 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -673,7 +673,7 @@ type mSwap, mIsNil, mArrToSeq, mNewString, mNewStringOfCap, mParseBiggestFloat, mMove, mWasMoved, mDestroy, mTrace, - mDefault, mUnown, mFinished, mIsolate, mAccessEnv, mReset, + mDefault, mUnown, mFinished, mIsolate, mAccessEnv, mAccessTypeField, mReset, mArray, mOpenArray, mRange, mSet, mSeq, mVarargs, mRef, mPtr, mVar, mDistinct, mVoid, mTuple, mOrdinal, mIterableType, diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index fc09fefc3..1f1035115 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1741,6 +1741,13 @@ proc genGetTypeInfoV2(p: BProc, e: PNode, d: var TLoc) = # use the dynamic type stored at offset 0: putIntoDest(p, d, e, rdMType(p, a, nilCheck)) +proc genAccessTypeField(p: BProc; e: PNode; d: var TLoc) = + var a: TLoc + initLocExpr(p, e[1], a) + var nilCheck = Rope(nil) + # use the dynamic type stored at offset 0: + putIntoDest(p, d, e, rdMType(p, a, nilCheck)) + template genDollar(p: BProc, n: PNode, d: var TLoc, frmt: string) = var a: TLoc initLocExpr(p, n[1], a) @@ -2449,6 +2456,7 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = of mMove: genMove(p, e, d) of mDestroy: genDestroy(p, e) of mAccessEnv: unaryExpr(p, e, d, "$1.ClE_0") + of mAccessTypeField: genAccessTypeField(p, e, d) of mSlice: genSlice(p, e, d) of mTrace: discard "no code to generate" else: diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index e7e099659..740ce5f19 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -583,7 +583,7 @@ proc getRecordDesc(m: BModule, typ: PType, name: Rope, if typ.kind == tyObject: if typ[0] == nil: - if (typ.sym != nil and sfPure in typ.sym.flags) or tfFinal in typ.flags: + if lacksMTypeField(typ): appcg(m, result, " {$n", []) else: if optTinyRtti in m.config.globalOptions: diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim index cc7e36cdb..51b4ddfb0 100644 --- a/compiler/liftdestructors.nim +++ b/compiler/liftdestructors.nim @@ -941,6 +941,12 @@ proc symPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttachedOp incl result.flags, sfFromGeneric incl result.flags, sfGeneratedOp +proc genTypeFieldCopy(c: var TLiftCtx; t: PType; body, x, y: PNode) = + let xx = genBuiltin(c, mAccessTypeField, "accessTypeField", x) + let yy = genBuiltin(c, mAccessTypeField, "accessTypeField", y) + xx.typ = getSysType(c.g, c.info, tyPointer) + yy.typ = xx.typ + body.add newAsgnStmt(xx, yy) proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp; info: TLineInfo; idgen: IdGenerator): PSym = @@ -980,6 +986,10 @@ proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp; fillStrOp(a, typ, result.ast[bodyPos], d, src) else: fillBody(a, typ, result.ast[bodyPos], d, src) + if tk == tyObject and a.kind in {attachedAsgn, attachedSink, attachedDeepCopy} and not lacksMTypeField(typ): + # bug #19205: Do not forget to also copy the hidden type field: + genTypeFieldCopy(a, typ, result.ast[bodyPos], d, src) + if not a.canRaise: incl result.flags, sfNeverRaises completePartialOp(g, idgen.module, typ, kind, result) diff --git a/compiler/types.nim b/compiler/types.nim index 7cf02c240..007a61356 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -1705,3 +1705,6 @@ proc isCharArrayPtr*(t: PType; allowPointerToChar: bool): bool = result = allowPointerToChar else: discard + +proc lacksMTypeField*(typ: PType): bool {.inline.} = + (typ.sym != nil and sfPure in typ.sym.flags) or tfFinal in typ.flags diff --git a/tests/arc/tarcmisc.nim b/tests/arc/tarcmisc.nim index d259e1197..45d1514cd 100644 --- a/tests/arc/tarcmisc.nim +++ b/tests/arc/tarcmisc.nim @@ -30,6 +30,7 @@ ok true copying 123 +42 closed destroying variable: 20 destroying variable: 10 @@ -482,3 +483,17 @@ method testMethod(self: BrokenObject) {.base.} = let mikasa = BrokenObject() mikasa.testMethod() + +# bug #19205 +type + InputSectionBase* = object of RootObj + relocations*: seq[int] # traced reference. string has a similar SIGSEGV. + InputSection* = object of InputSectionBase + +proc fooz(sec: var InputSectionBase) = + if sec of InputSection: # this line SIGSEGV. + echo 42 + +var sec = create(InputSection) +sec[] = InputSection(relocations: newSeq[int]()) +fooz sec[] From 69aabdab800077e9aaa08344494c83138a02f57c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Fri, 10 Dec 2021 06:31:29 +0100 Subject: [PATCH 1400/2472] nimRawSetjmp: support Windows (#19197) * nimRawSetjmp: support Windows Using `_setjmp()` directly is required to avoid some rare (but very annoying) exception-related stack corruption leading to segfaults on Windows, with Mingw-w64 and SEH. More details: https://github.com/status-im/nimbus-eth2/issues/3121 Also add "nimBuiltinSetjmp" - mostly for benchmarking. * fix for Apple's Clang++ --- compiler/ccgstmts.nim | 13 +++- doc/nimc.rst | 14 ++++ lib/system/ansi_c.nim | 46 +++++++++-- tests/exception/texceptions.nim | 6 +- tests/exception/texceptions2.nim | 130 +++++++++++++++++++++++++++++++ 5 files changed, 199 insertions(+), 10 deletions(-) create mode 100644 tests/exception/texceptions2.nim diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index 559849f0d..7fecc1475 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -1356,8 +1356,19 @@ proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) = linefmt(p, cpsStmts, "$1.status = setjmp($1.context);$n", [safePoint]) elif isDefined(p.config, "nimSigSetjmp"): linefmt(p, cpsStmts, "$1.status = sigsetjmp($1.context, 0);$n", [safePoint]) + elif isDefined(p.config, "nimBuiltinSetjmp"): + linefmt(p, cpsStmts, "$1.status = __builtin_setjmp($1.context);$n", [safePoint]) elif isDefined(p.config, "nimRawSetjmp"): - linefmt(p, cpsStmts, "$1.status = _setjmp($1.context);$n", [safePoint]) + if isDefined(p.config, "mswindows"): + # The Windows `_setjmp()` takes two arguments, with the second being an + # undocumented buffer used by the SEH mechanism for stack unwinding. + # Mingw-w64 has been trying to get it right for years, but it's still + # prone to stack corruption during unwinding, so we disable that by setting + # it to NULL. + # More details: https://github.com/status-im/nimbus-eth2/issues/3121 + linefmt(p, cpsStmts, "$1.status = _setjmp($1.context, 0);$n", [safePoint]) + else: + linefmt(p, cpsStmts, "$1.status = _setjmp($1.context);$n", [safePoint]) else: linefmt(p, cpsStmts, "$1.status = setjmp($1.context);$n", [safePoint]) lineCg(p, cpsStmts, "if ($1.status == 0) {$n", [safePoint]) diff --git a/doc/nimc.rst b/doc/nimc.rst index eb066e748..0ab1501ff 100644 --- a/doc/nimc.rst +++ b/doc/nimc.rst @@ -165,6 +165,20 @@ ignored too. `--define:FOO`:option: and `--define:foo`:option: are identical. Compile-time symbols starting with the `nim` prefix are reserved for the implementation and should not be used elsewhere. +========================== ============================================ +Name Description +========================== ============================================ +nimStdSetjmp Use the standard `setjmp()/longjmp()` library + functions for setjmp-based exceptions. This is + the default on most platforms. +nimSigSetjmp Use `sigsetjmp()/siglongjmp()` for setjmp-based exceptions. +nimRawSetjmp Use `_setjmp()/_longjmp()` on POSIX and `_setjmp()/longjmp()` + on Windows, for setjmp-based exceptions. It's the default on + BSDs and BSD-like platforms, where it's significantly faster + than the standard functions. +nimBuiltinSetjmp Use `__builtin_setjmp()/__builtin_longjmp()` for setjmp-based + exceptions. This will not work if an exception is being thrown + and caught inside the same procedure. Useful for benchmarking. Configuration files ------------------- diff --git a/lib/system/ansi_c.nim b/lib/system/ansi_c.nim index 259d36633..23fb9fdef 100644 --- a/lib/system/ansi_c.nim +++ b/lib/system/ansi_c.nim @@ -31,7 +31,10 @@ proc c_abort*() {. importc: "abort", header: "", noSideEffect, noreturn.} -when defined(linux) and defined(amd64): +when defined(nimBuiltinSetjmp): + type + C_JmpBuf* = array[5, pointer] +elif defined(linux) and defined(amd64): type C_JmpBuf* {.importc: "jmp_buf", header: "", bycopy.} = object abi: array[200 div sizeof(clong), clong] @@ -92,18 +95,47 @@ when defined(macosx): elif defined(haiku): const SIGBUS* = cint(30) -when defined(nimSigSetjmp) and not defined(nimStdSetjmp): +# "nimRawSetjmp" is defined by default for certain platforms, so we need the +# "nimStdSetjmp" escape hatch with it. +when defined(nimSigSetjmp): proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {. header: "", importc: "siglongjmp".} - template c_setjmp*(jmpb: C_JmpBuf): cint = + proc c_setjmp*(jmpb: C_JmpBuf): cint = proc c_sigsetjmp(jmpb: C_JmpBuf, savemask: cint): cint {. header: "", importc: "sigsetjmp".} c_sigsetjmp(jmpb, 0) +elif defined(nimBuiltinSetjmp): + proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) = + # Apple's Clang++ has trouble converting array names to pointers, so we need + # to be very explicit here. + proc c_builtin_longjmp(jmpb: ptr pointer, retval: cint) {. + importc: "__builtin_longjmp", nodecl.} + # The second parameter needs to be 1 and sometimes the C/C++ compiler checks it. + c_builtin_longjmp(unsafeAddr jmpb[0], 1) + proc c_setjmp*(jmpb: C_JmpBuf): cint = + proc c_builtin_setjmp(jmpb: ptr pointer): cint {. + importc: "__builtin_setjmp", nodecl.} + c_builtin_setjmp(unsafeAddr jmpb[0]) elif defined(nimRawSetjmp) and not defined(nimStdSetjmp): - proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {. - header: "", importc: "_longjmp".} - proc c_setjmp*(jmpb: C_JmpBuf): cint {. - header: "", importc: "_setjmp".} + when defined(windows): + # No `_longjmp()` on Windows. + proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {. + header: "", importc: "longjmp".} + # The Windows `_setjmp()` takes two arguments, with the second being an + # undocumented buffer used by the SEH mechanism for stack unwinding. + # Mingw-w64 has been trying to get it right for years, but it's still + # prone to stack corruption during unwinding, so we disable that by setting + # it to NULL. + # More details: https://github.com/status-im/nimbus-eth2/issues/3121 + proc c_setjmp*(jmpb: C_JmpBuf): cint = + proc c_setjmp_win(jmpb: C_JmpBuf, ctx: pointer): cint {. + header: "", importc: "_setjmp".} + c_setjmp_win(jmpb, nil) + else: + proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {. + header: "", importc: "_longjmp".} + proc c_setjmp*(jmpb: C_JmpBuf): cint {. + header: "", importc: "_setjmp".} else: proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {. header: "", importc: "longjmp".} diff --git a/tests/exception/texceptions.nim b/tests/exception/texceptions.nim index adee5d1d5..62d24c934 100644 --- a/tests/exception/texceptions.nim +++ b/tests/exception/texceptions.nim @@ -1,4 +1,6 @@ discard """ + disabled: "windows" # no sigsetjmp() there + matrix: "-d:nimStdSetjmp; -d:nimSigSetjmp; -d:nimRawSetjmp; -d:nimBuiltinSetjmp" output: ''' BEFORE @@ -17,7 +19,7 @@ FINALLY echo "" -proc no_expcetion = +proc no_exception = try: echo "BEFORE" @@ -28,7 +30,7 @@ proc no_expcetion = finally: echo "FINALLY" -try: no_expcetion() +try: no_exception() except: echo "RECOVER" echo "" diff --git a/tests/exception/texceptions2.nim b/tests/exception/texceptions2.nim new file mode 100644 index 000000000..97fd856a0 --- /dev/null +++ b/tests/exception/texceptions2.nim @@ -0,0 +1,130 @@ +discard """ + disabled: "posix" # already covered by texceptions.nim + matrix: "-d:nimStdSetjmp; -d:nimRawSetjmp; -d:nimBuiltinSetjmp" + output: ''' + +BEFORE +FINALLY + +BEFORE +EXCEPT +FINALLY +RECOVER + +BEFORE +EXCEPT: IOError: hi +FINALLY +''' +""" + +echo "" + +proc no_exception = + try: + echo "BEFORE" + + except: + echo "EXCEPT" + raise + + finally: + echo "FINALLY" + +try: no_exception() +except: echo "RECOVER" + +echo "" + +proc reraise_in_except = + try: + echo "BEFORE" + raise newException(IOError, "") + + except IOError: + echo "EXCEPT" + raise + + finally: + echo "FINALLY" + +try: reraise_in_except() +except: echo "RECOVER" + +echo "" + +proc return_in_except = + try: + echo "BEFORE" + raise newException(IOError, "hi") + + except: + echo "EXCEPT: ", getCurrentException().name, ": ", getCurrentExceptionMsg() + return + + finally: + echo "FINALLY" + +try: return_in_except() +except: echo "RECOVER" + +block: #10417 + proc moo() {.noreturn.} = discard + + let bar = + try: + 1 + except: + moo() + + doAssert(bar == 1) + +# Make sure the VM handles the exceptions correctly +block: + proc fun1(): seq[int] = + try: + try: + raise newException(ValueError, "xx") + except: + doAssert("xx" == getCurrentExceptionMsg()) + raise newException(KeyError, "yy") + except: + doAssert("yy" == getCurrentExceptionMsg()) + result.add(1212) + try: + try: + raise newException(AssertionDefect, "a") + finally: + result.add(42) + except AssertionDefect: + result.add(99) + finally: + result.add(10) + result.add(4) + result.add(0) + try: + result.add(1) + except KeyError: + result.add(-1) + except ValueError: + result.add(-1) + except IndexDefect: + result.add(2) + except: + result.add(3) + + try: + try: + result.add(1) + return + except: + result.add(-1) + finally: + result.add(2) + except KeyError: + doAssert(false) + finally: + result.add(3) + + let x1 = fun1() + const x2 = fun1() + doAssert(x1 == x2) From d39147219a41f54b45b585a103bfb04cce71c52c Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 9 Dec 2021 22:38:39 -0800 Subject: [PATCH 1401/2472] Revert "swap port to correct port order (#19177)" (#19234) This reverts commit 0d0c249074d6a1041de16108dc247396efef5513. --- lib/pure/net.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 06024e46d..7b0ff78e7 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -478,14 +478,14 @@ proc toSockAddr*(address: IpAddress, port: Port, sa: var Sockaddr_storage, sl = sizeof(Sockaddr_in).SockLen let s = cast[ptr Sockaddr_in](addr sa) s.sin_family = typeof(s.sin_family)(toInt(AF_INET)) - s.sin_port = htons(port) + s.sin_port = port copyMem(addr s.sin_addr, unsafeAddr address.address_v4[0], sizeof(s.sin_addr)) of IpAddressFamily.IPv6: sl = sizeof(Sockaddr_in6).SockLen let s = cast[ptr Sockaddr_in6](addr sa) s.sin6_family = typeof(s.sin6_family)(toInt(AF_INET6)) - s.sin6_port = htons(port) + s.sin6_port = port copyMem(addr s.sin6_addr, unsafeAddr address.address_v6[0], sizeof(s.sin6_addr)) From c98954233981cb30807bd8be8b805663d44964d1 Mon Sep 17 00:00:00 2001 From: MichalMarsalek Date: Fri, 10 Dec 2021 07:39:12 +0100 Subject: [PATCH 1402/2472] move toDeque to after addLast (#19233) [backport:1.0] Changes the order of procs definitions in order to avoid calling an undefined proc. --- lib/pure/collections/deques.nim | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/pure/collections/deques.nim b/lib/pure/collections/deques.nim index ecdf199d2..c4eb5331e 100644 --- a/lib/pure/collections/deques.nim +++ b/lib/pure/collections/deques.nim @@ -87,20 +87,6 @@ proc initDeque*[T](initialSize: int = defaultInitialSize): Deque[T] = ## * `toDeque proc <#toDeque,openArray[T]>`_ result.initImpl(initialSize) -proc toDeque*[T](x: openArray[T]): Deque[T] {.since: (1, 3).} = - ## Creates a new deque that contains the elements of `x` (in the same order). - ## - ## **See also:** - ## * `initDeque proc <#initDeque,int>`_ - runnableExamples: - let a = toDeque([7, 8, 9]) - assert len(a) == 3 - assert $a == "[7, 8, 9]" - - result.initImpl(x.len) - for item in items(x): - result.addLast(item) - proc len*[T](deq: Deque[T]): int {.inline.} = ## Returns the number of elements of `deq`. result = deq.count @@ -303,6 +289,20 @@ proc addLast*[T](deq: var Deque[T], item: sink T) = deq.data[deq.tail] = item deq.tail = (deq.tail + 1) and deq.mask +proc toDeque*[T](x: openArray[T]): Deque[T] {.since: (1, 3).} = + ## Creates a new deque that contains the elements of `x` (in the same order). + ## + ## **See also:** + ## * `initDeque proc <#initDeque,int>`_ + runnableExamples: + let a = toDeque([7, 8, 9]) + assert len(a) == 3 + assert $a == "[7, 8, 9]" + + result.initImpl(x.len) + for item in items(x): + result.addLast(item) + proc peekFirst*[T](deq: Deque[T]): lent T {.inline.} = ## Returns the first element of `deq`, but does not remove it from the deque. ## From 9338aa24977e84a33b9a7802eaff0777fcf4d9c3 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 10 Dec 2021 09:24:20 +0100 Subject: [PATCH 1403/2472] fixes a possible 'javascript:' protocol exploit [backport:1.0] (#19134) * fixes a possible 'javascript:' protocol exploit [backport:1.0] * add tests * Update tests/stdlib/trstgen.nim * add the same logic for hyperlinks * move the logic into a proc Co-authored-by: narimiran --- lib/packages/docutils/rstgen.nim | 15 +++++++++++++- tests/stdlib/trstgen.nim | 35 ++++++++++++++++++++++++++++---- 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index dc6ca25c1..9018087f7 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -40,7 +40,7 @@ ## can be done by simply searching for [footnoteName]. import strutils, os, hashes, strtabs, rstast, rst, highlite, tables, sequtils, - algorithm, parseutils, std/strbasics + algorithm, parseutils, std/strbasics, strscans import ../../std/private/since @@ -827,6 +827,16 @@ proc renderOverline(d: PDoc, n: PRstNode, result: var string) = rstnodeToRefname(n).idS, tmp, $chr(n.level - 1 + ord('A')), tocName]) +proc safeProtocol(linkStr: var string) = + var protocol = "" + if scanf(linkStr, "$w:", protocol): + # if it has a protocol at all, ensure that it's not 'javascript:' or worse: + if cmpIgnoreCase(protocol, "http") == 0 or cmpIgnoreCase(protocol, "https") == 0 or + cmpIgnoreCase(protocol, "ftp") == 0: + discard "it's fine" + else: + linkStr = "" + proc renderTocEntry(d: PDoc, e: TocEntry, result: var string) = dispA(d.target, result, "
                • $2
                  • \n", @@ -891,6 +901,8 @@ proc renderImage(d: PDoc, n: PRstNode, result: var string) = # support for `:target:` links for images: var target = esc(d.target, getFieldValue(n, "target").strip(), escMode=emUrl) + safeProtocol(target) + if target.len > 0: # `htmlOut` needs to be of the following format for link to work for images: # @@ -1192,6 +1204,7 @@ proc renderHyperlink(d: PDoc, text, link: PRstNode, result: var string, d.escMode = emUrl renderRstToOut(d, link, linkStr) d.escMode = mode + safeProtocol(linkStr) var textStr = "" renderRstToOut(d, text, textStr) let nimDocStr = if nimdoc: " nimdoc" else: "" diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index 45cf1a68e..2be65b35b 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -398,7 +398,7 @@ Some chapter Level2 ------ - + Level3 ~~~~~~ @@ -407,7 +407,7 @@ Some chapter More ~~~~ - + Another ------- @@ -683,7 +683,7 @@ Test1 test "RST line blocks": let input2 = dedent""" Paragraph1 - + | Paragraph2""" @@ -704,7 +704,7 @@ Test1 # check that '| ' with a few spaces is still parsed as new line let input4 = dedent""" | xxx - | + | | zzz""" let output4 = input4.toHtml @@ -1549,3 +1549,30 @@ suite "RST/Code highlight": check strip(rstToHtml(pythonCode, {}, newStringTable(modeCaseSensitive))) == strip(expected) + + +suite "invalid targets": + test "invalid image target": + let input1 = dedent """.. image:: /images/myimage.jpg + :target: https://bar.com + :alt: Alt text for the image""" + let output1 = input1.toHtml + check output1 == """Alt text for the image""" + + let input2 = dedent """.. image:: /images/myimage.jpg + :target: javascript://bar.com + :alt: Alt text for the image""" + let output2 = input2.toHtml + check output2 == """Alt text for the image""" + + let input3 = dedent """.. image:: /images/myimage.jpg + :target: bar.com + :alt: Alt text for the image""" + let output3 = input3.toHtml + check output3 == """Alt text for the image""" + + test "invalid links": + check("(([Nim](https://nim-lang.org/)))".toHtml == + """((Nim))""") + check("(([Nim](javascript://nim-lang.org/)))".toHtml == + """((Nim))""") From 7ff43d07b2b983271d666b1c654b1426cbbbf236 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 10 Dec 2021 14:28:59 +0100 Subject: [PATCH 1404/2472] added --nimMainPrefix switch; fixes #15955; refs #16945 [backport:1.6] (#19235) --- changelog.md | 2 ++ compiler/cgen.nim | 37 ++++++++++++++++++++----------------- compiler/commands.nim | 1 + compiler/options.nim | 1 + doc/advopt.txt | 2 ++ doc/backends.rst | 3 +++ doc/nimc.rst | 7 +++++++ 7 files changed, 36 insertions(+), 17 deletions(-) diff --git a/changelog.md b/changelog.md index ebad54fe4..df8051a16 100644 --- a/changelog.md +++ b/changelog.md @@ -71,3 +71,5 @@ - The `gc` switch has been renamed to `mm` ("memory management") in order to reflect the reality better. (Nim moved away from all techniques based on "tracing".) +- There is a new switch `--nimMainPrefix:prefix` to influence the `NimMain` that the + compiler produces. This is particularly useful for generating static libraries. diff --git a/compiler/cgen.nim b/compiler/cgen.nim index c862d4d30..032c22ec0 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -154,6 +154,11 @@ macro ropecg(m: BModule, frmt: static[FormatStr], args: untyped): Rope = inc(i) result.add newCall(formatValue, resVar, args[num]) inc(num) + of '^': + flushStrLit() + inc(i) + result.add newCall(formatValue, resVar, args[^1]) + inc(num) of '0'..'9': var j = 0 while true: @@ -1366,7 +1371,7 @@ proc genMainProc(m: BModule) = "}$N$N" MainProcs = - "\tNimMain();$N" + "\t$^NimMain();$N" MainProcsWithResult = MainProcs & ("\treturn $1nim_program_result;$N") @@ -1376,7 +1381,7 @@ proc genMainProc(m: BModule) = "}$N$N" NimMainProc = - "N_CDECL(void, NimMain)(void) {$N" & + "N_CDECL(void, $5NimMain)(void) {$N" & "\tvoid (*volatile inner)(void);$N" & "$4" & "\tinner = NimMainInner;$N" & @@ -1456,28 +1461,27 @@ proc genMainProc(m: BModule) = if optGenGuiApp in m.config.globalOptions: const nimMain = WinNimMain appcg(m, m.s[cfsProcs], nimMain, - [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode]) + [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix]) else: const nimMain = WinNimDllMain appcg(m, m.s[cfsProcs], nimMain, - [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode]) + [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix]) elif m.config.target.targetOS == osGenode: const nimMain = GenodeNimMain appcg(m, m.s[cfsProcs], nimMain, - [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode]) + [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix]) elif optGenDynLib in m.config.globalOptions: const nimMain = PosixNimDllMain appcg(m, m.s[cfsProcs], nimMain, - [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode]) + [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix]) elif m.config.target.targetOS == osStandalone: const nimMain = NimMainBody appcg(m, m.s[cfsProcs], nimMain, - [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode]) + [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix]) else: const nimMain = NimMainBody appcg(m, m.s[cfsProcs], nimMain, - [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode]) - + [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix]) if optNoMain notin m.config.globalOptions: if m.config.cppCustomNamespace.len > 0: @@ -1487,23 +1491,22 @@ proc genMainProc(m: BModule) = m.config.globalOptions * {optGenGuiApp, optGenDynLib} != {}: if optGenGuiApp in m.config.globalOptions: const otherMain = WinCMain - appcg(m, m.s[cfsProcs], otherMain, [if m.hcrOn: "*" else: ""]) + appcg(m, m.s[cfsProcs], otherMain, [if m.hcrOn: "*" else: "", m.config.nimMainPrefix]) else: const otherMain = WinCDllMain - appcg(m, m.s[cfsProcs], otherMain, []) + appcg(m, m.s[cfsProcs], otherMain, [m.config.nimMainPrefix]) elif m.config.target.targetOS == osGenode: const otherMain = ComponentConstruct - appcg(m, m.s[cfsProcs], otherMain, []) + appcg(m, m.s[cfsProcs], otherMain, [m.config.nimMainPrefix]) elif optGenDynLib in m.config.globalOptions: const otherMain = PosixCDllMain - appcg(m, m.s[cfsProcs], otherMain, []) + appcg(m, m.s[cfsProcs], otherMain, [m.config.nimMainPrefix]) elif m.config.target.targetOS == osStandalone: const otherMain = StandaloneCMain - appcg(m, m.s[cfsProcs], otherMain, []) + appcg(m, m.s[cfsProcs], otherMain, [m.config.nimMainPrefix]) else: const otherMain = PosixCMain - appcg(m, m.s[cfsProcs], otherMain, [if m.hcrOn: "*" else: ""]) - + appcg(m, m.s[cfsProcs], otherMain, [if m.hcrOn: "*" else: "", m.config.nimMainPrefix]) if m.config.cppCustomNamespace.len > 0: m.s[cfsProcs].add openNamespaceNim(m.config.cppCustomNamespace) @@ -1885,7 +1888,7 @@ proc writeHeader(m: BModule) = if optGenDynLib in m.config.globalOptions: result.add("N_LIB_IMPORT ") - result.addf("N_CDECL(void, NimMain)(void);$n", []) + result.addf("N_CDECL(void, $1NimMain)(void);$n", [rope m.config.nimMainPrefix]) if m.config.cppCustomNamespace.len > 0: result.add closeNamespaceNim() result.addf("#endif /* $1 */$n", [guard]) if not writeRope(result, m.filename): diff --git a/compiler/commands.nim b/compiler/commands.nim index d5c5f24e4..c4df46bc2 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -1052,6 +1052,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; of "": # comes from "-" in for example: `nim c -r -` (gets stripped from -) handleStdinInput(conf) of "nilseqs", "nilchecks", "mainmodule", "m", "symbol", "taintmode", "cs", "deadcodeelim": warningOptionNoop(switch) + of "nimmainprefix": conf.nimMainPrefix = arg else: if strutils.find(switch, '.') >= 0: options.setConfigVar(conf, switch, arg) else: invalidCmdLineOption(conf, pass, switch, info) diff --git a/compiler/options.nim b/compiler/options.nim index 383b6fa2a..9e957332f 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -390,6 +390,7 @@ type structuredErrorHook*: proc (config: ConfigRef; info: TLineInfo; msg: string; severity: Severity) {.closure, gcsafe.} cppCustomNamespace*: string + nimMainPrefix*: string vmProfileData*: ProfileData proc parseNimVersion*(a: string): NimVer = diff --git a/doc/advopt.txt b/doc/advopt.txt index 79e784fde..e27c75ada 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -135,6 +135,8 @@ Advanced options: --cppCompileToNamespace:namespace use the provided namespace for the generated C++ code, if no namespace is provided "Nim" will be used + --nimMainPrefix:prefix use `{prefix}NimMain` instead of `NimMain` in the produced + C/C++ code --expandMacro:MACRO dump every generated AST from MACRO --expandArc:PROCNAME show how PROCNAME looks like after diverse optimizations before the final backend phase (mostly ARC/ORC specific) diff --git a/doc/backends.rst b/doc/backends.rst index 3a3359fca..65dd8a6f2 100644 --- a/doc/backends.rst +++ b/doc/backends.rst @@ -246,6 +246,9 @@ Also, C code requires you to specify a forward declaration for functions or the compiler will assume certain types for the return value and parameters which will likely make your program crash at runtime. +The name `NimMain` can be influenced via the `--nimMainPrefix:prefix` switch. +Use `--nimMainPrefix:MyLib` and the function to call is named `MyLibNimMain`. + Nim invocation example from C ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/nimc.rst b/doc/nimc.rst index 0ab1501ff..6b4f3c9a1 100644 --- a/doc/nimc.rst +++ b/doc/nimc.rst @@ -385,6 +385,10 @@ of your program. NimMain() # initialize garbage collector memory, types and stack +The name `NimMain` can be influenced via the `--nimMainPrefix:prefix` switch. +Use `--nimMainPrefix:MyLib` and the function to call is named `MyLibNimMain`. + + Cross-compilation for iOS ========================= @@ -413,6 +417,9 @@ of your program. Note: XCode's "make clean" gets confused about the generated nim.c files, so you need to clean those files manually to do a clean build. +The name `NimMain` can be influenced via the `--nimMainPrefix:prefix` switch. +Use `--nimMainPrefix:MyLib` and the function to call is named `MyLibNimMain`. + Cross-compilation for Nintendo Switch ===================================== From 908fc2a22e7336670173bcd06ab30f440ca6d321 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 10 Dec 2021 17:13:48 +0100 Subject: [PATCH 1405/2472] let Nim support Nimble 0.14 with lock-file support [backport:1.6] (#19236) --- changelog.md | 4 ++++ config/nim.cfg | 2 ++ 2 files changed, 6 insertions(+) diff --git a/changelog.md b/changelog.md index df8051a16..6f47ef8a4 100644 --- a/changelog.md +++ b/changelog.md @@ -71,5 +71,9 @@ - The `gc` switch has been renamed to `mm` ("memory management") in order to reflect the reality better. (Nim moved away from all techniques based on "tracing".) +- Nim now supports Nimble version 0.14 which added support for lock-files. This is done by + a simple configuration change setting that you can do yourself too. In `$nim/config/nim.cfg` + replace `pkgs` by `pkgs2`. + - There is a new switch `--nimMainPrefix:prefix` to influence the `NimMain` that the compiler produces. This is particularly useful for generating static libraries. diff --git a/config/nim.cfg b/config/nim.cfg index 86c6e2141..59d7f48d2 100644 --- a/config/nim.cfg +++ b/config/nim.cfg @@ -44,10 +44,12 @@ path="$lib/core" path="$lib/pure" @if not windows: + nimblepath="/opt/nimble/pkgs2/" nimblepath="/opt/nimble/pkgs/" @else: # TODO: @end +nimblepath="$home/.nimble/pkgs2/" nimblepath="$home/.nimble/pkgs/" # Syncronize with compiler/commands.specialDefine From 1a92edeb894a10a13f60905a79304bd26e1c144e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Sat, 11 Dec 2021 05:15:44 +0100 Subject: [PATCH 1406/2472] nimc.rst: fix table markup (#19239) --- doc/nimc.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/nimc.rst b/doc/nimc.rst index 6b4f3c9a1..aa6650491 100644 --- a/doc/nimc.rst +++ b/doc/nimc.rst @@ -179,6 +179,8 @@ nimRawSetjmp Use `_setjmp()/_longjmp()` on POSIX and `_setjm nimBuiltinSetjmp Use `__builtin_setjmp()/__builtin_longjmp()` for setjmp-based exceptions. This will not work if an exception is being thrown and caught inside the same procedure. Useful for benchmarking. +========================== ============================================ + Configuration files ------------------- From a3ef5df680e55d9bf68027fcb0ec6358b4279d09 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sat, 11 Dec 2021 08:24:23 +0000 Subject: [PATCH 1407/2472] Update uri.nim (#19148) [backport:1.0] --- lib/pure/uri.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/pure/uri.nim b/lib/pure/uri.nim index a828298c2..5de333b8f 100644 --- a/lib/pure/uri.nim +++ b/lib/pure/uri.nim @@ -216,6 +216,8 @@ func parseAuthority(authority: string, result: var Uri) = result.isIpv6 = true of ']': inIPv6 = false + of '\0': + break else: if inPort: result.port.add(authority[i]) From 4b5cecd902cc4126ff9d6cda9edb78a13a421239 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Sun, 12 Dec 2021 13:39:56 -0800 Subject: [PATCH 1408/2472] Various std net improvements (#19132) * Variant of that works with raw IpAddresses. - Add doc tests for new net proc's. - Aadd recvFrom impl - Add recvFrom impl -- tweak handling data var - Update lib/pure/net.nim Co-authored-by: Dominik Picheta - cleaning up sendTo args - remove extra connect test - cleaning up sendTo args - fix inet_ntop test - fix test failing - byte len * fix test failing - byte len * debugging odd windows build failure * debugging odd windows build failure * more experiments to figure out the windows failure * try manual assigment on InAddr Co-authored-by: Jaremy Creechley --- lib/posix/posix.nim | 4 +- lib/pure/net.nim | 61 +++++++++++++++++++++++----- tests/stdlib/tnet_ll.nim | 88 ++++++++++++++++++++-------------------- 3 files changed, 97 insertions(+), 56 deletions(-) diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index c2504f994..57dd6e2c7 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -155,9 +155,9 @@ when not defined(zephyr): proc inet_addr*(a1: cstring): InAddrT {.importc, header: "".} proc inet_ntoa*(a1: InAddr): cstring {.importc, header: "".} -proc inet_ntop*(a1: cint, a2: pointer, a3: cstring, a4: int32): cstring {. +proc inet_ntop*(a1: cint, a2: pointer | ptr InAddr | ptr In6Addr, a3: cstring, a4: int32): cstring {. importc:"(char *)$1", header: "".} -proc inet_pton*(a1: cint, a2: cstring, a3: pointer): cint {. +proc inet_pton*(a1: cint, a2: cstring, a3: pointer | ptr InAddr | ptr In6Addr): cint {. importc, header: "".} var diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 7b0ff78e7..2d1bb0b33 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -65,6 +65,11 @@ runnableExamples("-r:off"): let socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) socket.sendTo("192.168.0.1", Port(27960), "status\n") +runnableExamples("-r:off"): + let socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) + let ip = parseIpAddress("192.168.0.1") + doAssert socket.sendTo(ip, Port(27960), "status\c\l") == 8 + ## Creating a server ## ----------------- ## @@ -1607,11 +1612,12 @@ proc recvLine*(socket: Socket, timeout = -1, result = "" readLine(socket, result, timeout, flags, maxLength) -proc recvFrom*(socket: Socket, data: var string, length: int, - address: var string, port: var Port, flags = 0'i32): int {. +proc recvFrom*[T: string | IpAddress](socket: Socket, data: var string, length: int, + address: var T, port: var Port, flags = 0'i32): int {. tags: [ReadIOEffect].} = ## Receives data from `socket`. This function should normally be used with - ## connection-less sockets (UDP sockets). + ## connection-less sockets (UDP sockets). The source address of the data + ## packet is stored in the `address` argument as either a string or an IpAddress. ## ## If an error occurs an OSError exception will be raised. Otherwise the return ## value will be the length of data received. @@ -1620,31 +1626,37 @@ proc recvFrom*(socket: Socket, data: var string, length: int, ## so when `socket` is buffered the non-buffered implementation will be ## used. Therefore if `socket` contains something in its buffer this ## function will make no effort to return it. - template adaptRecvFromToDomain(domain: Domain) = + template adaptRecvFromToDomain(sockAddress: untyped, domain: Domain) = var addrLen = sizeof(sockAddress).SockLen result = recvfrom(socket.fd, cstring(data), length.cint, flags.cint, cast[ptr SockAddr](addr(sockAddress)), addr(addrLen)) if result != -1: data.setLen(result) - address = getAddrString(cast[ptr SockAddr](addr(sockAddress))) - when domain == AF_INET6: - port = ntohs(sockAddress.sin6_port).Port + + when typeof(address) is string: + address = getAddrString(cast[ptr SockAddr](addr(sockAddress))) + when domain == AF_INET6: + port = ntohs(sockAddress.sin6_port).Port + else: + port = ntohs(sockAddress.sin_port).Port else: - port = ntohs(sockAddress.sin_port).Port + data.setLen(result) + sockAddress.fromSockAddr(addrLen, address, port) else: raiseOSError(osLastError()) assert(socket.protocol != IPPROTO_TCP, "Cannot `recvFrom` on a TCP socket") # TODO: Buffered sockets data.setLen(length) + case socket.domain of AF_INET6: var sockAddress: Sockaddr_in6 - adaptRecvFromToDomain(AF_INET6) + adaptRecvFromToDomain(sockAddress, AF_INET6) of AF_INET: var sockAddress: Sockaddr_in - adaptRecvFromToDomain(AF_INET) + adaptRecvFromToDomain(sockAddress, AF_INET) else: raise newException(ValueError, "Unknown socket address family") @@ -1707,7 +1719,8 @@ proc sendTo*(socket: Socket, address: string, port: Port, data: pointer, tags: [WriteIOEffect].} = ## This proc sends `data` to the specified `address`, ## which may be an IP address or a hostname, if a hostname is specified - ## this function will try each IP of that hostname. + ## this function will try each IP of that hostname. This function + ## should normally be used with connection-less sockets (UDP sockets). ## ## If an error occurs an OSError exception will be raised. ## @@ -1741,12 +1754,38 @@ proc sendTo*(socket: Socket, address: string, port: Port, ## This proc sends `data` to the specified `address`, ## which may be an IP address or a hostname, if a hostname is specified ## this function will try each IP of that hostname. + ## + ## Generally for use with connection-less (UDP) sockets. ## ## If an error occurs an OSError exception will be raised. ## ## This is the high-level version of the above `sendTo` function. socket.sendTo(address, port, cstring(data), data.len, socket.domain) +proc sendTo*(socket: Socket, address: IpAddress, port: Port, + data: string, flags = 0'i32): int {. + discardable, tags: [WriteIOEffect].} = + ## This proc sends `data` to the specified `IpAddress` and returns + ## the number of bytes written. + ## + ## Generally for use with connection-less (UDP) sockets. + ## + ## If an error occurs an OSError exception will be raised. + ## + ## This is the high-level version of the above `sendTo` function. + assert(socket.protocol != IPPROTO_TCP, "Cannot `sendTo` on a TCP socket") + assert(not socket.isClosed, "Cannot `sendTo` on a closed socket") + + var sa: Sockaddr_storage + var sl: Socklen + toSockAddr(address, port, sa, sl) + result = sendto(socket.fd, cstring(data), data.len().cint, flags.cint, + cast[ptr SockAddr](addr sa), sl) + + if result == -1'i32: + let osError = osLastError() + raiseOSError(osError) + proc isSsl*(socket: Socket): bool = ## Determines whether `socket` is a SSL socket. diff --git a/tests/stdlib/tnet_ll.nim b/tests/stdlib/tnet_ll.nim index affa21947..248dbb79b 100644 --- a/tests/stdlib/tnet_ll.nim +++ b/tests/stdlib/tnet_ll.nim @@ -1,43 +1,45 @@ -discard """ - action: run - output: ''' - -[Suite] inet_ntop tests -''' -""" - -when defined(windows): - import winlean -elif defined(posix): - import posix -else: - {.error: "Unsupported OS".} - -import unittest, strutils - -suite "inet_ntop tests": - - setup: - when defined(windows): - var wsa: WSAData - discard wsaStartup(0x101'i16, wsa.addr) - - test "IP V4": - var ip4 = 0x10111213 - var buff: array[0..255, char] - let r = inet_ntop(AF_INET, ip4.addr, buff[0].addr, buff.sizeof.int32) - let res = if r == nil: "" else: $r - check: res == "19.18.17.16" - - - test "IP V6": - when defined(windows): - let ipv6Support = (getVersion() and 0xff) > 0x5 - else: - let ipv6Support = true - - var ip6 = [0x1000'u16, 0x1001, 0x2000, 0x2001, 0x3000, 0x3001, 0x4000, 0x4001] - var buff: array[0..255, char] - let r = inet_ntop(AF_INET6, ip6[0].addr, buff[0].addr, buff.sizeof.int32) - let res = if r == nil: "" else: $r - check: not ipv6Support or res == "10:110:20:120:30:130:40:140" +discard """ + action: run + output: ''' + +[Suite] inet_ntop tests +''' +""" + +when defined(windows): + import winlean +elif defined(posix): + import posix +else: + {.error: "Unsupported OS".} + +import unittest, strutils + +suite "inet_ntop tests": + + setup: + when defined(windows): + var wsa: WSAData + discard wsaStartup(0x101'i16, wsa.addr) + + test "IP V4": + # regular + var ip4 = InAddr() + ip4.s_addr = 0x10111213'u32 + + var buff: array[0..255, char] + let r = inet_ntop(AF_INET, cast[pointer](ip4.s_addr.addr), buff[0].addr, buff.len.int32) + let res = if r == nil: "" else: $r + check: res == "19.18.17.16" + + test "IP V6": + when defined(windows): + let ipv6Support = (getVersion() and 0xff) > 0x5 + else: + let ipv6Support = true + + var ip6 = [0x1000'u16, 0x1001, 0x2000, 0x2001, 0x3000, 0x3001, 0x4000, 0x4001] + var buff: array[0..255, char] + let r = inet_ntop(AF_INET6, cast[pointer](ip6[0].addr), buff[0].addr, buff.len.int32) + let res = if r == nil: "" else: $r + check: not ipv6Support or res == "10:110:20:120:30:130:40:140" From 0ff4b2ba7ee74ed2758b0eb5992e4ccab7433952 Mon Sep 17 00:00:00 2001 From: Carlo Capocasa Date: Mon, 13 Dec 2021 07:29:22 +0100 Subject: [PATCH 1409/2472] fix bug #14468 zero-width split (#19248) --- lib/impure/re.nim | 11 +++++++---- tests/stdlib/tre.nim | 6 ++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/lib/impure/re.nim b/lib/impure/re.nim index 8e5efbb28..4eacf0091 100644 --- a/lib/impure/re.nim +++ b/lib/impure/re.nim @@ -556,19 +556,22 @@ iterator split*(s: string, sep: Regex; maxsplit = -1): string = @["", "this", "is", "an", "example", ""] var last = 0 var splits = maxsplit - var x: int + var x = -1 + if len(s) == 0: + last = 1 + if matchLen(s, sep, 0) == 0: + x = 0 while last <= len(s): var first = last var sepLen = 1 + if x == 0: + inc(last) while last < len(s): x = matchLen(s, sep, last) if x >= 0: sepLen = x break inc(last) - if x == 0: - if last >= len(s): break - inc last if splits == 0: last = len(s) yield substr(s, first, last-1) if splits == 0: break diff --git a/tests/stdlib/tre.nim b/tests/stdlib/tre.nim index 2857c6c9e..9f27f7db2 100644 --- a/tests/stdlib/tre.nim +++ b/tests/stdlib/tre.nim @@ -108,4 +108,10 @@ proc testAll() = doAssert replace("foo", re"", "-") == "-f-o-o-" doAssert replace("ooo", re"o", "-") == "---" + block: # bug #14468 + accum = @[] + for word in split("this is an example", re"\b"): + accum.add(word) + doAssert(accum == @["this", " ", "is", " ", "an", " ", "example"]) + testAll() From 78b86b79425f37683ce522dbdd567f52ea26aa35 Mon Sep 17 00:00:00 2001 From: Nan Xiao Date: Tue, 14 Dec 2021 16:54:01 +0800 Subject: [PATCH 1410/2472] basicopt.txt: Unify the format (#19251) --- doc/basicopt.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/basicopt.txt b/doc/basicopt.txt index a2db1c92e..390fc6fd2 100644 --- a/doc/basicopt.txt +++ b/doc/basicopt.txt @@ -27,11 +27,11 @@ Options: -a, --assertions:on|off turn assertions on|off --opt:none|speed|size optimize not at all or for speed|size Note: use -d:release for a release build! - --debugger:native Use native debugger (gdb) + --debugger:native use native debugger (gdb) --app:console|gui|lib|staticlib generate a console app|GUI app|DLL|static library -r, --run run the compiled program with given arguments - --eval:cmd evaluates nim code directly; e.g.: `nim --eval:"echo 1"` + --eval:cmd evaluate nim code directly; e.g.: `nim --eval:"echo 1"` defaults to `e` (nimscript) but customizable: `nim r --eval:'for a in stdin.lines: echo a'` --fullhelp show all command line switches From c55930f2e644fe04526eb4878e7e106229812fe4 Mon Sep 17 00:00:00 2001 From: Nick Wilburn Date: Tue, 14 Dec 2021 06:22:10 -0600 Subject: [PATCH 1411/2472] fix: fixes bug in CVerifyPeerUseEnvVars (#19247) Previously CVerifyPeerUseEnvVars was not being passed into scanSslCertificates, which meant that we weren't scanning additional certificate locations given via the SSL_CERT_FILE and SSL_CERT_DIR environment variables --- lib/pure/net.nim | 3 ++- tests/stdlib/thttpclient_ssl.nim | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 2d1bb0b33..ced6b2fb2 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -680,7 +680,8 @@ when defineSsl: # Scan for certs in known locations. For CVerifyPeerUseEnvVars also scan # the SSL_CERT_FILE and SSL_CERT_DIR env vars var found = false - for fn in scanSSLCertificates(): + let useEnvVars = (if verifyMode == CVerifyPeerUseEnvVars: true else: false) + for fn in scanSSLCertificates(useEnvVars = useEnvVars): if newCTX.SSL_CTX_load_verify_locations(fn, nil) == VerifySuccess: found = true break diff --git a/tests/stdlib/thttpclient_ssl.nim b/tests/stdlib/thttpclient_ssl.nim index 1c531eae9..3acdacfe3 100644 --- a/tests/stdlib/thttpclient_ssl.nim +++ b/tests/stdlib/thttpclient_ssl.nim @@ -129,3 +129,19 @@ when not defined(windows): msg.contains("certificate verify failed")): echo "CVerifyPeer exception: " & msg check(false) + + test "HttpClient with CVerifyPeerUseEnvVars": + const port = 12346.Port + let t = spawn runServer(port) + sleep(100) + + putEnv("SSL_CERT_FILE", getCurrentDir() / certFile) + var client = newHttpClient(sslContext=newContext(verifyMode=CVerifyPeerUseEnvVars)) + try: + log "client: connect" + discard client.getContent("https://127.0.0.1:12346") + except: + let msg = getCurrentExceptionMsg() + log "client: exception: " & msg + log "getContent should not have raised an exception" + fail() From 3e31f55aed4cd9fdcc289687fd0e31edfc0af805 Mon Sep 17 00:00:00 2001 From: PMunch Date: Tue, 14 Dec 2021 14:28:42 +0100 Subject: [PATCH 1412/2472] Fix #19253 (#19254) This fixes 19253 by marking the bodyStream as completed when no content was sent. --- lib/pure/httpclient.nim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 33da6ef6c..67ba8cf2b 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -869,6 +869,9 @@ proc parseResponse(client: HttpClient | AsyncHttpClient, client.parseBodyFut.addCallback do(): if client.parseBodyFut.failed: client.bodyStream.fail(client.parseBodyFut.error) + else: + when client is AsyncHttpClient: + result.bodyStream.complete() proc newConnection(client: HttpClient | AsyncHttpClient, url: Uri) {.multisync.} = From c17baaefbcff5c207a4e95242fa0790e64ca6c8c Mon Sep 17 00:00:00 2001 From: Miran Date: Tue, 14 Dec 2021 18:16:49 +0100 Subject: [PATCH 1413/2472] [backport:1.0] json: limit recursion depth (#19252) * json: limit recursion depth * do not run this check for JS backend --- lib/pure/json.nim | 12 +++++++++--- tests/stdlib/tjson.nim | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index c831bf85d..922cd4e2f 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -202,6 +202,8 @@ type of JArray: elems*: seq[JsonNode] +const DepthLimit = 1000 + proc newJString*(s: string): JsonNode = ## Creates a new `JString JsonNode`. result = JsonNode(kind: JString, str: s) @@ -849,7 +851,7 @@ iterator mpairs*(node: var JsonNode): tuple[key: string, val: var JsonNode] = for key, val in mpairs(node.fields): yield (key, val) -proc parseJson(p: var JsonParser; rawIntegers, rawFloats: bool): JsonNode = +proc parseJson(p: var JsonParser; rawIntegers, rawFloats: bool, depth = 0): JsonNode = ## Parses JSON from a JSON Parser `p`. case p.tok of tkString: @@ -885,6 +887,8 @@ proc parseJson(p: var JsonParser; rawIntegers, rawFloats: bool): JsonNode = result = newJNull() discard getTok(p) of tkCurlyLe: + if depth > DepthLimit: + raiseParseErr(p, "}") result = newJObject() discard getTok(p) while p.tok != tkCurlyRi: @@ -893,16 +897,18 @@ proc parseJson(p: var JsonParser; rawIntegers, rawFloats: bool): JsonNode = var key = p.a discard getTok(p) eat(p, tkColon) - var val = parseJson(p, rawIntegers, rawFloats) + var val = parseJson(p, rawIntegers, rawFloats, depth+1) result[key] = val if p.tok != tkComma: break discard getTok(p) eat(p, tkCurlyRi) of tkBracketLe: + if depth > DepthLimit: + raiseParseErr(p, "]") result = newJArray() discard getTok(p) while p.tok != tkBracketRi: - result.add(parseJson(p, rawIntegers, rawFloats)) + result.add(parseJson(p, rawIntegers, rawFloats, depth+1)) if p.tok != tkComma: break discard getTok(p) eat(p, tkBracketRi) diff --git a/tests/stdlib/tjson.nim b/tests/stdlib/tjson.nim index 289ef9d05..336558ff3 100644 --- a/tests/stdlib/tjson.nim +++ b/tests/stdlib/tjson.nim @@ -345,3 +345,35 @@ block: doAssert c == "18446744073709552000" else: doAssert c == "18446744073709551615" + +block: + let a = """ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ +""" + + when not defined(js): + try: + discard parseJson(a) + except JsonParsingError: + doAssert getCurrentExceptionMsg().contains("] expected") From 5d2bab7558b54813b212a26fbcdb51556ff37b05 Mon Sep 17 00:00:00 2001 From: Sven Keller Date: Thu, 16 Dec 2021 08:58:32 +0100 Subject: [PATCH 1414/2472] suggestion to respect typedarray type (#19257) * suggestion to respect typedarray * Update jssys.nim Co-authored-by: Sven Keller --- lib/system/jssys.nim | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/lib/system/jssys.nim b/lib/system/jssys.nim index 250bd069d..6608a2927 100644 --- a/lib/system/jssys.nim +++ b/lib/system/jssys.nim @@ -587,7 +587,33 @@ proc nimCopy(dest, src: JSRef, ti: PNimType): JSRef = else: asm "`result` = (`dest` === null || `dest` === undefined) ? {} : `dest`;" nimCopyAux(result, src, ti.node) - of tySequence, tyArrayConstr, tyOpenArray, tyArray: + of tyArrayConstr, tyArray: + # In order to prevent a type change (TypedArray -> Array) and to have better copying performance, + # arrays constructors are considered separately + asm """ + if(ArrayBuffer.isView(`src`)) { + if(`dest` === null || `dest` === undefined || `dest`.length != `src`.length) { + `dest` = new `src`.constructor(`src`); + } else { + `dest`.set(`src`, 0); + } + `result` = `dest`; + } else { + if (`src` === null) { + `result` = null; + } + else { + if (`dest` === null || `dest` === undefined || `dest`.length != `src`.length) { + `dest` = new Array(`src`.length); + } + `result` = `dest`; + for (var i = 0; i < `src`.length; ++i) { + `result`[i] = nimCopy(`result`[i], `src`[i], `ti`.base); + } + } + } + """ + of tySequence, tyOpenArray: asm """ if (`src` === null) { `result` = null; From 609388e33fb9a19d65b5dee8711d411916981802 Mon Sep 17 00:00:00 2001 From: snxx Date: Fri, 17 Dec 2021 13:58:05 +0600 Subject: [PATCH 1415/2472] Add icon for Nim (#19258) * Add Nim's file icon * Add installer icon * Add uninstaller icon --- tools/niminst/nim-file.ico | Bin 0 -> 3311 bytes tools/niminst/setup.ico | Bin 0 -> 2551 bytes tools/niminst/uninstall.ico | Bin 0 -> 2226 bytes 3 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 tools/niminst/nim-file.ico create mode 100644 tools/niminst/setup.ico create mode 100644 tools/niminst/uninstall.ico diff --git a/tools/niminst/nim-file.ico b/tools/niminst/nim-file.ico new file mode 100644 index 0000000000000000000000000000000000000000..0685fedcb0b7851d874bbe9606912b243e712e68 GIT binary patch literal 3311 zcmcImc{r47AAV=dVJZ$K8ctcWv{wMn{(~UoFwoaF+h5(Ea8~eT?Oosm0PHyi+L~7b#)y-_?!!MI9h;na7f`h} zXFk}JSzA3vZA#{%w7YC|(+9um$fjy{M#buOB%}P^+uxT{Ug&jKK!`AQP*HcO$LWs*MNm@IPit%3;KnR(&oYeK@_I zSh=1>ik^&tLGXN2Er&&)q$j97f^z{}5D_GRg#9^fTj56fJ=Xm$V4MOJJ9{EJ>l952 z3tBRglp{ME8tL1hKslps`^bHLZTz1k2+%9Hp3wP`Cp~{e)7m9?C|N9^OCBDc6J5vR zK~BV!x;7|ibOHheL~Lik=$3ZSIX+Sp3E-_n6B2^j&k8D|_pz+%sB7xRaUlSCte5;j z6b8t8fy)0D!_|d4L{*21>MqF}(D>%hm$L#?%^gICwrKDocpZ#ReTSeO4d z2(g9#g@!CF9Nhml>R?p}OwEY+3=gnB9^@FAL{HCTDH-guS>Iyjm@}Sx)+(8$g<98X zsN`)Mr%a@A`%xW^LWBTVPK|1r;iLMVw~0?HPZsuPNW*MN<=nEcz8*546B#wA5;%k> z+QJ~RRXYv$8Xa4@$V*gR9$xF@zA^iEi@JbkhxM(HC$|ffhLiTLoxiS*ySo^bdGUB8 z0FkAh#O$RNX?Ao_C@bc5O?6X&Yi+LV;#CjVM=AUZj`XXRp2npLv+3uak&WjW2$ZRhOjr|2RixqFRg_i$slK! zTJ$cZR&xg#t&QY;VPP`qr_lWTuI^Cw@|hzU4a$hE(zT!M-9lUd){UgI6n^A47O!aV zv}5J*0K4L7O@yXbq;da(0(;lDK=VZ@GD8H4UL-~i5l%YWkJ~VwPbj~ilK@@~H1WzN zJlbhc7V)Jzq%^5I4-6!m)&+@qBdhZ-V6fAMl%5|xdw9VT65;XXb8b4Y7NaPX`_r`f z)y@dwOwTc2Fztfr6r77?`rBl`35>l*pd2feYU%XO8|RZ)P^0zt&);pyWc{yL~qxG~pw& z!lmze=F8WV)SRyDdbRS|jiLOHqzEBk!HjO4Yp9gxuWj3*?xqJDli0H*eiJlYAJdX+Arakc=^FnZu-H z!Bbmm{;kgrsU+a-OqUPbq;ag z4%{0@t)?`u1m-qA06$z*jCYbw2lGO$$=<7A8ODd65UBu>kk8^F7Fz=30`jVEUQDGV z*1k$^Fff!lGe9?gy|cguMFMc7^M^hCA*?($_UEwb)lAZI*WxF{k7e++TAtL!uyL92(JM*mdVd<5i3=-OyiP$htOvEZHTx*xEkz9(Uz_$E?7#*bdqVpU_6w0riY?K$P8bD>cD z`?Thj?Kcw=m%Uo^*(%Qbz5@gNUCk4%`k9*z{6T%fdzQZi^)ap_TAAEoq|0*w6VETQ zt^Vk|VlLc>$(eaz60c^?L*io-0uVxJuXpb)8z z`YVQmHv-JCqGCO{BYrq;W3yuX{j;A$3pNoouX70yqh~6HVBMbCR`$B&tuwtfsEV8P z%i<2a@1-aNK`;?&N@vjW0+0FLD25L?hkRc4;5Wb1`(>5$7hwRriwF8#Ny+PGAKOh0 z`J7F?2XBIP9+|7E9@cqtcH`?ET0n+5-7i<#yk_#F*)y=YYsrC3O9Hm`u>WTx@XuZY zh8O`IcDL58x& zNTR4C`wRl7#+CN2(EO_Zn{uXOOjrS>UGa;2KcUA zbwo=3gM8mkXn@_}a0t{^*KKMl=bi}*3y8(DXyj(Plig?Bb#k)yXXxotG0E` z@e{Lc8=7tGzkV6DPUk~&rYnNoC;UGe_%DdNS!bQS&N+MU@B7X=>l^?8aoT}_09jy$ z9{`SX{6uR@xZq*Q!yH!-VQPHsVD+H$9pVhmA*Egbz+Z|m{>?6Y?AxS^|C6m>+Gx`k z_0L^%wWw{2@4XsdYJSV+h@$!*q5PLrJra+JXdltOo#4-ZvpNRiMiM}Z2e!XG=db85 zprWoIkF+aS6_@z4+_23BHr>&jViv%~&Mfx-U?3XIdApF>y2kux{u6 zSvars>y=p}Qb5EzQzKZWYO88(U9U7Cs&*jm{|ih4a`EybSKIM*nc9`xr*4YpQv3uP zfJeuRCL`eA8ZIDeK&nPZL4S$(f0X5mUPuBm)O;~q@oFREs4royN9=ulaXM~Ml5Q`e zt7S}93p5?qNE;V+!Q-W!K}gwGZ+WTYKJfyDoe`7fXPoVif9VW65=NN6kkKlv!!2)N31nU4}y_GNYOB_%GCZ&*2UsyEwBp+wCmJ^VKN-NH_vh2Ppb2-y_Mj_%SY<#i z$fkB7v*QJMNIvAo)|Q}+EgI=K{!kS!>PCzRlTMw8M@ONT#aqL7dPQvI-Pe4kFFXKH z60wn&qOF-%i-1bc|^kzKuITh7&e=u#AiWro#r0*K&aGN5yIcy69>CAFla|R%G9wZFP za05)uxW~}l=F!}Fj+$8kQZ;-WVH8^d>lDcO9iAAu)HeLbX?*i=n*iXOWC&+?WypY( z?Ug$956=A{O5!B`Bb!htFB8iF{GUJ{m2u$`v$gZ}V8betHr{)(z+Fkss-SqNWYxzs z;A@q@&e_<^>M%tZm{aC}D{12Xp)}W=qd5x@i5__J#S->xJsP3aqNa*9AXo#3HNaV< z=>U3K?1?A%Y#aPED2UG*fGnXvY2+;+p; zh*4GX)^C`tuUK}G&TfP?KyT-(NVM&V^n!KvfNWtKj{8e9b4oq^$VYD_Cqbjbj$A+Q z+M=hEiJx8&^KrK0l+7dN9UVU)ndMYyMx}{V5}8@`Ay=2BYevo)PHr}nN3bdibG@nB+nz|wb)16u*-POZ7(FTadnF4) z>z0a$W9rKN76hBHDc-887OgOB>+P|WvMmU0+YpASay``K8GTuA|3f1LLR%hJFQ5(3 zvwy$gXa3SBnNnlf9#6|%m{_yBsq%2hA`yo}g=J9f{Cp$#G8;2G$LVQertoM;lOqdZFtxuG6saLOIfUh2&s2Ce>o@^QfIPZ^7(p zcWjM!Zc0QT`%8{M;!U=e(iU}AI3}MX?)BgeQxoIwdQ=!~HI+uSNLE>_-Qy^V8jzXq zVn#P87;G=H*Oj(AHnrPdFhXH_@KLTjTzQ0sI3XH?(u-V=<<#!GI1OkxbL!DHGU__H zFfxgq#k9wbU;Tnd(#@M*7n~VFaYV=uexsG>jdhjA-digE&1#=#vpRSo{hd|DE8ThH zwe-r4vFmR~Z{)6hR0TwzxC~9IvFPUh(G42Md%BP6(6nJLk_6(XgH7SIx+1Owfi$)qzOSAs|(PiH0fs|pft*#7L5$qg2-7&~z z9b;wHswd*-Fy}r;S0MSXB-Jq(&%R((mMDR{bNb@}l+1l^EGTbrwS0}=(72u4L2+!* zfeI28Y|zSyOHc3yoMY~bM{a$>xeU3e`5Gy<+(m0CM&D|#oDU+OISmF**`P~xB@GaL z2DU`}pqaSPx5yWXa}Z{9$=4iqmTLb#WEfC3>i2&eF7jqR=$Ph%1|ERC{Hl49R^yiX zV?6Grll(IZ8SjAl6%Z|Q8Ip&_GGXTK9chw31L*=ko1K-~ znCY=szQ`<@$xzs@F3|nm9y8am!U8`Ebm){dkbViXRTSEW?rrl-khJCUkz7GndY`Y= zBx4*LJZG27%zH{#8yz;7mk}CL{lu&tZXjXNCm9i06pqR6 zD<1+sVj3jlv~45Dg#ne!sJ$tQ45aP^{}qDffB!;a;;2zT$`kD)?{GUo^-!<$LY=;d wL%Sh-oNRoYf7Z{m|A-R+ow1$5#O@i^ICKpvQHMNZ-`BJO;KT7(uw57M@WEOFY8<42S`WP!n3 z*&ln-C)|k!FC}Y#KifKdj$t>U#GEDGf9I~09&~lbuYtW$Sl39)~C^FGZ5o^>%}TKsq`5l1Z+@3QpwM zzcKhjCg_VdDjf44z%%7Vp6Sbb!!G^~)b~)J9k@x)Uj14-y2iLI2HY-L_#+ihD_#e` zDU=3TRy1(X1V2}t>eC)v;*wdpu(B5tw;(w(23LyXjw<|TIBiYaciKN823k1$&|jQ0 z+|A)phY1a>hAaqSnQCL^R+u$zB=nbe^*bdt-=!L&jM4s_aHLggzvMH>uUVYt>^_;! zpEXh2{blpW;)26Ps9$=8RdaacgD7rT2_<*Qjdx^AQsApzRnf zE?BR&I@8%T>nqQEsW2FDU@=&wsKC=%L?=`*lrmb<;PKf*Uoq38Qio+eoCL*vHzRa5yP*?X8Ei zZDUn~KFlj-XVi4@OQq#_Uy;X(4_IF`gWW`sJbB9hd-FThzWm5;2fOAZxn&t+!YbW` zw;$+3Z``oM&D|;w6}DDb#|S7~%W2f0t`SfJ^|+Jj=hTF+*aLjvz9kp(X_+Lm%&EznGb?ZqQX zgG_}0)G6N6*xPUbUwkLGSTl9qsGAm@+07QWLlPkGE<{A;chhV_Qn#|_0n7hx3DJky zbX1AIQ?~4e{aetJHcrzjUXO~};5K=65$B(KpZVL=&ly7f;b=e1=K!pdY5kYd>iw*3 z?I25M1)AVObU;R;d{qwT=cYxC%-wAHE)tcOsm54b_?eTJ0gq2HhRFpN?iO*4HRVf{ zP6VxYQ*=Nfm7Plo(}Q+38zO4>H4hWQoo3AIt|K&eZs^{-*x9RU<9pm^OL9AkgfmMx)dyX}&^I0)G$(X5h5Iz-|U>pHG2_ccrHayazRp}&W@I+s}5FOzk{PC9X z+{DTGkv!7+om&N_2b8cvPt56<%dOA3rwud?i2)sI_%(eKw%r|U<)Heul!K2 z<}6EpDjs_{&qFAE1zdQS6OH6R=;4QwALS&7fxwloR}vOpVjVX9JC3))oMzX`9(P%S z9#QG3On-eZb?TOtx_5Jvu>JpqkkCSs%JnVTHfg=x) z`I$;Li9hYO c(!hC-V4HpC^wmC3csCFYjFlt0+R`unUtQjos{jB1 literal 0 HcmV?d00001 From 5ddaf3e1fbff4c0519088612f9ec3b10d62d2316 Mon Sep 17 00:00:00 2001 From: rockcavera Date: Fri, 17 Dec 2021 05:01:33 -0300 Subject: [PATCH 1416/2472] fix #19244 - solves the problem of the InAddr object constructor in Windows. (#19259) * Update winlean.nim * Update tnet_ll.nim --- lib/windows/winlean.nim | 2 +- tests/stdlib/tnet_ll.nim | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index b8f3886d7..6e626e4ba 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -478,7 +478,7 @@ type PSockAddr = ptr SockAddr - InAddr* {.importc: "IN_ADDR", header: "winsock2.h".} = object + InAddr* {.importc: "IN_ADDR", header: "winsock2.h", union.} = object s_addr*: uint32 # IP address Sockaddr_in* {.importc: "SOCKADDR_IN", diff --git a/tests/stdlib/tnet_ll.nim b/tests/stdlib/tnet_ll.nim index 248dbb79b..2d340cea8 100644 --- a/tests/stdlib/tnet_ll.nim +++ b/tests/stdlib/tnet_ll.nim @@ -26,6 +26,7 @@ suite "inet_ntop tests": # regular var ip4 = InAddr() ip4.s_addr = 0x10111213'u32 + check: ip4.s_addr == 0x10111213'u32 var buff: array[0..255, char] let r = inet_ntop(AF_INET, cast[pointer](ip4.s_addr.addr), buff[0].addr, buff.len.int32) @@ -43,3 +44,8 @@ suite "inet_ntop tests": let r = inet_ntop(AF_INET6, cast[pointer](ip6[0].addr), buff[0].addr, buff.len.int32) let res = if r == nil: "" else: $r check: not ipv6Support or res == "10:110:20:120:30:130:40:140" + + test "InAddr": + # issue 19244 + var ip4 = InAddr(s_addr: 0x10111213'u32) + check: ip4.s_addr == 0x10111213'u32 From 3d9b6cfebce898dede2f63d0288050a106a7f5e4 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 17 Dec 2021 18:16:59 +0800 Subject: [PATCH 1417/2472] [formatter] async.nim uses two spaces (#19264) according to https://nim-lang.org/docs/nep1.html#introduction-spacing-and-whitespace-conventions, two spaces should be preferred in stdlib. --- lib/pure/async.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pure/async.nim b/lib/pure/async.nim index 97b29f81d..482ab32c6 100644 --- a/lib/pure/async.nim +++ b/lib/pure/async.nim @@ -1,6 +1,6 @@ when defined(js): - import asyncjs - export asyncjs + import asyncjs + export asyncjs else: - import asyncmacro, asyncfutures - export asyncmacro, asyncfutures + import asyncmacro, asyncfutures + export asyncmacro, asyncfutures From 610516e02756dd1447fc807077bacf86eb75acdd Mon Sep 17 00:00:00 2001 From: Don-Duong Quach Date: Fri, 17 Dec 2021 11:58:31 -0800 Subject: [PATCH 1418/2472] Fixed typo in manual.rst unsafeAssign->uncheckedAssign. Fixes part 1 of #19266 (#19267) --- doc/manual.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual.rst b/doc/manual.rst index 6705e673e..d9f5d4ea6 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -1838,7 +1838,7 @@ A small example: cast uncheckedAssign -------------------- -Some restrictions for case objects can be disabled via a `{.cast(unsafeAssign).}` section: +Some restrictions for case objects can be disabled via a `{.cast(uncheckedAssign).}` section: .. code-block:: nim :test: "nim c $1" From b812431f831748c9013a12074cc6a7a838b580cd Mon Sep 17 00:00:00 2001 From: xioren <40043405+xioren@users.noreply.github.com> Date: Sun, 19 Dec 2021 05:01:17 -0800 Subject: [PATCH 1419/2472] use uppercase "type" for Proxy-Authorization header (#19273) Some servers will reject authorization requests with a lowercase "basic" type. Changing to "Basic" seems to solve these issues. https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Proxy-Authorization --- lib/pure/httpclient.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 67ba8cf2b..7686d15bd 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -523,7 +523,7 @@ proc generateHeaders(requestUrl: Uri, httpMethod: HttpMethod, headers: HttpHeade # Proxy auth header. if not proxy.isNil and proxy.auth != "": let auth = base64.encode(proxy.auth) - add(result, "Proxy-Authorization: basic " & auth & httpNewLine) + add(result, "Proxy-Authorization: Basic " & auth & httpNewLine) for key, val in headers: add(result, key & ": " & val & httpNewLine) From ea475a4e98bc1cfc861f870baf6f2cc4c979ac7a Mon Sep 17 00:00:00 2001 From: MichalMarsalek Date: Mon, 20 Dec 2021 10:40:32 +0100 Subject: [PATCH 1420/2472] Update colors.nim (#19274) * Update colors.nim Added `lightgray` alias to `lightgrey` and `...grey`aliases for the rest of the gray colors. Added color `rebeccapurple`. Fixed the incorrect values for the `PaleVioletRed` and `MediumPurple` colors. This module should now be matching the CSS colors. I used the seq[tuple] syntax for defining the names. * Document colors changes. --- changelog.md | 2 + lib/pure/colors.nim | 307 +++++++++++++++++++++++--------------------- 2 files changed, 164 insertions(+), 145 deletions(-) diff --git a/changelog.md b/changelog.md index 6f47ef8a4..52159b865 100644 --- a/changelog.md +++ b/changelog.md @@ -15,11 +15,13 @@ `experimental:flexibleOptionalParams`. - `std/sharedstrings` module is removed. +- Constants `colors.colPaleVioletRed` and `colors.colMediumPurple` changed to match the CSS color standard. ## Standard library additions and changes - `macros.parseExpr` and `macros.parseStmt` now accept an optional filename argument for more informative errors. +- Module `colors` expanded with missing colors from the CSS color standard. ## `std/smtp` diff --git a/lib/pure/colors.nim b/lib/pure/colors.nim index 2193f69a6..8cda2fab8 100644 --- a/lib/pure/colors.nim +++ b/lib/pure/colors.nim @@ -6,7 +6,8 @@ # distribution, for details about the copyright. # -## This module implements color handling for Nim. +## This module implements color handling for Nim, +## namely color mixing and parsing the CSS color names. import strutils from algorithm import binarySearch @@ -170,6 +171,7 @@ const colDarkGoldenRod* = Color(0xB8860B) colDarkGray* = Color(0xA9A9A9) colDarkGreen* = Color(0x006400) + colDarkGrey* = Color(0xA9A9A9) colDarkKhaki* = Color(0xBDB76B) colDarkMagenta* = Color(0x8B008B) colDarkOliveGreen* = Color(0x556B2F) @@ -180,11 +182,13 @@ const colDarkSeaGreen* = Color(0x8FBC8F) colDarkSlateBlue* = Color(0x483D8B) colDarkSlateGray* = Color(0x2F4F4F) + colDarkSlateGrey* = Color(0x2F4F4F) colDarkTurquoise* = Color(0x00CED1) colDarkViolet* = Color(0x9400D3) colDeepPink* = Color(0xFF1493) colDeepSkyBlue* = Color(0x00BFFF) colDimGray* = Color(0x696969) + colDimGrey* = Color(0x696969) colDodgerBlue* = Color(0x1E90FF) colFireBrick* = Color(0xB22222) colFloralWhite* = Color(0xFFFAF0) @@ -196,6 +200,7 @@ const colGoldenRod* = Color(0xDAA520) colGray* = Color(0x808080) colGreen* = Color(0x008000) + colGrey* = Color(0x808080) colGreenYellow* = Color(0xADFF2F) colHoneyDew* = Color(0xF0FFF0) colHotPink* = Color(0xFF69B4) @@ -211,13 +216,15 @@ const colLightCoral* = Color(0xF08080) colLightCyan* = Color(0xE0FFFF) colLightGoldenRodYellow* = Color(0xFAFAD2) - colLightGrey* = Color(0xD3D3D3) + colLightGray* = Color(0xD3D3D3) colLightGreen* = Color(0x90EE90) + colLightGrey* = Color(0xD3D3D3) colLightPink* = Color(0xFFB6C1) colLightSalmon* = Color(0xFFA07A) colLightSeaGreen* = Color(0x20B2AA) colLightSkyBlue* = Color(0x87CEFA) colLightSlateGray* = Color(0x778899) + colLightSlateGrey* = Color(0x778899) colLightSteelBlue* = Color(0xB0C4DE) colLightYellow* = Color(0xFFFFE0) colLime* = Color(0x00FF00) @@ -228,7 +235,7 @@ const colMediumAquaMarine* = Color(0x66CDAA) colMediumBlue* = Color(0x0000CD) colMediumOrchid* = Color(0xBA55D3) - colMediumPurple* = Color(0x9370D8) + colMediumPurple* = Color(0x9370DB) colMediumSeaGreen* = Color(0x3CB371) colMediumSlateBlue* = Color(0x7B68EE) colMediumSpringGreen* = Color(0x00FA9A) @@ -249,7 +256,7 @@ const colPaleGoldenRod* = Color(0xEEE8AA) colPaleGreen* = Color(0x98FB98) colPaleTurquoise* = Color(0xAFEEEE) - colPaleVioletRed* = Color(0xD87093) + colPaleVioletRed* = Color(0xDB7093) colPapayaWhip* = Color(0xFFEFD5) colPeachPuff* = Color(0xFFDAB9) colPeru* = Color(0xCD853F) @@ -257,6 +264,7 @@ const colPlum* = Color(0xDDA0DD) colPowderBlue* = Color(0xB0E0E6) colPurple* = Color(0x800080) + colRebeccaPurple* = Color(0x663399) colRed* = Color(0xFF0000) colRosyBrown* = Color(0xBC8F8F) colRoyalBlue* = Color(0x4169E1) @@ -270,6 +278,7 @@ const colSkyBlue* = Color(0x87CEEB) colSlateBlue* = Color(0x6A5ACD) colSlateGray* = Color(0x708090) + colSlateGrey* = Color(0x708090) colSnow* = Color(0xFFFAFA) colSpringGreen* = Color(0x00FF7F) colSteelBlue* = Color(0x4682B4) @@ -285,147 +294,155 @@ const colYellow* = Color(0xFFFF00) colYellowGreen* = Color(0x9ACD32) - colorNames = [ - ("aliceblue", colAliceBlue), - ("antiquewhite", colAntiqueWhite), - ("aqua", colAqua), - ("aquamarine", colAquamarine), - ("azure", colAzure), - ("beige", colBeige), - ("bisque", colBisque), - ("black", colBlack), - ("blanchedalmond", colBlanchedAlmond), - ("blue", colBlue), - ("blueviolet", colBlueViolet), - ("brown", colBrown), - ("burlywood", colBurlyWood), - ("cadetblue", colCadetBlue), - ("chartreuse", colChartreuse), - ("chocolate", colChocolate), - ("coral", colCoral), - ("cornflowerblue", colCornflowerBlue), - ("cornsilk", colCornsilk), - ("crimson", colCrimson), - ("cyan", colCyan), - ("darkblue", colDarkBlue), - ("darkcyan", colDarkCyan), - ("darkgoldenrod", colDarkGoldenRod), - ("darkgray", colDarkGray), - ("darkgreen", colDarkGreen), - ("darkkhaki", colDarkKhaki), - ("darkmagenta", colDarkMagenta), - ("darkolivegreen", colDarkOliveGreen), - ("darkorange", colDarkorange), - ("darkorchid", colDarkOrchid), - ("darkred", colDarkRed), - ("darksalmon", colDarkSalmon), - ("darkseagreen", colDarkSeaGreen), - ("darkslateblue", colDarkSlateBlue), - ("darkslategray", colDarkSlateGray), - ("darkturquoise", colDarkTurquoise), - ("darkviolet", colDarkViolet), - ("deeppink", colDeepPink), - ("deepskyblue", colDeepSkyBlue), - ("dimgray", colDimGray), - ("dodgerblue", colDodgerBlue), - ("firebrick", colFireBrick), - ("floralwhite", colFloralWhite), - ("forestgreen", colForestGreen), - ("fuchsia", colFuchsia), - ("gainsboro", colGainsboro), - ("ghostwhite", colGhostWhite), - ("gold", colGold), - ("goldenrod", colGoldenRod), - ("gray", colGray), - ("green", colGreen), - ("greenyellow", colGreenYellow), - ("honeydew", colHoneyDew), - ("hotpink", colHotPink), - ("indianred", colIndianRed), - ("indigo", colIndigo), - ("ivory", colIvory), - ("khaki", colKhaki), - ("lavender", colLavender), - ("lavenderblush", colLavenderBlush), - ("lawngreen", colLawnGreen), - ("lemonchiffon", colLemonChiffon), - ("lightblue", colLightBlue), - ("lightcoral", colLightCoral), - ("lightcyan", colLightCyan), - ("lightgoldenrodyellow", colLightGoldenRodYellow), - ("lightgreen", colLightGreen), - ("lightgrey", colLightGrey), - ("lightpink", colLightPink), - ("lightsalmon", colLightSalmon), - ("lightseagreen", colLightSeaGreen), - ("lightskyblue", colLightSkyBlue), - ("lightslategray", colLightSlateGray), - ("lightsteelblue", colLightSteelBlue), - ("lightyellow", colLightYellow), - ("lime", colLime), - ("limegreen", colLimeGreen), - ("linen", colLinen), - ("magenta", colMagenta), - ("maroon", colMaroon), - ("mediumaquamarine", colMediumAquaMarine), - ("mediumblue", colMediumBlue), - ("mediumorchid", colMediumOrchid), - ("mediumpurple", colMediumPurple), - ("mediumseagreen", colMediumSeaGreen), - ("mediumslateblue", colMediumSlateBlue), - ("mediumspringgreen", colMediumSpringGreen), - ("mediumturquoise", colMediumTurquoise), - ("mediumvioletred", colMediumVioletRed), - ("midnightblue", colMidnightBlue), - ("mintcream", colMintCream), - ("mistyrose", colMistyRose), - ("moccasin", colMoccasin), - ("navajowhite", colNavajoWhite), - ("navy", colNavy), - ("oldlace", colOldLace), - ("olive", colOlive), - ("olivedrab", colOliveDrab), - ("orange", colOrange), - ("orangered", colOrangeRed), - ("orchid", colOrchid), - ("palegoldenrod", colPaleGoldenRod), - ("palegreen", colPaleGreen), - ("paleturquoise", colPaleTurquoise), - ("palevioletred", colPaleVioletRed), - ("papayawhip", colPapayaWhip), - ("peachpuff", colPeachPuff), - ("peru", colPeru), - ("pink", colPink), - ("plum", colPlum), - ("powderblue", colPowderBlue), - ("purple", colPurple), - ("red", colRed), - ("rosybrown", colRosyBrown), - ("royalblue", colRoyalBlue), - ("saddlebrown", colSaddleBrown), - ("salmon", colSalmon), - ("sandybrown", colSandyBrown), - ("seagreen", colSeaGreen), - ("seashell", colSeaShell), - ("sienna", colSienna), - ("silver", colSilver), - ("skyblue", colSkyBlue), - ("slateblue", colSlateBlue), - ("slategray", colSlateGray), - ("snow", colSnow), - ("springgreen", colSpringGreen), - ("steelblue", colSteelBlue), - ("tan", colTan), - ("teal", colTeal), - ("thistle", colThistle), - ("tomato", colTomato), - ("turquoise", colTurquoise), - ("violet", colViolet), - ("wheat", colWheat), - ("white", colWhite), - ("whitesmoke", colWhiteSmoke), - ("yellow", colYellow), - ("yellowgreen", colYellowGreen)] + colorNames = { + "aliceblue": colAliceBlue, + "antiquewhite": colAntiqueWhite, + "aqua": colAqua, + "aquamarine": colAquamarine, + "azure": colAzure, + "beige": colBeige, + "bisque": colBisque, + "black": colBlack, + "blanchedalmond": colBlanchedAlmond, + "blue": colBlue, + "blueviolet": colBlueViolet, + "brown": colBrown, + "burlywood": colBurlyWood, + "cadetblue": colCadetBlue, + "chartreuse": colChartreuse, + "chocolate": colChocolate, + "coral": colCoral, + "cornflowerblue": colCornflowerBlue, + "cornsilk": colCornsilk, + "crimson": colCrimson, + "cyan": colCyan, + "darkblue": colDarkBlue, + "darkcyan": colDarkCyan, + "darkgoldenrod": colDarkGoldenRod, + "darkgray": colDarkGray, + "darkgreen": colDarkGreen, + "darkgrey": colDarkGrey, + "darkkhaki": colDarkKhaki, + "darkmagenta": colDarkMagenta, + "darkolivegreen": colDarkOliveGreen, + "darkorange": colDarkorange, + "darkorchid": colDarkOrchid, + "darkred": colDarkRed, + "darksalmon": colDarkSalmon, + "darkseagreen": colDarkSeaGreen, + "darkslateblue": colDarkSlateBlue, + "darkslategray": colDarkSlateGray, + "darkslategrey": colDarkSlateGrey, + "darkturquoise": colDarkTurquoise, + "darkviolet": colDarkViolet, + "deeppink": colDeepPink, + "deepskyblue": colDeepSkyBlue, + "dimgray": colDimGray, + "dimgrey": colDimGrey, + "dodgerblue": colDodgerBlue, + "firebrick": colFireBrick, + "floralwhite": colFloralWhite, + "forestgreen": colForestGreen, + "fuchsia": colFuchsia, + "gainsboro": colGainsboro, + "ghostwhite": colGhostWhite, + "gold": colGold, + "goldenrod": colGoldenRod, + "gray": colGray, + "green": colGreen, + "grey": colGrey, + "greenyellow": colGreenYellow, + "honeydew": colHoneyDew, + "hotpink": colHotPink, + "indianred": colIndianRed, + "indigo": colIndigo, + "ivory": colIvory, + "khaki": colKhaki, + "lavender": colLavender, + "lavenderblush": colLavenderBlush, + "lawngreen": colLawnGreen, + "lemonchiffon": colLemonChiffon, + "lightblue": colLightBlue, + "lightcoral": colLightCoral, + "lightcyan": colLightCyan, + "lightgoldenrodyellow": colLightGoldenRodYellow, + "lightgray": colLightGray, + "lightgreen": colLightGreen, + "lightgrey": colLightGrey, + "lightpink": colLightPink, + "lightsalmon": colLightSalmon, + "lightseagreen": colLightSeaGreen, + "lightskyblue": colLightSkyBlue, + "lightslategray": colLightSlateGray, + "lightslategrey": colLightSlateGrey, + "lightsteelblue": colLightSteelBlue, + "lightyellow": colLightYellow, + "lime": colLime, + "limegreen": colLimeGreen, + "linen": colLinen, + "magenta": colMagenta, + "maroon": colMaroon, + "mediumaquamarine": colMediumAquaMarine, + "mediumblue": colMediumBlue, + "mediumorchid": colMediumOrchid, + "mediumpurple": colMediumPurple, + "mediumseagreen": colMediumSeaGreen, + "mediumslateblue": colMediumSlateBlue, + "mediumspringgreen": colMediumSpringGreen, + "mediumturquoise": colMediumTurquoise, + "mediumvioletred": colMediumVioletRed, + "midnightblue": colMidnightBlue, + "mintcream": colMintCream, + "mistyrose": colMistyRose, + "moccasin": colMoccasin, + "navajowhite": colNavajoWhite, + "navy": colNavy, + "oldlace": colOldLace, + "olive": colOlive, + "olivedrab": colOliveDrab, + "orange": colOrange, + "orangered": colOrangeRed, + "orchid": colOrchid, + "palegoldenrod": colPaleGoldenRod, + "palegreen": colPaleGreen, + "paleturquoise": colPaleTurquoise, + "palevioletred": colPaleVioletRed, + "papayawhip": colPapayaWhip, + "peachpuff": colPeachPuff, + "peru": colPeru, + "pink": colPink, + "plum": colPlum, + "powderblue": colPowderBlue, + "purple": colPurple, + "rebeccapurple": colRebeccaPurple, + "red": colRed, + "rosybrown": colRosyBrown, + "royalblue": colRoyalBlue, + "saddlebrown": colSaddleBrown, + "salmon": colSalmon, + "sandybrown": colSandyBrown, + "seagreen": colSeaGreen, + "seashell": colSeaShell, + "sienna": colSienna, + "silver": colSilver, + "skyblue": colSkyBlue, + "slateblue": colSlateBlue, + "slategray": colSlateGray, + "slategrey": colSlateGrey, + "snow": colSnow, + "springgreen": colSpringGreen, + "steelblue": colSteelBlue, + "tan": colTan, + "teal": colTeal, + "thistle": colThistle, + "tomato": colTomato, + "turquoise": colTurquoise, + "violet": colViolet, + "wheat": colWheat, + "white": colWhite, + "whitesmoke": colWhiteSmoke, + "yellow": colYellow, + "yellowgreen": colYellowGreen} proc `$`*(c: Color): string = ## Converts a color into its textual representation. From 4da7dbffc504db9159285ca59e2cdfe9b103afb8 Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Tue, 21 Dec 2021 03:29:03 +1100 Subject: [PATCH 1421/2472] Extract runnables that specify `doccmd` (#19275) [backport:1.6] --- lib/core/macros.nim | 4 ++-- tests/stdlib/tmacros.nim | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 444783b5a..ed7e85c64 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -1715,8 +1715,8 @@ proc extractDocCommentsAndRunnables*(n: NimNode): NimNode = case ni.kind of nnkCommentStmt: result.add ni - of nnkCall: - if ni[0].kind == nnkIdent and ni[0].strVal == "runnableExamples": + of nnkCall, nnkCommand: + if ni[0].kind == nnkIdent and ni[0].eqIdent "runnableExamples": result.add ni else: break else: break diff --git a/tests/stdlib/tmacros.nim b/tests/stdlib/tmacros.nim index 64a474743..299eac49b 100644 --- a/tests/stdlib/tmacros.nim +++ b/tests/stdlib/tmacros.nim @@ -124,3 +124,23 @@ block: # SameType assert not testTensorInt(x1) assert testTensorInt(x2) assert not testTensorInt(x3) + +block: # extractDocCommentsAndRunnables + macro checkRunnables(prc: untyped) = + let runnables = prc.body.extractDocCommentsAndRunnables() + doAssert runnables[0][0].eqIdent("runnableExamples") + + macro checkComments(comment: static[string], prc: untyped) = + let comments = prc.body.extractDocCommentsAndRunnables() + doAssert comments[0].strVal == comment + + proc a() {.checkRunnables.} = + runnableExamples: discard + discard + + proc b() {.checkRunnables.} = + runnableExamples "-d:ssl": discard + discard + + proc c() {.checkComments("Hello world").} = + ## Hello world From 19898e1225862560b216b765187fc0b98dca7608 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Mon, 20 Dec 2021 23:10:15 +0300 Subject: [PATCH 1422/2472] Fix group reference (with capital letters (#19196) in group name) --- compiler/docgen.nim | 3 +- lib/packages/docutils/rst.nim | 2 +- .../expected/subdir/subdir_b/utils.html | 28 +++++++++++++++++++ .../expected/subdir/subdir_b/utils.idx | 2 ++ nimdoc/testproject/expected/theindex.html | 6 ++++ nimdoc/testproject/subdir/subdir_b/utils.nim | 6 ++++ 6 files changed, 45 insertions(+), 2 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 7ba952083..90e0f54b3 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -1301,7 +1301,8 @@ proc finishGenerateDoc*(d: var PDoc) = let tooltip = "$1 ($2 overloads)" % [ k.toHumanStr & " " & plainName, $overloadChoices.len] addAnchorNim(d.sharedState, refn, tooltip, - LangSymbol(symKind: k.toHumanStr, name: plainName, + LangSymbol(symKind: k.toHumanStr, + name: nimIdentBackticksNormalize(plainName), isGroup: true), priority = symbolPriority(k), # select index `0` just to have any meaningful warning: diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 4c28894fb..a04eb696f 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -993,7 +993,7 @@ proc findMainAnchorNim(s: PRstSharedState, signature: PRstNode, result.add s else: # when there are many overloads a link like foo_ points to all # of them, so selecting the group - var foundGroup = true + var foundGroup = false for s in sList: if s.langSym.isGroup: result.add s diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html index c64039941..574ac5b9b 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html @@ -207,6 +207,13 @@ window.addEventListener('DOMContentLoaded', main);
                • fn10(a: int): int
                  • +
                +
                +
              + +
              + +
              +
              +
              func fN11() {....raises: [], tags: [].}
              +
              + + + +
              +
              +
              +
              func fN11(x: int) {....raises: [], tags: [].}
              +
              + + +
              diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.idx b/nimdoc/testproject/expected/subdir/subdir_b/utils.idx index 441bd8675..5c8fee527 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.idx +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.idx @@ -17,6 +17,8 @@ fn7 subdir/subdir_b/utils.html#fn7 utils: fn7() fn8 subdir/subdir_b/utils.html#fn8 utils: fn8(): auto fn9 subdir/subdir_b/utils.html#fn9,int utils: fn9(a: int): int fn10 subdir/subdir_b/utils.html#fn10,int utils: fn10(a: int): int +fN11 subdir/subdir_b/utils.html#fN11 utils: fN11() +fN11 subdir/subdir_b/utils.html#fN11,int utils: fN11(x: int) aEnum subdir/subdir_b/utils.html#aEnum.t utils: aEnum(): untyped bEnum subdir/subdir_b/utils.html#bEnum.t utils: bEnum(): untyped fromUtilsGen subdir/subdir_b/utils.html#fromUtilsGen.t utils: fromUtilsGen(): untyped diff --git a/nimdoc/testproject/expected/theindex.html b/nimdoc/testproject/expected/theindex.html index da76334d4..88d4b86aa 100644 --- a/nimdoc/testproject/expected/theindex.html +++ b/nimdoc/testproject/expected/theindex.html @@ -209,6 +209,12 @@ window.addEventListener('DOMContentLoaded', main);
            • utils: fn10(a: int): int
            +
            fN11:
            fn2:
            • utils: fn2()
              • diff --git a/nimdoc/testproject/subdir/subdir_b/utils.nim b/nimdoc/testproject/subdir/subdir_b/utils.nim index 5d881e620..e201a3d38 100644 --- a/nimdoc/testproject/subdir/subdir_b/utils.nim +++ b/nimdoc/testproject/subdir/subdir_b/utils.nim @@ -36,6 +36,7 @@ Note that `proc` can be used in postfix form: `binarySearch proc`_. Ref. type like G_ and `type G`_ and `G[T]`_ and `type G*[T]`_. +Group ref. with capital letters works: fN11_ or fn11_ ]## include ./utils_helpers @@ -77,6 +78,11 @@ proc fn8*(): auto = func fn9*(a: int): int = 42 ## comment func fn10*(a: int): int = a ## comment +# Note capital letter N will be handled correctly in +# group references like fN11_ or fn11_: +func fN11*() = discard +func fN11*(x: int) = discard + # bug #9235 template aEnum*(): untyped = From 7a5314c571ec2c370f63d5fab0afd336e04ec161 Mon Sep 17 00:00:00 2001 From: pkubaj Date: Thu, 23 Dec 2021 02:52:33 +0100 Subject: [PATCH 1423/2472] Fix build on FreeBSD/powerpc (#19282) It's currently misdetected as powerpc64. --- tools/niminst/buildsh.nimf | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/niminst/buildsh.nimf b/tools/niminst/buildsh.nimf index 464c70abf..5f1db9cd3 100644 --- a/tools/niminst/buildsh.nimf +++ b/tools/niminst/buildsh.nimf @@ -192,8 +192,10 @@ case $ucpu in mycpu="powerpc64" ;; *power*|*ppc* ) if [ "$myos" = "freebsd" ] ; then - COMP_FLAGS="$COMP_FLAGS -m64" - LINK_FLAGS="$LINK_FLAGS -m64" + if [ "$ucpu" != "powerpc" ] ; then + COMP_FLAGS="$COMP_FLAGS -m64" + LINK_FLAGS="$LINK_FLAGS -m64" + fi mycpu=`uname -p` case $mycpu in powerpc64le) From 81d32cf7e53b244776c4196098be154c1223d726 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 23 Dec 2021 07:03:40 +0100 Subject: [PATCH 1424/2472] Revert "Update uri.nim (#19148) [backport:1.0]" (#19280) This reverts commit a3ef5df680e55d9bf68027fcb0ec6358b4279d09. --- lib/pure/uri.nim | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/pure/uri.nim b/lib/pure/uri.nim index 5de333b8f..a828298c2 100644 --- a/lib/pure/uri.nim +++ b/lib/pure/uri.nim @@ -216,8 +216,6 @@ func parseAuthority(authority: string, result: var Uri) = result.isIpv6 = true of ']': inIPv6 = false - of '\0': - break else: if inPort: result.port.add(authority[i]) From fa96e56ad06f0562b5d608cc896a9b17c9512958 Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Wed, 22 Dec 2021 23:12:56 -0700 Subject: [PATCH 1425/2472] Fixed object field access of static objects in generics (#19283) [backport] --- compiler/semexprs.nim | 2 +- tests/system/tstatic.nim | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 0df332689..c593efe55 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1414,7 +1414,7 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode = if ty.kind in tyUserTypeClasses and ty.isResolvedUserTypeClass: ty = ty.lastSon - ty = skipTypes(ty, {tyGenericInst, tyVar, tyLent, tyPtr, tyRef, tyOwned, tyAlias, tySink}) + ty = skipTypes(ty, {tyGenericInst, tyVar, tyLent, tyPtr, tyRef, tyOwned, tyAlias, tySink, tyStatic}) while tfBorrowDot in ty.flags: ty = ty.skipTypes({tyDistinct, tyGenericInst, tyAlias}) var check: PNode = nil if ty.kind == tyObject: diff --git a/tests/system/tstatic.nim b/tests/system/tstatic.nim index 1f1b9dbe3..6e2893e2b 100644 --- a/tests/system/tstatic.nim +++ b/tests/system/tstatic.nim @@ -44,6 +44,14 @@ template main() = proc parseInt(f: static[bool]): int {.used.} = discard doAssert "123".parseInt == 123 + block: + type + MyType = object + field: float32 + AType[T: static MyType] = distinct range[0f32 .. T.field] + var a: AType[MyType(field: 5f32)] + proc n(S: static Slice[int]): range[S.a..S.b] = discard + assert typeof(n 1..2) is range[1..2] static: main() From fdbec969d8601af0f666426b523e8ebba644ab56 Mon Sep 17 00:00:00 2001 From: Tomohiro Date: Sat, 25 Dec 2021 18:31:35 +0900 Subject: [PATCH 1426/2472] Fix #19107 (#19286) [backport] --- lib/pure/strformat.nim | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim index 746f01ace..40a33951c 100644 --- a/lib/pure/strformat.nim +++ b/lib/pure/strformat.nim @@ -574,6 +574,9 @@ template formatValue(result: var string; value: cstring; specifier: string) = result.add value proc strformatImpl(f: string; openChar, closeChar: char): NimNode = + template missingCloseChar = + error("invalid format string: missing closing character '" & closeChar & "'") + if openChar == ':' or closeChar == ':': error "openChar and closeChar must not be ':'" var i = 0 @@ -618,6 +621,8 @@ proc strformatImpl(f: string; openChar, closeChar: char): NimNode = let start = i inc i i += f.skipWhitespace(i) + if i == f.len: + missingCloseChar if f[i] == closeChar or f[i] == ':': result.add newCall(bindSym"add", res, newLit(subexpr & f[start ..< i])) else: @@ -627,6 +632,9 @@ proc strformatImpl(f: string; openChar, closeChar: char): NimNode = subexpr.add f[i] inc i + if i == f.len: + missingCloseChar + var x: NimNode try: x = parseExpr(subexpr) @@ -639,10 +647,10 @@ proc strformatImpl(f: string; openChar, closeChar: char): NimNode = while i < f.len and f[i] != closeChar: options.add f[i] inc i + if i == f.len: + missingCloseChar if f[i] == closeChar: inc i - else: - doAssert false, "invalid format string: missing '}'" result.add newCall(formatSym, res, x, newLit(options)) elif f[i] == closeChar: if i Date: Mon, 27 Dec 2021 17:28:19 +0100 Subject: [PATCH 1427/2472] fixes grammar typos [backport] (#19289) --- compiler/parser.nim | 6 +++--- doc/grammar.txt | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/compiler/parser.nim b/compiler/parser.nim index 9ff847ef6..7bc2c78e0 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -582,10 +582,10 @@ proc parsePar(p: var Parser): PNode = #| | 'finally' | 'except' | 'for' | 'block' | 'const' | 'let' #| | 'when' | 'var' | 'mixin' #| par = '(' optInd - #| ( &parKeyw (ifExpr \ complexOrSimpleStmt) ^+ ';' - #| | ';' (ifExpr \ complexOrSimpleStmt) ^+ ';' + #| ( &parKeyw (ifExpr / complexOrSimpleStmt) ^+ ';' + #| | ';' (ifExpr / complexOrSimpleStmt) ^+ ';' #| | pragmaStmt - #| | simpleExpr ( ('=' expr (';' (ifExpr \ complexOrSimpleStmt) ^+ ';' )? ) + #| | simpleExpr ( ('=' expr (';' (ifExpr / complexOrSimpleStmt) ^+ ';' )? ) #| | (':' expr (',' exprColonEqExpr ^+ ',' )? ) ) ) #| optPar ')' # diff --git a/doc/grammar.txt b/doc/grammar.txt index 8f86dd98c..bf6d381de 100644 --- a/doc/grammar.txt +++ b/doc/grammar.txt @@ -37,10 +37,10 @@ parKeyw = 'discard' | 'include' | 'if' | 'while' | 'case' | 'try' | 'finally' | 'except' | 'for' | 'block' | 'const' | 'let' | 'when' | 'var' | 'mixin' par = '(' optInd - ( &parKeyw (ifExpr \ complexOrSimpleStmt) ^+ ';' - | ';' (ifExpr \ complexOrSimpleStmt) ^+ ';' + ( &parKeyw (ifExpr / complexOrSimpleStmt) ^+ ';' + | ';' (ifExpr / complexOrSimpleStmt) ^+ ';' | pragmaStmt - | simpleExpr ( ('=' expr (';' (ifExpr \ complexOrSimpleStmt) ^+ ';' )? ) + | simpleExpr ( ('=' expr (';' (ifExpr / complexOrSimpleStmt) ^+ ';' )? ) | (':' expr (',' exprColonEqExpr ^+ ',' )? ) ) ) optPar ')' literal = | INT_LIT | INT8_LIT | INT16_LIT | INT32_LIT | INT64_LIT From 77ad8b81e414088c3116bf33f96efacd27eeb471 Mon Sep 17 00:00:00 2001 From: rockcavera Date: Wed, 29 Dec 2021 03:26:54 -0300 Subject: [PATCH 1428/2472] fix 19292 (#19293) --- lib/std/private/win_setenv.nim | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/std/private/win_setenv.nim b/lib/std/private/win_setenv.nim index 067e656a3..91c3f15b0 100644 --- a/lib/std/private/win_setenv.nim +++ b/lib/std/private/win_setenv.nim @@ -23,20 +23,21 @@ check errno_t vs cint when not defined(windows): discard else: + type wchar_t {.importc: "wchar_t".} = int16 + proc setEnvironmentVariableA*(lpName, lpValue: cstring): int32 {. stdcall, dynlib: "kernel32", importc: "SetEnvironmentVariableA", sideEffect.} # same as winlean.setEnvironmentVariableA proc c_getenv(env: cstring): cstring {.importc: "getenv", header: "".} proc c_putenv_s(envname: cstring, envval: cstring): cint {.importc: "_putenv_s", header: "".} - proc c_wgetenv(varname: WideCString): WideCString {.importc: "_wgetenv", header: "".} + proc c_wgetenv(varname: ptr wchar_t): ptr wchar_t {.importc: "_wgetenv", header: "".} var errno {.importc, header: "".}: cint - type wchar_t {.importc: "wchar_t".} = int16 var gWenviron {.importc:"_wenviron".}: ptr ptr wchar_t # xxx `ptr UncheckedArray[WideCString]` did not work - proc mbstowcs_s(pReturnValue: ptr csize_t, wcstr: WideCString, sizeInWords: csize_t, mbstr: cstring, count: csize_t): cint {.importc: "mbstowcs_s", header: "".} + proc mbstowcs_s(pReturnValue: ptr csize_t, wcstr: ptr wchar_t, sizeInWords: csize_t, mbstr: cstring, count: csize_t): cint {.importc: "mbstowcs_s", header: "".} # xxx cint vs errno_t? proc setEnvImpl*(name: cstring, value: cstring, overwrite: cint): cint = @@ -77,13 +78,15 @@ else: if gWenviron != nil: # var buf: array[MAX_ENV + 1, WideCString] var buf: array[MAX_ENV + 1, Utf16Char] - let buf2 = cast[WideCString](buf[0].addr) + let buf2 = cast[ptr wchar_t](buf[0].addr) var len: csize_t if mbstowcs_s(len.addr, buf2, buf.len.csize_t, name, MAX_ENV) != 0: errno = EINVAL return -1 - c_wgetenv(buf2)[0] = '\0'.Utf16Char - c_wgetenv(buf2)[1] = '='.Utf16Char + let ptrToEnv = cast[WideCString](c_wgetenv(buf2)) + ptrToEnv[0] = '\0'.Utf16Char + let ptrToEnv2 = cast[WideCString](c_wgetenv(buf2)) + ptrToEnv2[1] = '='.Utf16Char # And now, we have to update the outer environment to have a proper empty value. if setEnvironmentVariableA(name, value) == 0: From a9223d4cdd2e6c23506b666a0958cd2ae3bf5dfd Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 29 Dec 2021 12:18:32 +0100 Subject: [PATCH 1429/2472] nep1: make header reflect reality (#19294) --- doc/nep1.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/nep1.rst b/doc/nep1.rst index fc69bab69..8eb31332f 100644 --- a/doc/nep1.rst +++ b/doc/nep1.rst @@ -1,6 +1,6 @@ -========================================================== -Nim Enhancement Proposal #1 - Standard Library Style Guide -========================================================== +============================ +Standard Library Style Guide +============================ :Author: Clay Sweetser, Dominik Picheta :Version: |nimversion| From 23d64be860958fb7149baed44f6e493394950d69 Mon Sep 17 00:00:00 2001 From: BarrOff <58253563+BarrOff@users.noreply.github.com> Date: Thu, 30 Dec 2021 20:10:36 +0000 Subject: [PATCH 1430/2472] enable `maxDescriptors` on Illumos/Solaris (#19295) --- lib/pure/asyncdispatch.nim | 4 ++-- lib/pure/selectors.nim | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 20ad60427..5e2a8d98e 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -1974,11 +1974,11 @@ when defined(posix): import posix when defined(linux) or defined(windows) or defined(macosx) or defined(bsd) or - defined(zephyr) or defined(freertos): + defined(solaris) or defined(zephyr) or defined(freertos): proc maxDescriptors*(): int {.raises: OSError.} = ## Returns the maximum number of active file descriptors for the current ## process. This involves a system call. For now `maxDescriptors` is - ## supported on the following OSes: Windows, Linux, OSX, BSD. + ## supported on the following OSes: Windows, Linux, OSX, BSD, Solaris. when defined(windows): result = 16_700_000 elif defined(zephyr) or defined(freertos): diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim index ec441f6da..313702370 100644 --- a/lib/pure/selectors.nim +++ b/lib/pure/selectors.nim @@ -324,11 +324,11 @@ else: doAssert(timeout >= -1, "Cannot select with a negative value, got: " & $timeout) when defined(linux) or defined(windows) or defined(macosx) or defined(bsd) or - defined(zephyr) or defined(freertos): + defined(solaris) or defined(zephyr) or defined(freertos): template maxDescriptors*(): int = ## Returns the maximum number of active file descriptors for the current ## process. This involves a system call. For now `maxDescriptors` is - ## supported on the following OSes: Windows, Linux, OSX, BSD. + ## supported on the following OSes: Windows, Linux, OSX, BSD, Solaris. when defined(windows): 16_700_000 elif defined(zephyr) or defined(freertos): From dc5c88ca79e871aefa76c7fefae4282c639a3283 Mon Sep 17 00:00:00 2001 From: rockcavera Date: Thu, 30 Dec 2021 17:52:48 -0300 Subject: [PATCH 1431/2472] Fix #19297 - fixing broken list after adding empty list (#19299) * Update lists.nim * Update tlists.nim * removed check `if b.tail != nil` The tail of the list being null it is still possible to retrieve its end by going through all nodes from the head. So checking for null from `b.tail` is unnecessary. However, setting `a.tail = b.tail` only if `a.head != nil`, so you don't break a good list with an already broken one. --- lib/pure/collections/lists.nim | 11 ++++++----- tests/stdlib/tlists.nim | 12 ++++++++++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/lib/pure/collections/lists.nim b/lib/pure/collections/lists.nim index 8b255fb60..de5550d6e 100644 --- a/lib/pure/collections/lists.nim +++ b/lib/pure/collections/lists.nim @@ -530,11 +530,12 @@ proc addMoved*[T](a, b: var SinglyLinkedList[T]) {.since: (1, 5, 1).} = ci assert s == [0, 1, 0, 1, 0, 1] - if a.tail != nil: - a.tail.next = b.head - a.tail = b.tail - if a.head == nil: - a.head = b.head + if b.head != nil: + if a.head == nil: + a.head = b.head + else: + a.tail.next = b.head + a.tail = b.tail if a.addr != b.addr: b.head = nil b.tail = nil diff --git a/tests/stdlib/tlists.nim b/tests/stdlib/tlists.nim index 54fa8b24f..2f6d5ee6a 100644 --- a/tests/stdlib/tlists.nim +++ b/tests/stdlib/tlists.nim @@ -233,6 +233,18 @@ template main = doAssert l.toSeq == [1] doAssert l.remove(l.head) == true doAssert l.toSeq == [] + + block issue19297: # add (appends a shallow copy) + var a: SinglyLinkedList[int] + var b: SinglyLinkedList[int] + + doAssert a.toSeq == @[] + a.add(1) + doAssert a.toSeq == @[1] + a.add(b) + doAssert a.toSeq == @[1] + a.add(2) + doAssert a.toSeq == @[1, 2] static: main() main() From ac37eed5a28042d50e23d9ae145f2d46b8dc5058 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 31 Dec 2021 09:21:30 +0100 Subject: [PATCH 1432/2472] fixes #16617 [backport] (#19300) --- compiler/typeallowed.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compiler/typeallowed.nim b/compiler/typeallowed.nim index d00aa538f..78cd74e56 100644 --- a/compiler/typeallowed.nim +++ b/compiler/typeallowed.nim @@ -57,6 +57,8 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, of tyVar, tyLent: if kind in {skProc, skFunc, skConst} and (views notin c.features): result = t + elif taIsOpenArray in flags: + result = t elif t.kind == tyLent and ((kind != skResult and views notin c.features) or kind == skParam): # lent can't be used as parameters. result = t From faaf271928acaa19b53c40d9c5b37e6441cb96c1 Mon Sep 17 00:00:00 2001 From: hlaaftana <10591326+hlaaftana@users.noreply.github.com> Date: Sat, 1 Jan 2022 15:33:16 +0300 Subject: [PATCH 1433/2472] Update JS and nimscript import tests (#19306) * add new modules, except experimental ones * remove deprecated modules mersenne and sharedlist * better describe why some modules fail and some modules don't --- tests/js/tstdlib_imports.nim | 43 +++++++++++++++++-------------- tests/test_nimscript.nims | 50 +++++++++++++++++++++--------------- 2 files changed, 53 insertions(+), 40 deletions(-) diff --git a/tests/js/tstdlib_imports.nim b/tests/js/tstdlib_imports.nim index 6fe3772d3..9986fa059 100644 --- a/tests/js/tstdlib_imports.nim +++ b/tests/js/tstdlib_imports.nim @@ -7,46 +7,48 @@ discard """ import std/[ # Core: bitops, typetraits, lenientops, macros, volatile, typeinfo, - # fails: endians, rlocks - # works but shouldn't: cpuinfo, locks + # fails due to FFI: rlocks + # fails due to cstring cast/copyMem: endians + # works but uses FFI: cpuinfo, locks # Algorithms: - algorithm, sequtils, + algorithm, enumutils, sequtils, setutils, # Collections: critbits, deques, heapqueue, intsets, lists, options, sets, - sharedlist, tables, - # fails: sharedtables + tables, packedsets, # Strings: cstrutils, editdistance, wordwrap, parseutils, ropes, pegs, punycode, strformat, strmisc, strscans, strtabs, strutils, unicode, unidecode, - # fails: encodings + # fails due to FFI: encodings # Time handling: monotimes, times, # Generic operator system services: os, streams, - # fails: distros, dynlib, marshal, memfiles, osproc, terminal + # fails intentionally: dynlib, marshal, memfiles + # fails due to FFI: osproc, terminal + # fails due to osproc import: distros # Math libraries: - complex, math, mersenne, random, rationals, stats, sums, - # works but shouldn't: fenv + complex, math, random, rationals, stats, sums, sysrand, + # works but uses FFI: fenv # Internet protocols: cookies, httpcore, mimetypes, uri, - # fails: asyncdispatch, asyncfile, asyncftpclient, asynchttpserver, + # fails due to FFI: asyncdispatch, asyncfile, asyncftpclient, asynchttpserver, # asyncnet, cgi, httpclient, nativesockets, net, selectors, smtp - # works but shouldn't test: asyncstreams, asyncfutures + # works but no need to test: asyncstreams, asyncfutures # Threading: - # fails: threadpool + # fails due to FFI: threadpool # Parsers: htmlparser, json, lexbase, parsecfg, parsecsv, parsesql, parsexml, - parseopt, + parseopt, jsonutils, # XML processing: xmltree, xmlparser, @@ -56,16 +58,19 @@ import std/[ # Hashing: base64, hashes, - # fails: md5, oids, sha1 + # fails due to cstring cast/zeroMem/copyMem/moveMem: md5 + # fails due to cstring cast/endians import: oids + # fails due to copyMem/endians import: sha1 # Miscellaneous: - colors, logging, sugar, unittest, varints, - # fails: browsers, coro - # works but shouldn't: segfaults + colors, logging, sugar, unittest, varints, enumerate, with, + # fails due to FFI: browsers, coro + # works but uses FFI: segfaults # Modules for JS backend: - asyncjs, dom, jsconsole, jscore, jsffi, + asyncjs, dom, jsconsole, jscore, jsffi, jsbigints, # Unlisted in lib.html: - decls, compilesettings, with, wrapnils + decls, compilesettings, wrapnils, exitprocs, effecttraits, + genasts, importutils, isolation, jsfetch, jsformdata, jsheaders ] diff --git a/tests/test_nimscript.nims b/tests/test_nimscript.nims index ecb409677..761ddafdf 100644 --- a/tests/test_nimscript.nims +++ b/tests/test_nimscript.nims @@ -8,47 +8,49 @@ from stdtest/specialpaths import buildDir import std/[ # Core: bitops, typetraits, lenientops, macros, volatile, - # fails: typeinfo, endians - # works but shouldn't: cpuinfo, rlocks, locks + # fails due to FFI: typeinfo + # fails due to cstring cast/copyMem: endians + # works but uses FFI: cpuinfo, rlocks, locks # Algorithms: - algorithm, sequtils, + algorithm, enumutils, sequtils, setutils, # Collections: critbits, deques, heapqueue, intsets, lists, options, sets, - sharedlist, tables, - # fails: sharedtables + tables, packedsets, # Strings: editdistance, wordwrap, parseutils, ropes, pegs, punycode, strformat, strmisc, strscans, strtabs, - strutils, unicode, unidecode, - # works but shouldn't: cstrutils, encodings + strutils, unicode, unidecode, cstrutils, + # works but uses FFI: encodings # Time handling: - # fails: monotimes, times + # fails due to FFI: monotimes, times # but times.getTime() implemented for VM # Generic operator system services: - os, streams, - # fails: distros, dynlib, marshal, memfiles, osproc, terminal + os, streams, distros, + # fails due to FFI: memfiles, osproc, terminal + # works but uses FFI: dynlib + # intentionally fails: marshal # Math libraries: - complex, math, mersenne, random, rationals, stats, sums, - # works but shouldn't: fenv + complex, math, random, rationals, stats, sums, + # works but uses FFI: fenv, sysrand # Internet protocols: httpcore, mimetypes, uri, - # fails: asyncdispatch, asyncfile, asyncftpclient, asynchttpserver, + # fails due to FFI: asyncdispatch, asyncfile, asyncftpclient, asynchttpserver, # asyncnet, cgi, cookies, httpclient, nativesockets, net, selectors, smtp - # works but shouldn't test: asyncstreams, asyncfutures + # works but no need to test: asyncstreams, asyncfutures # Threading: - # fails: threadpool + # fails due to FFI: threadpool # Parsers: htmlparser, json, lexbase, parsecfg, parsecsv, parsesql, parsexml, - parseopt, + parseopt, jsonutils, # XML processing: xmltree, xmlparser, @@ -58,17 +60,23 @@ import std/[ # Hashing: base64, hashes, - # fails: md5, oids, sha1 + # fails due to cstring cast/zeroMem/copyMem/moveMem: md5 + # fails due to cstring cast/times import/endians import: oids + # fails due to copyMem/endians import: sha1 # Miscellaneous: - colors, sugar, varints, - # fails: browsers, coro, logging (times), segfaults, unittest (uses methods) + colors, sugar, varints, enumerate, with, + # fails due to FFI: browsers, coro, segfaults + # fails due to times import/methods: logging + # fails due to methods: unittest # Modules for JS backend: - # fails: asyncjs, dom, jsconsole, jscore, jsffi, + # fails intentionally: asyncjs, dom, jsconsole, jscore, jsffi, jsbigints, + # jsfetch, jsformdata, jsheaders # Unlisted in lib.html: - decls, compilesettings, with, wrapnils + decls, compilesettings, wrapnils, effecttraits, genasts, + importutils, isolation ] # non-std imports From ef634cc2516ce81a05486f726e423ac458c4467a Mon Sep 17 00:00:00 2001 From: rockcavera Date: Sat, 1 Jan 2022 09:33:29 -0300 Subject: [PATCH 1434/2472] Update manual.rst (#19301) --- doc/manual.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual.rst b/doc/manual.rst index d9f5d4ea6..8788e53a3 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -5018,7 +5018,7 @@ be used: See also: -- `Shared heap memory management `_. +- `Shared heap memory management `_. From 61d682e733005534a16af72fc9b0c0e74d2f9ebf Mon Sep 17 00:00:00 2001 From: hlaaftana <10591326+hlaaftana@users.noreply.github.com> Date: Sun, 2 Jan 2022 09:29:55 +0300 Subject: [PATCH 1435/2472] Allow full commands and blocks in type sections (#19181) * allow full commands and blocks in type sections * update grammar * fix changelog [skip ci] * more tests * even more tests --- changelog.md | 3 + compiler/parser.nim | 28 ++- doc/grammar.txt | 5 +- tests/parser/ttypecommandcomma.nim | 9 + tests/parser/ttypecommandindent1.nim | 9 + tests/parser/ttypecommandindent2.nim | 11 + tests/parser/ttypecommandindent3.nim | 11 + tests/parser/ttypesectioncalls.nim | 328 +++++++++++++++++++++++++++ 8 files changed, 393 insertions(+), 11 deletions(-) create mode 100644 tests/parser/ttypecommandcomma.nim create mode 100644 tests/parser/ttypecommandindent1.nim create mode 100644 tests/parser/ttypecommandindent2.nim create mode 100644 tests/parser/ttypecommandindent3.nim create mode 100644 tests/parser/ttypesectioncalls.nim diff --git a/changelog.md b/changelog.md index 52159b865..89aaff664 100644 --- a/changelog.md +++ b/changelog.md @@ -61,6 +61,9 @@ ``` - [Case statement macros](manual.html#macros-case-statement-macros) are no longer experimental, meaning you no longer need to enable the experimental switch `caseStmtMacros` to use them. +- Full command syntax and block arguments i.e. `foo a, b: c` are now allowed + for the right-hand side of type definitions in type sections. Previously + they would error with "invalid indentation". ## Compiler changes diff --git a/compiler/parser.nim b/compiler/parser.nim index 7bc2c78e0..0fb60440c 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -811,7 +811,7 @@ proc primarySuffix(p: var Parser, r: PNode, #| | DOTLIKEOP optInd symbol generalizedLit? #| | '[' optInd exprColonEqExprList optPar ']' #| | '{' optInd exprColonEqExprList optPar '}' - #| | &( '`'|IDENT|literal|'cast'|'addr'|'type') expr # command syntax + #| | &( '`'|IDENT|literal|'cast'|'addr'|'type') expr (comma expr)* # command syntax result = r # progress guaranteed @@ -821,14 +821,14 @@ proc primarySuffix(p: var Parser, r: PNode, of tkParLe: # progress guaranteed if p.tok.strongSpaceA > 0: - # inside type sections, expressions such as `ref (int, bar)` - # are parsed as a nkCommand with a single tuple argument (nkPar) + result = commandExpr(p, result, mode) + # type sections allow full command syntax if mode == pmTypeDef: - result = newNodeP(nkCommand, p) - result.add r - result.add primary(p, pmNormal) - else: - result = commandExpr(p, result, mode) + var isFirstParam = false + while p.tok.tokType == tkComma: + getTok(p) + optInd(p, result) + result.add(commandParam(p, isFirstParam, mode)) break result = namedParams(p, result, nkCall, tkParRi) if result.len > 1 and result[1].kind == nkExprColonExpr: @@ -869,9 +869,18 @@ proc primarySuffix(p: var Parser, r: PNode, # actually parsing {.push hints:off.} as {.push(hints:off).} is a sweet # solution, but pragmas.nim can't handle that result = commandExpr(p, result, mode) + if mode == pmTypeDef: + var isFirstParam = false + while p.tok.tokType == tkComma: + getTok(p) + optInd(p, result) + result.add(commandParam(p, isFirstParam, mode)) break else: break + # type sections allow post-expr blocks + if mode == pmTypeDef: + result = postExprBlocks(p, result) proc parseOperators(p: var Parser, headNode: PNode, limit: int, mode: PrimaryMode): PNode = @@ -1342,7 +1351,8 @@ proc parseTypeDesc(p: var Parser): PNode = result = binaryNot(p, result) proc parseTypeDefAux(p: var Parser): PNode = - #| typeDefAux = simpleExpr ('not' expr)? + #| typeDefAux = simpleExpr ('not' expr + #| | postExprBlocks)? result = simpleExpr(p, pmTypeDef) result = binaryNot(p, result) diff --git a/doc/grammar.txt b/doc/grammar.txt index bf6d381de..1e41b9eeb 100644 --- a/doc/grammar.txt +++ b/doc/grammar.txt @@ -60,7 +60,7 @@ primarySuffix = '(' (exprColonEqExpr comma?)* ')' | DOTLIKEOP optInd symbol generalizedLit? | '[' optInd exprColonEqExprList optPar ']' | '{' optInd exprColonEqExprList optPar '}' - | &( '`'|IDENT|literal|'cast'|'addr'|'type') expr # command syntax + | &( '`'|IDENT|literal|'cast'|'addr'|'type') expr (comma expr)* # command syntax pragma = '{.' optInd (exprColonEqExpr comma?)* optPar ('.}' | '}') identVis = symbol OPR? # postfix position identVisDot = symbol '.' optInd symbol OPR? @@ -93,7 +93,8 @@ primary = operatorB primary primarySuffix* | ('var' | 'out' | 'ref' | 'ptr' | 'distinct') primary / prefixOperator* identOrLiteral primarySuffix* typeDesc = simpleExpr ('not' expr)? -typeDefAux = simpleExpr ('not' expr)? +typeDefAux = simpleExpr ('not' expr + | postExprBlocks)? postExprBlocks = ':' stmt? ( IND{=} doBlock | IND{=} 'of' exprList ':' stmt | IND{=} 'elif' expr ':' stmt diff --git a/tests/parser/ttypecommandcomma.nim b/tests/parser/ttypecommandcomma.nim new file mode 100644 index 000000000..7ca59a799 --- /dev/null +++ b/tests/parser/ttypecommandcomma.nim @@ -0,0 +1,9 @@ +discard """ + errormsg: "invalid indentation" + line: 8 + column: 19 +""" + +type + Foo = call(1, 2), 3: + 4 \ No newline at end of file diff --git a/tests/parser/ttypecommandindent1.nim b/tests/parser/ttypecommandindent1.nim new file mode 100644 index 000000000..9aa274e2a --- /dev/null +++ b/tests/parser/ttypecommandindent1.nim @@ -0,0 +1,9 @@ +discard """ + errormsg: "invalid indentation" + line: 9 + column: 3 +""" + +type + Foo = call x, y, z: + abc diff --git a/tests/parser/ttypecommandindent2.nim b/tests/parser/ttypecommandindent2.nim new file mode 100644 index 000000000..0883df90f --- /dev/null +++ b/tests/parser/ttypecommandindent2.nim @@ -0,0 +1,11 @@ +discard """ + errormsg: "invalid indentation" + line: 10 + column: 5 +""" + +type + Foo = call x, y, z: + abc + do: + def diff --git a/tests/parser/ttypecommandindent3.nim b/tests/parser/ttypecommandindent3.nim new file mode 100644 index 000000000..f9fdcc1e4 --- /dev/null +++ b/tests/parser/ttypecommandindent3.nim @@ -0,0 +1,11 @@ +discard """ + errormsg: "expression expected, but found 'keyword do'" + line: 10 + column: 1 +""" + +type + Foo = call x, y, z: + abc +do: + def diff --git a/tests/parser/ttypesectioncalls.nim b/tests/parser/ttypesectioncalls.nim new file mode 100644 index 000000000..003444fc5 --- /dev/null +++ b/tests/parser/ttypesectioncalls.nim @@ -0,0 +1,328 @@ +discard """ +nimout: ''' +StmtList + TypeSection + TypeDef + Ident "A" + Empty + Call + Ident "call" + IntLit 1 + TypeSection + TypeDef + Ident "B" + Empty + Command + Ident "call" + IntLit 2 + TypeDef + Ident "C" + Empty + Call + Ident "call" + StmtList + IntLit 3 + TypeDef + Ident "D" + Empty + Call + Ident "call" + StmtList + IntLit 4 + TypeSection + TypeDef + Ident "E" + Empty + Call + Ident "call" + IntLit 5 + IntLit 6 + TypeDef + Ident "F" + Empty + Command + Ident "call" + IntLit 7 + IntLit 8 + TypeDef + Ident "G" + Empty + Call + Ident "call" + IntLit 9 + StmtList + IntLit 10 + TypeDef + Ident "H" + Empty + Call + Ident "call" + IntLit 11 + StmtList + IntLit 12 + TypeDef + Ident "I" + Empty + Command + Ident "call" + IntLit 13 + StmtList + IntLit 14 + TypeDef + Ident "J" + Empty + Command + Ident "call" + IntLit 15 + StmtList + IntLit 16 + TypeSection + TypeDef + Ident "K" + Empty + Call + Ident "call" + IntLit 17 + IntLit 18 + IntLit 19 + TypeDef + Ident "L" + Empty + Command + Ident "call" + IntLit 20 + IntLit 21 + IntLit 22 + TypeDef + Ident "M" + Empty + Call + Ident "call" + IntLit 23 + IntLit 24 + StmtList + IntLit 25 + TypeDef + Ident "N" + Empty + Command + Ident "call" + IntLit 26 + IntLit 27 + StmtList + IntLit 28 + TypeDef + Ident "O" + Empty + Command + Ident "call" + IntLit 29 + IntLit 30 + StmtList + IntLit 31 + TypeSection + TypeDef + Ident "P" + Empty + Command + Ident "call" + TupleConstr + IntLit 32 + IntLit 33 + Infix + Ident "+" + Infix + Ident "*" + IntLit 34 + IntLit 35 + IntLit 36 + StmtList + IntLit 37 + TypeDef + Ident "R" + Empty + Command + Ident "call" + Infix + Ident "@" + TupleConstr + IntLit 38 + IntLit 39 + Infix + Ident "shl" + IntLit 40 + IntLit 41 + Infix + Ident "-" + Infix + Ident "*" + IntLit 42 + IntLit 43 + IntLit 44 + StmtList + IntLit 45 + TypeDef + Ident "S" + Empty + Command + Ident "call" + IntLit 46 + StmtList + IntLit 47 + StmtList + IntLit 48 + TypeDef + Ident "T" + Empty + Call + Ident "call" + StmtList + IntLit 49 + StmtList + IntLit 50 + StmtList + IntLit 51 +a: IntLit 1 +a: IntLit 2 +a: StmtList + IntLit 3 +a: StmtList + IntLit 4 +a: IntLit 5 +b: IntLit 6 +a: IntLit 7 +b: IntLit 8 +a: IntLit 9 +b: StmtList + IntLit 10 +a: IntLit 11 +b: StmtList + IntLit 12 +a: IntLit 13 +b: StmtList + IntLit 14 +a: IntLit 15 +b: StmtList + IntLit 16 +a: IntLit 17 +b: IntLit 18 +c: IntLit 19 +a: IntLit 20 +b: IntLit 21 +c: IntLit 22 +a: IntLit 23 +b: IntLit 24 +c: StmtList + IntLit 25 +a: IntLit 26 +b: IntLit 27 +c: StmtList + IntLit 28 +a: IntLit 29 +b: IntLit 30 +c: StmtList + IntLit 31 +a: TupleConstr + IntLit 32 + IntLit 33 +b: Infix + Ident "+" + Infix + Ident "*" + IntLit 34 + IntLit 35 + IntLit 36 +c: StmtList + IntLit 37 +a: Infix + Ident "@" + TupleConstr + IntLit 38 + IntLit 39 + Infix + Ident "shl" + IntLit 40 + IntLit 41 +b: Infix + Ident "-" + Infix + Ident "*" + IntLit 42 + IntLit 43 + IntLit 44 +c: StmtList + IntLit 45 +a: IntLit 46 +b: StmtList + IntLit 47 +c: StmtList + IntLit 48 +a: StmtList + IntLit 49 +b: StmtList + IntLit 50 +c: StmtList + IntLit 51 +''' +""" +import macros + +macro call(a): untyped = + echo "a: ", a.treeRepr + result = ident"int" +macro call(a, b): untyped = + echo "a: ", a.treeRepr + echo "b: ", b.treeRepr + result = ident"int" +macro call(a, b, c): untyped = + echo "a: ", a.treeRepr + echo "b: ", b.treeRepr + echo "c: ", c.treeRepr + result = ident"int" + +macro sections(x): untyped = + echo x.treeRepr + result = newStmtList(x) + for ts in x: + for td in ts: + let t = td[0] + result.add quote do: + doAssert `t` is int + +sections: + type A = call(1) + type + B = call 2 + C = call: 3 + D = call(): 4 + type + E = call(5, 6) + F = call 7, 8 + G = call(9): 10 + H = call(11): + 12 + I = call 13: 14 + J = call 15: + 16 + type + K = call(17, 18, 19) + L = call 20, 21, 22 + M = call(23, 24): 25 + N = call 26, 27: 28 + O = call 29, 30: + 31 + type + P = call (32, 33), 34 * 35 + 36: + 37 + R = call (38, 39) @ 40 shl 41, 42 * 43 - 44: + 45 + S = call 46: + 47 + do: + 48 + T = call: + 49 + do: + 50 + do: + 51 From 53e1d57419431050a54dcdf539a2a02bf6d28c5b Mon Sep 17 00:00:00 2001 From: BarrOff <58253563+BarrOff@users.noreply.github.com> Date: Sun, 2 Jan 2022 06:37:13 +0000 Subject: [PATCH 1436/2472] add compile time option for POSIX sigwait on Illumos/Solaris (#19296) * add compile time option for POSIX sigwait on Illumos/Solaris * fix link to documentation of `sigwait` on Illumos/Solaris --- lib/posix/posix.nim | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 57dd6e2c7..912e395a7 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -772,6 +772,13 @@ else: proc sigtimedwait*(a1: var Sigset, a2: var SigInfo, a3: var Timespec): cint {.importc, header: "".} +when defined(sunos) or defined(solaris): + # The following compile time flag is needed on Illumos/Solaris to use the POSIX + # `sigwait` implementation. See the documentation here: + # https://docs.oracle.com/cd/E19455-01/806-5257/6je9h033k/index.html + # https://www.illumos.org/man/2/sigwait + {.passc: "-D_POSIX_PTHREAD_SEMANTICS".} + proc sigwait*(a1: var Sigset, a2: var cint): cint {. importc, header: "".} proc sigwaitinfo*(a1: var Sigset, a2: var SigInfo): cint {. From bbd5086bc319dc22971a06ef24d02dc49ff308f3 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 3 Jan 2022 16:02:58 +0800 Subject: [PATCH 1437/2472] [docs] clarify the raised exception (#19308) * [docs] clarify the raised exception Lest developers wanna know what the exception is. * Apply suggestions from @konsumlamm Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- lib/pure/uri.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pure/uri.nim b/lib/pure/uri.nim index a828298c2..7583dbd1e 100644 --- a/lib/pure/uri.nim +++ b/lib/pure/uri.nim @@ -151,10 +151,10 @@ func encodeQuery*(query: openArray[(string, string)], usePlus = true, result.add(encodeUrl(val, usePlus)) iterator decodeQuery*(data: string): tuple[key, value: string] = - ## Reads and decodes query string `data` and yields the `(key, value)` pairs - ## the data consists of. If compiled with `-d:nimLegacyParseQueryStrict`, an - ## error is raised when there is an unencoded `=` character in a decoded - ## value, which was the behavior in Nim < 1.5.1 + ## Reads and decodes the query string `data` and yields the `(key, value)` pairs + ## the data consists of. If compiled with `-d:nimLegacyParseQueryStrict`, + ## a `UriParseError` is raised when there is an unencoded `=` character in a decoded + ## value, which was the behavior in Nim < 1.5.1. runnableExamples: import std/sequtils assert toSeq(decodeQuery("foo=1&bar=2=3")) == @[("foo", "1"), ("bar", "2=3")] From e49d52eb6174b2721293968e7811928c303c3776 Mon Sep 17 00:00:00 2001 From: Carlo Capocasa Date: Mon, 3 Jan 2022 09:11:23 +0100 Subject: [PATCH 1438/2472] Add Week-Of-Year Implementation to Times Module (#17223) * initial * more tests * Apply suggestions from code review idiomatize Co-authored-by: Timothee Cour * test iron age dates * add examples * fix typo * consistent param mention * add since pragrams * add changelog * Update lib/pure/times.nim Co-authored-by: Timothee Cour * fix examples * fix negative years * add getWeeksInYear tests * add back fix dropped by rebase * week-year tuple api * add changelog * fix doc tags * add docstrings * fix typos Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Co-authored-by: Timothee Cour Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- changelog.md | 6 +++ lib/pure/times.nim | 82 +++++++++++++++++++++++++++++++++++ tests/stdlib/ttimes.nim | 95 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 183 insertions(+) diff --git a/changelog.md b/changelog.md index 89aaff664..1b97fd99a 100644 --- a/changelog.md +++ b/changelog.md @@ -27,6 +27,12 @@ - Sends `ehlo` first. If the mail server does not understand, it sends `helo` as a fallback. +- Added `IsoWeekRange`, a range type to represent the number of weeks in an ISO week-based year. +- Added `IsoYear`, a distinct int type to prevent bugs from confusing the week-based year and the regular year. +- Added `initDateTime` in `times` to create a datetime from a weekday, and ISO 8601 week number and week-based year. +- Added `getIsoWeekAndYear` in `times` to get an ISO week number along with the corresponding ISO week-based year from a datetime. +- Added `getIsoWeeksInYear` in `times` to return the number of weeks in an ISO week-based year. + ## Language changes - Pragma macros on type definitions can now return `nnkTypeSection` nodes as well as `nnkTypeDef`, diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 113f73d2a..7851bf158 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -288,6 +288,13 @@ type YeardayRange* = range[0..365] NanosecondRange* = range[0..999_999_999] + IsoWeekRange* = range[1 .. 53] + ## An ISO 8601 calendar week number. + IsoYear* = distinct int + ## An ISO 8601 calendar year number. + ## + ## .. warning:: The ISO week-based year can correspond to the following or previous year from 29 December to January 3. + Time* = object ## Represents a point in time. seconds: int64 nanosecond: NanosecondRange @@ -529,6 +536,54 @@ proc getDaysInYear*(year: int): int = doAssert getDaysInYear(2001) == 365 result = 365 + (if isLeapYear(year): 1 else: 0) +proc `==`*(a, b: IsoYear): bool {.borrow.} +proc `$`*(p: IsoYear): string {.borrow.} + +proc getWeeksInIsoYear*(y: IsoYear): IsoWeekRange {.since: (1, 5).} = + ## Returns the number of weeks in the specified ISO 8601 week-based year, which can be + ## either 53 or 52. + runnableExamples: + assert getWeeksInIsoYear(IsoYear(2000)) == 52 + assert getWeeksInIsoYear(IsoYear(2001)) == 53 + + var y = int(y) + + # support negative years + y = if y < 0: 400 + y mod 400 else: y + + # source: https://webspace.science.uu.nl/~gent0113/calendar/isocalendar.htm + let p = (y + (y div 4) - (y div 100) + (y div 400)) mod 7 + let y1 = y - 1 + let p1 = (y1 + (y1 div 4) - (y1 div 100) + (y1 div 400)) mod 7 + if p == 4 or p1 == 3: 53 else: 52 + +proc getIsoWeekAndYear*(dt: DateTime): + tuple[isoweek: IsoWeekRange, isoyear: IsoYear] {.since: (1, 5).} = + ## Returns the ISO 8601 week and year. + ## + ## .. warning:: The ISO week-based year can correspond to the following or previous year from 29 December to January 3. + runnableExamples: + assert getIsoWeekAndYear(initDateTime(21, mApr, 2018, 00, 00, 00)) == (isoweek: 16.IsoWeekRange, isoyear: 2018.IsoYear) + block: + let (w, y) = getIsoWeekAndYear(initDateTime(30, mDec, 2019, 00, 00, 00)) + assert w == 01.IsoWeekRange + assert y == 2020.IsoYear + assert getIsoWeekAndYear(initDateTime(13, mSep, 2020, 00, 00, 00)) == (isoweek: 37.IsoWeekRange, isoyear: 2020.IsoYear) + block: + let (w, y) = getIsoWeekAndYear(initDateTime(2, mJan, 2021, 00, 00, 00)) + assert w.int > 52 + assert w.int < 54 + assert y.int mod 100 == 20 + + # source: https://webspace.science.uu.nl/~gent0113/calendar/isocalendar.htm + var w = (dt.yearday.int - dt.weekday.int + 10) div 7 + if w < 1: + (isoweek: getWeeksInIsoYear(IsoYear(dt.year - 1)), isoyear: IsoYear(dt.year - 1)) + elif (w > getWeeksInIsoYear(IsoYear(dt.year))): + (isoweek: IsoWeekRange(1), isoyear: IsoYear(dt.year + 1)) + else: + (isoweek: IsoWeekRange(w), isoyear: IsoYear(dt.year)) + proc stringifyUnit(value: int | int64, unit: TimeUnit): string = ## Stringify time unit with it's name, lowercased let strUnit = $unit @@ -2578,6 +2633,33 @@ proc `+=`*(t: var Time, b: TimeInterval) = proc `-=`*(t: var Time, b: TimeInterval) = t = t - b +# +# Day of year +# + +proc initDateTime*(weekday: WeekDay, isoweek: IsoWeekRange, isoyear: IsoYear, + hour: HourRange, minute: MinuteRange, second: SecondRange, + nanosecond: NanosecondRange, + zone: Timezone = local()): DateTime {.since: (1, 5).} = + ## Create a new `DateTime <#DateTime>`_ from a weekday and an ISO 8601 week number and year + ## in the specified timezone. + ## + ## .. warning:: The ISO week-based year can correspond to the following or previous year from 29 December to January 3. + runnableExamples: + assert initDateTime(21, mApr, 2018, 00, 00, 00) == initDateTime(dSat, 16, 2018.IsoYear, 00, 00, 00) + assert initDateTime(30, mDec, 2019, 00, 00, 00) == initDateTime(dMon, 01, 2020.IsoYear, 00, 00, 00) + assert initDateTime(13, mSep, 2020, 00, 00, 00) == initDateTime(dSun, 37, 2020.IsoYear, 00, 00, 00) + assert initDateTime(2, mJan, 2021, 00, 00, 00) == initDateTime(dSat, 53, 2020.IsoYear, 00, 00, 00) + + # source https://webspace.science.uu.nl/~gent0113/calendar/isocalendar.htm + let d = isoweek * 7 + weekday.int - initDateTime(4, mJan, isoyear.int, 00, 00, 00).weekday.int - 4 + initDateTime(1, mJan, isoyear.int, hour, minute, second, nanosecond, zone) + initTimeInterval(days=d) + +proc initDateTime*(weekday: WeekDay, isoweek: IsoWeekRange, isoyear: IsoYear, + hour: HourRange, minute: MinuteRange, second: SecondRange, + zone: Timezone = local()): DateTime {.since: (1, 5).} = + initDateTime(weekday, isoweek, isoyear, hour, minute, second, 0, zone) + # # Other # diff --git a/tests/stdlib/ttimes.nim b/tests/stdlib/ttimes.nim index 66157b91c..2cf945786 100644 --- a/tests/stdlib/ttimes.nim +++ b/tests/stdlib/ttimes.nim @@ -646,3 +646,98 @@ block: # ttimes doAssert initDuration(milliseconds = 500).inMilliseconds == 500 doAssert initDuration(milliseconds = -500).inMilliseconds == -500 doAssert initDuration(nanoseconds = -999999999).inMilliseconds == -999 + + block: # getIsoWeekAndYear + doAssert getIsoWeekAndYear(initDateTime(04, mNov, 2019, 00, 00, 00)) == (isoweek: 45.IsoWeekRange, isoyear: 2019.IsoYear) + doAssert initDateTime(dMon, 45, 2019.IsoYear, 00, 00, 00) == initDateTime(04, mNov, 2019, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(28, mDec, 2019, 00, 00, 00)) == (isoweek: 52.IsoWeekRange, isoyear: 2019.IsoYear) + doAssert initDateTime(dSat, 52, 2019.IsoYear, 00, 00, 00) == initDateTime(28, mDec, 2019, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(29, mDec, 2019, 00, 00, 00)) == (isoweek: 52.IsoWeekRange, isoyear: 2019.IsoYear) + doAssert initDateTime(dSun, 52, 2019.IsoYear, 00, 00, 00) == initDateTime(29, mDec, 2019, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(30, mDec, 2019, 00, 00, 00)) == (isoweek: 01.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dMon, 01, 2020.IsoYear, 00, 00, 00) == initDateTime(30, mDec, 2019, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(31, mDec, 2019, 00, 00, 00)) == (isoweek: 01.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dTue, 01, 2020.IsoYear, 00, 00, 00) == initDateTime(31, mDec, 2019, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(01, mJan, 2020, 00, 00, 00)) == (isoweek: 01.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dWed, 01, 2020.IsoYear, 00, 00, 00) == initDateTime(01, mJan, 2020, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(02, mJan, 2020, 00, 00, 00)) == (isoweek: 01.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dThu, 01, 2020.IsoYear, 00, 00, 00) == initDateTime(02, mJan, 2020, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(05, mApr, 2020, 00, 00, 00)) == (isoweek: 14.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dSun, 14, 2020.IsoYear, 00, 00, 00) == initDateTime(05, mApr, 2020, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(06, mApr, 2020, 00, 00, 00)) == (isoweek: 15.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dMon, 15, 2020.IsoYear, 00, 00, 00) == initDateTime(06, mApr, 2020, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(10, mApr, 2020, 00, 00, 00)) == (isoweek: 15.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dFri, 15, 2020.IsoYear, 00, 00, 00) == initDateTime(10, mApr, 2020, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(12, mApr, 2020, 00, 00, 00)) == (isoweek: 15.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dSun, 15, 2020.IsoYear, 00, 00, 00) == initDateTime(12, mApr, 2020, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(13, mApr, 2020, 00, 00, 00)) == (isoweek: 16.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dMon, 16, 2020.IsoYear, 00, 00, 00) == initDateTime(13, mApr, 2020, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(15, mApr, 2020, 00, 00, 00)) == (isoweek: 16.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dThu, 16, 2020.IsoYear, 00, 00, 00) == initDateTime(16, mApr, 2020, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(17, mJul, 2020, 00, 00, 00)) == (isoweek: 29.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dFri, 29, 2020.IsoYear, 00, 00, 00) == initDateTime(17, mJul, 2020, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(19, mJul, 2020, 00, 00, 00)) == (isoweek: 29.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dSun, 29, 2020.IsoYear, 00, 00, 00) == initDateTime(19, mJul, 2020, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(20, mJul, 2020, 00, 00, 00)) == (isoweek: 30.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dMon, 30, 2020.IsoYear, 00, 00, 00) == initDateTime(20, mJul, 2020, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(23, mJul, 2020, 00, 00, 00)) == (isoweek: 30.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dThu, 30, 2020.IsoYear, 00, 00, 00) == initDateTime(23, mJul, 2020, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(31, mDec, 2020, 00, 00, 00)) == (isoweek: 53.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dThu, 53, 2020.IsoYear, 00, 00, 00) == initDateTime(31, mDec, 2020, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(01, mJan, 2021, 00, 00, 00)) == (isoweek: 53.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dFri, 53, 2020.IsoYear, 00, 00, 00) == initDateTime(01, mJan, 2021, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(02, mJan, 2021, 00, 00, 00)) == (isoweek: 53.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dSat, 53, 2020.IsoYear, 00, 00, 00) == initDateTime(02, mJan, 2021, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(03, mJan, 2021, 00, 00, 00)) == (isoweek: 53.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dSun, 53, 2020.IsoYear, 00, 00, 00) == initDateTime(03, mJan, 2021, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(04, mJan, 2021, 00, 00, 00)) == (isoweek: 01.IsoWeekRange, isoyear: 2021.IsoYear) + doAssert initDateTime(dMon, 01, 2021.IsoYear, 00, 00, 00) == initDateTime(04, mJan, 2021, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(01, mFeb, 2021, 00, 00, 00)) == (isoweek: 05.IsoWeekRange, isoyear: 2021.IsoYear) + doAssert initDateTime(dMon, 05, 2021.IsoYear, 00, 00, 00) == initDateTime(01, mFeb, 2021, 00, 00, 00) + + doAssert getIsoWeekAndYear(initDateTime(01, mFeb, 2021, 01, 02, 03, 400_000_000, staticTz(hours=1))) == (isoweek: 05.IsoWeekRange, isoyear: 2021.IsoYear) + doAssert initDateTime(dMon, 05, 2021.IsoYear, 01, 02, 03, 400_000_000, staticTz(hours=1)) == initDateTime(01, mFeb, 2021, 01, 02, 03, 400_000_000, staticTz(hours=1)) + + doAssert getIsoWeekAndYear(initDateTime(01, mApr, +0001, 00, 00, 00)) == (isoweek: 13.IsoWeekRange, isoyear: 0001.IsoYear) + doAssert initDateTime(dSun, 13, 0001.IsoYear, 00, 00, 00) == initDateTime(01, mApr, 0001, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(01, mApr, +0000, 00, 00, 00)) == (isoweek: 13.IsoWeekRange, isoyear: 0000.IsoYear) + doAssert initDateTime(dSat, 13, 0000.IsoYear, 00, 00, 00) == initDateTime(01, mApr, 0000, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(01, mApr, -0001, 00, 00, 00)) == (isoweek: 13.IsoWeekRange, isoyear: (-0001).IsoYear) + doAssert initDateTime(dThu, 13, (-0001).IsoYear, 00, 00, 00) == initDateTime(01, mApr, -0001, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(01, mApr, -0002, 00, 00, 00)) == (isoweek: 14.IsoWeekRange, isoyear: (-0002).IsoYear) + doAssert initDateTime(dWed, 14, (-0002).IsoYear, 00, 00, 00) == initDateTime(01, mApr, -0002, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(01, mApr, -0753, 00, 00, 00)) == (isoweek: 14.IsoWeekRange, isoyear: (-0753).IsoYear) + doAssert initDateTime(dMon, 14, (-0753).IsoYear, 00, 00, 00) == initDateTime(01, mApr, -0753, 00, 00, 00) + + block: # getWeeksInIsoYear + doAssert getWeeksInIsoYear((-0014).IsoYear) == 52 + doAssert getWeeksInIsoYear((-0013).IsoYear) == 53 + doAssert getWeeksInIsoYear((-0012).IsoYear) == 52 + + doAssert getWeeksInIsoYear((-0009).IsoYear) == 52 + doAssert getWeeksInIsoYear((-0008).IsoYear) == 53 + doAssert getWeeksInIsoYear((-0007).IsoYear) == 52 + + doAssert getWeeksInIsoYear((-0003).IsoYear) == 52 + doAssert getWeeksInIsoYear((-0002).IsoYear) == 53 + doAssert getWeeksInIsoYear((-0001).IsoYear) == 52 + + doAssert getWeeksInIsoYear(0003.IsoYear) == 52 + doAssert getWeeksInIsoYear(0004.IsoYear) == 53 + doAssert getWeeksInIsoYear(0005.IsoYear) == 52 + + doAssert getWeeksInIsoYear(1653.IsoYear) == 52 + doAssert getWeeksInIsoYear(1654.IsoYear) == 53 + doAssert getWeeksInIsoYear(1655.IsoYear) == 52 + + doAssert getWeeksInIsoYear(1997.IsoYear) == 52 + doAssert getWeeksInIsoYear(1998.IsoYear) == 53 + doAssert getWeeksInIsoYear(1999.IsoYear) == 52 + + doAssert getWeeksInIsoYear(2008.IsoYear) == 52 + doAssert getWeeksInIsoYear(2009.IsoYear) == 53 + doAssert getWeeksInIsoYear(2010.IsoYear) == 52 + + doAssert getWeeksInIsoYear(2014.IsoYear) == 52 + doAssert getWeeksInIsoYear(2015.IsoYear) == 53 + doAssert getWeeksInIsoYear(2016.IsoYear) == 52 From bd4f76a2e3b9282548e62ba3b620b09d6c564373 Mon Sep 17 00:00:00 2001 From: Nan Xiao Date: Mon, 3 Jan 2022 17:30:26 +0800 Subject: [PATCH 1439/2472] docs: Fix typo in tut1.rst (#19309) --- doc/tut1.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/tut1.rst b/doc/tut1.rst index 2856625b7..d8e0ec8e4 100644 --- a/doc/tut1.rst +++ b/doc/tut1.rst @@ -913,7 +913,7 @@ Let's return to the simple counting example: echo i Can a `countup `_ proc be written that -supports this loop? Lets try: +supports this loop? Let's try: .. code-block:: nim proc countup(a, b: int): int = From 19bcb43a0e21dd222ffd3fad8c5fb2d8e85d59ea Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 3 Jan 2022 20:40:43 +0800 Subject: [PATCH 1440/2472] enable multiple packages (arraymancer, fidget ...) (#19311) The cause of arraymancer failure has been tracked here: https://github.com/mratsim/Arraymancer/issues/505 And it was fixed by https://github.com/mratsim/Arraymancer/pull/542 --- testament/important_packages.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 7e15e7c65..381aa27ac 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -35,7 +35,7 @@ proc pkg(name: string; cmd = "nimble test"; url = "", useHead = true, allowFailu pkg "alea", allowFailure = true pkg "argparse" -pkg "arraymancer", "nim c tests/tests_cpu.nim", allowFailure = true +pkg "arraymancer", "nim c tests/tests_cpu.nim" pkg "ast_pattern_matching", "nim c -r --oldgensym:on tests/test1.nim", allowFailure = true pkg "asyncthreadpool" pkg "awk" @@ -63,7 +63,7 @@ pkg "delaunay" pkg "docopt" pkg "easygl", "nim c -o:egl -r src/easygl.nim", "https://github.com/jackmott/easygl" pkg "elvis" -pkg "fidget", allowFailure = true +pkg "fidget" pkg "fragments", "nim c -r fragments/dsl.nim" pkg "fusion" pkg "gara" @@ -91,7 +91,7 @@ pkg "memo" pkg "msgpack4nim", "nim c -r tests/test_spec.nim" pkg "nake", "nim c nakefile.nim" pkg "neo", "nim c -d:blas=openblas tests/all.nim" -pkg "nesm", "nimble tests", allowFailure = true # notice plural 'tests' +pkg "nesm", "nimble tests" # notice plural 'tests' pkg "netty" pkg "nico", allowFailure = true pkg "nicy", "nim c -r src/nicy.nim" From 526a32e1697489b73e54c1b984ba5413842f1252 Mon Sep 17 00:00:00 2001 From: rockcavera Date: Mon, 3 Jan 2022 16:14:08 -0300 Subject: [PATCH 1441/2472] Fix #19314 - fixing broken `DoublyLinkedList` after adding empty `DoublyLinkedList` (#19315) [backport] * Update lists.nim * Update tlists.nim --- lib/pure/collections/lists.nim | 12 ++++++------ tests/stdlib/tlists.nim | 12 ++++++++++++ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/lib/pure/collections/lists.nim b/lib/pure/collections/lists.nim index de5550d6e..f7627ae67 100644 --- a/lib/pure/collections/lists.nim +++ b/lib/pure/collections/lists.nim @@ -675,12 +675,12 @@ proc addMoved*[T](a, b: var DoublyLinkedList[T]) {.since: (1, 5, 1).} = assert s == [0, 1, 0, 1, 0, 1] if b.head != nil: - b.head.prev = a.tail - if a.tail != nil: - a.tail.next = b.head - a.tail = b.tail - if a.head == nil: - a.head = b.head + if a.head == nil: + a.head = b.head + else: + b.head.prev = a.tail + a.tail.next = b.head + a.tail = b.tail if a.addr != b.addr: b.head = nil b.tail = nil diff --git a/tests/stdlib/tlists.nim b/tests/stdlib/tlists.nim index 2f6d5ee6a..14cbf2f9d 100644 --- a/tests/stdlib/tlists.nim +++ b/tests/stdlib/tlists.nim @@ -245,6 +245,18 @@ template main = doAssert a.toSeq == @[1] a.add(2) doAssert a.toSeq == @[1, 2] + + block issue19314: # add (appends a shallow copy) + var a: DoublyLinkedList[int] + var b: DoublyLinkedList[int] + + doAssert a.toSeq == @[] + a.add(1) + doAssert a.toSeq == @[1] + a.add(b) + doAssert a.toSeq == @[1] + a.add(2) + doAssert a.toSeq == @[1, 2] static: main() main() From fcf1df6e56934b1b5021a0898b9152b86e0cb5f7 Mon Sep 17 00:00:00 2001 From: Smarcy Date: Mon, 3 Jan 2022 21:04:15 +0100 Subject: [PATCH 1442/2472] fixed typos (#19316) --- nimpretty/nimpretty.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nimpretty/nimpretty.nim b/nimpretty/nimpretty.nim index a7940349d..e3b4c9a3a 100644 --- a/nimpretty/nimpretty.nim +++ b/nimpretty/nimpretty.nim @@ -69,8 +69,8 @@ proc main = var backup = false # when `on`, create a backup file of input in case - # `prettyPrint` could over-write it (note that the backup may happen even - # if input is not actually over-written, when nimpretty is a noop). + # `prettyPrint` could overwrite it (note that the backup may happen even + # if input is not actually overwritten, when nimpretty is a noop). # --backup was un-documented (rely on git instead). var opt = PrettyOptions(indWidth: 0, maxLineLen: 80) From 35cae73aa5c39b950386febfa5182bab920ddfbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Tue, 4 Jan 2022 07:56:10 +0100 Subject: [PATCH 1443/2472] devel: style fix (#19318) this allows "--styleCheck:usages --styleCheck:error" --- lib/pure/nativesockets.nim | 2 +- lib/pure/net.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index 4037d7181..eb0514a8f 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -810,7 +810,7 @@ proc accept*(fd: SocketHandle, inheritable = defined(nimInheritHandles)): (Socke ## child processes. ## ## Returns (osInvalidSocket, "") if an error occurred. - var sockAddress: Sockaddr + var sockAddress: SockAddr var addrLen = sizeof(sockAddress).SockLen var sock = when (defined(linux) or defined(bsd)) and not defined(nimdoc): diff --git a/lib/pure/net.nim b/lib/pure/net.nim index ced6b2fb2..8dc92e9d6 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -1778,7 +1778,7 @@ proc sendTo*(socket: Socket, address: IpAddress, port: Port, assert(not socket.isClosed, "Cannot `sendTo` on a closed socket") var sa: Sockaddr_storage - var sl: Socklen + var sl: SockLen toSockAddr(address, port, sa, sl) result = sendto(socket.fd, cstring(data), data.len().cint, flags.cint, cast[ptr SockAddr](addr sa), sl) From 39a27783fc0d43e9828bb031cc811042226407b9 Mon Sep 17 00:00:00 2001 From: Nan Xiao Date: Tue, 4 Jan 2022 17:42:21 +0800 Subject: [PATCH 1444/2472] docs: Fix typo in tut1.rst (#19324) --- doc/tut1.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/tut1.rst b/doc/tut1.rst index d8e0ec8e4..3107aef45 100644 --- a/doc/tut1.rst +++ b/doc/tut1.rst @@ -1052,7 +1052,7 @@ errors. Lossless `Automatic type conversion`:idx: is performed in expressions where different kinds of integer types are used. However, if the type conversion -would cause loss of information, the ``RangeDefect``:idx: is raised (if the error +would cause loss of information, the `RangeDefect`:idx: is raised (if the error cannot be detected at compile time). From 0bcd7062c6e7a4131d166dacea41d661b7a09748 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 4 Jan 2022 18:10:46 +0800 Subject: [PATCH 1445/2472] correct the comments (#19322) --expandArc ``` var a b a = matrix(5, 5, 1.0) b = matrix(5, 5, 2.0) `=sink`(b, - let blitTmp = b wasMoved(b) blitTmp + a) `=destroy`(b) `=destroy`(a) ``` --- compiler/optimizer.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/optimizer.nim b/compiler/optimizer.nim index 744c82ab5..f484fdbf5 100644 --- a/compiler/optimizer.nim +++ b/compiler/optimizer.nim @@ -127,11 +127,11 @@ proc analyse(c: var Con; b: var BasicBlock; n: PNode) = for i in 0 ..< n.len: analyse(c, b, n[i]) else: - #[ Test tmatrix.test3: + #[ Test destructor/tmatrix.test3: Prevent this from being elided. We should probably find a better solution... - `=sink`(b, - ( + `=sink`(b, - let blitTmp = b; wasMoved(b); blitTmp + a) From 1869826668e2e3a0fd69cc1b69fb12b07993e417 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 4 Jan 2022 18:49:54 +0800 Subject: [PATCH 1446/2472] add std/private/win_getsysteminfo; refactor the usage of `GetSystemInfo` (#19310) * add std/private/win_getsysteminfo * import at the top level * wrappers follow nep1 too * follow review comment --- lib/pure/concurrency/cpuinfo.nim | 24 ++++++------------------ lib/pure/reservedmem.nim | 19 ++----------------- lib/std/private/win_getsysteminfo.nim | 15 +++++++++++++++ 3 files changed, 23 insertions(+), 35 deletions(-) create mode 100644 lib/std/private/win_getsysteminfo.nim diff --git a/lib/pure/concurrency/cpuinfo.nim b/lib/pure/concurrency/cpuinfo.nim index ee43b8e11..16d32002d 100644 --- a/lib/pure/concurrency/cpuinfo.nim +++ b/lib/pure/concurrency/cpuinfo.nim @@ -18,6 +18,9 @@ include "system/inclrtl" when defined(posix) and not (defined(macosx) or defined(bsd)): import posix +when defined(windows): + import std/private/win_getsysteminfo + when defined(freebsd) or defined(macosx): {.emit: "#include ".} @@ -54,25 +57,10 @@ proc countProcessors*(): int {.rtl, extern: "ncpi$1".} = ## Returns the number of the processors/cores the machine has. ## Returns 0 if it cannot be detected. when defined(windows): - type - SYSTEM_INFO {.final, pure.} = object - u1: int32 - dwPageSize: int32 - lpMinimumApplicationAddress: pointer - lpMaximumApplicationAddress: pointer - dwActiveProcessorMask: ptr int32 - dwNumberOfProcessors: int32 - dwProcessorType: int32 - dwAllocationGranularity: int32 - wProcessorLevel: int16 - wProcessorRevision: int16 - - proc GetSystemInfo(lpSystemInfo: var SYSTEM_INFO) {.stdcall, dynlib: "kernel32", importc: "GetSystemInfo".} - var - si: SYSTEM_INFO - GetSystemInfo(si) - result = si.dwNumberOfProcessors + si: SystemInfo + getSystemInfo(addr si) + result = int(si.dwNumberOfProcessors) elif defined(macosx) or defined(bsd): var mib: array[0..3, cint] diff --git a/lib/pure/reservedmem.nim b/lib/pure/reservedmem.nim index 232a2b383..99fbe1429 100644 --- a/lib/pure/reservedmem.nim +++ b/lib/pure/reservedmem.nim @@ -42,25 +42,10 @@ type when defined(windows): import winlean - - type - SYSTEM_INFO {.final, pure.} = object - u1: uint32 - dwPageSize: uint32 - lpMinimumApplicationAddress: pointer - lpMaximumApplicationAddress: pointer - dwActiveProcessorMask: ptr uint32 - dwNumberOfProcessors: uint32 - dwProcessorType: uint32 - dwAllocationGranularity: uint32 - wProcessorLevel: uint16 - wProcessorRevision: uint16 - - proc getSystemInfo(lpSystemInfo: ptr SYSTEM_INFO) {.stdcall, - dynlib: "kernel32", importc: "GetSystemInfo".} + import std/private/win_getsysteminfo proc getAllocationGranularity: uint = - var sysInfo: SYSTEM_INFO + var sysInfo: SystemInfo getSystemInfo(addr sysInfo) return uint(sysInfo.dwAllocationGranularity) diff --git a/lib/std/private/win_getsysteminfo.nim b/lib/std/private/win_getsysteminfo.nim new file mode 100644 index 000000000..b98478231 --- /dev/null +++ b/lib/std/private/win_getsysteminfo.nim @@ -0,0 +1,15 @@ +type + SystemInfo* = object + u1: uint32 + dwPageSize: uint32 + lpMinimumApplicationAddress: pointer + lpMaximumApplicationAddress: pointer + dwActiveProcessorMask: ptr uint32 + dwNumberOfProcessors*: uint32 + dwProcessorType: uint32 + dwAllocationGranularity*: uint32 + wProcessorLevel: uint16 + wProcessorRevision: uint16 + +proc getSystemInfo*(lpSystemInfo: ptr SystemInfo) {.stdcall, + dynlib: "kernel32", importc: "GetSystemInfo".} From 9df195ef581879cf9e5c3c3154755bd1dba677c6 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 4 Jan 2022 20:29:50 +0800 Subject: [PATCH 1447/2472] style usages part one (openarray => openArray) (#19321) * style usages (openArray) * revert doc changes --- tests/arc/tmovebug.nim | 4 ++-- tests/arc/tmovebugcopy.nim | 4 ++-- tests/array/tarray.nim | 2 +- tests/async/tioselectors.nim | 4 ++-- tests/closure/tclosure.nim | 4 ++-- tests/compiles/trecursive_generic_in_compiles.nim | 2 +- tests/concepts/tconcepts_issues.nim | 8 ++++---- tests/converter/tconverter_with_varargs.nim | 2 +- tests/cpp/tcovariancerules.nim | 8 ++++---- tests/destructor/tmove_objconstr.nim | 2 +- tests/errmsgs/t17460.nim | 2 +- tests/errmsgs/tproper_stacktrace.nim | 2 +- tests/errmsgs/tsigmatch.nim | 4 ++-- tests/generics/tgenerics_issues.nim | 2 +- tests/generics/tgenerics_various.nim | 4 ++-- tests/iter/titer13.nim | 2 +- tests/iter/titer_issues.nim | 4 ++-- tests/iter/tmoditer.nim | 2 +- tests/iter/tpermutations.nim | 2 +- tests/js/t9410.nim | 4 ++-- tests/js/tbasics.nim | 2 +- tests/js/tbyvar.nim | 4 ++-- tests/lent/tbasic_lent_check.nim | 2 +- tests/magics/tmagics.nim | 2 +- tests/manyloc/named_argument_bug/tri_engine/gfx/gl/gl.nim | 2 +- .../named_argument_bug/tri_engine/gfx/gl/primitive.nim | 2 +- tests/misc/t9039.nim | 2 +- tests/misc/theaproots.nim | 2 +- tests/misc/tsimplesort.nim | 2 +- tests/openarray/topena1.nim | 2 +- tests/openarray/topenarrayrepr.nim | 4 ++-- tests/openarray/topenlen.nim | 2 +- tests/openarray/tptrarrayderef.nim | 6 +++--- tests/overload/toverload_various.nim | 4 ++-- tests/showoff/tdrdobbs_examples.nim | 4 ++-- tests/stdlib/tdochelpers.nim | 4 ++-- tests/stdlib/tparseipv6.nim | 4 ++-- tests/stdlib/trepr.nim | 2 +- tests/system/tostring.nim | 2 +- tests/system/tsystem_misc.nim | 4 ++-- tests/template/otests.nim | 4 ++-- tests/template/sunset.nimf | 2 +- tests/template/template_issues.nim | 2 +- tests/tuples/tuple_with_nil.nim | 4 ++-- tests/typerel/tcommontype.nim | 2 +- tests/typerel/texplicitcmp.nim | 2 +- tests/typerel/tstr_as_openarray.nim | 6 +++--- tests/views/tviews1.nim | 2 +- tests/vm/tyaytypedesc.nim | 2 +- 49 files changed, 76 insertions(+), 76 deletions(-) diff --git a/tests/arc/tmovebug.nim b/tests/arc/tmovebug.nim index 7977d330a..002bd6796 100644 --- a/tests/arc/tmovebug.nim +++ b/tests/arc/tmovebug.nim @@ -292,7 +292,7 @@ when false: # bug #13456 -iterator combinations[T](s: openarray[T], k: int): seq[T] = +iterator combinations[T](s: openArray[T], k: int): seq[T] = let n = len(s) assert k >= 0 and k <= n var pos = newSeq[int](k) @@ -455,7 +455,7 @@ initFoo7(2) # bug #14902 -iterator zip[T](s: openarray[T]): (T, T) = +iterator zip[T](s: openArray[T]): (T, T) = var i = 0 while i < 10: yield (s[i mod 2], s[i mod 2 + 1]) diff --git a/tests/arc/tmovebugcopy.nim b/tests/arc/tmovebugcopy.nim index 7c5228147..ec4315777 100644 --- a/tests/arc/tmovebugcopy.nim +++ b/tests/arc/tmovebugcopy.nim @@ -253,7 +253,7 @@ when false: # bug #13456 -iterator combinations[T](s: openarray[T], k: int): seq[T] = +iterator combinations[T](s: openArray[T], k: int): seq[T] = let n = len(s) assert k >= 0 and k <= n var pos = newSeq[int](k) @@ -416,7 +416,7 @@ initFoo7(2) # bug #14902 -iterator zip[T](s: openarray[T]): (T, T) = +iterator zip[T](s: openArray[T]): (T, T) = var i = 0 while i < 10: yield (s[i mod 2], s[i mod 2 + 1]) diff --git a/tests/array/tarray.nim b/tests/array/tarray.nim index d10011ef2..e9f330e3b 100644 --- a/tests/array/tarray.nim +++ b/tests/array/tarray.nim @@ -44,7 +44,7 @@ block tarray: arr: TMyarray - proc sum(a: openarray[int]): int = + proc sum(a: openArray[int]): int = result = 0 var i = 0 while i < len(a): diff --git a/tests/async/tioselectors.nim b/tests/async/tioselectors.nim index f7487525a..fdb8d9597 100644 --- a/tests/async/tioselectors.nim +++ b/tests/async/tioselectors.nim @@ -288,8 +288,8 @@ when not defined(windows): events: set[Event] proc vnode_test(): bool = - proc validate(test: openarray[ReadyKey], - check: openarray[valType]): bool = + proc validate(test: openArray[ReadyKey], + check: openArray[valType]): bool = result = false if len(test) == len(check): for checkItem in check: diff --git a/tests/closure/tclosure.nim b/tests/closure/tclosure.nim index 546d7026d..8ae6c44bb 100644 --- a/tests/closure/tclosure.nim +++ b/tests/closure/tclosure.nim @@ -38,10 +38,10 @@ joinable: false block tclosure: - proc map(n: var openarray[int], fn: proc (x: int): int {.closure}) = + proc map(n: var openArray[int], fn: proc (x: int): int {.closure}) = for i in 0..n.len-1: n[i] = fn(n[i]) - proc each(n: openarray[int], fn: proc(x: int) {.closure.}) = + proc each(n: openArray[int], fn: proc(x: int) {.closure.}) = for i in 0..n.len-1: fn(n[i]) diff --git a/tests/compiles/trecursive_generic_in_compiles.nim b/tests/compiles/trecursive_generic_in_compiles.nim index 88d3faeed..ab31ad0f5 100644 --- a/tests/compiles/trecursive_generic_in_compiles.nim +++ b/tests/compiles/trecursive_generic_in_compiles.nim @@ -50,7 +50,7 @@ proc `==`*[T](xs, ys: List[T]): bool = proc asList*[T](xs: varargs[T]): List[T] = ## Creates list from varargs - proc initListImpl(i: int, xs: openarray[T]): List[T] = + proc initListImpl(i: int, xs: openArray[T]): List[T] = if i > high(xs): Nil[T]() else: diff --git a/tests/concepts/tconcepts_issues.nim b/tests/concepts/tconcepts_issues.nim index c76049bdd..1a0dd8063 100644 --- a/tests/concepts/tconcepts_issues.nim +++ b/tests/concepts/tconcepts_issues.nim @@ -324,15 +324,15 @@ block t6691: block t6782: type Reader = concept c - c.read(openarray[byte], int, int) is int + c.read(openArray[byte], int, int) is int Rdr = concept c - c.rd(openarray[byte], int, int) is int + c.rd(openArray[byte], int, int) is int type TestFile = object - proc read(r: TestFile, dest: openarray[byte], offset: int, limit: int): int = + proc read(r: TestFile, dest: openArray[byte], offset: int, limit: int): int = result = 0 - proc rd(r: TestFile, dest: openarray[byte], offset: int, limit: int): int = + proc rd(r: TestFile, dest: openArray[byte], offset: int, limit: int): int = result = 0 doAssert TestFile is Reader diff --git a/tests/converter/tconverter_with_varargs.nim b/tests/converter/tconverter_with_varargs.nim index 6d7e31e85..fae83221b 100644 --- a/tests/converter/tconverter_with_varargs.nim +++ b/tests/converter/tconverter_with_varargs.nim @@ -8,7 +8,7 @@ type converter to_py*(i: int) : PPyRef = nil when false: - proc to_tuple*(vals: openarray[PPyRef]): PPyRef = + proc to_tuple*(vals: openArray[PPyRef]): PPyRef = discard proc abc(args: varargs[PPyRef]) = diff --git a/tests/cpp/tcovariancerules.nim b/tests/cpp/tcovariancerules.nim index 5ac7d8bac..9d49f2cbd 100644 --- a/tests/cpp/tcovariancerules.nim +++ b/tests/cpp/tcovariancerules.nim @@ -79,16 +79,16 @@ proc wantsCovariantSeq2(s: seq[AnimalRef]) = proc wantsCovariantSeq3(s: seq[RefAlias[Animal]]) = for a in s: echo a.x -proc wantsCovariantOpenArray(s: openarray[ref Animal]) = +proc wantsCovariantOpenArray(s: openArray[ref Animal]) = for a in s: echo a.x -proc modifiesCovariantOpenArray(s: var openarray[ref Animal]) = +proc modifiesCovariantOpenArray(s: var openArray[ref Animal]) = for a in s: echo a.x -proc modifiesDerivedOpenArray(s: var openarray[ref Dog]) = +proc modifiesDerivedOpenArray(s: var openArray[ref Dog]) = for a in s: echo a.x -proc wantsNonCovariantOpenArray(s: openarray[Animal]) = +proc wantsNonCovariantOpenArray(s: openArray[Animal]) = for a in s: echo a.x proc wantsCovariantArray(s: array[2, ref Animal]) = diff --git a/tests/destructor/tmove_objconstr.nim b/tests/destructor/tmove_objconstr.nim index 5faaabb8b..9eee4bfdb 100644 --- a/tests/destructor/tmove_objconstr.nim +++ b/tests/destructor/tmove_objconstr.nim @@ -178,7 +178,7 @@ proc myfuncLoop(x: int): MySeqNonCopyable = discard myfuncLoop(3) #------------------------------------------------------------ -# Move into table via openarray +# Move into table via openArray #------------------------------------------------------------ type diff --git a/tests/errmsgs/t17460.nim b/tests/errmsgs/t17460.nim index 8192cc4c8..e377bc48a 100644 --- a/tests/errmsgs/t17460.nim +++ b/tests/errmsgs/t17460.nim @@ -3,7 +3,7 @@ discard """ errormsg: "wrong number of variables" """ -iterator xclusters*[T](a: openarray[T]; s: static[int]): array[s, T] {.inline.} = +iterator xclusters*[T](a: openArray[T]; s: static[int]): array[s, T] {.inline.} = var result: array[s, T] # iterators have no default result variable var i = 0 while i < len(a): diff --git a/tests/errmsgs/tproper_stacktrace.nim b/tests/errmsgs/tproper_stacktrace.nim index c7dfbaf2a..8617984fb 100644 --- a/tests/errmsgs/tproper_stacktrace.nim +++ b/tests/errmsgs/tproper_stacktrace.nim @@ -6,7 +6,7 @@ import strscans, strutils proc raiseTestException*() = raise newException(Exception, "test") -proc matchStackTrace(actualEntries: openarray[StackTraceEntry], expected: string) = +proc matchStackTrace(actualEntries: openArray[StackTraceEntry], expected: string) = var expectedEntries = newSeq[StackTraceEntry]() var i = 0 diff --git a/tests/errmsgs/tsigmatch.nim b/tests/errmsgs/tsigmatch.nim index 023b7d518..8f32ef9e6 100644 --- a/tests/errmsgs/tsigmatch.nim +++ b/tests/errmsgs/tsigmatch.nim @@ -132,12 +132,12 @@ block: echo foo(fun) block: - # bug #10285 Function signature don't match when inside seq/array/openarray + # bug #10285 Function signature don't match when inside seq/array/openArray # Note: the error message now shows `closure` which helps debugging the issue # out why it doesn't match proc takesFunc(f: proc (x: int) {.gcsafe, locks: 0.}) = echo "takes single Func" - proc takesFuncs(fs: openarray[proc (x: int) {.gcsafe, locks: 0.}]) = + proc takesFuncs(fs: openArray[proc (x: int) {.gcsafe, locks: 0.}]) = echo "takes multiple Func" takesFunc(proc (x: int) {.gcsafe, locks: 0.} = echo x) # works takesFuncs([proc (x: int) {.gcsafe, locks: 0.} = echo x]) # fails diff --git a/tests/generics/tgenerics_issues.nim b/tests/generics/tgenerics_issues.nim index 9e70efcd2..092926c01 100644 --- a/tests/generics/tgenerics_issues.nim +++ b/tests/generics/tgenerics_issues.nim @@ -603,7 +603,7 @@ block t7854: block t5864: - proc defaultStatic(s: openarray, N: static[int] = 1): int = N + proc defaultStatic(s: openArray, N: static[int] = 1): int = N proc defaultGeneric[T](a: T = 2): int = a let a = [1, 2, 3, 4].defaultStatic() diff --git a/tests/generics/tgenerics_various.nim b/tests/generics/tgenerics_various.nim index 285108cd3..6c76502e1 100644 --- a/tests/generics/tgenerics_various.nim +++ b/tests/generics/tgenerics_various.nim @@ -230,8 +230,8 @@ block tvarargs_vs_generics: echo "direct" proc withDirectType[T](arg: T) = echo "generic" - proc withOpenArray(args: openarray[string]) = - echo "openarray" + proc withOpenArray(args: openArray[string]) = + echo "openArray" proc withOpenArray[T](arg: T) = echo "generic" proc withVarargs(args: varargs[string]) = diff --git a/tests/iter/titer13.nim b/tests/iter/titer13.nim index 0d4a399c5..086c40ca4 100644 --- a/tests/iter/titer13.nim +++ b/tests/iter/titer13.nim @@ -72,7 +72,7 @@ block: echo a block t5859: - proc flatIterator[T](s: openarray[T]): auto {.noSideEffect.}= + proc flatIterator[T](s: openArray[T]): auto {.noSideEffect.}= result = iterator(): auto = when (T is not seq|array): for item in s: diff --git a/tests/iter/titer_issues.nim b/tests/iter/titer_issues.nim index 4a76b2fb0..65f66ad26 100644 --- a/tests/iter/titer_issues.nim +++ b/tests/iter/titer_issues.nim @@ -135,7 +135,7 @@ block t3837_chained: block t3221_complex: - iterator permutations[T](ys: openarray[T]): seq[T] = + iterator permutations[T](ys: openArray[T]): seq[T] = var d = 1 c = newSeq[int](ys.len) @@ -228,7 +228,7 @@ block t2023_objiter: block: # bug #13739 - iterator myIter(arg: openarray[int]): int = + iterator myIter(arg: openArray[int]): int = var tmp = 0 let len = arg.len while tmp < len: diff --git a/tests/iter/tmoditer.nim b/tests/iter/tmoditer.nim index b92a416fb..99e5b642d 100644 --- a/tests/iter/tmoditer.nim +++ b/tests/iter/tmoditer.nim @@ -43,7 +43,7 @@ proc `=copy`(dst: var NonCopyable, src: NonCopyable) {.error.} proc `=sink`(dst: var NonCopyable, src: NonCopyable) = dst.x = src.x -iterator lentItems[T](a: openarray[T]): lent T = +iterator lentItems[T](a: openArray[T]): lent T = for i in 0..a.high: yield a[i] diff --git a/tests/iter/tpermutations.nim b/tests/iter/tpermutations.nim index c5067ba31..30a66460f 100644 --- a/tests/iter/tpermutations.nim +++ b/tests/iter/tpermutations.nim @@ -12,7 +12,7 @@ perm: 6778800.0 det: 0.0 import sequtils, sugar -iterator permutations*[T](ys: openarray[T]): tuple[perm: seq[T], sign: int] = +iterator permutations*[T](ys: openArray[T]): tuple[perm: seq[T], sign: int] = var d = 1 c = newSeq[int](ys.len) diff --git a/tests/js/t9410.nim b/tests/js/t9410.nim index 78c329a24..042520dc5 100644 --- a/tests/js/t9410.nim +++ b/tests/js/t9410.nim @@ -447,10 +447,10 @@ template tests = doAssert seqOfSeqs == @[@[10, 2], @[30, 4]] when false: - block: # openarray + block: # openArray # Error: internal error: genAddr: nkStmtListExpr var calls = 0 - proc getvarint(x: var openarray[int]): var int = + proc getvarint(x: var openArray[int]): var int = calls += 1 if true: x[1] diff --git a/tests/js/tbasics.nim b/tests/js/tbasics.nim index b297bb037..ef84f8bb5 100644 --- a/tests/js/tbasics.nim +++ b/tests/js/tbasics.nim @@ -54,7 +54,7 @@ for x in someGlobal: doAssert(x == 0) proc tdefault = var x = default(int) doAssert(x == 0) - proc inner(v: openarray[string]) = + proc inner(v: openArray[string]) = doAssert(v.len == 0) inner(default(seq[string])) diff --git a/tests/js/tbyvar.nim b/tests/js/tbyvar.nim index a4c60b0b3..93724a2f1 100644 --- a/tests/js/tbyvar.nim +++ b/tests/js/tbyvar.nim @@ -39,9 +39,9 @@ proc main = main() -# Test: pass var seq to var openarray +# Test: pass var seq to var openArray var s = @[2, 1] -proc foo(a: var openarray[int]) = a[0] = 123 +proc foo(a: var openArray[int]) = a[0] = 123 proc bar(s: var seq[int], a: int) = doAssert(a == 5) diff --git a/tests/lent/tbasic_lent_check.nim b/tests/lent/tbasic_lent_check.nim index 049fa276a..e33da06f8 100644 --- a/tests/lent/tbasic_lent_check.nim +++ b/tests/lent/tbasic_lent_check.nim @@ -39,7 +39,7 @@ template main2 = # bug #15958 p[] = 20.0 doAssert byLent(p)[] == 20.0 - proc byLent2[T](a: openarray[T]): lent T = a[0] + proc byLent2[T](a: openArray[T]): lent T = a[0] doAssert byLent2(a) == 11 doAssert byLent2(a).unsafeAddr == a[0].unsafeAddr doAssert byLent2(b) == 21 diff --git a/tests/magics/tmagics.nim b/tests/magics/tmagics.nim index 70926a0d7..0e412940c 100644 --- a/tests/magics/tmagics.nim +++ b/tests/magics/tmagics.nim @@ -18,7 +18,7 @@ block tlowhigh: for i in low(a) .. high(a): a[i] = 0 - proc sum(a: openarray[int]): int = + proc sum(a: openArray[int]): int = result = 0 for i in low(a)..high(a): inc(result, a[i]) diff --git a/tests/manyloc/named_argument_bug/tri_engine/gfx/gl/gl.nim b/tests/manyloc/named_argument_bug/tri_engine/gfx/gl/gl.nim index 7d787c07b..29e23f9d0 100644 --- a/tests/manyloc/named_argument_bug/tri_engine/gfx/gl/gl.nim +++ b/tests/manyloc/named_argument_bug/tri_engine/gfx/gl/gl.nim @@ -53,7 +53,7 @@ else: const glRealType* = cGLfloat -proc setUniformV4*[T](loc: GLint, vecs: var openarray[TV4[T]]) = +proc setUniformV4*[T](loc: GLint, vecs: var openArray[TV4[T]]) = glUniform4fv(loc, vecs.len.GLsizei, cast[ptr GLfloat](vecs[0].addr)) proc setUniformV4*[T](loc: GLint, vec: TV4[T]) = diff --git a/tests/manyloc/named_argument_bug/tri_engine/gfx/gl/primitive.nim b/tests/manyloc/named_argument_bug/tri_engine/gfx/gl/primitive.nim index 3bfaf1cbc..accc2d96b 100644 --- a/tests/manyloc/named_argument_bug/tri_engine/gfx/gl/primitive.nim +++ b/tests/manyloc/named_argument_bug/tri_engine/gfx/gl/primitive.nim @@ -47,7 +47,7 @@ proc newVert*(rect: rect.TRect): seq[TVert] = proc newVertAttrib(i: GLuint, size: GLint, stride: GLsizei, offset: GLvoid): TVertAttrib = TVertAttrib(i: i, size: size, stride: stride, offset: offset) -proc genBuf*[T](vboTarget, objUsage: GLenum, data: var openarray[T]): GLuint = +proc genBuf*[T](vboTarget, objUsage: GLenum, data: var openArray[T]): GLuint = result = 0.GLuint ?glGenBuffers(1, result.addr) ?glBindBuffer(vboTarget, result) diff --git a/tests/misc/t9039.nim b/tests/misc/t9039.nim index ba636d1db..3271cd34e 100644 --- a/tests/misc/t9039.nim +++ b/tests/misc/t9039.nim @@ -15,7 +15,7 @@ but expression 'nesting + 1' is of type: int # line 15 func default(T: typedesc[array]): T = discard doAssert default(array[3, int]) == [0, 0, 0] -func shapeBad*[T: not char](s: openarray[T], rank: static[int], nesting = 0, parent_shape = default(array[rank, int])): array[rank, int] = +func shapeBad*[T: not char](s: openArray[T], rank: static[int], nesting = 0, parent_shape = default(array[rank, int])): array[rank, int] = result = parent_shape result[nesting] = s.len when (T is seq|array): diff --git a/tests/misc/theaproots.nim b/tests/misc/theaproots.nim index 1ea3c86b9..2dd345254 100644 --- a/tests/misc/theaproots.nim +++ b/tests/misc/theaproots.nim @@ -28,7 +28,7 @@ proc acc(x: var Foo): var ref Bar = proc test(maybeFoo: var Foo, maybeSeq: var seq[ref Bar], - bars: var openarray[ref Bar], + bars: var openArray[ref Bar], maybeTup: var Tup) = var bb: ref Bar maybeFoo.rmaybe = bb diff --git a/tests/misc/tsimplesort.nim b/tests/misc/tsimplesort.nim index e4a8e0b37..4793c63df 100644 --- a/tests/misc/tsimplesort.nim +++ b/tests/misc/tsimplesort.nim @@ -112,7 +112,7 @@ proc initTable*[A, B](initialSize=64): TTable[A, B] = result.counter = 0 newSeq(result.data, initialSize) -proc toTable*[A, B](pairs: openarray[tuple[key: A, +proc toTable*[A, B](pairs: openArray[tuple[key: A, val: B]]): TTable[A, B] = ## creates a new hash table that contains the given `pairs`. result = initTable[A, B](nextPowerOfTwo(pairs.len+10)) diff --git a/tests/openarray/topena1.nim b/tests/openarray/topena1.nim index ed3a0cedb..380c57f2a 100644 --- a/tests/openarray/topena1.nim +++ b/tests/openarray/topena1.nim @@ -6,4 +6,4 @@ discard """ # Tests a special bug var - x: ref openarray[string] #ERROR_MSG invalid type + x: ref openArray[string] #ERROR_MSG invalid type diff --git a/tests/openarray/topenarrayrepr.nim b/tests/openarray/topenarrayrepr.nim index 3784d4bbb..fc40d88c3 100644 --- a/tests/openarray/topenarrayrepr.nim +++ b/tests/openarray/topenarrayrepr.nim @@ -2,12 +2,12 @@ discard """ output: "5 - [1]" """ type - TProc = proc (n: int, m: openarray[int64]) {.nimcall.} + TProc = proc (n: int, m: openArray[int64]) {.nimcall.} proc Foo(x: int, P: TProc) = P(x, [ 1'i64 ]) -proc Bar(n: int, m: openarray[int64]) = +proc Bar(n: int, m: openArray[int64]) = echo($n & " - " & repr(m)) Foo(5, Bar) #OUT 5 - [1] diff --git a/tests/openarray/topenlen.nim b/tests/openarray/topenlen.nim index 83d58ac5c..164241b85 100644 --- a/tests/openarray/topenlen.nim +++ b/tests/openarray/topenlen.nim @@ -5,7 +5,7 @@ discard """ proc choose(b: openArray[string]): string = return b[0] -proc p(a, b: openarray[string]): int = +proc p(a, b: openArray[string]): int = result = a.len + b.len - 1 for j in 0 .. a.len: inc(result) discard choose(a) diff --git a/tests/openarray/tptrarrayderef.nim b/tests/openarray/tptrarrayderef.nim index 5e77430d1..1b7ef0df0 100644 --- a/tests/openarray/tptrarrayderef.nim +++ b/tests/openarray/tptrarrayderef.nim @@ -15,14 +15,14 @@ var raa = [11,12,13] #bug #3586 -proc mutate[T](arr:openarray[T], brr: openArray[T]) = +proc mutate[T](arr:openArray[T], brr: openArray[T]) = for i in 0..arr.len-1: doAssert(arr[i] == brr[i]) mutate(arr, arr) #bug #2240 -proc f(a: openarray[int], b: openArray[int]) = +proc f(a: openArray[int], b: openArray[int]) = for i in 0..a.len-1: doAssert(a[i] == b[i]) @@ -37,7 +37,7 @@ ra[2] = 13 f(ra[], raa) #bug #2240b -proc fillBuffer(buf: var openarray[char]) = +proc fillBuffer(buf: var openArray[char]) = for i in 0..buf.len-1: buf[i] = chr(i) diff --git a/tests/overload/toverload_various.nim b/tests/overload/toverload_various.nim index 55a8d17e6..0741fce60 100644 --- a/tests/overload/toverload_various.nim +++ b/tests/overload/toverload_various.nim @@ -264,8 +264,8 @@ proc init*[T](hmctx: HMAC[T], key: ptr byte, ulen: uint) = const sizeBlock = hmctx.sizeBlock echo sizeBlock -proc hmac*[A, B](HashType: typedesc, key: openarray[A], - data: openarray[B]) = +proc hmac*[A, B](HashType: typedesc, key: openArray[A], + data: openArray[B]) = var ctx: HMAC[HashType] ctx.init(nil, 0) diff --git a/tests/showoff/tdrdobbs_examples.nim b/tests/showoff/tdrdobbs_examples.nim index 0e10c6dd8..c61e177dc 100644 --- a/tests/showoff/tdrdobbs_examples.nim +++ b/tests/showoff/tdrdobbs_examples.nim @@ -40,12 +40,12 @@ const msb3999 = mostSignificantBit(3999) echo msb3999, " ", mostSignificantBit(0), " ", square(44) -proc filter[T](a: openarray[T], predicate: proc (x: T): bool): seq[T] = +proc filter[T](a: openArray[T], predicate: proc (x: T): bool): seq[T] = result = @[] # @[] constructs the empty seq for x in a: if predicate(x): result.add(x) -proc map[T, S](a: openarray[T], fn: proc (x: T): S): seq[S] = +proc map[T, S](a: openArray[T], fn: proc (x: T): S): seq[S] = newSeq(result, a.len) for i in 0 ..< a.len: result[i] = fn(a[i]) diff --git a/tests/stdlib/tdochelpers.nim b/tests/stdlib/tdochelpers.nim index 9d440a1d6..8dcb158ca 100644 --- a/tests/stdlib/tdochelpers.nim +++ b/tests/stdlib/tdochelpers.nim @@ -76,7 +76,7 @@ suite "Integration with Nim": check(input3.toLangSymbol == expected) test "advanced proc parsing with Nim identifier normalization": - let input = """`proc binarySearch*[T, K](a: openArray[T]; key: K; + let input = """`proc binarySearch*[T, K](a: openarray[T]; key: K; cmp: proc (x: T; y: K): int)`_""".rstParseTest let expected = LangSymbol(symKind: "proc", name: "binarysearch", @@ -90,7 +90,7 @@ suite "Integration with Nim": check(input.toLangSymbol == expected) test "the same without proc": - let input = """`binarySearch*[T, K](a: openArray[T]; key: K; + let input = """`binarySearch*[T, K](a: openarray[T]; key: K; cmp: proc (x: T; y: K): int {.closure.})`_""".rstParseTest let expected = LangSymbol(symKind: "", name: "binarysearch", diff --git a/tests/stdlib/tparseipv6.nim b/tests/stdlib/tparseipv6.nim index dd9abc511..9b9c464c7 100644 --- a/tests/stdlib/tparseipv6.nim +++ b/tests/stdlib/tparseipv6.nim @@ -210,12 +210,12 @@ const "::a:b:85:e4d9:252.9.229.056", ] -proc ok(pos: openarray[string]) = +proc ok(pos: openArray[string]) = for p in pos: if not isIpAddress(p): echo "failure ", p -proc notok(neg: openarray[string]) = +proc notok(neg: openArray[string]) = for n in neg: if isIpAddress(n): echo "failure ", n diff --git a/tests/stdlib/trepr.nim b/tests/stdlib/trepr.nim index 83ae7b119..82c991805 100644 --- a/tests/stdlib/trepr.nim +++ b/tests/stdlib/trepr.nim @@ -64,7 +64,7 @@ template main() = doAssert repr(arr) == "[1, 2, 3]" block: # bug #7878 - proc reprOpenarray(variable: var openarray[int]): string = repr(variable) + proc reprOpenarray(variable: var openArray[int]): string = repr(variable) when defined(js): discard # BUG: doesn't work else: doAssert reprOpenarray(arr) == "[1, 2, 3]" diff --git a/tests/system/tostring.nim b/tests/system/tostring.nim index fa82acc3b..bdbaa2ce6 100644 --- a/tests/system/tostring.nim +++ b/tests/system/tostring.nim @@ -68,7 +68,7 @@ doAssert yy == "" proc bar(arg: cstring) = doAssert arg[0] == '\0' -proc baz(arg: openarray[char]) = +proc baz(arg: openArray[char]) = doAssert arg.len == 0 proc stringCompare() = diff --git a/tests/system/tsystem_misc.nim b/tests/system/tsystem_misc.nim index 13b8c18a9..7f5914725 100644 --- a/tests/system/tsystem_misc.nim +++ b/tests/system/tsystem_misc.nim @@ -131,12 +131,12 @@ let str = "0123456789" foo(toOpenArrayByte(str, 0, str.high)) -template boundedOpenArray[T](x: seq[T], first, last: int): openarray[T] = +template boundedOpenArray[T](x: seq[T], first, last: int): openArray[T] = toOpenarray(x, max(0, first), min(x.high, last)) # bug #9281 -proc foo[T](x: openarray[T]) = +proc foo[T](x: openArray[T]) = echo x.len let a = @[1, 2, 3] diff --git a/tests/template/otests.nim b/tests/template/otests.nim index 120146343..9cb9d7fcb 100644 --- a/tests/template/otests.nim +++ b/tests/template/otests.nim @@ -163,7 +163,7 @@ when true: #embeddingTest """ # Expression template - proc test_expression(nums: openarray[int] = []): string = + proc test_expression(nums: openArray[int] = []): string = var i = 2 tmpli html""" $(no_substitution()) @@ -171,7 +171,7 @@ when true: #embeddingTest
              Age: $($nums[i] & "!!")
              """ - proc test_statements(nums: openarray[int] = []): string = + proc test_statements(nums: openArray[int] = []): string = tmpli html""" $(test_expression(nums)) $if true { diff --git a/tests/template/sunset.nimf b/tests/template/sunset.nimf index 465b12a5e..bd57bb8e1 100644 --- a/tests/template/sunset.nimf +++ b/tests/template/sunset.nimf @@ -1,6 +1,6 @@ #? stdtmpl #proc sunsetTemplate*(current, ticker, content: string, -# tabs: openarray[array[0..1, string]]): string = +# tabs: openArray[array[0..1, string]]): string = # result = "" diff --git a/tests/template/template_issues.nim b/tests/template/template_issues.nim index 502c6d1ee..1fed694ef 100644 --- a/tests/template/template_issues.nim +++ b/tests/template/template_issues.nim @@ -173,7 +173,7 @@ block t2585: st echo "a ", $fb - proc render(rdat: var RenderData; passes: var openarray[RenderPass]; proj: Mat2; + proc render(rdat: var RenderData; passes: var openArray[RenderPass]; proj: Mat2; indexType = 1) = for i in 0 ..< len(passes): echo "blah ", repr(passes[i]) diff --git a/tests/tuples/tuple_with_nil.nim b/tests/tuples/tuple_with_nil.nim index 7136c8cab..9cad6eccd 100644 --- a/tests/tuples/tuple_with_nil.nim +++ b/tests/tuples/tuple_with_nil.nim @@ -477,7 +477,7 @@ proc writeformat(o: var Writer; b: bool; fmt: Format) = else: raise newException(FormatError, "Boolean values must of one of the following types: s,b,o,x,X,d,n") -proc writeformat(o: var Writer; ary: openarray[system.any]; fmt: Format) = +proc writeformat(o: var Writer; ary: openArray[system.any]; fmt: Format) = ## Write array `ary` according to format `fmt` using output object ## `o` and output function `add`. if ary.len == 0: return @@ -657,7 +657,7 @@ proc literal[T](x: T): NimNode {.compiletime, nosideeffect.} = result = newLit(x) proc generatefmt(fmtstr: string; - args: var openarray[tuple[arg:NimNode, cnt:int]]; + args: var openArray[tuple[arg:NimNode, cnt:int]]; arg: var int;): seq[tuple[val, fmt:NimNode]] {.compiletime.} = ## fmtstr ## the format string diff --git a/tests/typerel/tcommontype.nim b/tests/typerel/tcommontype.nim index 8215ebd5e..488047fd1 100644 --- a/tests/typerel/tcommontype.nim +++ b/tests/typerel/tcommontype.nim @@ -12,7 +12,7 @@ type proc newDog():PDog = new(result) proc newCat():PCat = new(result) -proc test(a:openarray[PAnimal])= +proc test(a:openArray[PAnimal])= echo("dummy") #test(newDog(),newCat()) #does not work diff --git a/tests/typerel/texplicitcmp.nim b/tests/typerel/texplicitcmp.nim index b11aa2f4e..57f1e81b6 100644 --- a/tests/typerel/texplicitcmp.nim +++ b/tests/typerel/texplicitcmp.nim @@ -8,7 +8,7 @@ discard """ import json, tables, algorithm -proc outp(a: openarray[int]) = +proc outp(a: openArray[int]) = stdout.write "[" for i in a: stdout.write($i & " ") stdout.write "]\n" diff --git a/tests/typerel/tstr_as_openarray.nim b/tests/typerel/tstr_as_openarray.nim index fc28d6c93..14cd21479 100644 --- a/tests/typerel/tstr_as_openarray.nim +++ b/tests/typerel/tstr_as_openarray.nim @@ -3,18 +3,18 @@ discard """ """ var s = "HI" -proc x (zz: openarray[char]) = +proc x (zz: openArray[char]) = discard x s -proc z [T] (zz: openarray[T]) = +proc z [T] (zz: openArray[T]) = discard z s z([s,s,s]) -proc y [T] (arg: var openarray[T]) = +proc y [T] (arg: var openArray[T]) = arg[0] = 'X' y s doAssert s == "XI" diff --git a/tests/views/tviews1.nim b/tests/views/tviews1.nim index ced487ce8..249058eb6 100644 --- a/tests/views/tviews1.nim +++ b/tests/views/tviews1.nim @@ -44,7 +44,7 @@ echo foo(x) type F = object - oa: openarray[int] + oa: openArray[int] let s1 = @[1,3,4,5,6] var test = F(oa: toOpenArray(s1, 0, 2)) diff --git a/tests/vm/tyaytypedesc.nim b/tests/vm/tyaytypedesc.nim index a3ad9b707..aab94d643 100644 --- a/tests/vm/tyaytypedesc.nim +++ b/tests/vm/tyaytypedesc.nim @@ -10,7 +10,7 @@ type NodeType* = enum type TokenType* = enum ttWhitespace -proc enumTable*[A, B, C](a: openarray[tuple[key: A, val: B]], ret: typedesc[C]): C = +proc enumTable*[A, B, C](a: openArray[tuple[key: A, val: B]], ret: typedesc[C]): C = for item in a: result[item.key] = item.val From 5ec8b609421a2e74e7d5f8d00293194faf4437d4 Mon Sep 17 00:00:00 2001 From: rockcavera Date: Thu, 6 Jan 2022 08:19:32 -0300 Subject: [PATCH 1448/2472] Update net.nim (#19327) [backport] --- lib/pure/net.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 8dc92e9d6..c1d4c6228 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -1628,7 +1628,7 @@ proc recvFrom*[T: string | IpAddress](socket: Socket, data: var string, length: ## used. Therefore if `socket` contains something in its buffer this ## function will make no effort to return it. template adaptRecvFromToDomain(sockAddress: untyped, domain: Domain) = - var addrLen = sizeof(sockAddress).SockLen + var addrLen = SockLen(sizeof(sockAddress)) result = recvfrom(socket.fd, cstring(data), length.cint, flags.cint, cast[ptr SockAddr](addr(sockAddress)), addr(addrLen)) From c7d5b8c83ddf204c6e0cd7b6937d33318320ed91 Mon Sep 17 00:00:00 2001 From: rockcavera Date: Fri, 7 Jan 2022 04:26:55 -0300 Subject: [PATCH 1449/2472] Fix #19038 - making the Nim compiler work again on Windows XP (#19331) * Update osenv.nim * Update win_setenv.nim * Update lib/pure/includes/osenv.nim * Update lib/pure/includes/osenv.nim * fixing cstring Co-authored-by: Andreas Rumpf --- lib/pure/includes/osenv.nim | 5 +++-- lib/std/private/win_setenv.nim | 41 +++++++++++++++++----------------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/lib/pure/includes/osenv.nim b/lib/pure/includes/osenv.nim index 1a01ab9bc..4a776eb78 100644 --- a/lib/pure/includes/osenv.nim +++ b/lib/pure/includes/osenv.nim @@ -45,7 +45,7 @@ when not defined(nimscript): proc c_getenv(env: cstring): cstring {. importc: "getenv", header: "".} when defined(windows): - proc c_putenv_s(envname: cstring, envval: cstring): cint {.importc: "_putenv_s", header: "".} + proc c_putenv(envstring: cstring): cint {.importc: "_putenv", header: "".} from std/private/win_setenv import setEnvImpl else: proc c_setenv(envname: cstring, envval: cstring, overwrite: cint): cint {.importc: "setenv", header: "".} @@ -121,7 +121,8 @@ when not defined(nimscript): ]# if key.len == 0 or '=' in key: raise newException(OSError, "invalid key, got: " & key) - if c_putenv_s(key, "") != 0'i32: bail + let envToDel = key & "=" + if c_putenv(cstring envToDel) != 0'i32: bail else: if c_unsetenv(key) != 0'i32: bail diff --git a/lib/std/private/win_setenv.nim b/lib/std/private/win_setenv.nim index 91c3f15b0..0dfe0ed46 100644 --- a/lib/std/private/win_setenv.nim +++ b/lib/std/private/win_setenv.nim @@ -30,44 +30,45 @@ else: # same as winlean.setEnvironmentVariableA proc c_getenv(env: cstring): cstring {.importc: "getenv", header: "".} - proc c_putenv_s(envname: cstring, envval: cstring): cint {.importc: "_putenv_s", header: "".} + proc c_putenv(envstring: cstring): cint {.importc: "_putenv", header: "".} proc c_wgetenv(varname: ptr wchar_t): ptr wchar_t {.importc: "_wgetenv", header: "".} var errno {.importc, header: "".}: cint - var gWenviron {.importc:"_wenviron".}: ptr ptr wchar_t + var gWenviron {.importc: "_wenviron".}: ptr ptr wchar_t # xxx `ptr UncheckedArray[WideCString]` did not work - proc mbstowcs_s(pReturnValue: ptr csize_t, wcstr: ptr wchar_t, sizeInWords: csize_t, mbstr: cstring, count: csize_t): cint {.importc: "mbstowcs_s", header: "".} + proc mbstowcs(wcstr: ptr wchar_t, mbstr: cstring, count: csize_t): csize_t {.importc: "mbstowcs", header: "".} # xxx cint vs errno_t? - proc setEnvImpl*(name: cstring, value: cstring, overwrite: cint): cint = + proc setEnvImpl*(name: string, value: string, overwrite: cint): cint = const EINVAL = cint(22) - const MAX_ENV = 32767 - # xxx get it from: `var MAX_ENV {.importc: "_MAX_ENV", header:"".}: cint` - if overwrite == 0 and c_getenv(name) != nil: return 0 - if value[0] != '\0': - let e = c_putenv_s(name, value) + if overwrite == 0 and c_getenv(cstring(name)) != nil: return 0 + if value != "": + let envstring = name & "=" & value + let e = c_putenv(cstring(envstring)) if e != 0: - errno = e + errno = EINVAL return -1 return 0 #[ - We are trying to set the value to an empty string, but `_putenv_s` deletes + We are trying to set the value to an empty string, but `_putenv` deletes entries if the value is an empty string, and just calling SetEnvironmentVariableA doesn't update `_environ`, so we have to do these terrible things. ]# - if c_putenv_s(name, " ") != 0: + let envstring = name & "= " + if c_putenv(cstring(envstring)) != 0: errno = EINVAL return -1 # Here lies the documentation we blatently ignore to make this work. - var s = c_getenv(name) + var s = c_getenv(cstring(name)) s[0] = '\0' #[ This would result in a double null termination, which normally signifies the end of the environment variable list, so we stick a completely empty environment variable into the list instead. ]# + s = c_getenv(cstring(name)) s[1] = '=' #[ If gWenviron is null, the wide environment has not been initialized @@ -77,19 +78,19 @@ else: ]# if gWenviron != nil: # var buf: array[MAX_ENV + 1, WideCString] - var buf: array[MAX_ENV + 1, Utf16Char] + let requiredSize = mbstowcs(nil, cstring(name), 0).int + var buf = newSeq[Utf16Char](requiredSize + 1) let buf2 = cast[ptr wchar_t](buf[0].addr) - var len: csize_t - if mbstowcs_s(len.addr, buf2, buf.len.csize_t, name, MAX_ENV) != 0: + if mbstowcs(buf2, cstring(name), csize_t(requiredSize + 1)) == csize_t(high(uint)): errno = EINVAL return -1 - let ptrToEnv = cast[WideCString](c_wgetenv(buf2)) + var ptrToEnv = cast[WideCString](c_wgetenv(buf2)) ptrToEnv[0] = '\0'.Utf16Char - let ptrToEnv2 = cast[WideCString](c_wgetenv(buf2)) - ptrToEnv2[1] = '='.Utf16Char + ptrToEnv = cast[WideCString](c_wgetenv(buf2)) + ptrToEnv[1] = '='.Utf16Char # And now, we have to update the outer environment to have a proper empty value. - if setEnvironmentVariableA(name, value) == 0: + if setEnvironmentVariableA(cstring(name), cstring(value)) == 0: errno = EINVAL return -1 return 0 From 4306d8fb36d3810b8d38a665f98dc89ce107740e Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 8 Jan 2022 12:06:28 +0800 Subject: [PATCH 1450/2472] disable fragments (#19341) The original repo has moved to Rust, while the package path stay unchanged. And it causes troubles to https://github.com/nim-lang/Nim/pull/19338 --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 381aa27ac..853a58698 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -64,7 +64,7 @@ pkg "docopt" pkg "easygl", "nim c -o:egl -r src/easygl.nim", "https://github.com/jackmott/easygl" pkg "elvis" pkg "fidget" -pkg "fragments", "nim c -r fragments/dsl.nim" +pkg "fragments", "nim c -r fragments/dsl.nim", allowFailure = true # pending https://github.com/nim-lang/packages/issues/2115 pkg "fusion" pkg "gara" pkg "glob" From 58656aa5bba572672f093499280b69bb0f0d4c06 Mon Sep 17 00:00:00 2001 From: Zachary Marquez Date: Sat, 8 Jan 2022 03:22:56 -0600 Subject: [PATCH 1451/2472] fix nim-lang#19343 (#19344) [backport] Ensure HttpClient onProgress is called once per second Ensure that reported speed is accurate --- lib/pure/httpclient.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 7686d15bd..5b6e439db 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -673,7 +673,7 @@ proc reportProgress(client: HttpClient | AsyncHttpClient, progress: BiggestInt) {.multisync.} = client.contentProgress += progress client.oneSecondProgress += progress - if (getMonoTime() - client.lastProgressReport).inSeconds > 1: + if (getMonoTime() - client.lastProgressReport).inSeconds >= 1: if not client.onProgressChanged.isNil: await client.onProgressChanged(client.contentTotal, client.contentProgress, From ae92eac06031ed57a755c01efdbb9cc5b72e5dbf Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 9 Jan 2022 04:34:22 +0800 Subject: [PATCH 1452/2472] stylecheck usages part two: stdlib cleanup (#19338) * stylecheck usages part two: stdlib cleanup typeinfo.nim: importCompilerProc => importcompilerproc nre.nim: newLineFlags => newlineFlags system.nim: JSRoot => JsRoot ref #19319 * prefer importCompilerProc --- compiler/wordrecg.nim | 2 +- lib/impure/nre.nim | 2 +- lib/pure/os.nim | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/wordrecg.nim b/compiler/wordrecg.nim index 22f6cc71d..c819f4306 100644 --- a/compiler/wordrecg.nim +++ b/compiler/wordrecg.nim @@ -41,7 +41,7 @@ type wImmediate = "immediate", wConstructor = "constructor", wDestructor = "destructor", wDelegator = "delegator", wOverride = "override", wImportCpp = "importcpp", wCppNonPod = "cppNonPod", - wImportObjC = "importobjc", wImportCompilerProc = "importcompilerproc", + wImportObjC = "importobjc", wImportCompilerProc = "importCompilerProc", wImportc = "importc", wImportJs = "importjs", wExportc = "exportc", wExportCpp = "exportcpp", wExportNims = "exportnims", wIncompleteStruct = "incompleteStruct", # deprecated diff --git a/lib/impure/nre.nim b/lib/impure/nre.nim index f346d84cb..03cbf5220 100644 --- a/lib/impure/nre.nim +++ b/lib/impure/nre.nim @@ -281,7 +281,7 @@ proc matchesCrLf(pattern: Regex): bool = let newlineFlags = flags and (pcre.NEWLINE_CRLF or pcre.NEWLINE_ANY or pcre.NEWLINE_ANYCRLF) - if newLineFlags > 0u32: + if newlineFlags > 0u32: return true # get flags from build config diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 9b08fe2e1..81b432d7f 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -2888,7 +2888,7 @@ when defined(nimdoc): elif defined(nimscript): discard elif defined(nodejs): - type Argv = object of JSRoot + type Argv = object of JsRoot let argv {.importjs: "process.argv".} : Argv proc len(argv: Argv): int {.importjs: "#.length".} proc `[]`(argv: Argv, i: int): cstring {.importjs: "#[#]".} From b098546da051ed026e85a9f398e74c396c79a8ad Mon Sep 17 00:00:00 2001 From: Smarcy Date: Sun, 9 Jan 2022 14:02:08 +0100 Subject: [PATCH 1453/2472] added filemode docs (#19346) --- lib/system/io.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/system/io.nim b/lib/system/io.nim index 661d1a9ba..04a43328d 100644 --- a/lib/system/io.nim +++ b/lib/system/io.nim @@ -22,6 +22,8 @@ type FileMode* = enum ## The file mode when opening a file. fmRead, ## Open the file for read access only. + ## If the file does not exist, it will not + ## be created. fmWrite, ## Open the file for write access only. ## If the file does not exist, it will be ## created. Existing files will be cleared! @@ -32,7 +34,8 @@ type ## If the file does not exist, it will not be ## created. The existing file will not be cleared. fmAppend ## Open the file for writing only; append data - ## at the end. + ## at the end. If the file does not exist, it + ## will be created. FileHandle* = cint ## type that represents an OS file handle; this is ## useful for low-level file access From 955040f0f1d4f9b2eab68c385fab9d4cfbef8ea5 Mon Sep 17 00:00:00 2001 From: gecko Date: Mon, 10 Jan 2022 09:27:59 +0000 Subject: [PATCH 1454/2472] Fix `remove` on last node of singly-linked list [backport:1.6] (#19353) --- changelog.md | 1 + lib/pure/collections/lists.nim | 2 ++ tests/stdlib/tlists.nim | 13 +++++++++++++ 3 files changed, 16 insertions(+) diff --git a/changelog.md b/changelog.md index 1b97fd99a..d07f80623 100644 --- a/changelog.md +++ b/changelog.md @@ -22,6 +22,7 @@ - `macros.parseExpr` and `macros.parseStmt` now accept an optional filename argument for more informative errors. - Module `colors` expanded with missing colors from the CSS color standard. +- Fixed `lists.SinglyLinkedList` being broken after removing the last node ([#19353](https://github.com/nim-lang/Nim/pull/19353)). ## `std/smtp` diff --git a/lib/pure/collections/lists.nim b/lib/pure/collections/lists.nim index f7627ae67..d1de0ea67 100644 --- a/lib/pure/collections/lists.nim +++ b/lib/pure/collections/lists.nim @@ -739,6 +739,8 @@ proc remove*[T](L: var SinglyLinkedList[T], n: SinglyLinkedNode[T]): bool {.disc if prev.next == nil: return false prev.next = n.next + if L.tail == n: + L.tail = prev # update tail if we removed the last node true proc remove*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = diff --git a/tests/stdlib/tlists.nim b/tests/stdlib/tlists.nim index 14cbf2f9d..00c5b1a27 100644 --- a/tests/stdlib/tlists.nim +++ b/tests/stdlib/tlists.nim @@ -258,5 +258,18 @@ template main = a.add(2) doAssert a.toSeq == @[1, 2] + block RemoveLastNodeFromSinglyLinkedList: + var list = initSinglyLinkedList[string]() + let n1 = newSinglyLinkedNode("sonic") + let n2 = newSinglyLinkedNode("the") + let n3 = newSinglyLinkedNode("tiger") + let n4 = newSinglyLinkedNode("hedgehog") + list.add(n1) + list.add(n2) + list.add(n3) + list.remove(n3) + list.add(n4) + doAssert list.toSeq == @["sonic", "the", "hedgehog"] + static: main() main() From 92e5573b20f30f4114a4eb1f35eddc7064bfcd71 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 10 Jan 2022 17:29:04 +0800 Subject: [PATCH 1455/2472] fix stylecheck error with asyncdispatch (#19350) * stylecheck usages part two: stdlib cleanup typeinfo.nim: importCompilerProc => importcompilerproc nre.nim: newLineFlags => newlineFlags system.nim: JSRoot => JsRoot ref #19319 * prefer importCompilerProc * fix stylecheck error with asyncdispatch it is a partial regression since #12842 * add tests * don't use echo in tests --- compiler/semtempl.nim | 3 ++- tests/stylecheck/taccept.nim | 17 +++++++++++++++++ tests/stylecheck/treject.nim | 17 +++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 tests/stylecheck/taccept.nim create mode 100644 tests/stylecheck/treject.nim diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim index 49ce55a64..b921fda2c 100644 --- a/compiler/semtempl.nim +++ b/compiler/semtempl.nim @@ -270,7 +270,8 @@ proc semTemplSymbol(c: PContext, n: PNode, s: PSym; isField: bool): PNode = # Issue #12832 when defined(nimsuggest): suggestSym(c.graph, n.info, s, c.graph.usageSym, false) - if {optStyleHint, optStyleError} * c.config.globalOptions != {}: + # field access (dot expr) will be handled by builtinFieldAccess + if not isField and {optStyleHint, optStyleError} * c.config.globalOptions != {}: styleCheckUse(c.config, n.info, s) proc semRoutineInTemplName(c: var TemplCtx, n: PNode): PNode = diff --git a/tests/stylecheck/taccept.nim b/tests/stylecheck/taccept.nim new file mode 100644 index 000000000..6fb55f835 --- /dev/null +++ b/tests/stylecheck/taccept.nim @@ -0,0 +1,17 @@ +discard """ + matrix: "--styleCheck:error --styleCheck:usages" +""" + +import asyncdispatch + +type + Name = object + id: int + +template hello = + var iD = "string" + var name: Name + doAssert name.id == 0 + doAssert iD == "string" + +hello() diff --git a/tests/stylecheck/treject.nim b/tests/stylecheck/treject.nim new file mode 100644 index 000000000..b9d97a58d --- /dev/null +++ b/tests/stylecheck/treject.nim @@ -0,0 +1,17 @@ +discard """ + action: reject + nimout: '''treject.nim(14, 13) Error: 'iD' should be: 'id' [field declared in treject.nim(9, 5)]''' + matrix: "--styleCheck:error --styleCheck:usages" +""" + +type + Name = object + id: int + +template hello = + var iD = "string" + var name: Name + echo name.iD + echo iD + +hello() \ No newline at end of file From bcbeff36321aa203cd258fd02c82d9276badf31f Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 10 Jan 2022 18:59:44 +0800 Subject: [PATCH 1456/2472] remove spaces between an identifier and a star (#19355) It makes search easier by searching `+`* instead of `+` which filter lots of unexported versions. Follow https://github.com/nim-lang/Nim/pull/18681 --- lib/js/jsffi.nim | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/lib/js/jsffi.nim b/lib/js/jsffi.nim index 937e3727b..ac963eb89 100644 --- a/lib/js/jsffi.nim +++ b/lib/js/jsffi.nim @@ -163,28 +163,28 @@ macro jsFromAst*(n: untyped): untyped = proc `&`*(a, b: cstring): cstring {.importjs: "(# + #)".} ## Concatenation operator for JavaScript strings. -proc `+` *(x, y: JsObject): JsObject {.importjs: "(# + #)".} -proc `-` *(x, y: JsObject): JsObject {.importjs: "(# - #)".} -proc `*` *(x, y: JsObject): JsObject {.importjs: "(# * #)".} -proc `/` *(x, y: JsObject): JsObject {.importjs: "(# / #)".} -proc `%` *(x, y: JsObject): JsObject {.importjs: "(# % #)".} -proc `+=` *(x, y: JsObject): JsObject {.importjs: "(# += #)", discardable.} -proc `-=` *(x, y: JsObject): JsObject {.importjs: "(# -= #)", discardable.} -proc `*=` *(x, y: JsObject): JsObject {.importjs: "(# *= #)", discardable.} -proc `/=` *(x, y: JsObject): JsObject {.importjs: "(# /= #)", discardable.} -proc `%=` *(x, y: JsObject): JsObject {.importjs: "(# %= #)", discardable.} -proc `++` *(x: JsObject): JsObject {.importjs: "(++#)".} -proc `--` *(x: JsObject): JsObject {.importjs: "(--#)".} -proc `>` *(x, y: JsObject): JsObject {.importjs: "(# > #)".} -proc `<` *(x, y: JsObject): JsObject {.importjs: "(# < #)".} -proc `>=` *(x, y: JsObject): JsObject {.importjs: "(# >= #)".} -proc `<=` *(x, y: JsObject): JsObject {.importjs: "(# <= #)".} -proc `**` *(x, y: JsObject): JsObject {.importjs: "((#) ** #)".} +proc `+`*(x, y: JsObject): JsObject {.importjs: "(# + #)".} +proc `-`*(x, y: JsObject): JsObject {.importjs: "(# - #)".} +proc `*`*(x, y: JsObject): JsObject {.importjs: "(# * #)".} +proc `/`*(x, y: JsObject): JsObject {.importjs: "(# / #)".} +proc `%`*(x, y: JsObject): JsObject {.importjs: "(# % #)".} +proc `+=`*(x, y: JsObject): JsObject {.importjs: "(# += #)", discardable.} +proc `-=`*(x, y: JsObject): JsObject {.importjs: "(# -= #)", discardable.} +proc `*=`*(x, y: JsObject): JsObject {.importjs: "(# *= #)", discardable.} +proc `/=`*(x, y: JsObject): JsObject {.importjs: "(# /= #)", discardable.} +proc `%=`*(x, y: JsObject): JsObject {.importjs: "(# %= #)", discardable.} +proc `++`*(x: JsObject): JsObject {.importjs: "(++#)".} +proc `--`*(x: JsObject): JsObject {.importjs: "(--#)".} +proc `>`*(x, y: JsObject): JsObject {.importjs: "(# > #)".} +proc `<`*(x, y: JsObject): JsObject {.importjs: "(# < #)".} +proc `>=`*(x, y: JsObject): JsObject {.importjs: "(# >= #)".} +proc `<=`*(x, y: JsObject): JsObject {.importjs: "(# <= #)".} +proc `**`*(x, y: JsObject): JsObject {.importjs: "((#) ** #)".} # (#) needed, refs https://github.com/nim-lang/Nim/pull/16409#issuecomment-760550812 proc `and`*(x, y: JsObject): JsObject {.importjs: "(# && #)".} -proc `or` *(x, y: JsObject): JsObject {.importjs: "(# || #)".} +proc `or`*(x, y: JsObject): JsObject {.importjs: "(# || #)".} proc `not`*(x: JsObject): JsObject {.importjs: "(!#)".} -proc `in` *(x, y: JsObject): JsObject {.importjs: "(# in #)".} +proc `in`*(x, y: JsObject): JsObject {.importjs: "(# in #)".} proc `[]`*(obj: JsObject, field: cstring): JsObject {.importjs: getImpl.} ## Returns the value of a property of name `field` from a JsObject `obj`. From 9888a29c3de09c73202b2a955e09e2a7b5e56ea4 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 11 Jan 2022 10:16:51 +0100 Subject: [PATCH 1457/2472] bitsets.nim: cleanup (#19361) --- compiler/bitsets.nim | 35 +++++++++++------------------------ 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/compiler/bitsets.nim b/compiler/bitsets.nim index 2e8aa1db6..e0d94409e 100644 --- a/compiler/bitsets.nim +++ b/compiler/bitsets.nim @@ -23,53 +23,40 @@ const template modElemSize(arg: untyped): untyped = arg and 7 template divElemSize(arg: untyped): untyped = arg shr 3 -proc bitSetInit*(b: var TBitSet, length: int) -proc bitSetUnion*(x: var TBitSet, y: TBitSet) -proc bitSetDiff*(x: var TBitSet, y: TBitSet) -proc bitSetSymDiff*(x: var TBitSet, y: TBitSet) -proc bitSetIntersect*(x: var TBitSet, y: TBitSet) -proc bitSetIncl*(x: var TBitSet, elem: BiggestInt) -proc bitSetExcl*(x: var TBitSet, elem: BiggestInt) -proc bitSetIn*(x: TBitSet, e: BiggestInt): bool -proc bitSetEquals*(x, y: TBitSet): bool -proc bitSetContains*(x, y: TBitSet): bool -proc bitSetCard*(x: TBitSet): BiggestInt -# implementation - -proc bitSetIn(x: TBitSet, e: BiggestInt): bool = +proc bitSetIn*(x: TBitSet, e: BiggestInt): bool = result = (x[int(e.divElemSize)] and (One shl e.modElemSize)) != Zero -proc bitSetIncl(x: var TBitSet, elem: BiggestInt) = +proc bitSetIncl*(x: var TBitSet, elem: BiggestInt) = assert(elem >= 0) x[int(elem.divElemSize)] = x[int(elem.divElemSize)] or (One shl elem.modElemSize) -proc bitSetExcl(x: var TBitSet, elem: BiggestInt) = +proc bitSetExcl*(x: var TBitSet, elem: BiggestInt) = x[int(elem.divElemSize)] = x[int(elem.divElemSize)] and not(One shl elem.modElemSize) -proc bitSetInit(b: var TBitSet, length: int) = +proc bitSetInit*(b: var TBitSet, length: int) = newSeq(b, length) -proc bitSetUnion(x: var TBitSet, y: TBitSet) = +proc bitSetUnion*(x: var TBitSet, y: TBitSet) = for i in 0..high(x): x[i] = x[i] or y[i] -proc bitSetDiff(x: var TBitSet, y: TBitSet) = +proc bitSetDiff*(x: var TBitSet, y: TBitSet) = for i in 0..high(x): x[i] = x[i] and not y[i] -proc bitSetSymDiff(x: var TBitSet, y: TBitSet) = +proc bitSetSymDiff*(x: var TBitSet, y: TBitSet) = for i in 0..high(x): x[i] = x[i] xor y[i] -proc bitSetIntersect(x: var TBitSet, y: TBitSet) = +proc bitSetIntersect*(x: var TBitSet, y: TBitSet) = for i in 0..high(x): x[i] = x[i] and y[i] -proc bitSetEquals(x, y: TBitSet): bool = +proc bitSetEquals*(x, y: TBitSet): bool = for i in 0..high(x): if x[i] != y[i]: return false result = true -proc bitSetContains(x, y: TBitSet): bool = +proc bitSetContains*(x, y: TBitSet): bool = for i in 0..high(x): if (x[i] and not y[i]) != Zero: return false @@ -96,6 +83,6 @@ const populationCount: array[uint8, uint8] = block: arr -proc bitSetCard(x: TBitSet): BiggestInt = +proc bitSetCard*(x: TBitSet): BiggestInt = for it in x: result.inc int(populationCount[it]) From 9a32905d89d23e423bca2d0ce643b9331a6741c0 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 13 Jan 2022 15:02:41 +0800 Subject: [PATCH 1458/2472] nimscript ignore the threads option (#19370) because nimscript doesn't support threads and causes troubles when the threads option is on --- compiler/commands.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index c4df46bc2..8e79a4477 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -717,7 +717,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; of "linedir": processOnOffSwitch(conf, {optLineDir}, arg, pass, info) of "assertions", "a": processOnOffSwitch(conf, {optAssert}, arg, pass, info) of "threads": - if conf.backend == backendJs: discard + if conf.backend == backendJs or conf.cmd == cmdNimscript: discard else: processOnOffSwitchG(conf, {optThreads}, arg, pass, info) #if optThreads in conf.globalOptions: conf.setNote(warnGcUnsafe) of "tlsemulation": processOnOffSwitchG(conf, {optTlsEmulation}, arg, pass, info) From ada815e45957e5675a9ac978b85b8467d68c5945 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 13 Jan 2022 15:03:14 +0800 Subject: [PATCH 1459/2472] make rst thread safe (#19369) split for the convenience of review --- lib/packages/docutils/rst.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index a04eb696f..791953767 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -2822,7 +2822,7 @@ type DirFlag = enum hasArg, hasOptions, argIsFile, argIsWord DirFlags = set[DirFlag] - SectionParser = proc (p: var RstParser): PRstNode {.nimcall.} + SectionParser = proc (p: var RstParser): PRstNode {.nimcall, gcsafe.} proc parseDirective(p: var RstParser, k: RstNodeKind, flags: DirFlags): PRstNode = ## Parses arguments and options for a directive block. @@ -2869,7 +2869,7 @@ proc indFollows(p: RstParser): bool = result = currentTok(p).kind == tkIndent and currentTok(p).ival > currInd(p) proc parseBlockContent(p: var RstParser, father: var PRstNode, - contentParser: SectionParser): bool = + contentParser: SectionParser): bool {.gcsafe.} = ## parse the final content part of explicit markup blocks (directives, ## footnotes, etc). Returns true if succeeded. if currentTok(p).kind != tkIndent or indFollows(p): @@ -3123,7 +3123,7 @@ proc prefix(ftnType: FootnoteType): string = of fnAutoSymbol: result = "footnotesym-" of fnCitation: result = "citation-" -proc parseFootnote(p: var RstParser): PRstNode = +proc parseFootnote(p: var RstParser): PRstNode {.gcsafe.} = ## Parses footnotes and citations, always returns 2 sons: ## ## 1) footnote label, always containing rnInner with 1 or more sons From a9135095f02587430b179a33ae03910d0d4201cd Mon Sep 17 00:00:00 2001 From: Leon Date: Fri, 14 Jan 2022 00:40:27 +1100 Subject: [PATCH 1460/2472] docs: Mention `import foo {.all.}` syntax (#19377) Mention the `import foo {.all.}` syntax in the manual, with a caveat about private imports. Also link to the experimental importutils module. Co-authored-by: adigitoleo --- doc/manual_experimental.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/manual_experimental.rst b/doc/manual_experimental.rst index 4ed8439df..0f82b9950 100644 --- a/doc/manual_experimental.rst +++ b/doc/manual_experimental.rst @@ -168,6 +168,17 @@ This feature will likely be superseded in the future by support for recursive module dependencies. +Importing private symbols +========================= + +In some situations, it may be useful to import all symbols (public or private) +from a module. The syntax `import foo {.all.}` can be used to import all +symbols from the module `foo`. Note that importing private symbols is +generally not recommended. + +See also the experimental `importutils `_ module. + + Code reordering =============== From 40a9c33eff099a37be236826f84eb2064d0ec247 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 13 Jan 2022 21:43:35 +0800 Subject: [PATCH 1461/2472] update copyright year (#19381) --- compiler/options.nim | 2 +- copying.txt | 2 +- readme.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/options.nim b/compiler/options.nim index 9e957332f..5cfe58119 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -19,7 +19,7 @@ const useEffectSystem* = true useWriteTracking* = false hasFFI* = defined(nimHasLibFFI) - copyrightYear* = "2021" + copyrightYear* = "2022" nimEnableCovariance* = defined(nimEnableCovariance) diff --git a/copying.txt b/copying.txt index a498a9525..60b6a0217 100644 --- a/copying.txt +++ b/copying.txt @@ -1,7 +1,7 @@ ===================================================== Nim -- a Compiler for Nim. https://nim-lang.org/ -Copyright (C) 2006-2021 Andreas Rumpf. All rights reserved. +Copyright (C) 2006-2022 Andreas Rumpf. All rights reserved. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/readme.md b/readme.md index 7d45556db..6bc7a2385 100644 --- a/readme.md +++ b/readme.md @@ -203,7 +203,7 @@ Nim. You are explicitly permitted to develop commercial applications using Nim. Please read the [copying.txt](copying.txt) file for more details. -Copyright © 2006-2021 Andreas Rumpf, all rights reserved. +Copyright © 2006-2022 Andreas Rumpf, all rights reserved. [nim-site]: https://nim-lang.org [nim-forum]: https://forum.nim-lang.org From 5853303be09464832ce0f189513c90c9dd15f4a0 Mon Sep 17 00:00:00 2001 From: Leon Date: Fri, 14 Jan 2022 00:45:47 +1100 Subject: [PATCH 1462/2472] docs: Fix broken cross references to `rfind` in strutils (#19382) [backport] Fixes three broken cross references to `rfind` in strutils. Breakage due to signature changes of the `rfind` methods. Co-authored-by: adigitoleo --- lib/pure/strutils.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index e4a4a5ec1..5e505ec3a 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -1865,7 +1865,7 @@ func find*(s: string, sub: char, start: Natural = 0, last = 0): int {.rtl, ## Use `s[start..last].rfind` for a `start`-origin index. ## ## See also: - ## * `rfind func<#rfind,string,char,Natural>`_ + ## * `rfind func<#rfind,string,char,Natural,int>`_ ## * `replace func<#replace,string,char,char>`_ let last = if last == 0: s.high else: last when nimvm: @@ -1893,7 +1893,7 @@ func find*(s: string, chars: set[char], start: Natural = 0, last = 0): int {. ## Use `s[start..last].find` for a `start`-origin index. ## ## See also: - ## * `rfind func<#rfind,string,set[char],Natural>`_ + ## * `rfind func<#rfind,string,set[char],Natural,int>`_ ## * `multiReplace func<#multiReplace,string,varargs[]>`_ let last = if last == 0: s.high else: last for i in int(start)..last: @@ -1910,7 +1910,7 @@ func find*(s, sub: string, start: Natural = 0, last = 0): int {.rtl, ## Use `s[start..last].find` for a `start`-origin index. ## ## See also: - ## * `rfind func<#rfind,string,string,Natural>`_ + ## * `rfind func<#rfind,string,string,Natural,int>`_ ## * `replace func<#replace,string,string,string>`_ if sub.len > s.len - start: return -1 if sub.len == 1: return find(s, sub[0], start, last) From 9b9ae8a487c6fbf77c8c72196e2b74f3371382b2 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 14 Jan 2022 00:39:17 +0800 Subject: [PATCH 1463/2472] nrvo shouldn't touch bycopy object[backport:1.2] (#19385) fix nim-lang#19342 --- compiler/ccgtypes.nim | 2 +- tests/objects/m19342.c | 12 ++++++++++++ tests/objects/t19342.nim | 18 ++++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 tests/objects/m19342.c create mode 100644 tests/objects/t19342.nim diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 740ce5f19..df1874b26 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -220,7 +220,7 @@ proc isInvalidReturnType(conf: ConfigRef; rettype: PType): bool = # such a poor programming language. # We exclude records with refs too. This enhances efficiency and # is necessary for proper code generation of assignments. - if rettype == nil or getSize(conf, rettype) > conf.target.floatSize*3: + if rettype == nil or (tfByCopy notin rettype.flags and getSize(conf, rettype) > conf.target.floatSize*3): result = true else: case mapType(conf, rettype, skResult) diff --git a/tests/objects/m19342.c b/tests/objects/m19342.c new file mode 100644 index 000000000..113f65309 --- /dev/null +++ b/tests/objects/m19342.c @@ -0,0 +1,12 @@ +struct Node +{ + int data[25]; +}; + + +struct Node hello(int name) { + struct Node x = {999, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 0, 1, 2, 3, 4, 5, 6, 7 ,8, 9, + 1, 2, 3, 4, 5}; + return x; +} \ No newline at end of file diff --git a/tests/objects/t19342.nim b/tests/objects/t19342.nim new file mode 100644 index 000000000..d40d15a4b --- /dev/null +++ b/tests/objects/t19342.nim @@ -0,0 +1,18 @@ +discard """ + targets: "c cpp" +""" + +{.compile: "m19342.c".} + +# bug #19342 +type + Node* {.bycopy.} = object + data: array[25, cint] + +proc myproc(name: cint): Node {.importc: "hello", cdecl.} + +proc parse = + let node = myproc(10) + doAssert node.data[0] == 999 + +parse() From a93f6e7acc9e02deb40a864d345e4c715346a98c Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Thu, 13 Jan 2022 09:39:55 -0700 Subject: [PATCH 1464/2472] Generic parameters now can constrain statics in type definitions (#19362) * Parameters now can constrain static in type definitions resolved regression with generic procedures * Update compiler/sigmatch.nim Co-authored-by: Andreas Rumpf --- compiler/sigmatch.nim | 14 +++++++-- tests/generics/tstatic_constrained.nim | 42 ++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 2 deletions(-) create mode 100644 tests/generics/tstatic_constrained.nim diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 93be92676..4d8e03fe9 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -1739,11 +1739,22 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, let prev = PType(idTableGet(c.bindings, f)) if prev == nil: if aOrig.kind == tyStatic: - if f.base.kind != tyNone: + if f.base.kind notin {tyNone, tyGenericParam}: result = typeRel(c, f.base, a, flags) if result != isNone and f.n != nil: if not exprStructuralEquivalent(f.n, aOrig.n): result = isNone + elif f.base.kind == tyGenericParam: + # Handling things like `type A[T; Y: static T] = object` + if f.base.len > 0: # There is a constraint, handle it + result = typeRel(c, f.base.lastSon, a, flags) + else: + # No constraint + if tfGenericTypeParam in f.flags: + result = isGeneric + else: + # for things like `proc fun[T](a: static[T])` + result = typeRel(c, f.base, a, flags) else: result = isGeneric if result != isNone: put(c, f, aOrig) @@ -1993,7 +2004,6 @@ proc paramTypesMatchAux(m: var TCandidate, f, a: PType, arg = argSemantized a = a c = m.c - if tfHasStatic in fMaybeStatic.flags: # XXX: When implicit statics are the default # this will be done earlier - we just have to diff --git a/tests/generics/tstatic_constrained.nim b/tests/generics/tstatic_constrained.nim new file mode 100644 index 000000000..07318d1bd --- /dev/null +++ b/tests/generics/tstatic_constrained.nim @@ -0,0 +1,42 @@ +discard """ + cmd: "nim check --hints:off --warnings:off $file" + action: "reject" + nimout:''' +tstatic_constrained.nim(41, 20) Error: cannot instantiate MyOtherType [type declared in tstatic_constrained.nim(27, 3)] +got: +but expected: +tstatic_constrained.nim(41, 20) Error: cannot instantiate MyOtherType [type declared in tstatic_constrained.nim(27, 3)] +got: +but expected: +tstatic_constrained.nim(41, 29) Error: object constructor needs an object type [proxy] +tstatic_constrained.nim(41, 29) Error: expression '' has no type (or is ambiguous) +tstatic_constrained.nim(42, 20) Error: cannot instantiate MyOtherType [type declared in tstatic_constrained.nim(27, 3)] +got: +but expected: +tstatic_constrained.nim(42, 20) Error: cannot instantiate MyOtherType [type declared in tstatic_constrained.nim(27, 3)] +got: +but expected: +tstatic_constrained.nim(42, 32) Error: object constructor needs an object type [proxy] +tstatic_constrained.nim(42, 32) Error: expression '' has no type (or is ambiguous) +''' +""" + +type + MyType[T; X: static T] = object + data: T + MyOtherType[T: float or string, Y: static T] = object + +func f[T,X](a: MyType[T,X]): MyType[T,X] = + when T is string: + MyType[T,X](data: a.data & X) + else: + MyType[T,X](data: a.data + X) + +discard MyType[int, 2](data: 1) +discard MyType[string, "Helelello"](data: "Hmmm") +discard MyType[int, 2](data: 1).f() +discard MyType[string, "Helelello"](data: "Hmmm").f() +discard MyOtherType[float, 1.3]() +discard MyOtherType[string, "Hello"]() +discard MyOtherType[int, 10]() +discard MyOtherType[byte, 10u8]() \ No newline at end of file From 7bdfeb78190ca9ff6a0cf702c6ec202f379bff2f Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Sat, 15 Jan 2022 01:24:23 -0700 Subject: [PATCH 1465/2472] Fixed concept constraints for static types (#19391) --- compiler/sigmatch.nim | 9 +++ tests/generics/tstatic_constrained.nim | 87 ++++++++++++++++++-------- 2 files changed, 71 insertions(+), 25 deletions(-) diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 4d8e03fe9..e1195d04a 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -1634,6 +1634,15 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, bindConcreteTypeToUserTypeClass(matched, a) if doBind: put(c, f, matched) result = isGeneric + elif a.len > 0 and a.lastSon == f: + # Needed for checking `Y` == `Addable` in the following + #[ + type + Addable = concept a, type A + a + a is A + MyType[T: Addable; Y: static T] = object + ]# + result = isGeneric else: result = isNone diff --git a/tests/generics/tstatic_constrained.nim b/tests/generics/tstatic_constrained.nim index 07318d1bd..3c9201548 100644 --- a/tests/generics/tstatic_constrained.nim +++ b/tests/generics/tstatic_constrained.nim @@ -2,41 +2,78 @@ discard """ cmd: "nim check --hints:off --warnings:off $file" action: "reject" nimout:''' -tstatic_constrained.nim(41, 20) Error: cannot instantiate MyOtherType [type declared in tstatic_constrained.nim(27, 3)] +tstatic_constrained.nim(44, 22) Error: cannot instantiate MyOtherType [type declared in tstatic_constrained.nim(30, 5)] got: but expected: -tstatic_constrained.nim(41, 20) Error: cannot instantiate MyOtherType [type declared in tstatic_constrained.nim(27, 3)] +tstatic_constrained.nim(44, 22) Error: cannot instantiate MyOtherType [type declared in tstatic_constrained.nim(30, 5)] got: but expected: -tstatic_constrained.nim(41, 29) Error: object constructor needs an object type [proxy] -tstatic_constrained.nim(41, 29) Error: expression '' has no type (or is ambiguous) -tstatic_constrained.nim(42, 20) Error: cannot instantiate MyOtherType [type declared in tstatic_constrained.nim(27, 3)] +tstatic_constrained.nim(44, 31) Error: object constructor needs an object type [proxy] +tstatic_constrained.nim(44, 31) Error: expression '' has no type (or is ambiguous) +tstatic_constrained.nim(45, 22) Error: cannot instantiate MyOtherType [type declared in tstatic_constrained.nim(30, 5)] got: but expected: -tstatic_constrained.nim(42, 20) Error: cannot instantiate MyOtherType [type declared in tstatic_constrained.nim(27, 3)] +tstatic_constrained.nim(45, 22) Error: cannot instantiate MyOtherType [type declared in tstatic_constrained.nim(30, 5)] got: but expected: -tstatic_constrained.nim(42, 32) Error: object constructor needs an object type [proxy] -tstatic_constrained.nim(42, 32) Error: expression '' has no type (or is ambiguous) +tstatic_constrained.nim(45, 34) Error: object constructor needs an object type [proxy] +tstatic_constrained.nim(45, 34) Error: expression '' has no type (or is ambiguous) +tstatic_constrained.nim(77, 14) Error: cannot instantiate MyType [type declared in tstatic_constrained.nim(71, 5)] +got: +but expected: ''' """ +block: + type + MyType[T; X: static T] = object + data: T + MyOtherType[T: float or string, Y: static T] = object -type - MyType[T; X: static T] = object - data: T - MyOtherType[T: float or string, Y: static T] = object + func f[T,X](a: MyType[T,X]): MyType[T,X] = + when T is string: + MyType[T,X](data: a.data & X) + else: + MyType[T,X](data: a.data + X) -func f[T,X](a: MyType[T,X]): MyType[T,X] = - when T is string: - MyType[T,X](data: a.data & X) - else: - MyType[T,X](data: a.data + X) + discard MyType[int, 2](data: 1) + discard MyType[string, "Helelello"](data: "Hmmm") + discard MyType[int, 2](data: 1).f() + discard MyType[string, "Helelello"](data: "Hmmm").f() + discard MyOtherType[float, 1.3]() + discard MyOtherType[string, "Hello"]() + discard MyOtherType[int, 10]() + discard MyOtherType[byte, 10u8]() -discard MyType[int, 2](data: 1) -discard MyType[string, "Helelello"](data: "Hmmm") -discard MyType[int, 2](data: 1).f() -discard MyType[string, "Helelello"](data: "Hmmm").f() -discard MyOtherType[float, 1.3]() -discard MyOtherType[string, "Hello"]() -discard MyOtherType[int, 10]() -discard MyOtherType[byte, 10u8]() \ No newline at end of file +block: + type + Moduloable = concept m, type M + m mod m is M + Addable = concept a, type A + a + a is A + Modulo[T: Moduloable; Mod: static T] = distinct T + ModuloAdd[T: Moduloable or Addable; Mod: static T] = distinct T + ModuAddable = Addable or Moduloable + ModdAddClass[T: ModuAddable; Mod: static T] = distinct T + + proc toMod[T](val: T, modVal: static T): Modulo[T, modVal] = + mixin `mod` + Modulo[T, modVal](val mod modVal) + var + a = 3231.toMod(10) + b = 5483.toMod(10) + discard ModuloAdd[int, 3](0) + discard ModdAddClass[int, 3](0) + +block: + type + MyConstraint = int or string + MyOtherConstraint[T] = object + MyType[T: MyConstraint; Y: static T] = object + MyOtherType[T: MyOtherConstraint; Y: static T] = object + + var + a: MyType[int, 10] + b: MyType[string, "hello"] + c: MyType[float, 10d] + d: MyOtherType[MyOtherConstraint[float],MyOtherConstraint[float]()] + e: MyOtherType[MyOtherConstraint[int], MyOtherConstraint[int]()] \ No newline at end of file From 342b74ef70e6a71c09ba536d20e23a7e887d2fb8 Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 15 Jan 2022 18:25:09 +0800 Subject: [PATCH 1466/2472] move type operation section and remove deepcopy document (#19389) ref #19173; because deepcopy is not fit for ORC/ARC which was used for spawn and spawn will be removed from compiler --- doc/manual.rst | 12 +++++++++ doc/manual_experimental.rst | 54 ++++++++++--------------------------- 2 files changed, 26 insertions(+), 40 deletions(-) diff --git a/doc/manual.rst b/doc/manual.rst index 8788e53a3..007b14b2d 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -3902,6 +3902,18 @@ the operator is in scope (including if it is private). Type bound operators are: `=destroy`, `=copy`, `=sink`, `=trace`, `=deepcopy`. +These operations can be *overridden* instead of *overloaded*. This means that +the implementation is automatically lifted to structured types. For instance, +if the type `T` has an overridden assignment operator `=`, this operator is +also used for assignments of the type `seq[T]`. + +Since these operations are bound to a type, they have to be bound to a +nominal type for reasons of simplicity of implementation; this means an +overridden `deepCopy` for `ref T` is really bound to `T` and not to `ref T`. +This also means that one cannot override `deepCopy` for both `ptr T` and +`ref T` at the same time, instead a distinct or object helper type has to be +used for one pointer type. + For more details on some of those procs, see `Lifetime-tracking hooks `_. diff --git a/doc/manual_experimental.rst b/doc/manual_experimental.rst index 0f82b9950..10958c29a 100644 --- a/doc/manual_experimental.rst +++ b/doc/manual_experimental.rst @@ -1192,51 +1192,25 @@ object inheritance syntax involving the `of` keyword: the `vtptr` magic produced types bound to `ptr` types. -Type bound operations -===================== +.. + deepCopy + -------- + `=deepCopy` is a builtin that is invoked whenever data is passed to + a `spawn`'ed proc to ensure memory safety. The programmer can override its + behaviour for a specific `ref` or `ptr` type `T`. (Later versions of the + language may weaken this restriction.) -There are 4 operations that are bound to a type: + The signature has to be: -1. Assignment -2. Moves -3. Destruction -4. Deep copying for communication between threads + .. code-block:: nim -These operations can be *overridden* instead of *overloaded*. This means that -the implementation is automatically lifted to structured types. For instance, -if the type `T` has an overridden assignment operator `=`, this operator is -also used for assignments of the type `seq[T]`. + proc `=deepCopy`(x: T): T -Since these operations are bound to a type, they have to be bound to a -nominal type for reasons of simplicity of implementation; this means an -overridden `deepCopy` for `ref T` is really bound to `T` and not to `ref T`. -This also means that one cannot override `deepCopy` for both `ptr T` and -`ref T` at the same time, instead a distinct or object helper type has to be -used for one pointer type. + This mechanism will be used by most data structures that support shared memory, + like channels, to implement thread safe automatic memory management. -Assignments, moves and destruction are specified in -the `destructors `_ document. - - -deepCopy --------- - -`=deepCopy` is a builtin that is invoked whenever data is passed to -a `spawn`'ed proc to ensure memory safety. The programmer can override its -behaviour for a specific `ref` or `ptr` type `T`. (Later versions of the -language may weaken this restriction.) - -The signature has to be: - -.. code-block:: nim - - proc `=deepCopy`(x: T): T - -This mechanism will be used by most data structures that support shared memory, -like channels, to implement thread safe automatic memory management. - -The builtin `deepCopy` can even clone closures and their environments. See -the documentation of `spawn <#parallel-amp-spawn-spawn-statement>`_ for details. + The builtin `deepCopy` can even clone closures and their environments. See + the documentation of `spawn <#parallel-amp-spawn-spawn-statement>`_ for details. Dynamic arguments for bindSym From a95399143fdbd518f4d5fe33487b656c4cde7d6d Mon Sep 17 00:00:00 2001 From: hlaaftana <10591326+hlaaftana@users.noreply.github.com> Date: Sat, 15 Jan 2022 15:19:05 +0300 Subject: [PATCH 1467/2472] Use openarray of bytes in md5 (#19307) * Use openarray of bytes in md5 * fix CI * cleanup * use noSideEffect for bootstrapping * fix CI again * actually fix CI by checking if it works * this is getting ridiculous * put old md5 version in compiler, remove vmop --- changelog.md | 1 + compiler/md5_old.nim | 297 +++++++++++++++++++++++++++++++++++ compiler/modulegraphs.nim | 2 +- compiler/sighashes.nim | 2 +- compiler/vmops.nim | 6 - lib/pure/md5.nim | 131 +++++++++++---- tests/js/tstdlib_imports.nim | 3 +- tests/stdlib/tmd5.nim | 19 ++- tests/test_nimscript.nims | 3 +- 9 files changed, 413 insertions(+), 51 deletions(-) create mode 100644 compiler/md5_old.nim diff --git a/changelog.md b/changelog.md index d07f80623..4d27be3a0 100644 --- a/changelog.md +++ b/changelog.md @@ -23,6 +23,7 @@ filename argument for more informative errors. - Module `colors` expanded with missing colors from the CSS color standard. - Fixed `lists.SinglyLinkedList` being broken after removing the last node ([#19353](https://github.com/nim-lang/Nim/pull/19353)). +- `md5` now works at compile time and in JavaScript. ## `std/smtp` diff --git a/compiler/md5_old.nim b/compiler/md5_old.nim new file mode 100644 index 000000000..da4a51198 --- /dev/null +++ b/compiler/md5_old.nim @@ -0,0 +1,297 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2010 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# `std/md5` without VM and JavaScript support, to circumvent a bug with +# openarrays on Nim < 1.4. + +when defined(nimHasStyleChecks): + {.push styleChecks: off.} + +type + MD5State = array[0..3, uint32] + MD5Block = array[0..15, uint32] + MD5CBits = array[0..7, uint8] + MD5Digest* = array[0..15, uint8] + ## MD5 checksum of a string, obtained with the `toMD5 proc <#toMD5,string>`_. + MD5Buffer = array[0..63, uint8] + MD5Context* {.final.} = object + state: MD5State + count: array[0..1, uint32] + buffer: MD5Buffer + +const + padding: array[0..63, uint8] = [ + 0x80'u8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 + ] + +proc F(x, y, z: uint32): uint32 {.inline.} = + result = (x and y) or ((not x) and z) + +proc G(x, y, z: uint32): uint32 {.inline.} = + result = (x and z) or (y and (not z)) + +proc H(x, y, z: uint32): uint32 {.inline.} = + result = x xor y xor z + +proc I(x, y, z: uint32): uint32 {.inline.} = + result = y xor (x or (not z)) + +proc rot(x: var uint32, n: uint8) {.inline.} = + x = (x shl n) or (x shr (32'u32 - n)) + +proc FF(a: var uint32, b, c, d, x: uint32, s: uint8, ac: uint32) = + a = a + F(b, c, d) + x + ac + rot(a, s) + a = a + b + +proc GG(a: var uint32, b, c, d, x: uint32, s: uint8, ac: uint32) = + a = a + G(b, c, d) + x + ac + rot(a, s) + a = a + b + +proc HH(a: var uint32, b, c, d, x: uint32, s: uint8, ac: uint32) = + a = a + H(b, c, d) + x + ac + rot(a, s) + a = a + b + +proc II(a: var uint32, b, c, d, x: uint32, s: uint8, ac: uint32) = + a = a + I(b, c, d) + x + ac + rot(a, s) + a = a + b + +proc encode(dest: var MD5Block, src: openArray[uint8]) = + var j = 0 + for i in 0..high(dest): + dest[i] = uint32(ord(src[j])) or + uint32(ord(src[j+1])) shl 8 or + uint32(ord(src[j+2])) shl 16 or + uint32(ord(src[j+3])) shl 24 + inc(j, 4) + +proc decode(dest: var openArray[uint8], src: openArray[uint32]) = + var i = 0 + for j in 0..high(src): + dest[i] = uint8(src[j] and 0xff'u32) + dest[i+1] = uint8(src[j] shr 8 and 0xff'u32) + dest[i+2] = uint8(src[j] shr 16 and 0xff'u32) + dest[i+3] = uint8(src[j] shr 24 and 0xff'u32) + inc(i, 4) + +template slice(s: string | cstring, a, b): openArray[uint8] = + s.toOpenArrayByte(a, b) + +template slice(s: openArray[uint8], a, b): openArray[uint8] = + s.toOpenArray(a, b) + +proc transform(buffer: openArray[uint8], state: var MD5State) = + var + myBlock: MD5Block + encode(myBlock, buffer) + var a = state[0] + var b = state[1] + var c = state[2] + var d = state[3] + FF(a, b, c, d, myBlock[0], 7'u8, 0xD76AA478'u32) + FF(d, a, b, c, myBlock[1], 12'u8, 0xE8C7B756'u32) + FF(c, d, a, b, myBlock[2], 17'u8, 0x242070DB'u32) + FF(b, c, d, a, myBlock[3], 22'u8, 0xC1BDCEEE'u32) + FF(a, b, c, d, myBlock[4], 7'u8, 0xF57C0FAF'u32) + FF(d, a, b, c, myBlock[5], 12'u8, 0x4787C62A'u32) + FF(c, d, a, b, myBlock[6], 17'u8, 0xA8304613'u32) + FF(b, c, d, a, myBlock[7], 22'u8, 0xFD469501'u32) + FF(a, b, c, d, myBlock[8], 7'u8, 0x698098D8'u32) + FF(d, a, b, c, myBlock[9], 12'u8, 0x8B44F7AF'u32) + FF(c, d, a, b, myBlock[10], 17'u8, 0xFFFF5BB1'u32) + FF(b, c, d, a, myBlock[11], 22'u8, 0x895CD7BE'u32) + FF(a, b, c, d, myBlock[12], 7'u8, 0x6B901122'u32) + FF(d, a, b, c, myBlock[13], 12'u8, 0xFD987193'u32) + FF(c, d, a, b, myBlock[14], 17'u8, 0xA679438E'u32) + FF(b, c, d, a, myBlock[15], 22'u8, 0x49B40821'u32) + GG(a, b, c, d, myBlock[1], 5'u8, 0xF61E2562'u32) + GG(d, a, b, c, myBlock[6], 9'u8, 0xC040B340'u32) + GG(c, d, a, b, myBlock[11], 14'u8, 0x265E5A51'u32) + GG(b, c, d, a, myBlock[0], 20'u8, 0xE9B6C7AA'u32) + GG(a, b, c, d, myBlock[5], 5'u8, 0xD62F105D'u32) + GG(d, a, b, c, myBlock[10], 9'u8, 0x02441453'u32) + GG(c, d, a, b, myBlock[15], 14'u8, 0xD8A1E681'u32) + GG(b, c, d, a, myBlock[4], 20'u8, 0xE7D3FBC8'u32) + GG(a, b, c, d, myBlock[9], 5'u8, 0x21E1CDE6'u32) + GG(d, a, b, c, myBlock[14], 9'u8, 0xC33707D6'u32) + GG(c, d, a, b, myBlock[3], 14'u8, 0xF4D50D87'u32) + GG(b, c, d, a, myBlock[8], 20'u8, 0x455A14ED'u32) + GG(a, b, c, d, myBlock[13], 5'u8, 0xA9E3E905'u32) + GG(d, a, b, c, myBlock[2], 9'u8, 0xFCEFA3F8'u32) + GG(c, d, a, b, myBlock[7], 14'u8, 0x676F02D9'u32) + GG(b, c, d, a, myBlock[12], 20'u8, 0x8D2A4C8A'u32) + HH(a, b, c, d, myBlock[5], 4'u8, 0xFFFA3942'u32) + HH(d, a, b, c, myBlock[8], 11'u8, 0x8771F681'u32) + HH(c, d, a, b, myBlock[11], 16'u8, 0x6D9D6122'u32) + HH(b, c, d, a, myBlock[14], 23'u8, 0xFDE5380C'u32) + HH(a, b, c, d, myBlock[1], 4'u8, 0xA4BEEA44'u32) + HH(d, a, b, c, myBlock[4], 11'u8, 0x4BDECFA9'u32) + HH(c, d, a, b, myBlock[7], 16'u8, 0xF6BB4B60'u32) + HH(b, c, d, a, myBlock[10], 23'u8, 0xBEBFBC70'u32) + HH(a, b, c, d, myBlock[13], 4'u8, 0x289B7EC6'u32) + HH(d, a, b, c, myBlock[0], 11'u8, 0xEAA127FA'u32) + HH(c, d, a, b, myBlock[3], 16'u8, 0xD4EF3085'u32) + HH(b, c, d, a, myBlock[6], 23'u8, 0x04881D05'u32) + HH(a, b, c, d, myBlock[9], 4'u8, 0xD9D4D039'u32) + HH(d, a, b, c, myBlock[12], 11'u8, 0xE6DB99E5'u32) + HH(c, d, a, b, myBlock[15], 16'u8, 0x1FA27CF8'u32) + HH(b, c, d, a, myBlock[2], 23'u8, 0xC4AC5665'u32) + II(a, b, c, d, myBlock[0], 6'u8, 0xF4292244'u32) + II(d, a, b, c, myBlock[7], 10'u8, 0x432AFF97'u32) + II(c, d, a, b, myBlock[14], 15'u8, 0xAB9423A7'u32) + II(b, c, d, a, myBlock[5], 21'u8, 0xFC93A039'u32) + II(a, b, c, d, myBlock[12], 6'u8, 0x655B59C3'u32) + II(d, a, b, c, myBlock[3], 10'u8, 0x8F0CCC92'u32) + II(c, d, a, b, myBlock[10], 15'u8, 0xFFEFF47D'u32) + II(b, c, d, a, myBlock[1], 21'u8, 0x85845DD1'u32) + II(a, b, c, d, myBlock[8], 6'u8, 0x6FA87E4F'u32) + II(d, a, b, c, myBlock[15], 10'u8, 0xFE2CE6E0'u32) + II(c, d, a, b, myBlock[6], 15'u8, 0xA3014314'u32) + II(b, c, d, a, myBlock[13], 21'u8, 0x4E0811A1'u32) + II(a, b, c, d, myBlock[4], 6'u8, 0xF7537E82'u32) + II(d, a, b, c, myBlock[11], 10'u8, 0xBD3AF235'u32) + II(c, d, a, b, myBlock[2], 15'u8, 0x2AD7D2BB'u32) + II(b, c, d, a, myBlock[9], 21'u8, 0xEB86D391'u32) + state[0] = state[0] + a + state[1] = state[1] + b + state[2] = state[2] + c + state[3] = state[3] + d + +proc md5Init*(c: var MD5Context) {.raises: [], tags: [], gcsafe.} +proc md5Update*(c: var MD5Context, input: openArray[uint8]) {.raises: [], + tags: [], gcsafe.} +proc md5Final*(c: var MD5Context, digest: var MD5Digest) {.raises: [], tags: [], gcsafe.} + +proc md5Update*(c: var MD5Context, input: cstring, len: int) {.raises: [], + tags: [], gcsafe.} = + ## Updates the `MD5Context` with the `input` data of length `len`. + ## + ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this + ## function explicitly. + md5Update(c, input.slice(0, len - 1)) + + +proc toMD5*(s: string): MD5Digest = + ## Computes the `MD5Digest` value for a string `s`. + ## + ## **See also:** + ## * `getMD5 proc <#getMD5,string>`_ which returns a string representation + ## of the `MD5Digest` + ## * `$ proc <#$,MD5Digest>`_ for converting MD5Digest to string + runnableExamples: + assert $toMD5("abc") == "900150983cd24fb0d6963f7d28e17f72" + + var c: MD5Context + md5Init(c) + md5Update(c, s.slice(0, s.len - 1)) + md5Final(c, result) + +proc `$`*(d: MD5Digest): string = + ## Converts a `MD5Digest` value into its string representation. + const digits = "0123456789abcdef" + result = "" + for i in 0..15: + add(result, digits[(d[i].int shr 4) and 0xF]) + add(result, digits[d[i].int and 0xF]) + +proc getMD5*(s: string): string = + ## Computes an MD5 value of `s` and returns its string representation. + ## + ## **See also:** + ## * `toMD5 proc <#toMD5,string>`_ which returns the `MD5Digest` of a string + runnableExamples: + assert getMD5("abc") == "900150983cd24fb0d6963f7d28e17f72" + + var + c: MD5Context + d: MD5Digest + md5Init(c) + md5Update(c, s.slice(0, s.len - 1)) + md5Final(c, d) + result = $d + +proc `==`*(D1, D2: MD5Digest): bool = + ## Checks if two `MD5Digest` values are identical. + for i in 0..15: + if D1[i] != D2[i]: return false + return true + + +proc clearBuffer(c: var MD5Context) {.inline.} = + zeroMem(addr(c.buffer), sizeof(MD5Buffer)) + +proc md5Init*(c: var MD5Context) = + ## Initializes an `MD5Context`. + ## + ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this + ## function explicitly. + c.state[0] = 0x67452301'u32 + c.state[1] = 0xEFCDAB89'u32 + c.state[2] = 0x98BADCFE'u32 + c.state[3] = 0x10325476'u32 + c.count[0] = 0'u32 + c.count[1] = 0'u32 + clearBuffer(c) + +proc writeBuffer(c: var MD5Context, index: int, + input: openArray[uint8], inputIndex, len: int) {.inline.} = + copyMem(addr(c.buffer[index]), unsafeAddr(input[inputIndex]), len) + +proc md5Update*(c: var MD5Context, input: openArray[uint8]) = + ## Updates the `MD5Context` with the `input` data. + ## + ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this + ## function explicitly. + var Index = int((c.count[0] shr 3) and 0x3F) + c.count[0] = c.count[0] + (uint32(input.len) shl 3) + if c.count[0] < (uint32(input.len) shl 3): c.count[1] = c.count[1] + 1'u32 + c.count[1] = c.count[1] + (uint32(input.len) shr 29) + var PartLen = 64 - Index + if input.len >= PartLen: + writeBuffer(c, Index, input, 0, PartLen) + transform(c.buffer, c.state) + var i = PartLen + while i + 63 < input.len: + transform(input.slice(i, i + 63), c.state) + inc(i, 64) + if i < input.len: + writeBuffer(c, 0, input, i, input.len - i) + elif input.len > 0: + writeBuffer(c, Index, input, 0, input.len) + +proc md5Final*(c: var MD5Context, digest: var MD5Digest) = + ## Finishes the `MD5Context` and stores the result in `digest`. + ## + ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this + ## function explicitly. + var + Bits: MD5CBits + PadLen: int + decode(Bits, c.count) + var Index = int((c.count[0] shr 3) and 0x3F) + if Index < 56: PadLen = 56 - Index + else: PadLen = 120 - Index + md5Update(c, padding.slice(0, PadLen - 1)) + md5Update(c, Bits) + decode(digest, c.state) + clearBuffer(c) + + +when defined(nimHasStyleChecks): + {.pop.} #{.push styleChecks: off.} diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index 90b130e92..28e0ddb82 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -11,7 +11,7 @@ ## represents a complete Nim project. Single modules can either be kept in RAM ## or stored in a rod-file. -import intsets, tables, hashes, md5 +import intsets, tables, hashes, md5_old import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils import ic / [packed_ast, ic] diff --git a/compiler/sighashes.nim b/compiler/sighashes.nim index d02728627..3d36d2533 100644 --- a/compiler/sighashes.nim +++ b/compiler/sighashes.nim @@ -9,7 +9,7 @@ ## Computes hash values for routine (proc, method etc) signatures. -import ast, tables, ropes, md5, modulegraphs +import ast, tables, ropes, md5_old, modulegraphs from hashes import Hash import types diff --git a/compiler/vmops.nim b/compiler/vmops.nim index b9801234d..22a632ee9 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -25,7 +25,6 @@ when declared(math.signbit): from std/os import getEnv, existsEnv, delEnv, putEnv, envPairs, dirExists, fileExists, walkDir, getAppFilename, raiseOSError, osLastError -from std/md5 import getMD5 from std/times import cpuTime from std/hashes import hash from std/osproc import nil @@ -53,9 +52,6 @@ template ioop(op) {.dirty.} = template macrosop(op) {.dirty.} = registerCallback(c, "stdlib.macros." & astToStr(op), `op Wrapper`) -template md5op(op) {.dirty.} = - registerCallback(c, "stdlib.md5." & astToStr(op), `op Wrapper`) - template wrap1f_math(op) {.dirty.} = proc `op Wrapper`(a: VmArgs) {.nimcall.} = doAssert a.numArgs == 1 @@ -212,8 +208,6 @@ proc registerAdditionalOps*(c: PCtx) = of 2: setResult(a, round(getFloat(a, 0), getInt(a, 1).int)) else: doAssert false, $n - wrap1s(getMD5, md5op) - proc `mod Wrapper`(a: VmArgs) {.nimcall.} = setResult(a, `mod`(getFloat(a, 0), getFloat(a, 1))) registerCallback(c, "stdlib.math.mod", `mod Wrapper`) diff --git a/lib/pure/md5.nim b/lib/pure/md5.nim index 11c324548..06c5608a4 100644 --- a/lib/pure/md5.nim +++ b/lib/pure/md5.nim @@ -9,7 +9,7 @@ ## Module for computing [MD5 checksums](https://en.wikipedia.org/wiki/MD5). ## -## **Note:** The procs in this module can be used at compile time. +## This module also works at compile time and in JavaScript. ## ## See also ## ======== @@ -34,15 +34,16 @@ type buffer: MD5Buffer const - padding: cstring = "\x80\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0" + padding: array[0..63, uint8] = [ + 0x80'u8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 + ] proc F(x, y, z: uint32): uint32 {.inline.} = result = (x and y) or ((not x) and z) @@ -79,7 +80,7 @@ proc II(a: var uint32, b, c, d, x: uint32, s: uint8, ac: uint32) = rot(a, s) a = a + b -proc encode(dest: var MD5Block, src: cstring) = +proc encode(dest: var MD5Block, src: openArray[uint8]) = var j = 0 for i in 0..high(dest): dest[i] = uint32(ord(src[j])) or @@ -97,10 +98,48 @@ proc decode(dest: var openArray[uint8], src: openArray[uint32]) = dest[i+3] = uint8(src[j] shr 24 and 0xff'u32) inc(i, 4) -proc transform(buffer: pointer, state: var MD5State) = +template slice(s: string, a, b): openArray[uint8] = + when nimvm: + # toOpenArray is not implemented in VM + var s2 = newSeq[uint8](s.len) + for i in 0 ..< s2.len: + s2[i] = uint8(s[i]) + s2 + else: + s.toOpenArrayByte(a, b) + +template slice(s: cstring, a, b): openArray[uint8] = + when nimvm: + # toOpenArray is not implemented in VM + slice($s, a, b) + else: + when defined(js): + # toOpenArrayByte for cstring is not implemented in JS + slice($s, a, b) + else: + s.toOpenArrayByte(a, b) + +template slice(s: openArray[uint8], a, b): openArray[uint8] = + when nimvm: + s[a .. b] + else: + s.toOpenArray(a, b) + +const useMem = declared(copyMem) + +template memOrNot(withMem, withoutMem): untyped = + when nimvm: + withoutMem + else: + when useMem: + withMem + else: + withoutMem + +proc transform(buffer: openArray[uint8], state: var MD5State) = var myBlock: MD5Block - encode(myBlock, cast[cstring](buffer)) + encode(myBlock, buffer) var a = state[0] var b = state[1] var c = state[2] @@ -175,10 +214,18 @@ proc transform(buffer: pointer, state: var MD5State) = state[3] = state[3] + d proc md5Init*(c: var MD5Context) {.raises: [], tags: [], gcsafe.} -proc md5Update*(c: var MD5Context, input: cstring, len: int) {.raises: [], +proc md5Update*(c: var MD5Context, input: openArray[uint8]) {.raises: [], tags: [], gcsafe.} proc md5Final*(c: var MD5Context, digest: var MD5Digest) {.raises: [], tags: [], gcsafe.} +proc md5Update*(c: var MD5Context, input: cstring, len: int) {.raises: [], + tags: [], gcsafe.} = + ## Updates the `MD5Context` with the `input` data of length `len`. + ## + ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this + ## function explicitly. + md5Update(c, input.slice(0, len - 1)) + proc toMD5*(s: string): MD5Digest = ## Computes the `MD5Digest` value for a string `s`. @@ -192,7 +239,7 @@ proc toMD5*(s: string): MD5Digest = var c: MD5Context md5Init(c) - md5Update(c, cstring(s), len(s)) + md5Update(c, s.slice(0, s.len - 1)) md5Final(c, result) proc `$`*(d: MD5Digest): string = @@ -215,7 +262,7 @@ proc getMD5*(s: string): string = c: MD5Context d: MD5Digest md5Init(c) - md5Update(c, cstring(s), len(s)) + md5Update(c, s.slice(0, s.len - 1)) md5Final(c, d) result = $d @@ -226,6 +273,12 @@ proc `==`*(D1, D2: MD5Digest): bool = return true +proc clearBuffer(c: var MD5Context) {.inline.} = + memOrNot: + zeroMem(addr(c.buffer), sizeof(MD5Buffer)) + do: + reset(c.buffer) + proc md5Init*(c: var MD5Context) = ## Initializes an `MD5Context`. ## @@ -237,29 +290,39 @@ proc md5Init*(c: var MD5Context) = c.state[3] = 0x10325476'u32 c.count[0] = 0'u32 c.count[1] = 0'u32 - zeroMem(addr(c.buffer), sizeof(MD5Buffer)) + clearBuffer(c) -proc md5Update*(c: var MD5Context, input: cstring, len: int) = - ## Updates the `MD5Context` with the `input` data of length `len`. +proc writeBuffer(c: var MD5Context, index: int, + input: openArray[uint8], inputIndex, len: int) {.inline.} = + memOrNot: + copyMem(addr(c.buffer[index]), unsafeAddr(input[inputIndex]), len) + do: + # cannot use system.`[]=` for arrays and openarrays as + # it can raise RangeDefect which gets tracked + for i in 0..`_, there's no need to call this ## function explicitly. - var input = input var Index = int((c.count[0] shr 3) and 0x3F) - c.count[0] = c.count[0] + (uint32(len) shl 3) - if c.count[0] < (uint32(len) shl 3): c.count[1] = c.count[1] + 1'u32 - c.count[1] = c.count[1] + (uint32(len) shr 29) + c.count[0] = c.count[0] + (uint32(input.len) shl 3) + if c.count[0] < (uint32(input.len) shl 3): c.count[1] = c.count[1] + 1'u32 + c.count[1] = c.count[1] + (uint32(input.len) shr 29) var PartLen = 64 - Index - if len >= PartLen: - copyMem(addr(c.buffer[Index]), input, PartLen) - transform(addr(c.buffer), c.state) + if input.len >= PartLen: + writeBuffer(c, Index, input, 0, PartLen) + transform(c.buffer, c.state) var i = PartLen - while i + 63 < len: - transform(addr(input[i]), c.state) + while i + 63 < input.len: + transform(input.slice(i, i + 63), c.state) inc(i, 64) - copyMem(addr(c.buffer[0]), addr(input[i]), len-i) - else: - copyMem(addr(c.buffer[Index]), addr(input[0]), len) + if i < input.len: + writeBuffer(c, 0, input, i, input.len - i) + elif input.len > 0: + writeBuffer(c, Index, input, 0, input.len) proc md5Final*(c: var MD5Context, digest: var MD5Digest) = ## Finishes the `MD5Context` and stores the result in `digest`. @@ -273,10 +336,10 @@ proc md5Final*(c: var MD5Context, digest: var MD5Digest) = var Index = int((c.count[0] shr 3) and 0x3F) if Index < 56: PadLen = 56 - Index else: PadLen = 120 - Index - md5Update(c, padding, PadLen) - md5Update(c, cast[cstring](addr(Bits)), 8) + md5Update(c, padding.slice(0, PadLen - 1)) + md5Update(c, Bits) decode(digest, c.state) - zeroMem(addr(c), sizeof(MD5Context)) + clearBuffer(c) when defined(nimHasStyleChecks): diff --git a/tests/js/tstdlib_imports.nim b/tests/js/tstdlib_imports.nim index 9986fa059..30aca7561 100644 --- a/tests/js/tstdlib_imports.nim +++ b/tests/js/tstdlib_imports.nim @@ -57,8 +57,7 @@ import std/[ htmlgen, # Hashing: - base64, hashes, - # fails due to cstring cast/zeroMem/copyMem/moveMem: md5 + base64, hashes, md5, # fails due to cstring cast/endians import: oids # fails due to copyMem/endians import: sha1 diff --git a/tests/stdlib/tmd5.nim b/tests/stdlib/tmd5.nim index 88a7b8d37..4017ac677 100644 --- a/tests/stdlib/tmd5.nim +++ b/tests/stdlib/tmd5.nim @@ -1,7 +1,16 @@ +discard """ + targets: "c cpp js" +""" + import md5 -doAssert(getMD5("Franz jagt im komplett verwahrlosten Taxi quer durch Bayern") == - "a3cca2b2aa1e3b5b3b5aad99a8529074") -doAssert(getMD5("Frank jagt im komplett verwahrlosten Taxi quer durch Bayern") == - "7e716d0e702df0505fc72e2b89467910") -doAssert($toMD5("") == "d41d8cd98f00b204e9800998ecf8427e") +proc main() {.raises: [].} = + doAssert(getMD5("Franz jagt im komplett verwahrlosten Taxi quer durch Bayern") == + "a3cca2b2aa1e3b5b3b5aad99a8529074") + doAssert(getMD5("Frank jagt im komplett verwahrlosten Taxi quer durch Bayern") == + "7e716d0e702df0505fc72e2b89467910") + doAssert($toMD5("") == "d41d8cd98f00b204e9800998ecf8427e") + +main() + +static: main() diff --git a/tests/test_nimscript.nims b/tests/test_nimscript.nims index 761ddafdf..22a566307 100644 --- a/tests/test_nimscript.nims +++ b/tests/test_nimscript.nims @@ -59,8 +59,7 @@ import std/[ htmlgen, # Hashing: - base64, hashes, - # fails due to cstring cast/zeroMem/copyMem/moveMem: md5 + base64, hashes, md5, # fails due to cstring cast/times import/endians import: oids # fails due to copyMem/endians import: sha1 From d102b2f54c41f19a0545f28109d0a93550b5d886 Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 16 Jan 2022 18:08:38 +0800 Subject: [PATCH 1468/2472] deprecate unsafeAddr; extend addr (#19373) * deprecate unsafeAddr; extend addr addr is now available for all addressable locations, unsafeAddr is deprecated and become an alias for addr * follow @Vindaar's advice * change the signature of addr * unsafeAddr => addr (stdlib) * Update changelog.md * unsafeAddr => addr (tests) * Revert "unsafeAddr => addr (stdlib)" This reverts commit ab83c99c507048a8396e636bf22d55fdd84d7d1c. * doc changes; thanks to @konsumlamm Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- changelog.md | 3 +++ compiler/isolation_check.nim | 2 +- compiler/semmagic.nim | 9 ++----- doc/manual.rst | 15 ++++------- doc/manual_experimental.rst | 2 +- lib/system.nim | 26 +++++++++++++------ tests/arc/tcustomtrace.nim | 2 +- tests/arc/thard_alignment.nim | 2 +- tests/async/tioselectors.nim | 2 +- tests/ccgbugs/tnoalias.nim | 2 +- tests/compiles/t8630.nim | 2 +- tests/concepts/tconcepts_issues.nim | 4 +-- tests/destructor/t12037.nim | 6 ++--- tests/destructor/tarray_indexing.nim | 2 +- tests/destructor/tcustomseqs.nim | 2 +- tests/destructor/tcustomstrings.nim | 2 +- .../destructor/tuse_result_prevents_sinks.nim | 2 +- tests/errmsgs/t10594.nim | 2 +- tests/errmsgs/tnon_concrete_cast.nim | 2 +- tests/js/tfieldchecks.nim | 2 +- tests/lent/tbasic_lent_check.nim | 10 +++---- tests/misc/taddr.nim | 2 +- tests/overload/t8829.nim | 4 +-- tests/overload/tstatic_with_converter.nim | 2 +- tests/stdlib/tdecls.nim | 14 +++++----- tests/stdlib/tmemory.nim | 2 +- tests/stdlib/tsqlitebindatas.nim | 4 +-- tests/strictnotnil/tnilcheck.nim | 2 +- tests/system/tostring.nim | 2 +- tests/types/tlent_var.nim | 2 +- 30 files changed, 69 insertions(+), 66 deletions(-) diff --git a/changelog.md b/changelog.md index 4d27be3a0..9b74d6d9f 100644 --- a/changelog.md +++ b/changelog.md @@ -17,6 +17,9 @@ - `std/sharedstrings` module is removed. - Constants `colors.colPaleVioletRed` and `colors.colMediumPurple` changed to match the CSS color standard. +- `addr` is now available for all addressable locations, `unsafeAddr` is deprecated and +becomes an alias for `addr`. + ## Standard library additions and changes - `macros.parseExpr` and `macros.parseStmt` now accept an optional diff --git a/compiler/isolation_check.nim b/compiler/isolation_check.nim index a8c5a3651..68a212794 100644 --- a/compiler/isolation_check.nim +++ b/compiler/isolation_check.nim @@ -71,7 +71,7 @@ proc isValueOnlyType(t: PType): bool = proc canAlias*(arg, ret: PType): bool = if isValueOnlyType(arg): - # can alias only with unsafeAddr(arg.x) and we don't care if it is not safe + # can alias only with addr(arg.x) and we don't care if it is not safe result = false else: var marker = initIntSet() diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim index 874098294..c7fc75620 100644 --- a/compiler/semmagic.nim +++ b/compiler/semmagic.nim @@ -14,13 +14,8 @@ proc semAddrArg(c: PContext; n: PNode; isUnsafeAddr = false): PNode = let x = semExprWithType(c, n) if x.kind == nkSym: x.sym.flags.incl(sfAddrTaken) - if isAssignable(c, x, isUnsafeAddr) notin {arLValue, arLocalLValue}: - # Do not suggest the use of unsafeAddr if this expression already is a - # unsafeAddr - if isUnsafeAddr: - localError(c.config, n.info, errExprHasNoAddress) - else: - localError(c.config, n.info, errExprHasNoAddress & "; maybe use 'unsafeAddr'") + if isAssignable(c, x, true) notin {arLValue, arLocalLValue}: + localError(c.config, n.info, errExprHasNoAddress) result = x proc semTypeOf(c: PContext; n: PNode): PNode = diff --git a/doc/manual.rst b/doc/manual.rst index 007b14b2d..4eabd0225 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -3502,8 +3502,9 @@ location is `T`, the `addr` operator result is of the type `ptr T`. An address is always an untraced reference. Taking the address of an object that resides on the stack is **unsafe**, as the pointer may live longer than the object on the stack and can thus reference a non-existing object. One can get -the address of variables, but one can't use it on variables declared through -`let` statements: +the address of variables. For easier interoperability with other compiled languages +such as C, retrieving the address of a `let` variable, a parameter, +or a `for` loop variable can be accomplished too: .. code-block:: nim @@ -3515,24 +3516,18 @@ the address of variables, but one can't use it on variables declared through # --> ref 0x7fff6b71b670 --> 0x10bb81050"Hello" echo cast[ptr string](t3)[] # --> Hello - # The following line doesn't compile: + # The following line also works echo repr(addr(t1)) - # Error: expression has no address - The unsafeAddr operator ----------------------- -For easier interoperability with other compiled languages such as C, retrieving -the address of a `let` variable, a parameter, or a `for` loop variable can -be accomplished by using the `unsafeAddr` operation: +The `unsafeAddr` operator is a deprecated alias for the `addr` operator: .. code-block:: nim - let myArray = [1, 2, 3] foreignProcThatTakesAnAddr(unsafeAddr myArray) - Procedures ========== diff --git a/doc/manual_experimental.rst b/doc/manual_experimental.rst index 10958c29a..407924a13 100644 --- a/doc/manual_experimental.rst +++ b/doc/manual_experimental.rst @@ -603,7 +603,7 @@ has `source` as the owner. A path expression `e` is defined recursively: - Object field access `e.field` is a path expression. - `system.toOpenArray(e, ...)` is a path expression. - Pointer dereference `e[]` is a path expression. -- An address `addr e`, `unsafeAddr e` is a path expression. +- An address `addr e` is a path expression. - A type conversion `T(e)` is a path expression. - A cast expression `cast[T](e)` is a path expression. - `f(e, ...)` is a path expression if `f`'s return type is a view type. diff --git a/lib/system.nim b/lib/system.nim index c424cbc1b..3b3e28344 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -190,8 +190,16 @@ when defined(nimHasDeclaredMagic): else: proc declaredInScope*(x: untyped): bool {.magic: "DefinedInScope", noSideEffect, compileTime.} -proc `addr`*[T](x: var T): ptr T {.magic: "Addr", noSideEffect.} = +proc `addr`*[T](x: T): ptr T {.magic: "Addr", noSideEffect.} = ## Builtin `addr` operator for taking the address of a memory location. + ## + ## .. note:: This works for `let` variables or parameters + ## for better interop with C. When you use it to write a wrapper + ## for a C library and take the address of `let` variables or parameters, + ## you should always check that the original library + ## does never write to data behind the pointer that is returned from + ## this procedure. + ## ## Cannot be overloaded. ## ## See also: @@ -205,15 +213,17 @@ proc `addr`*[T](x: var T): ptr T {.magic: "Addr", noSideEffect.} = ## echo p[] # b discard -proc unsafeAddr*[T](x: T): ptr T {.magic: "Addr", noSideEffect.} = +proc unsafeAddr*[T](x: T): ptr T {.magic: "Addr", noSideEffect, + deprecated: "'unsafeAddr' is a deprecated alias for 'addr'".} = ## Builtin `addr` operator for taking the address of a memory - ## location. This works even for `let` variables or parameters - ## for better interop with C and so it is considered even more - ## unsafe than the ordinary `addr <#addr,T>`_. + ## location. ## - ## **Note**: When you use it to write a wrapper for a C library, you should - ## always check that the original library does never write to data behind the - ## pointer that is returned from this procedure. + ## .. note:: This works for `let` variables or parameters + ## for better interop with C. When you use it to write a wrapper + ## for a C library and take the address of `let` variables or parameters, + ## you should always check that the original library + ## does never write to data behind the pointer that is returned from + ## this procedure. ## ## Cannot be overloaded. discard diff --git a/tests/arc/tcustomtrace.nim b/tests/arc/tcustomtrace.nim index 3977194d9..5e0ecfb24 100644 --- a/tests/arc/tcustomtrace.nim +++ b/tests/arc/tcustomtrace.nim @@ -130,7 +130,7 @@ proc createSeq*[T](elems: varargs[T]): myseq[T] = result.data = cast[type(result.data)](alloc0(result.cap * sizeof(T))) inc allocCount when supportsCopyMem(T): - copyMem(result.data, unsafeAddr(elems[0]), result.cap * sizeof(T)) + copyMem(result.data, addr(elems[0]), result.cap * sizeof(T)) else: for i in 0.. 0 doAssert cmpMem(b.addr, a.addr, sizeof(SomeHash)) < 0 - doAssert cmpMem(a.addr, c.unsafeAddr, sizeof(SomeHash)) == 0 + doAssert cmpMem(a.addr, c.addr, sizeof(SomeHash)) == 0 diff --git a/tests/stdlib/tsqlitebindatas.nim b/tests/stdlib/tsqlitebindatas.nim index 754c80ae1..b2c3247fa 100644 --- a/tests/stdlib/tsqlitebindatas.nim +++ b/tests/stdlib/tsqlitebindatas.nim @@ -27,7 +27,7 @@ block tsqlitebindatas: ## db_sqlite binary data db.exec(createTableStr) var dbuf = newSeq[byte](orig.len*sizeof(float64)) - copyMem(unsafeAddr(dbuf[0]), unsafeAddr(orig[0]), dbuf.len) + copyMem(addr(dbuf[0]), addr(orig[0]), dbuf.len) var insertStmt = db.prepare("INSERT INTO test (id, name, data) VALUES (?, ?, ?)") insertStmt.bindParams(1, origName, dbuf) @@ -42,7 +42,7 @@ block tsqlitebindatas: ## db_sqlite binary data var dataTest = db.getValue(sql"SELECT data FROM test WHERE id = ?", 1) let seqSize = int(dataTest.len*sizeof(byte)/sizeof(float64)) var res: seq[float64] = newSeq[float64](seqSize) - copyMem(unsafeAddr(res[0]), addr(dataTest[0]), dataTest.len) + copyMem(addr(res[0]), addr(dataTest[0]), dataTest.len) doAssert res.len == orig.len doAssert res == orig diff --git a/tests/strictnotnil/tnilcheck.nim b/tests/strictnotnil/tnilcheck.nim index 5b9292522..c2d009b70 100644 --- a/tests/strictnotnil/tnilcheck.nim +++ b/tests/strictnotnil/tnilcheck.nim @@ -194,7 +194,7 @@ proc testAliasChanging(a: Nilable) = # # proc testPtrAlias(a: Nilable) = # # # pointer to a: hm. # # # alias to a? -# # var ptrA = a.unsafeAddr # {0, 1} +# # var ptrA = a.addr # {0, 1} # # if not a.isNil: # {0, 1} # # ptrA[] = nil # {0, 1} 0: MaybeNil 1: MaybeNil # # echo a.a #[ tt.Warning diff --git a/tests/system/tostring.nim b/tests/system/tostring.nim index bdbaa2ce6..cae20865e 100644 --- a/tests/system/tostring.nim +++ b/tests/system/tostring.nim @@ -47,7 +47,7 @@ import strutils let arr = ['H','e','l','l','o',' ','W','o','r','l','d','!','\0'] doAssert $arr == "['H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd', '!', '\\x00']" -doAssert $cstring(unsafeAddr arr) == "Hello World!" +doAssert $cstring(addr arr) == "Hello World!" proc takes(c: cstring) = doAssert c == cstring"" diff --git a/tests/types/tlent_var.nim b/tests/types/tlent_var.nim index 491f6fde8..73b5bef9b 100644 --- a/tests/types/tlent_var.nim +++ b/tests/types/tlent_var.nim @@ -15,7 +15,7 @@ proc test_var(x: var MyObj): var int = var x = MyObj(a: 5) doAssert: test_var(x).addr == x.a.addr -doAssert: test_lent(x).unsafeAddr == x.a.addr +doAssert: test_lent(x).addr == x.a.addr proc varProc(x: var int) = x = 100 From 15e3813d96ad5646476b629f3a20ece89dff064d Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 17 Jan 2022 04:10:35 +0800 Subject: [PATCH 1469/2472] add mm to compilesettings; deprecate gc (#19394) --- compiler/vmops.nim | 1 + lib/std/compilesettings.nim | 3 ++- tests/vm/tcompilesetting.nim | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/compiler/vmops.nim b/compiler/vmops.nim index 22a632ee9..018e7b9c6 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -135,6 +135,7 @@ when defined(nimHasInvariant): of backend: result = $conf.backend of libPath: result = conf.libpath.string of gc: result = $conf.selectedGC + of mm: result = $conf.selectedGC proc querySettingSeqImpl(conf: ConfigRef, switch: BiggestInt): seq[string] = template copySeq(field: untyped): untyped = diff --git a/lib/std/compilesettings.nim b/lib/std/compilesettings.nim index 5bb0bd673..6d8bd22f4 100644 --- a/lib/std/compilesettings.nim +++ b/lib/std/compilesettings.nim @@ -32,7 +32,8 @@ type backend ## the backend (eg: c|cpp|objc|js); both `nim doc --backend:js` ## and `nim js` would imply backend=js libPath ## the absolute path to the stdlib library, i.e. nim's `--lib`, since 1.5.1 - gc ## gc selected + gc {.deprecated.} ## gc selected + mm ## memory management selected MultipleValueSetting* {.pure.} = enum ## \ ## settings resulting in a seq of string values diff --git a/tests/vm/tcompilesetting.nim b/tests/vm/tcompilesetting.nim index 5bf559c74..d6c08e70f 100644 --- a/tests/vm/tcompilesetting.nim +++ b/tests/vm/tcompilesetting.nim @@ -1,5 +1,5 @@ discard """ -cmd: "nim c --nimcache:build/myNimCache --nimblePath:myNimblePath $file" +cmd: "nim c --nimcache:build/myNimCache --nimblePath:myNimblePath --gc:arc $file" joinable: false """ @@ -12,6 +12,7 @@ template main = doAssert "myNimblePath" in nimblePaths.querySettingSeq[0] doAssert querySetting(backend) == "c" doAssert fileExists(libPath.querySetting / "system.nim") + doAssert querySetting(mm) == "arc" static: main() main() From 2c5b367001cc53859048d9adf450a5db3a949b92 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 17 Jan 2022 11:54:27 +0800 Subject: [PATCH 1470/2472] improve changelog a bit (#19400) --- changelog.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/changelog.md b/changelog.md index 9b74d6d9f..16b526f85 100644 --- a/changelog.md +++ b/changelog.md @@ -28,9 +28,7 @@ becomes an alias for `addr`. - Fixed `lists.SinglyLinkedList` being broken after removing the last node ([#19353](https://github.com/nim-lang/Nim/pull/19353)). - `md5` now works at compile time and in JavaScript. -## `std/smtp` - -- Sends `ehlo` first. If the mail server does not understand, it sends `helo` as a fallback. +- `std/smtp` sends `ehlo` first. If the mail server does not understand, it sends `helo` as a fallback. - Added `IsoWeekRange`, a range type to represent the number of weeks in an ISO week-based year. - Added `IsoYear`, a distinct int type to prevent bugs from confusing the week-based year and the regular year. From 4f6b59de9698d0c29a2170ecc23c8d009b3181b7 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 17 Jan 2022 18:14:13 +0800 Subject: [PATCH 1471/2472] mangle names in nimbase.h using cppDefine (#19395) [backport] mangle names in nimbase.h fix comments --- config/config.nims | 5 +++++ tests/ccgbugs/tmangle.nim | 16 ++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 tests/ccgbugs/tmangle.nim diff --git a/config/config.nims b/config/config.nims index 893d6960f..88c344642 100644 --- a/config/config.nims +++ b/config/config.nims @@ -3,6 +3,11 @@ cppDefine "errno" cppDefine "unix" +# mangle the macro names in nimbase.h +cppDefine "NAN_INFINITY" +cppDefine "INF" +cppDefine "NAN" + when defined(nimStrictMode): # xxx add more flags here, and use `-d:nimStrictMode` in more contexts in CI. diff --git a/tests/ccgbugs/tmangle.nim b/tests/ccgbugs/tmangle.nim new file mode 100644 index 000000000..0050cef92 --- /dev/null +++ b/tests/ccgbugs/tmangle.nim @@ -0,0 +1,16 @@ +block: + proc hello() = + let NAN_INFINITY = 12 + doAssert NAN_INFINITY == 12 + let INF = "2.0" + doAssert INF == "2.0" + let NAN = 2.3 + doAssert NAN == 2.3 + + hello() + +block: + proc hello(NAN: float) = + doAssert NAN == 2.0 + + hello(2.0) From dc8ac66873d5189f2e5cf5dbf56a12ab64a38140 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 17 Jan 2022 04:02:31 -0800 Subject: [PATCH 1472/2472] fix no net compilation on zephyr (#19399) Co-authored-by: Jaremy J. Creechley --- lib/posix/posix_other.nim | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/posix/posix_other.nim b/lib/posix/posix_other.nim index 688867b98..941e13192 100644 --- a/lib/posix/posix_other.nim +++ b/lib/posix/posix_other.nim @@ -399,16 +399,20 @@ elif defined(zephyr): const Sockaddr_max_length* = 24 elif defined(net_raw): const Sockaddr_max_length* = 20 - else: + elif defined(net_ipv4): const Sockaddr_max_length* = 8 + else: + const Sockaddr_max_length* = 255 # just for compilation purposes + const Sockaddr_un_path_length* = Sockaddr_max_length # Zephyr is heavily customizable so it's easy to get to a state # where Nim & Zephyr IPv6 settings are out of sync, causing painful runtime failures. - {.emit: ["NIM_STATIC_ASSERT(NET_SOCKADDR_MAX_SIZE == ", - Sockaddr_max_length, - ",\"NET_SOCKADDR_MAX_SIZE and Sockaddr_max_length size mismatch!", - " Check that Nim and Zephyr IPv4/IPv6 settings match.", - " Try adding -d:net_ipv6 to enable IPv6 for Nim on Zephyr.\" );"].} + when defined(net_ipv4) or defined(net_ipv6) or defined(net_raw): + {.emit: ["NIM_STATIC_ASSERT(NET_SOCKADDR_MAX_SIZE == ", + Sockaddr_max_length, + ",\"NET_SOCKADDR_MAX_SIZE and Sockaddr_max_length size mismatch!", + " Check that Nim and Zephyr IPv4/IPv6 settings match.", + " Try adding -d:net_ipv6 to enable IPv6 for Nim on Zephyr.\" );"].} elif defined(freertos) or defined(lwip): const Sockaddr_max_length* = 14 const Sockaddr_un_path_length* = 108 From 07c7a8a526cf4fd12c96192cda7a750fbea3382f Mon Sep 17 00:00:00 2001 From: hlaaftana <10591326+hlaaftana@users.noreply.github.com> Date: Mon, 17 Jan 2022 15:03:40 +0300 Subject: [PATCH 1473/2472] Optimize lent in JS [backport:1.6] (#19393) * Optimize lent in JS [backport:1.6] * addr on lent doesn't work anymore, don't use it * use unsafeAddr in test again for older versions --- compiler/jsgen.nim | 51 ++++++++++++++++++++++---------- tests/js/tlent.nim | 33 +++++++++++++++++++++ tests/lent/tbasic_lent_check.nim | 20 ++++++++----- 3 files changed, 81 insertions(+), 23 deletions(-) create mode 100644 tests/js/tlent.nim diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 7e8c25173..0a7036b46 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -178,7 +178,7 @@ const proc mapType(typ: PType): TJSTypeKind = let t = skipTypes(typ, abstractInst) case t.kind - of tyVar, tyRef, tyPtr, tyLent: + of tyVar, tyRef, tyPtr: if skipTypes(t.lastSon, abstractInst).kind in MappedToObject: result = etyObject else: @@ -186,7 +186,8 @@ proc mapType(typ: PType): TJSTypeKind = of tyPointer: # treat a tyPointer like a typed pointer to an array of bytes result = etyBaseIndex - of tyRange, tyDistinct, tyOrdinal, tyProxy: + of tyRange, tyDistinct, tyOrdinal, tyProxy, tyLent: + # tyLent is no-op as JS has pass-by-reference semantics result = mapType(t[0]) of tyInt..tyInt64, tyUInt..tyUInt64, tyEnum, tyChar: result = etyInt of tyBool: result = etyBool @@ -1060,14 +1061,14 @@ proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) = xtyp = etySeq case xtyp of etySeq: - if (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded: + if x.typ.kind in {tyVar, tyLent} or (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded: lineF(p, "$1 = $2;$n", [a.rdLoc, b.rdLoc]) else: useMagic(p, "nimCopy") lineF(p, "$1 = nimCopy(null, $2, $3);$n", [a.rdLoc, b.res, genTypeInfo(p, y.typ)]) of etyObject: - if x.typ.kind in {tyVar} or (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded: + if x.typ.kind in {tyVar, tyLent} or (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded: lineF(p, "$1 = $2;$n", [a.rdLoc, b.rdLoc]) else: useMagic(p, "nimCopy") @@ -1092,10 +1093,18 @@ proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) = lineF(p, "$# = [$#, $#];$n", [a.res, b.address, b.res]) lineF(p, "$1 = $2;$n", [a.address, b.res]) lineF(p, "$1 = $2;$n", [a.rdLoc, b.rdLoc]) + elif a.typ == etyBaseIndex: + # array indexing may not map to var type + if b.address != nil: + lineF(p, "$1 = $2; $3 = $4;$n", [a.address, b.address, a.res, b.res]) + else: + lineF(p, "$1 = $2;$n", [a.address, b.res]) else: internalError(p.config, x.info, $("genAsgn", b.typ, a.typ)) - else: + elif b.address != nil: lineF(p, "$1 = $2; $3 = $4;$n", [a.address, b.address, a.res, b.res]) + else: + lineF(p, "$1 = $2;$n", [a.address, b.res]) else: lineF(p, "$1 = $2;$n", [a.rdLoc, b.rdLoc]) @@ -1288,7 +1297,7 @@ template isIndirect(x: PSym): bool = v.kind notin {skProc, skFunc, skConverter, skMethod, skIterator, skConst, skTemp, skLet}) -proc genSymAddr(p: PProc, n: PNode, typ: PType, r: var TCompRes) = +proc genSymAddr(p: PProc, n: PNode, typ: PType, r: var TCompRes) = let s = n.sym if s.loc.r == nil: internalError(p.config, n.info, "genAddr: 3") case s.kind @@ -1456,13 +1465,17 @@ proc genSym(p: PProc, n: PNode, r: var TCompRes) = else: if s.loc.r == nil: internalError(p.config, n.info, "symbol has no generated name: " & s.name.s) - r.res = s.loc.r + if mapType(p, s.typ) == etyBaseIndex: + r.address = s.loc.r + r.res = s.loc.r & "_Idx" + else: + r.res = s.loc.r r.kind = resVal proc genDeref(p: PProc, n: PNode, r: var TCompRes) = let it = n[0] let t = mapType(p, it.typ) - if t == etyObject: + if t == etyObject or it.typ.kind == tyLent: gen(p, it, r) else: var a: TCompRes @@ -1703,7 +1716,7 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope = result = putToSeq("0", indirect) of tyFloat..tyFloat128: result = putToSeq("0.0", indirect) - of tyRange, tyGenericInst, tyAlias, tySink, tyOwned: + of tyRange, tyGenericInst, tyAlias, tySink, tyOwned, tyLent: result = createVar(p, lastSon(typ), indirect) of tySet: result = putToSeq("{}", indirect) @@ -1745,7 +1758,7 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope = createObjInitList(p, t, initIntSet(), initList) result = ("({$1})") % [initList] if indirect: result = "[$1]" % [result] - of tyVar, tyPtr, tyLent, tyRef, tyPointer: + of tyVar, tyPtr, tyRef, tyPointer: if mapType(p, t) == etyBaseIndex: result = putToSeq("[null, 0]", indirect) else: @@ -2394,16 +2407,17 @@ proc genProc(oldProc: PProc, prc: PSym): Rope = if prc.typ[0] != nil and sfPure notin prc.flags: resultSym = prc.ast[resultPos].sym let mname = mangleName(p.module, resultSym) - if not isIndirect(resultSym) and + let returnAddress = not isIndirect(resultSym) and resultSym.typ.kind in {tyVar, tyPtr, tyLent, tyRef, tyOwned} and - mapType(p, resultSym.typ) == etyBaseIndex: + mapType(p, resultSym.typ) == etyBaseIndex + if returnAddress: resultAsgn = p.indentLine(("var $# = null;$n") % [mname]) resultAsgn.add p.indentLine("var $#_Idx = 0;$n" % [mname]) else: let resVar = createVar(p, resultSym.typ, isIndirect(resultSym)) resultAsgn = p.indentLine(("var $# = $#;$n") % [mname, resVar]) gen(p, prc.ast[resultPos], a) - if mapType(p, resultSym.typ) == etyBaseIndex: + if returnAddress: returnStmt = "return [$#, $#];$n" % [a.address, a.res] else: returnStmt = "return $#;$n" % [a.res] @@ -2579,8 +2593,15 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) = of nkObjConstr: genObjConstr(p, n, r) of nkHiddenStdConv, nkHiddenSubConv, nkConv: genConv(p, n, r) of nkAddr, nkHiddenAddr: - genAddr(p, n, r) - of nkDerefExpr, nkHiddenDeref: genDeref(p, n, r) + if n.typ.kind in {tyLent}: + gen(p, n[0], r) + else: + genAddr(p, n, r) + of nkDerefExpr, nkHiddenDeref: + if n.typ.kind in {tyLent}: + gen(p, n[0], r) + else: + genDeref(p, n, r) of nkBracketExpr: genArrayAccess(p, n, r) of nkDotExpr: genFieldAccess(p, n, r) of nkCheckedFieldExpr: genCheckedFieldOp(p, n, nil, r) diff --git a/tests/js/tlent.nim b/tests/js/tlent.nim new file mode 100644 index 000000000..2546e5b1d --- /dev/null +++ b/tests/js/tlent.nim @@ -0,0 +1,33 @@ +discard """ + output: ''' +hmm +100 +hmm +100 +''' +""" + +# #16800 + +type A = object + b: int +var t = A(b: 100) +block: + proc getValues: lent int = + echo "hmm" + result = t.b + echo getValues() +block: + proc getValues: lent int = + echo "hmm" + t.b + echo getValues() + +when false: # still an issue, #16908 + template main = + iterator fn[T](a:T): lent T = yield a + let a = @[10] + for b in fn(a): echo b + + static: main() + main() diff --git a/tests/lent/tbasic_lent_check.nim b/tests/lent/tbasic_lent_check.nim index 0787e36d9..92d731451 100644 --- a/tests/lent/tbasic_lent_check.nim +++ b/tests/lent/tbasic_lent_check.nim @@ -17,22 +17,26 @@ proc main = main() template main2 = # bug #15958 + when defined(js): + proc sameAddress[T](a, b: T): bool {.importjs: "(# === #)".} + else: + template sameAddress(a, b): bool = a.unsafeAddr == b.unsafeAddr proc byLent[T](a: T): lent T = a let a = [11,12] let b = @[21,23] let ss = {1, 2, 3, 5} doAssert byLent(a) == [11,12] - doAssert byLent(a).addr == a.addr + doAssert sameAddress(byLent(a), a) doAssert byLent(b) == @[21,23] - when not defined(js): # pending bug #16073 - doAssert byLent(b).addr == b.addr + # pending bug #16073 + doAssert sameAddress(byLent(b), b) doAssert byLent(ss) == {1, 2, 3, 5} - doAssert byLent(ss).addr == ss.addr + doAssert sameAddress(byLent(ss), ss) let r = new(float) r[] = 10.0 - when not defined(js): # pending bug #16073 - doAssert byLent(r)[] == 10.0 + # bug #16073 + doAssert byLent(r)[] == 10.0 when not defined(js): # pending bug https://github.com/timotheecour/Nim/issues/372 let p = create(float) @@ -41,9 +45,9 @@ template main2 = # bug #15958 proc byLent2[T](a: openArray[T]): lent T = a[0] doAssert byLent2(a) == 11 - doAssert byLent2(a).addr == a[0].addr + doAssert sameAddress(byLent2(a), a[0]) doAssert byLent2(b) == 21 - doAssert byLent2(b).addr == b[0].addr + doAssert sameAddress(byLent2(b), b[0]) proc byLent3[T](a: varargs[T]): lent T = a[1] let From bc3bee54698b108977766b6b2daf8a33bd00d1ef Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 17 Jan 2022 20:06:31 +0800 Subject: [PATCH 1474/2472] Added `std/oserrors` for OS error reporting (#19390) * Added 'std/oserrors' for OS error reporting * add a simple test * New code should not support -d:useWinAnsi anymore thanks to @Araq --- changelog.md | 2 + lib/std/oserrors.nim | 118 +++++++++++++++++++++++++++++++++++++ tests/stdlib/toserrors.nim | 9 +++ 3 files changed, 129 insertions(+) create mode 100644 lib/std/oserrors.nim create mode 100644 tests/stdlib/toserrors.nim diff --git a/changelog.md b/changelog.md index 16b526f85..82f9daa1e 100644 --- a/changelog.md +++ b/changelog.md @@ -74,6 +74,8 @@ becomes an alias for `addr`. for the right-hand side of type definitions in type sections. Previously they would error with "invalid indentation". +- Added `std/oserrors` for OS error reporting. + ## Compiler changes - `nim` can now compile version 1.4.0 as follows: `nim c --lib:lib --stylecheck:off compiler/nim`, diff --git a/lib/std/oserrors.nim b/lib/std/oserrors.nim new file mode 100644 index 000000000..9c2649eab --- /dev/null +++ b/lib/std/oserrors.nim @@ -0,0 +1,118 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + + +## The `std/oserrors` module implements OS error reporting. + +type + OSErrorCode* = distinct int32 ## Specifies an OS Error Code. + +when not defined(nimscript): + when defined(windows): + import winlean + else: + var errno {.importc, header: "".}: cint + + proc c_strerror(errnum: cint): cstring {. + importc: "strerror", header: "".} + +proc `==`*(err1, err2: OSErrorCode): bool {.borrow.} +proc `$`*(err: OSErrorCode): string {.borrow.} + +proc osErrorMsg*(errorCode: OSErrorCode): string = + ## Converts an OS error code into a human readable string. + ## + ## The error code can be retrieved using the `osLastError proc`_. + ## + ## If conversion fails, or `errorCode` is `0` then `""` will be + ## returned. + ## + ## See also: + ## * `raiseOSError proc`_ + ## * `osLastError proc`_ + runnableExamples: + when defined(linux): + assert osErrorMsg(OSErrorCode(0)) == "" + assert osErrorMsg(OSErrorCode(1)) == "Operation not permitted" + assert osErrorMsg(OSErrorCode(2)) == "No such file or directory" + + result = "" + when defined(nimscript): + discard + elif defined(windows): + if errorCode != OSErrorCode(0'i32): + var msgbuf: WideCString + if formatMessageW(0x00000100 or 0x00001000 or 0x00000200, + nil, errorCode.int32, 0, addr(msgbuf), 0, nil) != 0'i32: + result = $msgbuf + if msgbuf != nil: localFree(cast[pointer](msgbuf)) + else: + if errorCode != OSErrorCode(0'i32): + result = $c_strerror(errorCode.int32) + +proc newOSError*( + errorCode: OSErrorCode, additionalInfo = "" +): owned(ref OSError) {.noinline.} = + ## Creates a new `OSError exception `_. + ## + ## The `errorCode` will determine the + ## message, `osErrorMsg proc`_ will be used + ## to get this message. + ## + ## The error code can be retrieved using the `osLastError proc`_. + ## + ## If the error code is `0` or an error message could not be retrieved, + ## the message `unknown OS error` will be used. + ## + ## See also: + ## * `osErrorMsg proc`_ + ## * `osLastError proc`_ + var e: owned(ref OSError); new(e) + e.errorCode = errorCode.int32 + e.msg = osErrorMsg(errorCode) + if additionalInfo.len > 0: + if e.msg.len > 0 and e.msg[^1] != '\n': e.msg.add '\n' + e.msg.add "Additional info: " + e.msg.add additionalInfo + # don't add trailing `.` etc, which negatively impacts "jump to file" in IDEs. + if e.msg == "": + e.msg = "unknown OS error" + return e + +proc raiseOSError*(errorCode: OSErrorCode, additionalInfo = "") {.noinline.} = + ## Raises an `OSError exception `_. + ## + ## Read the description of the `newOSError proc`_ to learn + ## how the exception object is created. + raise newOSError(errorCode, additionalInfo) + +{.push stackTrace:off.} +proc osLastError*(): OSErrorCode {.sideEffect.} = + ## Retrieves the last operating system error code. + ## + ## This procedure is useful in the event when an OS call fails. In that case + ## this procedure will return the error code describing the reason why the + ## OS call failed. The `OSErrorMsg` procedure can then be used to convert + ## this code into a string. + ## + ## .. warning:: The behaviour of this procedure varies between Windows and POSIX systems. + ## On Windows some OS calls can reset the error code to `0` causing this + ## procedure to return `0`. It is therefore advised to call this procedure + ## immediately after an OS call fails. On POSIX systems this is not a problem. + ## + ## See also: + ## * `osErrorMsg proc`_ + ## * `raiseOSError proc`_ + when defined(nimscript): + discard + elif defined(windows): + result = cast[OSErrorCode](getLastError()) + else: + result = OSErrorCode(errno) +{.pop.} diff --git a/tests/stdlib/toserrors.nim b/tests/stdlib/toserrors.nim new file mode 100644 index 000000000..e907dfe63 --- /dev/null +++ b/tests/stdlib/toserrors.nim @@ -0,0 +1,9 @@ +discard """ + action: compile +""" + +import std/oserrors + +let x1 = osLastError() +raiseOSError(x1) +echo osErrorMsg(x1) From 5d303762f156a276cb37162229e6b82b01e08253 Mon Sep 17 00:00:00 2001 From: Hamid Bluri Date: Tue, 18 Jan 2022 12:55:39 -0800 Subject: [PATCH 1475/2472] update deprecated example (#19415) `toNimIdent` proc is deprecated, so I replaced it with `ident` proc --- lib/core/macros.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index ed7e85c64..8ab893d21 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -1373,7 +1373,7 @@ template findChild*(n: NimNode; cond: untyped): NimNode {.dirty.} = ## ## .. code-block:: nim ## var res = findChild(n, it.kind == nnkPostfix and - ## it.basename.ident == toNimIdent"foo") + ## it.basename.ident == ident"foo") block: var res: NimNode for it in n.children: From d7869a80092354468594f7d5c5dfdd91f564fec7 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Tue, 18 Jan 2022 23:58:18 +0300 Subject: [PATCH 1476/2472] Improve Zshell completion (#19354) --- tools/nim.zsh-completion | 258 +++++++++++++++++---------------------- 1 file changed, 114 insertions(+), 144 deletions(-) diff --git a/tools/nim.zsh-completion b/tools/nim.zsh-completion index 45477d860..1c3670fd9 100644 --- a/tools/nim.zsh-completion +++ b/tools/nim.zsh-completion @@ -1,150 +1,120 @@ #compdef nim +# Installation note: +# Please name this file as _nim (with underscore!) and copy it to a +# completions directory, either: +# - system wide one, like /usr/share/zsh/functions/Completion/Unix/ on Linux +# - or to a user directory like ~/scripts. Then you also need to add +# that directory in your ~/.zshrc to `fpath` array like so: +# fpath=( ~/scripts "${fpath[@]}" ) + _nim() { - _arguments -C \ - ':command:(( - {compile,c}\:compile\ project\ with\ default\ code\ generator\ C - doc\:generate\ the\ documentation\ for\ inputfile - {compileToC,cc}\:compile\ project\ with\ C\ code\ generator - {compileToCpp,cpp}\:compile\ project\ to\ C++\ code - {compileToOC,objc}\:compile\ project\ to\ Objective\ C\ code - js\:compile\ project\ to\ Javascript - e\:run\ a\ Nimscript\ file - rst2html\:convert\ a\ reStructuredText\ file\ to\ HTML - rst2tex\:convert\ a\ reStructuredText\ file\ to\ TeX - jsondoc\:extract\ the\ documentation\ to\ a\ json\ file - buildIndex\:build\ an\ index\ for\ the\ whole\ documentation - genDepend\:generate\ a\ DOT\ file\ containing\ the\ module\ dependency\ graph - dump\:dump\ all\ defined\ conditionals\ and\ search\ paths - check\:checks\ the\ project\ for\ syntax\ and\ semantic - ))' \ - '*-r[run the application]' \ - '*--run[run the application]' \ - '*-p=[add path to search paths]' \ - '*--path=[add path to search paths]' \ - '*-d=[define a conditional symbol]' \ - '*--define=[define a conditional symbol]' \ - '*-u=[undefine a conditional symbol]' \ - '*--undef=[undefine a conditional symbol]' \ - '*-f[force rebuilding of all modules]' \ - '*--forceBuild[force rebuilding of all modules]' \ - '*--stackTrace=on[turn stack tracing on]' \ - '*--stackTrace=off[turn stack tracing off]' \ - '*--lineTrace=on[turn line tracing on]' \ - '*--lineTrace=off[turn line tracing off]' \ - '*--threads=on[turn support for multi-threading on]' \ - '*--threads=off[turn support for multi-threading off]' \ - '*-x=on[turn all runtime checks on]' \ - '*-x=off[turn all runtime checks off]' \ - '*--checks=on[turn all runtime checks on]' \ - '*--checks=off[turn all runtime checks off]' \ - '*--objChecks=on[turn obj conversion checks on]' \ - '*--objChecks=off[turn obj conversion checks off]' \ - '*--fieldChecks=on[turn case variant field checks on]' \ - '*--fieldChecks=off[turn case variant field checks off]' \ - '*--rangeChecks=on[turn range checks on]' \ - '*--rangeChecks=off[turn range checks off]' \ - '*--boundChecks=on[turn bound checks on]' \ - '*--boundChecks=off[turn bound checks off]' \ - '*--overflowChecks=on[turn int over-/underflow checks on]' \ - '*--overflowChecks=off[turn int over-/underflow checks off]' \ - '*-a[turn assertions on]' \ - '*-a[turn assertions off]' \ - '*--assertions=on[turn assertions on]' \ - '*--assertions=off[turn assertions off]' \ - '*--floatChecks=on[turn all floating point (NaN/Inf) checks on]' \ - '*--floatChecks=off[turn all floating point (NaN/Inf) checks off]' \ - '*--nanChecks=on[turn NaN checks on]' \ - '*--nanChecks=off[turn NaN checks off]' \ - '*--infChecks=on[turn Inf checks on]' \ - '*--infChecks=off[turn Inf checks off]' \ - '*--nilChecks=on[turn nil checks on]' \ - '*--nilChecks=off[turn nil checks off]' \ - '*--opt=none[do not optimize]' \ - '*--opt=speed[optimize for speed|size - use -d:release for a release build]' \ - '*--opt=size[optimize for size]' \ - '*--debugger:native[use native debugger (gdb)]' \ - '*--app=console[generate a console app]' \ - '*--app=gui[generate a GUI app]' \ - '*--app=lib[generate a dynamic library]' \ - '*--app=staticlib[generate a static library]' \ - '*--cpu=alpha[compile for Alpha architecture]' \ - '*--cpu=amd64[compile for x86_64 architecture]' \ - '*--cpu=arm[compile for ARM architecture]' \ - '*--cpu=arm64[compile for ARM64 architecture]' \ - '*--cpu=avr[compile for AVR architecture]' \ - '*--cpu=esp[compile for ESP architecture]' \ - '*--cpu=e2k[compile for Elbrus 2000 architecture]' \ - '*--cpu=hppa[compile for HPPA architecture]' \ - '*--cpu=i386[compile for i386 architecture]' \ - '*--cpu=ia64[compile for ia64 architecture]' \ - '*--cpu=js[compile to JavaScript]' \ - '*--cpu=m68k[compile for m68k architecture]' \ - '*--cpu=mips[compile for MIPS architecture]' \ - '*--cpu=mipsel[compile for MIPS EL architecture]' \ - '*--cpu=mips64[compile for MIPS64 architecture]' \ - '*--cpu=mips64el[compile for MIPS64 EL architecture]' \ - '*--cpu=msp430[compile for msp430 architecture]' \ - '*--cpu=nimvm[compile for Nim VM]' \ - '*--cpu=powerpc[compile for PowerPC architecture]' \ - '*--cpu=powerpc64[compile for PowerPC64 architecture]' \ - '*--cpu=powerpc64el[compile for PowerPC64 EL architecture]' \ - '*--cpu=riscv32[compile for RISC-V 32 architecture]' \ - '*--cpu=riscv64[compile for RISC-V 64 architecture]' \ - '*--cpu=sparc[compile for SPARC architecture]' \ - '*--cpu=sparc64[compile for SPARC64 architecture]' \ - '*--cpu=vm[compile for Nim VM]' \ - '*--cpu=wasm32[compile to WASM 32]' \ - '*--cpu=loongarch64[compile for LoongArch64 architecture]' \ - '*--gc=refc[use reference counting garbage collection]' \ - '*--gc=arc[use ARC garbage collection]' \ - '*--gc=orc[use ORC garbage collection]' \ - '*--gc=markAndSweep[use mark-and-sweep garbage collection]' \ - '*--gc=boehm[use Boehm garbage collection]' \ - '*--gc=go[use Go garbage collection]' \ - '*--gc=regions[use region-based memory management]' \ - '*--gc=none[disable garbage collection]' \ - '*--os=Standalone[generate a stand-alone executable]' \ - '*--os=AIX[compile for AIX]' \ - '*--os=Amiga[compile for Amiga OS]' \ - '*--os=Android[compile for Android]' \ - '*--os=Any[compile for any OS]' \ - '*--os=Atari[compile for Atari]' \ - '*--os=DOS[compile for DOS]' \ - '*--os=DragonFly[compile for DragonFly]' \ - '*--os=FreeBSD[compile for FreeBSD]' \ - '*--os=FreeRTOS[compile for FreeRTOS]' \ - '*--os=Genode[compile for Genode]' \ - '*--os=Haiku[compile for Haiku]' \ - '*--os=iOS[compile for iOS]' \ - '*--os=Irix[compile for Irix]' \ - '*--os=Linux[compile for Linux]' \ - '*--os=MacOS[compile for MacOS]' \ - '*--os=MacOSX[compile for MacOSX]' \ - '*--os=MorphOS[compile for MorphOS]' \ - '*--os=NetBSD[compile for NetBSD]' \ - '*--os=Netware[compile for Netware]' \ - '*--os=NimVM[compile for NimVM]' \ - '*--os=NintendoSwitch[compile for NintendoSwitch]' \ - '*--os=OS2[compile for OS2]' \ - '*--os=OpenBSD[compile for OpenBSD]' \ - '*--os=PalmOS[compile for PalmOS]' \ - '*--os=QNX[compile for QNX]' \ - '*--os=SkyOS[compile for SkyOS]' \ - '*--os=Solaris[compile for Solaris]' \ - '*--os=VxWorks[compile for VxWorks]' \ - '*--os=Windows[compile for Windows]' \ - '*--os=JS[generate javascript]' \ - '*--panics=off[turn panics into process terminations: off by default]' \ - '*--panics=on[turn panics into process terminations]' \ - '*--verbosity=0[set verbosity to 0]' \ - '*--verbosity=1[set verbosity to 1 (default)]' \ - '*--verbosity=2[set verbosity to 2]' \ - '*--verbosity=3[set verbosity to 3]' \ - '*--hints=on[print compilation hints]' \ - '*--hints=off[disable compilation hints]' \ - '*--hints=list[print compilation hints list]' \ - ':filename:_files -g"*.nim"' + local -a nimCommands=( + {compile,c}:'compile project with default code generator C' + {compileToC,cc}:'compile project with C code generator' + {compileToCpp,cpp}:'compile project to C++ code' + {compileToOC,objc}:'compile project to Objective C code' + 'js:compile project to Javascript' + 'e:run a Nimscript file' + 'doc:generate the HTML documentation for inputfile' + 'rst2html:convert a reStructuredText file to HTML' + 'doc2tex:generate the documentation for inputfile to LaTeX' + 'rst2tex:convert a reStructuredText file to TeX' + 'jsondoc:extract the documentation to a json file' + 'buildIndex:build an index for the whole documentation' + 'genDepend:generate a DOT file containing the module dependency graph' + 'dump:dump all defined conditionals and search paths' + 'check:checks the project for syntax and semantic' + {--help,-h}:'basic help' + '--fullhelp:show all switches' + {-v,--version}:'show version' + ) + + _arguments '*:: :->anyState' && return 0 + + if (( CURRENT == 1 )); then + _describe -t commands "Nim commands" nimCommands -V1 + return + fi + + local -a conditionalSymbols=( + release danger mingw androidNDK useNimRtl useMalloc noSignalHandler ssl + debug leanCompiler gcDestructors) + local -a sharedOpts=( + {--define\\:-,-d\\:-}'[define a conditional symbol]:x:($conditionalSymbols)' + {--undef\\:-,-u\\:-}'[undefine a conditional symbol]:x:($conditionalSymbols)' + {--path\\:-,-p\\:-}'[add path to search paths]:x:_files' + '--verbosity\:-[set verbosity level (default\: 1)]:x:(0 1 2 3)' + '--hints\:-[print compilation hints? (or `list`)]:x:(on off list)' + ) + local runOpts=( + {--run,-r}'[run the application]' + ) + local docOpts=( + '--index\:-[enable index .idx files?]:x:(on off)' + '--project\:-[output any dependency for doc?]:x:(on off)' + '--docInternal\:-[generate module-private documentation?]:x:(on off)' + ) + local -a codeOpts=( + {--forceBuild,-f}'[force rebuilding of all modules]' + '--stackTrace\:-[enable stack tracing?]:x:(on off)' + '--lineTrace\:-[enable line tracing?]:x:(on off)' + '--threads\:-[enable support for multi-threading?]:x:(on off)' + {--checks\\:-,-x\\:-}'[enable/disable all runtime checks?]:x:(on off)' + '--objChecks\:-[enable obj conversion checks]:x:(on off)' + '--fieldChecks\:-[enable case variant field checks?]:x:(on off)' + '--rangeChecks\:-[enable range checks?]:x:(on off)' + '--boundChecks\:-[enable bound checks?]:x:(on off)' + '--overflowChecks\:-[enable integer over-/underflow checks?]:x:(on off)' + {--assertions\\:-,-a\\:-}'[enable assertions?]:x:(on off)' + '--floatChecks\:-[enable floating point (NaN/Inf) checks?]:x:(on off)' + '--nanChecks\:-[enable NaN checks?]:x:(on off)' + '--infChecks\:-[enable Inf checks?]:x:(on off)' + '--nilChecks\:-[enable nil checks?]:x:(on off)' + '--expandArc\:-[show how given proc looks before final backend pass]' + '--expandMacro\:-[dump every generated AST from given macro]' + ) + local -a nativeOpts=( + '--opt\:-[optimization mode]:x:(none speed size)' + '--debugger\:native[use native debugger (gdb)]' + '--app\:-[generate this type of app (lib=dynamic)]:x:(console gui lib staticlib)' + '--cpu\:-[target architecture]:x:(alpha amd64 arm arm64 avr e2k esp hppa i386 ia64 js loongarch64 m68k mips mipsel mips64 mips64el msp430 nimvm powerpc powerpc64 powerpc64el riscv32 riscv64 sparc sparc64 vm wasm32)' + '--gc\:-[memory management algorithm to use (default\: refc)]:x:(refc arc orc markAndSweep boehm go regions none)' + '--os\:-[operating system to compile for]:x:(AIX Amiga Android Any Atari DOS DragonFly FreeBSD FreeRTOS Genode Haiku iOS Irix JS Linux MacOS MacOSX MorphOS NetBSD Netware NimVM NintendoSwitch OS2 OpenBSD PalmOS Standalone QNX SkyOS Solaris VxWorks Windows)' + '--panics\:-[turn panics into process termination (default\: off)]:x:(off on)' + ) + + case "$words[1]" in + compile|c|compileToC|cpp|compileToCpp|compileToOC|objc) + _arguments $codeOpts $runOpts $sharedOpts $nativeOpts \ + '*:filename:_files -g"*.nim"' + ;; + js) + _arguments $codeOpts $runOpts $sharedOpts \ + '*:filename:_files -g"*.nim"' + ;; + e) + _arguments $codeOpts $runOpts $sharedOpts \ + '*:filename:_files -g"*.nims"' + ;; + doc|doc2tex|jsondoc) + _arguments $runOpts $sharedOpts '*:filename:_files -g"*.nim"' + ;; + rst2html|rst2tex) + _arguments $runOpts $sharedOpts $docOpts '*:filename:_files -g"*.rst"' + ;; + buildIndex|genDepend|check) + _arguments $sharedOpts '*:filename:_files -g"*.nim"' + ;; + dump) + _arguments $sharedOpts + ;; + *) + _arguments '*:filename:_files -g"*"' + ;; + esac + + return 1 } _nim "$@" From aac54b9c7ffb75c9aab446574171b33bf23a29ed Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 19 Jan 2022 05:02:35 +0800 Subject: [PATCH 1477/2472] fix stricteffects (nimsuggest/sexp) (#19405) * fix stricteffects (nimsuggest/sexp) * Update tstrict_effects3.nim * Update tests/effects/tstrict_effects3.nim --- nimsuggest/sexp.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimsuggest/sexp.nim b/nimsuggest/sexp.nim index cee538b6e..31f4988e1 100644 --- a/nimsuggest/sexp.nim +++ b/nimsuggest/sexp.nim @@ -409,7 +409,7 @@ macro convertSexp*(x: untyped): untyped = ## `%` for every element. result = toSexp(x) -proc `==`* (a,b: SexpNode): bool = +proc `==`* (a, b: SexpNode): bool {.noSideEffect.} = ## Check two nodes for equality if a.isNil: if b.isNil: return true From b3c178c2024f679a83275f63ca9e21a8b5ba73d7 Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 19 Jan 2022 15:39:22 +0800 Subject: [PATCH 1478/2472] suppress deprecated warnings (#19408) * suppress deprecated warnings once bump version to 1.7.3 enable deprecated messages * deprecate later --- lib/system.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/system.nim b/lib/system.nim index 3b3e28344..02da048e4 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -213,8 +213,7 @@ proc `addr`*[T](x: T): ptr T {.magic: "Addr", noSideEffect.} = ## echo p[] # b discard -proc unsafeAddr*[T](x: T): ptr T {.magic: "Addr", noSideEffect, - deprecated: "'unsafeAddr' is a deprecated alias for 'addr'".} = +proc unsafeAddr*[T](x: T): ptr T {.magic: "Addr", noSideEffect.} = ## Builtin `addr` operator for taking the address of a memory ## location. ## From 23c4bbedcbf12a7ab87baefe589b472837d89dbf Mon Sep 17 00:00:00 2001 From: nblaxall <43897322+nblaxall@users.noreply.github.com> Date: Thu, 20 Jan 2022 00:25:21 +1300 Subject: [PATCH 1479/2472] add an example to setControlCHook (#19416) * add an example to setControlCHook * [skip CI] format example for setControlCHook Co-authored-by: Nathan Blaxall --- lib/system.nim | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/system.nim b/lib/system.nim index 02da048e4..09de9fcde 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2319,6 +2319,15 @@ when notJSnotNims: proc setControlCHook*(hook: proc () {.noconv.}) ## Allows you to override the behaviour of your application when CTRL+C ## is pressed. Only one such hook is supported. + ## Example: + ## + ## .. code-block:: Nim + ## proc ctrlc() {.noconv.} = + ## echo "Ctrl+C fired!" + ## # do clean up stuff + ## quit() + ## + ## setControlCHook(ctrlc) when not defined(noSignalHandler) and not defined(useNimRtl): proc unsetControlCHook*() From 5d34e81f2363e1de97916ac60b028b9a8b287a4b Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 19 Jan 2022 19:37:30 +0800 Subject: [PATCH 1480/2472] fix term rewriting with sideeffect (#19410) * fix term rewriting with sideeffect fix #6217 * add tests * Update tests/template/template_various.nim --- compiler/parampatterns.nim | 7 +++++-- tests/template/t6217.nim | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 tests/template/t6217.nim diff --git a/compiler/parampatterns.nim b/compiler/parampatterns.nim index eb99004ab..bfbc72675 100644 --- a/compiler/parampatterns.nim +++ b/compiler/parampatterns.nim @@ -143,8 +143,11 @@ proc checkForSideEffects*(n: PNode): TSideEffectAnalysis = let s = op.sym if sfSideEffect in s.flags: return seSideEffect - # assume no side effect: - result = seNoSideEffect + elif tfNoSideEffect in op.typ.flags: + result = seNoSideEffect + else: + # assume side effect: + result = seSideEffect elif tfNoSideEffect in op.typ.flags: # indirect call without side effects: result = seNoSideEffect diff --git a/tests/template/t6217.nim b/tests/template/t6217.nim new file mode 100644 index 000000000..b27b61881 --- /dev/null +++ b/tests/template/t6217.nim @@ -0,0 +1,19 @@ +discard """ + output: ''' +start +side effect! +end +''' +""" + +# bug #6217 + +template optMul{`*`(a, 2)}(a: int{noSideEffect}): int = a+a + +proc f(): int = + echo "side effect!" + result = 55 + +echo "start" +doAssert f() * 2 == 110 +echo "end" From 08261cb9e33445553144219023900b4ced0b0f55 Mon Sep 17 00:00:00 2001 From: metagn <10591326+metagn@users.noreply.github.com> Date: Wed, 19 Jan 2022 14:38:14 +0300 Subject: [PATCH 1481/2472] Don't reject types directly on AST (#19407) Instead of rejecting type expressions based on node kind, evaluate the expression as a type. This is already the behavior for call results, and it has its own error for non-types, which is the same error you would normally get with 2 words swapped. --- compiler/semtypes.nim | 6 ++++-- tests/types/tnontype.nim | 9 +++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 tests/types/tnontype.nim diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index d03fa88a8..b4f385fe6 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -1993,8 +1993,10 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = of nkStmtListType: result = semStmtListType(c, n, prev) of nkBlockType: result = semBlockType(c, n, prev) else: - localError(c.config, n.info, "type expected, but got: " & renderTree(n)) - result = newOrPrevType(tyError, prev, c) + result = semTypeExpr(c, n, prev) + when false: + localError(c.config, n.info, "type expected, but got: " & renderTree(n)) + result = newOrPrevType(tyError, prev, c) n.typ = result dec c.inTypeContext diff --git a/tests/types/tnontype.nim b/tests/types/tnontype.nim new file mode 100644 index 000000000..4e2bafb32 --- /dev/null +++ b/tests/types/tnontype.nim @@ -0,0 +1,9 @@ +discard """ + errormsg: "expected type, but got: 3" +""" + +type + Foo = (block: + int) + + Bar = 3 From ce44cf03cc4a78741c423b2b3963b48b6d9e6755 Mon Sep 17 00:00:00 2001 From: Tom Date: Wed, 19 Jan 2022 09:41:11 -0700 Subject: [PATCH 1482/2472] Add noQuit option (#19419) [backport:1.6] * Add noQuit option * Add nim prefix in case of conflicts Co-authored-by: flywind Co-authored-by: flywind --- lib/system.nim | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/system.nim b/lib/system.nim index 09de9fcde..63cdc3ca8 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1198,8 +1198,8 @@ proc align(address, alignment: int): int = else: result = (address + (alignment - 1)) and not (alignment - 1) -when defined(nimdoc): - proc quit*(errorcode: int = QuitSuccess) {.magic: "Exit", noreturn.} +when defined(nimNoQuit): + proc quit*(errorcode: int = QuitSuccess) = discard "ignoring quit" ## Stops the program immediately with an exit code. ## ## Before stopping the program the "exit procedures" are called in the @@ -1223,6 +1223,9 @@ when defined(nimdoc): ## It does *not* call the garbage collector to free all the memory, ## unless an `addExitProc` proc calls `GC_fullCollect <#GC_fullCollect>`_. +elif defined(nimdoc): + proc quit*(errorcode: int = QuitSuccess) {.magic: "Exit", noreturn.} + elif defined(genode): include genode/env From 4a38092ac1f8368cb1ebb0245c99c701963662f8 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 20 Jan 2022 20:55:19 +0800 Subject: [PATCH 1483/2472] Added `std/envvars` for env vars handling (#19421) --- changelog.md | 4 +- lib/std/envvars.nim | 212 ++++++++++++++++++++++++++++++++++++++ tests/stdlib/tenvvars.nim | 59 +++++++++++ 3 files changed, 273 insertions(+), 2 deletions(-) create mode 100644 lib/std/envvars.nim create mode 100644 tests/stdlib/tenvvars.nim diff --git a/changelog.md b/changelog.md index 82f9daa1e..bbbaf6e97 100644 --- a/changelog.md +++ b/changelog.md @@ -36,6 +36,8 @@ becomes an alias for `addr`. - Added `getIsoWeekAndYear` in `times` to get an ISO week number along with the corresponding ISO week-based year from a datetime. - Added `getIsoWeeksInYear` in `times` to return the number of weeks in an ISO week-based year. +- Added `std/oserrors` for OS error reporting. Added `std/envvars` for environment variables handling. + ## Language changes - Pragma macros on type definitions can now return `nnkTypeSection` nodes as well as `nnkTypeDef`, @@ -74,8 +76,6 @@ becomes an alias for `addr`. for the right-hand side of type definitions in type sections. Previously they would error with "invalid indentation". -- Added `std/oserrors` for OS error reporting. - ## Compiler changes - `nim` can now compile version 1.4.0 as follows: `nim c --lib:lib --stylecheck:off compiler/nim`, diff --git a/lib/std/envvars.nim b/lib/std/envvars.nim new file mode 100644 index 000000000..5b135cbd3 --- /dev/null +++ b/lib/std/envvars.nim @@ -0,0 +1,212 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + + +## The `std/envvars` module implements environment variables handling. +import std/oserrors + +type + ReadEnvEffect* = object of ReadIOEffect ## Effect that denotes a read + ## from an environment variable. + WriteEnvEffect* = object of WriteIOEffect ## Effect that denotes a write + ## to an environment variable. + + +when not defined(nimscript): + when defined(nodejs): + proc getEnv*(key: string, default = ""): string {.tags: [ReadEnvEffect].} = + var ret = default.cstring + let key2 = key.cstring + {.emit: "const value = process.env[`key2`];".} + {.emit: "if (value !== undefined) { `ret` = value };".} + result = $ret + + proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = + var key2 = key.cstring + var ret: bool + {.emit: "`ret` = `key2` in process.env;".} + result = ret + + proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = + var key2 = key.cstring + var val2 = val.cstring + {.emit: "process.env[`key2`] = `val2`;".} + + proc delEnv*(key: string) {.tags: [WriteEnvEffect].} = + var key2 = key.cstring + {.emit: "delete process.env[`key2`];".} + + iterator envPairsImpl(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = + var num: int + var keys: RootObj + {.emit: "`keys` = Object.keys(process.env); `num` = `keys`.length;".} + for i in 0..".} + when defined(windows): + proc c_putenv(envstring: cstring): cint {.importc: "_putenv", header: "".} + from std/private/win_setenv import setEnvImpl + import winlean + else: + proc c_setenv(envname: cstring, envval: cstring, overwrite: cint): cint {.importc: "setenv", header: "".} + proc c_unsetenv(env: cstring): cint {.importc: "unsetenv", header: "".} + + proc getEnv*(key: string, default = ""): string {.tags: [ReadEnvEffect].} = + ## Returns the value of the `environment variable`:idx: named `key`. + ## + ## If the variable does not exist, `""` is returned. To distinguish + ## whether a variable exists or it's value is just `""`, call + ## `existsEnv(key) proc`_. + ## + ## See also: + ## * `existsEnv proc`_ + ## * `putEnv proc`_ + ## * `delEnv proc`_ + ## * `envPairs iterator`_ + runnableExamples: + assert getEnv("unknownEnv") == "" + assert getEnv("unknownEnv", "doesn't exist") == "doesn't exist" + + let env = c_getenv(key) + if env == nil: return default + result = $env + + proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = + ## Checks whether the environment variable named `key` exists. + ## Returns true if it exists, false otherwise. + ## + ## See also: + ## * `getEnv proc`_ + ## * `putEnv proc`_ + ## * `delEnv proc`_ + ## * `envPairs iterator`_ + runnableExamples: + assert not existsEnv("unknownEnv") + + return c_getenv(key) != nil + + proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = + ## Sets the value of the `environment variable`:idx: named `key` to `val`. + ## If an error occurs, `OSError` is raised. + ## + ## See also: + ## * `getEnv proc`_ + ## * `existsEnv proc`_ + ## * `delEnv proc`_ + ## * `envPairs iterator`_ + when defined(windows): + if key.len == 0 or '=' in key: + raise newException(OSError, "invalid key, got: " & $(key, val)) + if setEnvImpl(key, val, 1'i32) != 0'i32: + raiseOSError(osLastError(), $(key, val)) + else: + if c_setenv(key, val, 1'i32) != 0'i32: + raiseOSError(osLastError(), $(key, val)) + + proc delEnv*(key: string) {.tags: [WriteEnvEffect].} = + ## Deletes the `environment variable`:idx: named `key`. + ## If an error occurs, `OSError` is raised. + ## + ## See also:ven + ## * `getEnv proc`_ + ## * `existsEnv proc`_ + ## * `putEnv proc`_ + ## * `envPairs iterator`_ + template bail = raiseOSError(osLastError(), key) + when defined(windows): + #[ + # https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/putenv-s-wputenv-s?view=msvc-160 + > You can remove a variable from the environment by specifying an empty string (that is, "") for value_string + note that nil is not legal + ]# + if key.len == 0 or '=' in key: + raise newException(OSError, "invalid key, got: " & key) + let envToDel = key & "=" + if c_putenv(cstring envToDel) != 0'i32: bail + else: + if c_unsetenv(key) != 0'i32: bail + + when defined(windows): + when defined(cpp): + proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.importcpp: "(NI16*)wcschr((const wchar_t *)#, #)", + header: "".} + else: + proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.importc: "wcschr", + header: "".} + elif defined(macosx) and not defined(ios) and not defined(emscripten): + # From the manual: + # Shared libraries and bundles don't have direct access to environ, + # which is only available to the loader ld(1) when a complete program + # is being linked. + # The environment routines can still be used, but if direct access to + # environ is needed, the _NSGetEnviron() routine, defined in + # , can be used to retrieve the address of environ + # at runtime. + proc NSGetEnviron(): ptr cstringArray {.importc: "_NSGetEnviron", + header: "".} + elif defined(haiku): + var gEnv {.importc: "environ", header: "".}: cstringArray + else: + var gEnv {.importc: "environ".}: cstringArray + + iterator envPairsImpl(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = + when defined(windows): + block: + template impl(get_fun, typ, size, zero, free_fun) = + let env = get_fun() + var e = env + if e == nil: break + while true: + let eend = strEnd(e) + let kv = $e + let p = find(kv, '=') + yield (substr(kv, 0, p-1), substr(kv, p+1)) + e = cast[typ](cast[ByteAddress](eend)+size) + if typeof(zero)(eend[1]) == zero: break + discard free_fun(env) + impl(getEnvironmentStringsW, WideCString, 2, 0, freeEnvironmentStringsW) + else: + var i = 0 + when defined(macosx) and not defined(ios) and not defined(emscripten): + var gEnv = NSGetEnviron()[] + while gEnv[i] != nil: + let kv = $gEnv[i] + inc(i) + let p = find(kv, '=') + yield (substr(kv, 0, p-1), substr(kv, p+1)) + +proc envPairsImplSeq(): seq[tuple[key, value: string]] = discard # vmops + +iterator envPairs*(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = + ## Iterate over all `environments variables`:idx:. + ## + ## In the first component of the tuple is the name of the current variable stored, + ## in the second its value. + ## + ## Works in native backends, nodejs and vm, like the following APIs: + ## * `getEnv proc`_ + ## * `existsEnv proc`_ + ## * `putEnv proc`_ + ## * `delEnv proc`_ + when nimvm: + for ai in envPairsImplSeq(): yield ai + else: + when defined(nimscript): discard + else: + for ai in envPairsImpl(): yield ai diff --git a/tests/stdlib/tenvvars.nim b/tests/stdlib/tenvvars.nim new file mode 100644 index 000000000..406aa3230 --- /dev/null +++ b/tests/stdlib/tenvvars.nim @@ -0,0 +1,59 @@ +discard """ + matrix: "--threads:on" + joinable: false + targets: "c js cpp" +""" + +import std/envvars +from std/sequtils import toSeq +import stdtest/testutils + +template main = + block: # delEnv, existsEnv, getEnv, envPairs + for val in ["val", ""]: # ensures empty val works too + const key = "NIM_TESTS_TOSENV_KEY" + doAssert not existsEnv(key) + + putEnv(key, "tempval") + doAssert existsEnv(key) + doAssert getEnv(key) == "tempval" + + putEnv(key, val) # change a key that already exists + doAssert existsEnv(key) + doAssert getEnv(key) == val + + doAssert (key, val) in toSeq(envPairs()) + delEnv(key) + doAssert (key, val) notin toSeq(envPairs()) + doAssert not existsEnv(key) + delEnv(key) # deleting an already deleted env var + doAssert not existsEnv(key) + + block: + doAssert getEnv("NIM_TESTS_TOSENV_NONEXISTENT", "") == "" + doAssert getEnv("NIM_TESTS_TOSENV_NONEXISTENT", " ") == " " + doAssert getEnv("NIM_TESTS_TOSENV_NONEXISTENT", "defval") == "defval" + + whenVMorJs: discard # xxx improve + do: + doAssertRaises(OSError, putEnv("NIM_TESTS_TOSENV_PUT=DUMMY_VALUE", "NEW_DUMMY_VALUE")) + doAssertRaises(OSError, putEnv("", "NEW_DUMMY_VALUE")) + doAssert not existsEnv("") + doAssert not existsEnv("NIM_TESTS_TOSENV_PUT=DUMMY_VALUE") + doAssert not existsEnv("NIM_TESTS_TOSENV_PUT") + +main() + +when not defined(js) and not defined(nimscript): + block: # bug #18533 + proc c_getenv(env: cstring): cstring {.importc: "getenv", header: "".} + var thr: Thread[void] + proc threadFunc {.thread.} = putEnv("foo", "fooVal2") + + putEnv("foo", "fooVal1") + doAssert getEnv("foo") == "fooVal1" + createThread(thr, threadFunc) + joinThreads(thr) + doAssert getEnv("foo") == $c_getenv("foo") + + doAssertRaises(OSError): delEnv("foo=bar") From 851e515bbae6edf6b28cf6723b2c7a45acce1869 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 20 Jan 2022 04:58:59 -0800 Subject: [PATCH 1484/2472] Resolve cross file resolution errors in atomics (#19422) [backport:1.6] * Resolve call undeclared routine testAndSet * Fix undeclared field atomicType --- lib/pure/concurrency/atomics.nim | 17 +++++++---------- tests/stdlib/concurrency/atomicSample.nim | 9 +++++++++ tests/stdlib/concurrency/tatomic_import.nim | 11 +++++++++++ 3 files changed, 27 insertions(+), 10 deletions(-) create mode 100644 tests/stdlib/concurrency/atomicSample.nim create mode 100644 tests/stdlib/concurrency/tatomic_import.nim diff --git a/lib/pure/concurrency/atomics.nim b/lib/pure/concurrency/atomics.nim index b252789fb..c0a122bf9 100644 --- a/lib/pure/concurrency/atomics.nim +++ b/lib/pure/concurrency/atomics.nim @@ -293,19 +293,16 @@ else: AtomicInt32 {.importc: "_Atomic NI32".} = int32 AtomicInt64 {.importc: "_Atomic NI64".} = int64 - template atomicType*(T: typedesc[Trivial]): untyped = - # Maps the size of a trivial type to it's internal atomic type - when sizeof(T) == 1: AtomicInt8 - elif sizeof(T) == 2: AtomicInt16 - elif sizeof(T) == 4: AtomicInt32 - elif sizeof(T) == 8: AtomicInt64 - type AtomicFlag* {.importc: "atomic_flag", size: 1.} = object Atomic*[T] = object when T is Trivial: - value: T.atomicType + # Maps the size of a trivial type to it's internal atomic type + when sizeof(T) == 1: value: AtomicInt8 + elif sizeof(T) == 2: value: AtomicInt16 + elif sizeof(T) == 4: value: AtomicInt32 + elif sizeof(T) == 8: value: AtomicInt64 else: nonAtomicValue: T guard: AtomicFlag @@ -364,11 +361,11 @@ else: cast[T](atomic_fetch_xor_explicit(addr(location.value), cast[nonAtomicType(T)](value), order)) template withLock[T: not Trivial](location: var Atomic[T]; order: MemoryOrder; body: untyped): untyped = - while location.guard.testAndSet(moAcquire): discard + while testAndSet(location.guard, moAcquire): discard try: body finally: - location.guard.clear(moRelease) + clear(location.guard, moRelease) proc load*[T: not Trivial](location: var Atomic[T]; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = withLock(location, order): diff --git a/tests/stdlib/concurrency/atomicSample.nim b/tests/stdlib/concurrency/atomicSample.nim new file mode 100644 index 000000000..d56d867df --- /dev/null +++ b/tests/stdlib/concurrency/atomicSample.nim @@ -0,0 +1,9 @@ +import atomics + +type + AtomicWithGeneric*[T] = object + value: Atomic[T] + +proc initAtomicWithGeneric*[T](value: T): AtomicWithGeneric[T] = + result.value.store(value) + diff --git a/tests/stdlib/concurrency/tatomic_import.nim b/tests/stdlib/concurrency/tatomic_import.nim new file mode 100644 index 000000000..e8faaae20 --- /dev/null +++ b/tests/stdlib/concurrency/tatomic_import.nim @@ -0,0 +1,11 @@ +import atomicSample + +block crossFileObjectContainingAGenericWithAComplexObject: + discard initAtomicWithGeneric[string]("foo") + +block crossFileObjectContainingAGenericWithAnInteger: + discard initAtomicWithGeneric[int](1) + discard initAtomicWithGeneric[int8](1) + discard initAtomicWithGeneric[int16](1) + discard initAtomicWithGeneric[int32](1) + discard initAtomicWithGeneric[int64](1) From 927fa890ec91cf13d7e3aa87267b994c2812bdc8 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 20 Jan 2022 20:59:36 +0800 Subject: [PATCH 1485/2472] enable weave (#19363) [backport:1.6] * enable weave * workaround CI --- compiler/cgen.nim | 9 +++++++-- testament/important_packages.nim | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 032c22ec0..246b8c0e9 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -48,8 +48,13 @@ proc addForwardedProc(m: BModule, prc: PSym) = m.g.forwardedProcs.add(prc) proc findPendingModule(m: BModule, s: PSym): BModule = - let ms = s.itemId.module #getModule(s) - result = m.g.modules[ms] + # TODO fixme + if m.config.symbolFiles == v2Sf: + let ms = s.itemId.module #getModule(s) + result = m.g.modules[ms] + else: + var ms = getModule(s) + result = m.g.modules[ms.position] proc initLoc(result: var TLoc, k: TLocKind, lode: PNode, s: TStorageLoc) = result.k = k diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 853a58698..5f6e19b52 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -157,7 +157,7 @@ pkg "tiny_sqlite" pkg "unicodedb", "nim c -d:release -r tests/tests.nim" pkg "unicodeplus", "nim c -d:release -r tests/tests.nim" pkg "unpack" -pkg "weave", "nimble test_gc_arc", allowFailure = true +pkg "weave", "nimble test_gc_arc" pkg "websocket", "nim c websocket.nim" pkg "winim", "nim c winim.nim" pkg "with" From 1563cb2f6e37f07c303d095dabde74955be1e523 Mon Sep 17 00:00:00 2001 From: Regis Caillaud <35006197+Clonkk@users.noreply.github.com> Date: Thu, 20 Jan 2022 20:50:36 +0100 Subject: [PATCH 1486/2472] Fix #11923 (#19427) * Apply commit https://github.com/nim-lang/Nim/commit/5da931fe811717a45f2dd272ea6281979c3e8f0b that was never merged (was part of a bigger PR). Should fix issue #11932 * add a generic object for custom pragma --- lib/core/macros.nim | 7 ++++++- tests/pragmas/tcustom_pragma.nim | 10 ++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 8ab893d21..28106493f 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -1523,7 +1523,12 @@ proc customPragmaNode(n: NimNode): NimNode = if n.kind in {nnkDotExpr, nnkCheckedFieldExpr}: let name = $(if n.kind == nnkCheckedFieldExpr: n[0][1] else: n[1]) let typInst = getTypeInst(if n.kind == nnkCheckedFieldExpr or n[0].kind == nnkHiddenDeref: n[0][0] else: n[0]) - var typDef = getImpl(if typInst.kind == nnkVarTy: typInst[0] else: typInst) + var typDef = getImpl( + if typInst.kind == nnkVarTy or + typInst.kind == nnkBracketExpr: + typInst[0] + else: typInst + ) while typDef != nil: typDef.expectKind(nnkTypeDef) let typ = typDef[2] diff --git a/tests/pragmas/tcustom_pragma.nim b/tests/pragmas/tcustom_pragma.nim index b197a7c55..1c3709b26 100644 --- a/tests/pragmas/tcustom_pragma.nim +++ b/tests/pragmas/tcustom_pragma.nim @@ -17,11 +17,21 @@ block: MyObj = object myField1, myField2 {.myAttr: "hi".}: int + MyGenericObj[T] = object + myField1, myField2 {.myAttr: "hi".}: int + + var o: MyObj static: doAssert o.myField2.hasCustomPragma(myAttr) doAssert(not o.myField1.hasCustomPragma(myAttr)) + var ogen: MyGenericObj[int] + static: + doAssert ogen.myField2.hasCustomPragma(myAttr) + doAssert(not ogen.myField1.hasCustomPragma(myAttr)) + + import custom_pragma block: # A bit more advanced case type From 2bd1aa186e09565b2103394bd281478fa1b10ef1 Mon Sep 17 00:00:00 2001 From: metagn <10591326+metagn@users.noreply.github.com> Date: Thu, 20 Jan 2022 22:57:50 +0300 Subject: [PATCH 1487/2472] New/better macro pragmas, mark some as experimental (#19406) * New/better macro pragmas, make some experimental fix #15920, close #18212, close #14781, close #6696, close https://github.com/nim-lang/RFCs/issues/220 Variable macro pragmas have been changed to only take a unary section node. They can now also be applied in sections with multiple variables, as well as `const` sections. They also accept arguments. Templates now support macro pragmas, mirroring other routine types. Type and variable macro pragmas have been made experimental. Symbols without parentheses instatiating nullary macros or templates has also been documented in the experimental manual. A check for a redefinition error based on the left hand side of variable definitions when using variable macro pragmas was disabled. This nerfs `byaddr` specifically, however this has been documented as a consequence of the experimental features `byaddr` uses. Given how simple these changes are I'm worried if I'm missing something. * accomodate compiler boot * allow weird pragmas * add test for #10994 * remove some control flow, try remove some logic --- compiler/semstmts.nim | 280 +++++++++++++++++++------------- compiler/semtempl.nim | 5 + doc/manual.rst | 28 ++-- doc/manual_experimental.rst | 72 +++++++- lib/std/decls.nim | 39 +++-- tests/pragmas/tpragmas_misc.nim | 24 ++- tests/pragmas/tvar_macro.nim | 128 +++++++++++++++ tests/stdlib/tdecls.nim | 64 ++------ 8 files changed, 435 insertions(+), 205 deletions(-) create mode 100644 tests/pragmas/tvar_macro.nim diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index f302dd4c3..33e304ab2 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -341,8 +341,12 @@ proc checkNilable(c: PContext; v: PSym) = #include liftdestructors -proc addToVarSection(c: PContext; result: PNode; orig, identDefs: PNode) = - let value = identDefs[^1] +proc addToVarSection(c: PContext; result: var PNode; n: PNode) = + if result.kind != nkStmtList: + result = makeStmtList(result) + result.add n + +proc addToVarSection(c: PContext; result: var PNode; orig, identDefs: PNode) = if result.kind == nkStmtList: let o = copyNode(orig) o.add identDefs @@ -445,55 +449,115 @@ proc setVarType(c: PContext; v: PSym, typ: PType) = "; new type is: " & typeToString(typ, preferDesc)) v.typ = typ -proc semLowerLetVarCustomPragma(c: PContext, a: PNode, n: PNode): PNode = - var b = a[0] - if b.kind == nkPragmaExpr: - if b[1].len != 1: - # we could in future support pragmas w args e.g.: `var foo {.bar:"goo".} = expr` - return nil - let nodePragma = b[1][0] - # see: `singlePragma` +proc isPossibleMacroPragma(c: PContext, it: PNode, key: PNode): bool = + # make sure it's not a normal pragma, and calls an identifier + # considerQuotedIdent below will fail on non-identifiers + result = whichPragma(it) == wInvalid and key.kind in nkIdentKinds + if result: + # make sure it's not a user pragma + let ident = considerQuotedIdent(c, key) + result = strTableGet(c.userPragmas, ident) == nil + if result: + # make sure it's not a custom pragma + var amb = false + let sym = searchInScopes(c, ident, amb) + result = sym == nil or sfCustomPragma notin sym.flags - var amb = false - var sym: PSym = nil - case nodePragma.kind - of nkIdent, nkAccQuoted: - let ident = considerQuotedIdent(c, nodePragma) - var userPragma = strTableGet(c.userPragmas, ident) - if userPragma != nil: return nil - let w = nodePragma.whichPragma - if n.kind == nkVarSection and w in varPragmas or - n.kind == nkLetSection and w in letPragmas or - n.kind == nkConstSection and w in constPragmas: - return nil - sym = searchInScopes(c, ident, amb) - # XXX what if amb is true? - # CHECKME: should that test also apply to `nkSym` case? - if sym == nil or sfCustomPragma in sym.flags: return nil - of nkSym: - sym = nodePragma.sym - else: - return nil - # skip if not in scope; skip `template myAttr() {.pragma.}` +proc copyExcept(n: PNode, i: int): PNode = + result = copyNode(n) + for j in 0..= 1: it[0] else: it + + when false: + let lhs = b[0] + let clash = strTableGet(c.currentScope.symbols, lhs.ident) + if clash != nil: + # refs https://github.com/nim-lang/Nim/issues/8275 + wrongRedefinition(c, lhs.info, lhs.ident.s, clash.info) - result = newTree(nkCall) - result.add nodePragma - result.add lhs - if a[1].kind != nkEmpty: - result.add a[1] - else: - result.add newNodeIT(nkNilLit, a.info, c.graph.sysTypes[tyNil]) - result.add a[2] - result.info = a.info - let ret = newNodeI(nkStmtList, a.info) - ret.add result - result = semExprNoType(c, ret) + if isPossibleMacroPragma(c, it, key): + # we transform ``var p {.m, rest.}`` into ``m(do: var p {.rest.})`` and + # let the semantic checker deal with it: + var x = newNodeI(nkCall, key.info) + x.add(key) + + if it.kind in nkPragmaCallKinds and it.len > 1: + # pass pragma arguments to the macro too: + for i in 1..= 1: it[0] else: it - if whichPragma(it) != wInvalid: - # Not a custom pragma - continue - else: - let ident = considerQuotedIdent(c, key) - if strTableGet(c.userPragmas, ident) != nil: - continue # User defined pragma + if isPossibleMacroPragma(c, it, key): + # we transform ``proc p {.m, rest.}`` into ``m(do: proc p {.rest.})`` and + # let the semantic checker deal with it: + var x = newNodeI(nkCall, key.info) + x.add(key) + + if it.kind in nkPragmaCallKinds and it.len > 1: + # pass pragma arguments to the macro too: + for i in 1.. 1: - # pass pragma arguments to the macro too: - for i in 1..` instead. Foreign function interface diff --git a/doc/manual_experimental.rst b/doc/manual_experimental.rst index 407924a13..3089755cb 100644 --- a/doc/manual_experimental.rst +++ b/doc/manual_experimental.rst @@ -404,6 +404,76 @@ to use this operator. doAssert (a.b)(c) == `()`(a.b, c) +Extended macro pragmas +====================== + +Macro pragmas as described in `the manual `_ +can also be applied to type, variable and constant declarations. + +For types: + +.. code-block:: nim + type + MyObject {.schema: "schema.protobuf".} = object + +This is translated to a call to the `schema` macro with a `nnkTypeDef` +AST node capturing the left-hand side, remaining pragmas and the right-hand +side of the definition. The macro can return either a type section or +another `nnkTypeDef` node, both of which will replace the original row +in the type section. + +In the future, this `nnkTypeDef` argument may be replaced with a unary +type section node containing the type definition, or some other node that may +be more convenient to work with. The ability to return nodes other than type +definitions may also be supported, however currently this is not convenient +when dealing with mutual type recursion. For now, macros can return an unused +type definition where the right-hand node is of kind `nnkStmtListType`. +Declarations in this node will be attached to the same scope as +the parent scope of the type section. + +------ + +For variables and constants, it is largely the same, except a unary node with +the same kind as the section containing a single definition is passed to macros, +and macros can return any expression. + +.. code-block:: nim + var + a = ... + b {.importc, foo, nodecl.} = ... + c = ... + +Assuming `foo` is a macro or a template, this is roughly equivalent to: + +.. code-block:: nim + var a = ... + foo: + var b {.importc, nodecl.} = ... + var c = ... + + +Symbols as template/macro calls +=============================== + +Templates and macros that take no arguments can be called as lone symbols, +i.e. without parentheses. This is useful for repeated uses of complex +expressions that cannot conveniently be represented as runtime values. + +.. code-block:: nim + type Foo = object + bar: int + + var foo = Foo(bar: 10) + template bar: untyped = foo.bar + assert bar == 10 + bar = 15 + assert bar == 15 + +In the future, this may require more specific information on template or macro +signatures to be used. Specializations for some applications of this may also +be introduced to guarantee consistency and circumvent bugs. + + Not nil annotation ================== @@ -613,7 +683,7 @@ has `source` as the owner. A path expression `e` is defined recursively: If a view type is used as a return type, the location must borrow from a location that is derived from the first parameter that is passed to the proc. -See `the manual `_ +See `the manual `_ for details about how this is done for `var T`. A mutable view can borrow from a mutable location, an immutable view can borrow diff --git a/lib/std/decls.nim b/lib/std/decls.nim index dd7d19da7..7b907f5e1 100644 --- a/lib/std/decls.nim +++ b/lib/std/decls.nim @@ -1,19 +1,32 @@ # see `semLowerLetVarCustomPragma` for compiler support that enables these # lowerings -template byaddr*(lhs, typ, ex) = - ## Allows a syntax for lvalue reference, exact analog to - ## `auto& a = ex;` in C++ +import macros + +macro byaddr*(sect) = + ## Allows a syntax for l-value references, being an exact analog to + ## `auto& a = ex;` in C++. + ## + ## Warning: This makes use of 2 experimental features, namely nullary + ## templates instantiated as symbols and variable macro pragmas. + ## For this reason, its behavior is not stable. The current implementation + ## allows redefinition, but this is not an intended consequence. runnableExamples: - var s = @[10,11,12] + var s = @[10, 11, 12] var a {.byaddr.} = s[0] - a+=100 - doAssert s == @[110,11,12] - doAssert a is int + a += 100 + assert s == @[110, 11, 12] + assert a is int var b {.byaddr.}: int = s[0] - doAssert a.addr == b.addr - when typ is typeof(nil): - let tmp = addr(ex) - else: - let tmp: ptr typ = addr(ex) - template lhs: untyped = tmp[] + assert a.addr == b.addr + expectLen sect, 1 + let def = sect[0] + let + lhs = def[0] + typ = def[1] + ex = def[2] + addrTyp = if typ.kind == nnkEmpty: typ else: newTree(nnkPtrTy, typ) + result = quote do: + let tmp: `addrTyp` = addr(`ex`) + template `lhs`: untyped = tmp[] + result.copyLineInfo(def) diff --git a/tests/pragmas/tpragmas_misc.nim b/tests/pragmas/tpragmas_misc.nim index 8cab74053..6dc2e6b80 100644 --- a/tests/pragmas/tpragmas_misc.nim +++ b/tests/pragmas/tpragmas_misc.nim @@ -13,8 +13,8 @@ block: block: # (partial fix) bug #15920 block: # var template pragmas don't work in templates - template foo(lhs, typ, expr) = - let lhs = expr + template foo(expr) = + expr proc fun1()= let a {.foo.} = 1 template fun2()= @@ -24,23 +24,22 @@ block: # (partial fix) bug #15920 template foo2() = discard # distractor (template or other symbol kind) block: - template foo2(lhs, typ, expr) = - let lhs = expr + template foo2(expr) = + expr proc fun1()= let a {.foo2.} = 1 template fun2()= let a {.foo2.} = 1 fun1() # ok - when false: # bug: Error: invalid pragma: foo2 - fun2() + fun2() # bug: Error: invalid pragma: foo2 - block: # proc template pragmas don't work in templates + block: # template pragmas don't work for templates, #18212 # adapted from $nim/lib/std/private/since.nim # case without overload template since3(version: (int, int), body: untyped) {.dirty.} = when (NimMajor, NimMinor) >= version: body - when false: # bug + when true: # bug template fun3(): int {.since3: (1, 3).} = 12 block: # ditto, w @@ -51,7 +50,7 @@ block: # (partial fix) bug #15920 template since2(version: (int, int, int), body: untyped) {.dirty.} = when (NimMajor, NimMinor, NimPatch) >= version: body - when false: # bug + when true: # bug template fun3(): int {.since2: (1, 3).} = 12 when true: # D20210801T100514:here @@ -62,3 +61,10 @@ when true: # D20210801T100514:here discard ret fn() static: discard genSym() + +block: # issue #10994 + macro foo(x): untyped = x + template bar {.pragma.} + + proc a {.bar.} = discard # works + proc b {.bar, foo.} = discard # doesn't diff --git a/tests/pragmas/tvar_macro.nim b/tests/pragmas/tvar_macro.nim new file mode 100644 index 000000000..d6a4ff983 --- /dev/null +++ b/tests/pragmas/tvar_macro.nim @@ -0,0 +1,128 @@ +import macros + +block: # test usage + macro modify(sec) = + result = copy sec + result[0][0] = ident(repr(result[0][0]) & "Modified") + + block: + let foo {.modify.} = 3 + doAssert fooModified == 3 + + block: # in section + let + a = 1 + b {.modify.} = 2 + c = 3 + doAssert (a, bModified, c) == (1, 2, 3) + +block: # with single argument + macro appendToName(name: static string, sec) = + result = sec + result[0][0] = ident(repr(result[0][0]) & name) + + block: + let foo {.appendToName: "Bar".} = 3 + doAssert fooBar == 3 + + block: + let + a = 1 + b {.appendToName("").} = 2 + c = 3 + doAssert (a, b, c) == (1, 2, 3) + +macro appendToNameAndAdd(name: static string, incr: static int, sec) = + result = sec + result[0][0] = ident(repr(result[0][0]) & name) + result[0][2] = infix(result[0][2], "+", newLit(incr)) + +block: # with multiple arguments + block: + let foo {.appendToNameAndAdd("Bar", 5).} = 3 + doAssert fooBar == 8 + + block: + let + a = 1 + b {.appendToNameAndAdd("", 15).} = 2 + c = 3 + doAssert (a, b, c) == (1, 17, 3) + +block: # in other kinds of sections + block: + const + a = 1 + b {.appendToNameAndAdd("", 15).} = 2 + c = 3 + doAssert (a, b, c) == (1, 17, 3) + doAssert static(b) == b + + block: + var + a = 1 + b {.appendToNameAndAdd("", 15).} = 2 + c = 3 + doAssert (a, b, c) == (1, 17, 3) + b += a + c += b + doAssert (a, b, c) == (1, 18, 21) + +block: # with other pragmas + macro appendToNameAndAdd(name: static string, incr, sec) = + result = sec + result[0][0][0] = ident(repr(result[0][0][0]) & name) + result[0][0][1].add(ident"deprecated") + result[0][2] = infix(result[0][2], "+", incr) + + var + a = 1 + foo {.exportc: "exportedFooBar", appendToNameAndAdd("Bar", {'0'..'9'}), used.} = {'a'..'z', 'A'..'Z'} + b = 2 + + doAssert (a, b) == (1, 2) + + let importedFooBar {.importc: "exportedFooBar", nodecl.}: set[char] + + doAssert importedFooBar == fooBar #[tt.Warning + ^ fooBar is deprecated + ]# + + +block: # with stropping + macro `cast`(def) = + let def = def[0] + let + lhs = def[0] + typ = def[1] + ex = def[2] + addrTyp = if typ.kind == nnkEmpty: typ else: newTree(nnkPtrTy, typ) + result = quote do: + let tmp: `addrTyp` = unsafeAddr(`ex`) + template `lhs`: untyped = tmp[] + + macro assign(def) = + result = getAst(`cast`(def)) + + block: + let s = @["foo", "bar"] + let a {.`assign`.} = s[0] + doAssert a == "foo" + doAssert a[0].addr == s[0][0].addr + + block: + let + s = @["foo", "bar"] + a {.`cast`.} = s[0] + doAssert a == "foo" + doAssert a[0].addr == s[0][0].addr + +block: # bug #15920 + macro foo(def) = + result = def + proc fun1()= + let a {.foo.} = 1 + template fun2()= + let a {.foo.} = 1 + fun1() # ok + fun2() # BUG diff --git a/tests/stdlib/tdecls.nim b/tests/stdlib/tdecls.nim index c0d6f8a08..4e7407045 100644 --- a/tests/stdlib/tdecls.nim +++ b/tests/stdlib/tdecls.nim @@ -13,15 +13,18 @@ template fun() = var b {.byaddr.}: int = s[0] doAssert a.addr == b.addr - doAssert not compiles(block: - # redeclaration not allowed - var foo = 0 - var foo {.byaddr.} = s[0]) + when false: + # template specific redeclaration issue + # see https://github.com/nim-lang/Nim/issues/8275 + doAssert not compiles(block: + # redeclaration not allowed + var foo = 0 + var foo {.byaddr.} = s[0]) - doAssert not compiles(block: - # ditto - var foo {.byaddr.} = s[0] - var foo {.byaddr.} = s[0]) + doAssert not compiles(block: + # ditto + var foo {.byaddr.} = s[0] + var foo {.byaddr.} = s[0]) block: var b {.byaddr.} = s[1] # redeclaration ok in sub scope @@ -44,48 +47,3 @@ fun2() static: fun2() when false: # pending bug #13887 static: fun() - -## We can define custom pragmas in user code -template byUnsafeAddr(lhs, typ, expr) = - when typ is type(nil): - let tmp = addr(expr) - else: - let tmp: ptr typ = addr(expr) - template lhs: untyped = tmp[] - -block: - let s = @["foo", "bar"] - let a {.byUnsafeAddr.} = s[0] - doAssert a == "foo" - doAssert a[0].addr == s[0][0].addr - -block: # nkAccQuoted - # shows using a keyword, which requires nkAccQuoted - template `cast`(lhs, typ, expr) = - when typ is type(nil): - let tmp = addr(expr) - else: - let tmp: ptr typ = addr(expr) - template lhs: untyped = tmp[] - - block: - let s = @["foo", "bar"] - let a {.`byUnsafeAddr`.} = s[0] - doAssert a == "foo" - doAssert a[0].addr == s[0][0].addr - - block: - let s = @["foo", "bar"] - let a {.`cast`.} = s[0] - doAssert a == "foo" - doAssert a[0].addr == s[0][0].addr - -block: # bug #15920 - template foo(lhs, typ, expr) = - let lhs = expr - proc fun1()= - let a {.foo.} = 1 - template fun2()= - let a {.foo.} = 1 - fun1() # ok - fun2() # BUG From 15f54de5c4ee2f70e1304201ebc8c8634b731c9a Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 22 Jan 2022 20:33:55 +0100 Subject: [PATCH 1488/2472] RST: allow empty number-lines directives just like it was done for a decade; all my documents rely on this feature [backport (#19431) --- lib/packages/docutils/rstgen.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index 9018087f7..d2180cb91 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -931,7 +931,8 @@ proc getField1Int(d: PDoc, n: PRstNode, fieldName: string): int = let nChars = parseInt(value, number) if nChars == 0: if value.len == 0: - err("field $1 requires an argument" % [fieldName]) + # use a good default value: + result = 1 else: err("field $1 requires an integer, but '$2' was given" % [fieldName, value]) From f7c4fb0014f320824d8f05e2e4d9776c09f51711 Mon Sep 17 00:00:00 2001 From: ehmry Date: Mon, 24 Jan 2022 20:57:37 +0100 Subject: [PATCH 1489/2472] os: faster getFileSize (#19438) Use "stat" rather than "open", "seek", and "close" system calls. The Windows implementation remains the same. --- lib/pure/os.nim | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 81b432d7f..07040d611 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -3216,11 +3216,10 @@ proc getFileSize*(file: string): BiggestInt {.rtl, extern: "nos$1", result = rdFileSize(a) findClose(resA) else: - var f: File - if open(f, file): - result = getFileSize(f) - close(f) - else: raiseOSError(osLastError(), file) + var rawInfo: Stat + if stat(file, rawInfo) < 0'i32: + raiseOSError(osLastError(), file) + rawInfo.st_size when defined(windows) or weirdTarget: type From 6ea622552345fc9a6c451615079d2a0ebbc6c4ca Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 25 Jan 2022 08:08:22 +0100 Subject: [PATCH 1490/2472] bugfix: varargs count as open arrays (#19447) --- compiler/typeallowed.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/typeallowed.nim b/compiler/typeallowed.nim index 78cd74e56..3d6ea0edb 100644 --- a/compiler/typeallowed.nim +++ b/compiler/typeallowed.nim @@ -233,7 +233,7 @@ proc classifyViewTypeAux(marker: var IntSet, t: PType): ViewTypeKind = case t.kind of tyVar: result = mutableView - of tyLent, tyOpenArray: + of tyLent, tyOpenArray, tyVarargs: result = immutableView of tyGenericInst, tyDistinct, tyAlias, tyInferred, tySink, tyOwned, tyUncheckedArray, tySequence, tyArray, tyRef, tyStatic: From 4b723c0f5350c5bfb9722dace7a40c66a4630a68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Granstr=C3=B6m?= <5092565+HugoGranstrom@users.noreply.github.com> Date: Tue, 25 Jan 2022 10:31:24 +0100 Subject: [PATCH 1491/2472] change run command for numericalnim (#19448) Now it makes runs the custom `nimCI` task that installs the external dependencies --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 5f6e19b52..40a3aef36 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -117,7 +117,7 @@ pkg "nimx", "nim c --threads:on test/main.nim", allowFailure = true pkg "nitter", "nim c src/nitter.nim", "https://github.com/zedeus/nitter" pkg "norm", "testament r tests/sqlite/trows.nim" pkg "npeg", "nimble testarc" -pkg "numericalnim", "nim c -r tests/test_integrate.nim" +pkg "numericalnim", "nimble nimCI" pkg "optionsutils" pkg "ormin", "nim c -o:orminn ormin.nim" pkg "parsetoml" From 6319b00ef7f93fa8d7d44c981f850262d8e0ed6a Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 27 Jan 2022 22:23:34 +0800 Subject: [PATCH 1492/2472] support set other GCs after `mm = orc` or `mm = arc` is set in the global config (#19455) * support set other GCs after arc/orc in global config fix #15535 * set before --- compiler/commands.nim | 54 +++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index 8e79a4477..a1fa55e74 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -497,6 +497,32 @@ proc specialDefine(conf: ConfigRef, key: string; pass: TCmdLinePass) = optOverflowCheck, optAssert, optStackTrace, optLineTrace, optLineDir} conf.globalOptions.excl {optCDebug} +proc registerArcOrc(pass: TCmdLinePass, conf: ConfigRef, isOrc: bool) = + if isOrc: + conf.selectedGC = gcOrc + defineSymbol(conf.symbols, "gcorc") + else: + conf.selectedGC = gcArc + defineSymbol(conf.symbols, "gcarc") + + defineSymbol(conf.symbols, "gcdestructors") + incl conf.globalOptions, optSeqDestructors + incl conf.globalOptions, optTinyRtti + if pass in {passCmd2, passPP}: + defineSymbol(conf.symbols, "nimSeqsV2") + defineSymbol(conf.symbols, "nimV2") + if conf.exc == excNone and conf.backend != backendCpp: + conf.exc = excGoto + +proc unregisterArcOrc(conf: ConfigRef) = + undefSymbol(conf.symbols, "gcdestructors") + undefSymbol(conf.symbols, "gcarc") + undefSymbol(conf.symbols, "gcorc") + undefSymbol(conf.symbols, "nimSeqsV2") + undefSymbol(conf.symbols, "nimV2") + excl conf.globalOptions, optSeqDestructors + excl conf.globalOptions, optTinyRtti + proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; conf: ConfigRef) = var @@ -602,36 +628,21 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; if pass in {passCmd2, passPP}: case arg.normalize of "boehm": + unregisterArcOrc(conf) conf.selectedGC = gcBoehm defineSymbol(conf.symbols, "boehmgc") incl conf.globalOptions, optTlsEmulation # Boehm GC doesn't scan the real TLS of "refc": + unregisterArcOrc(conf) conf.selectedGC = gcRefc of "markandsweep": + unregisterArcOrc(conf) conf.selectedGC = gcMarkAndSweep defineSymbol(conf.symbols, "gcmarkandsweep") of "destructors", "arc": - conf.selectedGC = gcArc - defineSymbol(conf.symbols, "gcdestructors") - defineSymbol(conf.symbols, "gcarc") - incl conf.globalOptions, optSeqDestructors - incl conf.globalOptions, optTinyRtti - if pass in {passCmd2, passPP}: - defineSymbol(conf.symbols, "nimSeqsV2") - defineSymbol(conf.symbols, "nimV2") - if conf.exc == excNone and conf.backend != backendCpp: - conf.exc = excGoto + registerArcOrc(pass, conf, false) of "orc": - conf.selectedGC = gcOrc - defineSymbol(conf.symbols, "gcdestructors") - defineSymbol(conf.symbols, "gcorc") - incl conf.globalOptions, optSeqDestructors - incl conf.globalOptions, optTinyRtti - if pass in {passCmd2, passPP}: - defineSymbol(conf.symbols, "nimSeqsV2") - defineSymbol(conf.symbols, "nimV2") - if conf.exc == excNone and conf.backend != backendCpp: - conf.exc = excGoto + registerArcOrc(pass, conf, true) of "hooks": conf.selectedGC = gcHooks defineSymbol(conf.symbols, "gchooks") @@ -640,12 +651,15 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; if pass in {passCmd2, passPP}: defineSymbol(conf.symbols, "nimSeqsV2") of "go": + unregisterArcOrc(conf) conf.selectedGC = gcGo defineSymbol(conf.symbols, "gogc") of "none": + unregisterArcOrc(conf) conf.selectedGC = gcNone defineSymbol(conf.symbols, "nogc") of "stack", "regions": + unregisterArcOrc(conf) conf.selectedGC = gcRegions defineSymbol(conf.symbols, "gcregions") of "v2": warningOptionNoop(arg) From 0c3892c3c707e75520d6c8b5a65a0eaf8c15fcd4 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 28 Jan 2022 16:53:42 +0800 Subject: [PATCH 1493/2472] nvro don't touch cdecl types [backport: 1.6] (#19461) * nvro don't touch cdecl types; fix #19342 again --- compiler/ast.nim | 2 +- compiler/ccgcalls.nim | 6 +++--- compiler/ccgstmts.nim | 21 ++++++++++++++------- compiler/ccgtypes.nim | 18 ++++++++++++------ compiler/cgen.nim | 4 ++-- compiler/semstmts.nim | 1 + tests/objects/t19342_2.nim | 18 ++++++++++++++++++ 7 files changed, 51 insertions(+), 19 deletions(-) create mode 100644 tests/objects/t19342_2.nim diff --git a/compiler/ast.nim b/compiler/ast.nim index 4286f940f..ad5ec12f5 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -501,7 +501,7 @@ type nfHasComment # node has a comment TNodeFlags* = set[TNodeFlag] - TTypeFlag* = enum # keep below 32 for efficiency reasons (now: 43) + TTypeFlag* = enum # keep below 32 for efficiency reasons (now: 45) tfVarargs, # procedure has C styled varargs # tyArray type represeting a varargs list tfNoSideEffect, # procedure type does not allow side effects diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index fe0516f9a..197728ccf 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -76,7 +76,7 @@ proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc, # getUniqueType() is too expensive here: var typ = skipTypes(ri[0].typ, abstractInst) if typ[0] != nil: - if isInvalidReturnType(p.config, typ[0]): + if isInvalidReturnType(p.config, typ): if params != nil: pl.add(~", ") # beware of 'result = p(result)'. We may need to allocate a temporary: if d.k in {locTemp, locNone} or not preventNrvo(p, le, ri): @@ -439,7 +439,7 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) = let rawProc = getClosureType(p.module, typ, clHalf) let canRaise = p.config.exc == excGoto and canRaiseDisp(p, ri[0]) if typ[0] != nil: - if isInvalidReturnType(p.config, typ[0]): + if isInvalidReturnType(p.config, typ): if ri.len > 1: pl.add(~", ") # beware of 'result = p(result)'. We may need to allocate a temporary: if d.k in {locTemp, locNone} or not preventNrvo(p, le, ri): @@ -737,7 +737,7 @@ proc genNamedParamCall(p: BProc, ri: PNode, d: var TLoc) = pl.add(~": ") pl.add(genArg(p, ri[i], param, ri)) if typ[0] != nil: - if isInvalidReturnType(p.config, typ[0]): + if isInvalidReturnType(p.config, typ): if ri.len > 1: pl.add(~" ") # beware of 'result = p(result)'. We always allocate a temporary: if d.k in {locTemp, locNone}: diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index 7fecc1475..58a6dd015 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -32,13 +32,20 @@ proc registerTraverseProc(p: BProc, v: PSym, traverseProc: Rope) = "$n\t#nimRegisterGlobalMarker($1);$n$n", [traverseProc]) proc isAssignedImmediately(conf: ConfigRef; n: PNode): bool {.inline.} = - if n.kind == nkEmpty: return false - if isInvalidReturnType(conf, n.typ): - # var v = f() - # is transformed into: var v; f(addr v) - # where 'f' **does not** initialize the result! - return false - result = true + if n.kind == nkEmpty: + result = false + elif n.kind in nkCallKinds and n[0] != nil and n[0].typ != nil and n[0].typ.skipTypes(abstractInst).kind == tyProc: + if isInvalidReturnType(conf, n[0].typ, true): + # var v = f() + # is transformed into: var v; f(addr v) + # where 'f' **does not** initialize the result! + result = false + else: + result = true + elif isInvalidReturnType(conf, n.typ, false): + result = false + else: + result = true proc inExceptBlockLen(p: BProc): int = for x in p.nestedTryStmts: diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index df1874b26..f010b25c9 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -215,12 +215,18 @@ proc isObjLackingTypeField(typ: PType): bool {.inline.} = result = (typ.kind == tyObject) and ((tfFinal in typ.flags) and (typ[0] == nil) or isPureObject(typ)) -proc isInvalidReturnType(conf: ConfigRef; rettype: PType): bool = +proc isInvalidReturnType(conf: ConfigRef; typ: PType, isProc = true): bool = # Arrays and sets cannot be returned by a C procedure, because C is # such a poor programming language. # We exclude records with refs too. This enhances efficiency and # is necessary for proper code generation of assignments. - if rettype == nil or (tfByCopy notin rettype.flags and getSize(conf, rettype) > conf.target.floatSize*3): + var rettype = typ + var isAllowedCall = true + if isProc: + rettype = rettype[0] + isAllowedCall = typ.callConv in {ccClosure, ccInline, ccNimCall} + if rettype == nil or (isAllowedCall and + getSize(conf, rettype) > conf.target.floatSize*3): result = true else: case mapType(conf, rettype, skResult) @@ -257,11 +263,11 @@ proc addAbiCheck(m: BModule, t: PType, name: Rope) = # see `testCodegenABICheck` for example error message it generates -proc fillResult(conf: ConfigRef; param: PNode) = +proc fillResult(conf: ConfigRef; param: PNode, proctype: PType) = fillLoc(param.sym.loc, locParam, param, ~"Result", OnStack) let t = param.sym.typ - if mapReturnType(conf, t) != ctArray and isInvalidReturnType(conf, t): + if mapReturnType(conf, t) != ctArray and isInvalidReturnType(conf, proctype): incl(param.sym.loc.flags, lfIndirect) param.sym.loc.storage = OnUnknown @@ -426,7 +432,7 @@ proc genProcParams(m: BModule, t: PType, rettype, params: var Rope, check: var IntSet, declareEnvironment=true; weakDep=false) = params = nil - if t[0] == nil or isInvalidReturnType(m.config, t[0]): + if t[0] == nil or isInvalidReturnType(m.config, t): rettype = ~"void" else: rettype = getTypeDescAux(m, t[0], check, skResult) @@ -461,7 +467,7 @@ proc genProcParams(m: BModule, t: PType, rettype, params: var Rope, params.addf(", NI $1Len_$2", [param.loc.r, j.rope]) inc(j) arr = arr[0].skipTypes({tySink}) - if t[0] != nil and isInvalidReturnType(m.config, t[0]): + if t[0] != nil and isInvalidReturnType(m.config, t): var arr = t[0] if params != nil: params.add(", ") if mapReturnType(m.config, t[0]) != ctArray: diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 246b8c0e9..10b4b55d0 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1039,7 +1039,7 @@ proc genProcAux(m: BModule, prc: PSym) = internalError(m.config, prc.info, "proc has no result symbol") let resNode = prc.ast[resultPos] let res = resNode.sym # get result symbol - if not isInvalidReturnType(m.config, prc.typ[0]): + if not isInvalidReturnType(m.config, prc.typ): if sfNoInit in prc.flags: incl(res.flags, sfNoInit) if sfNoInit in prc.flags and p.module.compileToCpp and (let val = easyResultAsgn(procBody); val != nil): var decl = localVarDecl(p, resNode) @@ -1053,7 +1053,7 @@ proc genProcAux(m: BModule, prc: PSym) = initLocalVar(p, res, immediateAsgn=false) returnStmt = ropecg(p.module, "\treturn $1;$n", [rdLoc(res.loc)]) else: - fillResult(p.config, resNode) + fillResult(p.config, resNode, prc.typ) assignParam(p, res, prc.typ[0]) # We simplify 'unsureAsgn(result, nil); unsureAsgn(result, x)' # to 'unsureAsgn(result, x)' diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 33e304ab2..c2b385c80 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -2137,6 +2137,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, incl(s.flags, sfWasForwarded) elif sfBorrow in s.flags: semBorrow(c, n, s) sideEffectsCheck(c, s) + closeScope(c) # close scope for parameters # c.currentScope = oldScope popOwner(c) diff --git a/tests/objects/t19342_2.nim b/tests/objects/t19342_2.nim new file mode 100644 index 000000000..6f6d0f2b3 --- /dev/null +++ b/tests/objects/t19342_2.nim @@ -0,0 +1,18 @@ +discard """ + targets: "c cpp" +""" + +{.compile: "m19342.c".} + +# bug #19342 +type + Node* {.byRef.} = object + data: array[25, cint] + +proc myproc(name: cint): Node {.importc: "hello", cdecl.} + +proc parse = + let node = myproc(10) + doAssert node.data[0] == 999 + +parse() \ No newline at end of file From 520881af9add94b26ef7d0630352ad425bb84490 Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 29 Jan 2022 10:08:38 +0800 Subject: [PATCH 1494/2472] update outdated link (#19465) Ref https://github.com/nim-lang/Nim/issues/19463 --- doc/intern.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/intern.rst b/doc/intern.rst index e8fb4ac86..a8846ae02 100644 --- a/doc/intern.rst +++ b/doc/intern.rst @@ -166,7 +166,7 @@ current commit: You can also bisect using custom options to build the compiler, for example if you don't need a debug version of the compiler (which runs slower), you can replace -`./koch temp`:cmd: by explicit compilation command, see `Rebuilding the compiler`_. +`./koch temp`:cmd: by explicit compilation command, see `Bootstrapping the compiler`_. Runtimes From cb894c7094fb49014f85815a9dafc38b5dda743e Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sat, 29 Jan 2022 13:03:01 +0000 Subject: [PATCH 1495/2472] Merge pull request from GHSA-ggrq-h43f-3w7m This fixes a CVE (currently https://github.com/nim-lang/Nim/security/advisories/GHSA-ggrq-h43f-3w7m) --- compiler/docgen.nim | 2 +- lib/packages/docutils/rst.nim | 22 ++++++++++++++--- tests/stdlib/trstgen.nim | 45 +++++++++++++++++++++++++++++------ 3 files changed, 58 insertions(+), 11 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 90e0f54b3..235922349 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -282,7 +282,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, result.cache = cache result.outDir = conf.outDir.string result.isPureRst = isPureRst - var options= {roSupportRawDirective, roSupportMarkdown, roPreferMarkdown} + var options= {roSupportRawDirective, roSupportMarkdown, roPreferMarkdown, roSandboxDisabled} if not isPureRst: options.incl roNimFile result.sharedState = newRstSharedState( options, filename.string, diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 791953767..93c2231dc 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -235,6 +235,12 @@ type ## to Markdown) -- implies `roSupportMarkdown` roNimFile ## set for Nim files where default interpreted ## text role should be :nim: + roSandboxDisabled ## this option enables certain options + ## (e.g. raw, include) + ## which are disabled by default as they can + ## enable users to read arbitrary data and + ## perform XSS if the parser is used in a web + ## app. RstParseOptions* = set[RstParseOption] @@ -260,7 +266,8 @@ type mwBrokenLink = "broken link '$1'", mwUnsupportedLanguage = "language '$1' not supported", mwUnsupportedField = "field '$1' not supported", - mwRstStyle = "RST style: $1" + mwRstStyle = "RST style: $1", + meSandboxedDirective = "disabled directive: '$1'", MsgHandler* = proc (filename: string, line, col: int, msgKind: MsgKind, arg: string) {.closure, gcsafe.} ## what to do in case of an error @@ -315,6 +322,7 @@ const ":geek:": "icon_e_geek", ":ugeek:": "icon_e_ugeek" } + SandboxDirAllowlist = ["image", "code", "code-block"] type TokType = enum @@ -2987,10 +2995,14 @@ proc dirCodeBlock(p: var RstParser, nimExtension = false): PRstNode = ## ## As an extension this proc will process the ``file`` extension field and if ## present will replace the code block with the contents of the referenced - ## file. + ## file. This behaviour is disabled in sandboxed mode and can be re-enabled + ## with the `roSandboxDisabled` flag. result = parseDirective(p, rnCodeBlock, {hasArg, hasOptions}, parseLiteralBlock) var filename = strip(getFieldValue(result, "file")) if filename != "": + if roSandboxDisabled notin p.s.options: + let tok = p.tok[p.idx-2] + rstMessage(p, meSandboxedDirective, "file", tok.line, tok.col) var path = p.findRelativeFile(filename) if path == "": rstMessage(p, meCannotOpenFile, filename) var n = newRstNode(rnLiteralBlock) @@ -3086,6 +3098,11 @@ proc dirRaw(p: var RstParser): PRstNode = proc selectDir(p: var RstParser, d: string): PRstNode = result = nil + let tok = p.tok[p.idx-2] # report on directive in ".. directive::" + if roSandboxDisabled notin p.s.options: + if d notin SandboxDirAllowlist: + rstMessage(p, meSandboxedDirective, d, tok.line, tok.col) + case d of "admonition", "attention", "caution": result = dirAdmonition(p, d) of "code": result = dirCodeBlock(p) @@ -3112,7 +3129,6 @@ proc selectDir(p: var RstParser, d: string): PRstNode = of "title": result = dirTitle(p) of "warning": result = dirAdmonition(p, d) else: - let tok = p.tok[p.idx-2] # report on directive in ".. directive::" rstMessage(p, meInvalidDirective, d, tok.line, tok.col) proc prefix(ftnType: FootnoteType): string = diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index 2be65b35b..995c0a151 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -47,6 +47,10 @@ proc optionListLabel(opt: string): string = opt & "              " +const + NoSandboxOpts = {roPreferMarkdown, roSupportMarkdown, roNimFile, roSandboxDisabled} + + suite "YAML syntax highlighting": test "Basics": let input = """.. code-block:: yaml @@ -631,7 +635,9 @@ let x = 1 let p3 = """

              Par3 """ & id"value3" & ".

              " let p4 = """

              Par4 value4.

              """ let expected = p1 & p2 & "\n" & p3 & "\n" & p4 - check(input.toHtml == expected) + check( + input.toHtml(NoSandboxOpts) == expected + ) test "role directive": let input = dedent""" @@ -642,7 +648,10 @@ let x = 1 :language: brainhelp """ var warnings = new seq[string] - let output = input.toHtml(warnings=warnings) + let output = input.toHtml( + NoSandboxOpts, + warnings=warnings + ) check(warnings[].len == 1 and "language 'brainhelp' not supported" in warnings[0]) test "RST comments": @@ -1187,7 +1196,9 @@ Test1 .. tip:: endOf tip .. warning:: endOf warning """ - let output0 = input0.toHtml + let output0 = input0.toHtml( + NoSandboxOpts + ) for a in ["admonition", "attention", "caution", "danger", "error", "hint", "important", "note", "tip", "warning" ]: doAssert "endOf " & a & "" in output0 @@ -1198,7 +1209,9 @@ Test1 Test paragraph. """ - let output1 = input1.toHtml + let output1 = input1.toHtml( + NoSandboxOpts + ) doAssert "endOfError" in output1 doAssert "

              Test paragraph.

              " in output1 doAssert "class=\"admonition admonition-error\"" in output1 @@ -1210,7 +1223,9 @@ Test1 Test paragraph. """ - let output2 = input2.toHtml + let output2 = input2.toHtml( + NoSandboxOpts + ) doAssert "endOfError Test2p." in output2 doAssert "

              Test paragraph.

              " in output2 doAssert "class=\"admonition admonition-error\"" in output2 @@ -1218,7 +1233,9 @@ Test1 let input3 = dedent """ .. note:: endOfNote """ - let output3 = input3.toHtml + let output3 = input3.toHtml( + NoSandboxOpts + ) doAssert "endOfNote" in output3 doAssert "class=\"admonition admonition-info\"" in output3 @@ -1303,7 +1320,9 @@ Test1 That was a transition. """ - let output1 = input1.toHtml + let output1 = input1.toHtml( + NoSandboxOpts + ) doAssert "

              Nim))""") check("(([Nim](javascript://nim-lang.org/)))".toHtml == """((Nim))""") + +suite "local file inclusion": + test "cannot include files in sandboxed mode": + var error = new string + discard ".. include:: ./readme.md".toHtml(error=error) + check(error[] == "input(1, 11) Error: disabled directive: 'include'") + + test "code-block file directive is disabled": + var error = new string + discard ".. code-block:: nim\n :file: ./readme.md".toHtml(error=error) + check(error[] == "input(2, 20) Error: disabled directive: 'file'") + From 33cd883e27f7ba7564cbeb59d2d38fe02bb507bf Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 29 Jan 2022 21:06:54 +0800 Subject: [PATCH 1496/2472] [add testcase] NRVO does not occur with init procedures (#19462) * [add testcase] NRVO does not occur with init procedures close #19094 * Update tests/ccgbugs2/tcodegen.nim --- tests/ccgbugs2/tcodegen.nim | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 tests/ccgbugs2/tcodegen.nim diff --git a/tests/ccgbugs2/tcodegen.nim b/tests/ccgbugs2/tcodegen.nim new file mode 100644 index 000000000..84cd76e2f --- /dev/null +++ b/tests/ccgbugs2/tcodegen.nim @@ -0,0 +1,28 @@ +discard """ + targets: "c cpp" +""" + +# bug #19094 +type + X = object + filler: array[2048, int] + innerAddress: uint + +proc initX(): X = + result.innerAddress = cast[uint](result.addr) + +proc initXInPlace(x: var X) = + x.innerAddress = cast[uint](x.addr) + +block: # NRVO1 + var x = initX() + let innerAddress = x.innerAddress + let outerAddress = cast[uint](x.addr) + doAssert(innerAddress == outerAddress) # [OK] + +block: # NRVO2 + var x: X + initXInPlace(x) + let innerAddress = x.innerAddress + let outerAddress = cast[uint](x.addr) + doAssert(innerAddress == outerAddress) # [OK] From 56a901f9e197dd03f8c70ab4ef1042dc33548af0 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 31 Jan 2022 20:49:20 +0800 Subject: [PATCH 1497/2472] fix broken CI (#19472) * fix broken CI * fix * fix tests --- compiler/docgen.nim | 1 + compiler/lineinfos.nim | 2 ++ tests/stdlib/trst.nim | 12 ++++++------ 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 235922349..3e61032d8 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -237,6 +237,7 @@ template declareClosures = of meInvalidDirective: k = errRstInvalidDirectiveX of meInvalidField: k = errRstInvalidField of meFootnoteMismatch: k = errRstFootnoteMismatch + of meSandboxedDirective: k = errRstSandboxedDirective of mwRedefinitionOfLabel: k = warnRstRedefinitionOfLabel of mwUnknownSubstitution: k = warnRstUnknownSubstitutionX of mwAmbiguousLink: k = warnRstAmbiguousLink diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index e13387be6..575be6196 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -39,6 +39,7 @@ type errRstInvalidDirectiveX, errRstInvalidField, errRstFootnoteMismatch, + errRstSandboxedDirective, errProveInit, # deadcode errGenerated, errUser, @@ -110,6 +111,7 @@ const errRstInvalidDirectiveX: "invalid directive: '$1'", errRstInvalidField: "invalid field: $1", errRstFootnoteMismatch: "number of footnotes and their references don't match: $1", + errRstSandboxedDirective: "disabled directive: '$1'", errProveInit: "Cannot prove that '$1' is initialized.", # deadcode errGenerated: "$1", errUser: "$1", diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index 0a9eb8038..771e02477 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -23,7 +23,7 @@ import std/private/miscdollars import os proc toAst(input: string, - rstOptions: RstParseOptions = {roPreferMarkdown, roSupportMarkdown, roNimFile}, + rstOptions: RstParseOptions = {roPreferMarkdown, roSupportMarkdown, roNimFile, roSandboxDisabled}, error: ref string = nil, warnings: ref seq[string] = nil): string = ## If `error` is nil then no errors should be generated. @@ -866,7 +866,7 @@ suite "RST include directive": test "Include whole": "other.rst".writeFile("**test1**") let input = ".. include:: other.rst" - doAssert "test1" == rstTohtml(input, {}, defaultConfig()) + doAssert "test1" == rstTohtml(input, {roSandboxDisabled}, defaultConfig()) removeFile("other.rst") test "Include starting from": @@ -880,7 +880,7 @@ OtherStart .. include:: other.rst :start-after: OtherStart """ - check "Visible" == rstTohtml(input, {}, defaultConfig()) + check "Visible" == rstTohtml(input, {roSandboxDisabled}, defaultConfig()) removeFile("other.rst") test "Include everything before": @@ -894,7 +894,7 @@ And this should **NOT** be visible in `docs.html` .. include:: other.rst :end-before: OtherEnd """ - doAssert "Visible" == rstTohtml(input, {}, defaultConfig()) + doAssert "Visible" == rstTohtml(input, {roSandboxDisabled}, defaultConfig()) removeFile("other.rst") @@ -912,7 +912,7 @@ And this should **NOT** be visible in `docs.html` :start-after: OtherStart :end-before: OtherEnd """ - check "Visible" == rstTohtml(input, {}, defaultConfig()) + check "Visible" == rstTohtml(input, {roSandboxDisabled}, defaultConfig()) removeFile("other.rst") @@ -932,7 +932,7 @@ And this should **NOT** be visible in `docs.html` :start-after: OtherStart :end-before: OtherEnd """ - doAssert "Visible" == rstTohtml(input, {}, defaultConfig()) + doAssert "Visible" == rstTohtml(input, {roSandboxDisabled}, defaultConfig()) removeFile("other.rst") suite "RST escaping": From 22ae0bef63189dff28676b0dd5f6089864151120 Mon Sep 17 00:00:00 2001 From: tandy1000 Date: Tue, 1 Feb 2022 11:12:55 +0000 Subject: [PATCH 1498/2472] Update jsfetch with latest API and fix missing bindings (#19473) * Update with latest API and fix missing bindings remove deprecated `Body` remove implicit `cstring` convs add `Headers` to `FetchOptions` add `Request` init proc which takes `FetchOptions` * Update lib/std/jsfetch.nim Co-authored-by: Juan Carlos * Update lib/std/jsfetch.nim Co-authored-by: Juan Carlos * remove experimental flag Co-authored-by: Juan Carlos --- lib/std/jsfetch.nim | 46 ++++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/lib/std/jsfetch.nim b/lib/std/jsfetch.nim index a25fe7ca8..808d699b0 100644 --- a/lib/std/jsfetch.nim +++ b/lib/std/jsfetch.nim @@ -1,5 +1,4 @@ ## - Fetch for the JavaScript target: https://developer.mozilla.org/docs/Web/API/Fetch_API -## .. Note:: jsfetch is Experimental. when not defined(js): {.fatal: "Module jsfetch is designed to be used with the JavaScript backend.".} @@ -12,6 +11,7 @@ type keepalive*: bool metod* {.importjs: "method".}: cstring body*, integrity*, referrer*, mode*, credentials*, cache*, redirect*, referrerPolicy*: cstring + headers*: Headers FetchModes* = enum ## Mode options. fmCors = "cors" @@ -42,16 +42,13 @@ type frpOriginWhenCrossOrigin = "origin-when-cross-origin" frpUnsafeUrl = "unsafe-url" - Body* = ref object of JsRoot ## https://developer.mozilla.org/en-US/docs/Web/API/Body - bodyUsed*: bool - Response* = ref object of JsRoot ## https://developer.mozilla.org/en-US/docs/Web/API/Response bodyUsed*, ok*, redirected*: bool typ* {.importjs: "type".}: cstring url*, statusText*: cstring status*: cint headers*: Headers - body*: Body + body*: cstring Request* = ref object of JsRoot ## https://developer.mozilla.org/en-US/docs/Web/API/Request bodyUsed*, ok*, redirected*: bool @@ -59,8 +56,7 @@ type url*, statusText*: cstring status*: cint headers*: Headers - body*: Body - + body*: cstring func newResponse*(body: cstring | FormData): Response {.importjs: "(new Response(#))".} ## Constructor for `Response`. This does *not* call `fetch()`. Same as `new Response()`. @@ -68,30 +64,34 @@ func newResponse*(body: cstring | FormData): Response {.importjs: "(new Response func newRequest*(url: cstring): Request {.importjs: "(new Request(#))".} ## Constructor for `Request`. This does *not* call `fetch()`. Same as `new Request()`. +func newRequest*(url: cstring; fetchOptions: FetchOptions): Request {.importjs: "(new Request(#, #))".} + ## Constructor for `Request` with `fetchOptions`. Same as `fetch(url, fetchOptions)`. + func clone*(self: Response | Request): Response {.importjs: "#.$1()".} ## https://developer.mozilla.org/en-US/docs/Web/API/Response/clone proc text*(self: Response): Future[cstring] {.importjs: "#.$1()".} - ## https://developer.mozilla.org/en-US/docs/Web/API/Body/text + ## https://developer.mozilla.org/en-US/docs/Web/API/Response/text proc json*(self: Response): Future[JsObject] {.importjs: "#.$1()".} - ## https://developer.mozilla.org/en-US/docs/Web/API/Body/json + ## https://developer.mozilla.org/en-US/docs/Web/API/Response/json -proc formData*(self: Body): Future[FormData] {.importjs: "#.$1()".} - ## https://developer.mozilla.org/en-US/docs/Web/API/Body/formData +proc formData*(self: Response): Future[FormData] {.importjs: "#.$1()".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Response/formData proc unsafeNewFetchOptions*(metod, body, mode, credentials, cache, referrerPolicy: cstring; - keepalive: bool; redirect = "follow".cstring; referrer = "client".cstring; integrity = "".cstring): FetchOptions {.importjs: - "{method: #, body: #, mode: #, credentials: #, cache: #, referrerPolicy: #, keepalive: #, redirect: #, referrer: #, integrity: #}".} + keepalive: bool; redirect = "follow".cstring; referrer = "client".cstring; integrity = "".cstring; headers: Headers = newHeaders()): FetchOptions {.importjs: + "{method: #, body: #, mode: #, credentials: #, cache: #, referrerPolicy: #, keepalive: #, redirect: #, referrer: #, integrity: #, headers: #}".} ## .. warning:: Unsafe `newfetchOptions`. func newfetchOptions*(metod: HttpMethod; body: cstring; mode: FetchModes; credentials: FetchCredentials; cache: FetchCaches; referrerPolicy: FetchReferrerPolicies; - keepalive: bool; redirect = frFollow; referrer = "client".cstring; integrity = "".cstring): FetchOptions = + keepalive: bool; redirect = frFollow; referrer = "client".cstring; integrity = "".cstring, + headers: Headers = newHeaders()): FetchOptions = ## Constructor for `FetchOptions`. result = FetchOptions( - body: body, mode: $mode, credentials: $credentials, cache: $cache, referrerPolicy: $referrerPolicy, - keepalive: keepalive, redirect: $redirect, referrer: referrer, integrity: integrity, + body: body, mode: cstring($mode), credentials: cstring($credentials), cache: cstring($cache), referrerPolicy: cstring($referrerPolicy), + keepalive: keepalive, redirect: cstring($redirect), referrer: referrer, integrity: integrity, metod: (case metod of HttpHead: "HEAD".cstring of HttpGet: "GET".cstring @@ -109,9 +109,9 @@ proc fetch*(url: cstring | Request): Future[Response] {.importjs: "$1(#)".} proc fetch*(url: cstring | Request; options: FetchOptions): Future[Response] {.importjs: "$1(#, #)".} ## `fetch()` API that takes a `FetchOptions`, returns a `Future[Response]`. -func toCstring*(self: Request | Response | Body | FetchOptions): cstring {.importjs: "JSON.stringify(#)".} +func toCstring*(self: Request | Response | FetchOptions): cstring {.importjs: "JSON.stringify(#)".} -func `$`*(self: Request | Response | Body | FetchOptions): string = $toCstring(self) +func `$`*(self: Request | Response | FetchOptions): string = $toCstring(self) runnableExamples("-r:off"): @@ -131,7 +131,8 @@ runnableExamples("-r:off"): keepalive = false, redirect = "follow".cstring, referrer = "client".cstring, - integrity = "".cstring + integrity = "".cstring, + headers = newHeaders() ) assert options0.keepalive == false assert options0.metod == "POST".cstring @@ -143,6 +144,7 @@ runnableExamples("-r:off"): assert options0.redirect == "follow".cstring assert options0.referrer == "client".cstring assert options0.integrity == "".cstring + assert options0.headers.len == 0 block: let options1: FetchOptions = newFetchOptions( @@ -155,7 +157,8 @@ runnableExamples("-r:off"): keepalive = false, redirect = frFollow, referrer = "client".cstring, - integrity = "".cstring + integrity = "".cstring, + headers = newHeaders() ) assert options1.keepalive == false assert options1.metod == $HttpPost @@ -167,6 +170,7 @@ runnableExamples("-r:off"): assert options1.redirect == $frFollow assert options1.referrer == "client".cstring assert options1.integrity == "".cstring + assert options1.headers.len == 0 block: let response: Response = newResponse(body = "-. .. --".cstring) @@ -182,7 +186,7 @@ runnableExamples("-r:off"): assert response.ok assert response.status == 200.cint assert response.headers is Headers - assert response.body is Body + assert response.body is cstring discard example() From 1830a3b505c33be5fc3b9e311c5e7720d9b37b32 Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Wed, 2 Feb 2022 01:38:21 -0700 Subject: [PATCH 1499/2472] No longer segfault when using a typeclass with a self referencing type (#19467) --- compiler/sigmatch.nim | 3 ++- tests/generics/tgenerics_issues.nim | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index e1195d04a..14f4c5e19 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -1688,7 +1688,8 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, var aa = a while aa.kind in {tyTypeDesc, tyGenericParam} and aa.len > 0: aa = lastSon(aa) - if aa.kind == tyGenericParam: + if aa.kind in {tyGenericParam} + tyTypeClasses: + # If the constraint is a genericParam or typeClass this isGeneric return isGeneric result = typeRel(c, f.base, aa, flags) if result > isGeneric: result = isGeneric diff --git a/tests/generics/tgenerics_issues.nim b/tests/generics/tgenerics_issues.nim index 092926c01..db7a16569 100644 --- a/tests/generics/tgenerics_issues.nim +++ b/tests/generics/tgenerics_issues.nim @@ -860,3 +860,15 @@ type var a: Tile3[int] +block: # Ensure no segfault from constraint + type + Regex[A: SomeOrdinal] = ref object + val: Regex[A] + MyConstraint = (seq or enum or set) + MyOtherType[A: MyConstraint] = ref object + val: MyOtherType[A] + + var + a = Regex[int]() + b = Regex[bool]() + c = MyOtherType[seq[int]]() From 486cb09ec2caef60011b3d182bfd188dadafdf62 Mon Sep 17 00:00:00 2001 From: Regis Caillaud <35006197+Clonkk@users.noreply.github.com> Date: Wed, 2 Feb 2022 09:44:51 +0100 Subject: [PATCH 1500/2472] Clonkk fix2 11923 (#19451) * fix nnkBracketExpr not compiling for getImpl on customPragmaNode * fix test import * fix alias not working with hasCustomPragmas --- lib/core/macros.nim | 27 ++++++++++++++++++++++----- tests/pragmas/tcustom_pragma.nim | 6 ++++++ 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 28106493f..c0e6e5154 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -1493,6 +1493,22 @@ macro expandMacros*(body: typed): untyped = echo body.toStrLit result = body +proc extractTypeImpl(n: NimNode): NimNode = + ## attempts to extract the type definition of the given symbol + case n.kind + of nnkSym: # can extract an impl + result = n.getImpl.extractTypeImpl() + of nnkObjectTy, nnkRefTy, nnkPtrTy: result = n + of nnkBracketExpr: + if n.typeKind == ntyTypeDesc: + result = n[1].extractTypeImpl() + else: + doAssert n.typeKind == ntyGenericInst + result = n[0].getImpl() + of nnkTypeDef: + result = n[2] + else: error("Invalid node to retrieve type implementation of: " & $n.kind) + proc customPragmaNode(n: NimNode): NimNode = expectKind(n, {nnkSym, nnkDotExpr, nnkBracketExpr, nnkTypeOfExpr, nnkCheckedFieldExpr}) let @@ -1501,7 +1517,10 @@ proc customPragmaNode(n: NimNode): NimNode = if typ.kind == nnkBracketExpr and typ.len > 1 and typ[1].kind == nnkProcTy: return typ[1][1] elif typ.typeKind == ntyTypeDesc: - let impl = typ[1].getImpl() + let impl = getImpl( + if kind(typ[1]) == nnkBracketExpr: typ[1][0] + else: typ[1] + ) if impl[0].kind == nnkPragmaExpr: return impl[0][1] else: @@ -1524,14 +1543,12 @@ proc customPragmaNode(n: NimNode): NimNode = let name = $(if n.kind == nnkCheckedFieldExpr: n[0][1] else: n[1]) let typInst = getTypeInst(if n.kind == nnkCheckedFieldExpr or n[0].kind == nnkHiddenDeref: n[0][0] else: n[0]) var typDef = getImpl( - if typInst.kind == nnkVarTy or - typInst.kind == nnkBracketExpr: - typInst[0] + if typInst.kind in {nnkVarTy, nnkBracketExpr}: typInst[0] else: typInst ) while typDef != nil: typDef.expectKind(nnkTypeDef) - let typ = typDef[2] + let typ = typDef[2].extractTypeImpl() typ.expectKind({nnkRefTy, nnkPtrTy, nnkObjectTy}) let isRef = typ.kind in {nnkRefTy, nnkPtrTy} if isRef and typ[0].kind in {nnkSym, nnkBracketExpr}: # defines ref type for another object(e.g. X = ref X) diff --git a/tests/pragmas/tcustom_pragma.nim b/tests/pragmas/tcustom_pragma.nim index 1c3709b26..db2536188 100644 --- a/tests/pragmas/tcustom_pragma.nim +++ b/tests/pragmas/tcustom_pragma.nim @@ -20,16 +20,22 @@ block: MyGenericObj[T] = object myField1, myField2 {.myAttr: "hi".}: int + MyOtherObj = MyObj + var o: MyObj static: doAssert o.myField2.hasCustomPragma(myAttr) doAssert(not o.myField1.hasCustomPragma(myAttr)) + doAssert(not o.myField1.hasCustomPragma(MyObj)) + doAssert(not o.myField1.hasCustomPragma(MyOtherObj)) var ogen: MyGenericObj[int] static: doAssert ogen.myField2.hasCustomPragma(myAttr) doAssert(not ogen.myField1.hasCustomPragma(myAttr)) + doAssert(not ogen.myField1.hasCustomPragma(MyGenericObj)) + doAssert(not ogen.myField1.hasCustomPragma(MyGenericObj)) import custom_pragma From 891329cd4b3e7e3b0d995cf342c1018239ebcf81 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 3 Feb 2022 00:10:11 +0800 Subject: [PATCH 1501/2472] move io out of system (#19442) * move io out of system * fix tests * fix tests * next step * rename to syncio * rename * fix nimscript * comma * fix * fix parts of errors * good for now * fix test --- changelog.md | 2 ++ compiler/pathutils.nim | 3 ++ compiler/vmops.nim | 2 +- doc/tut1.rst | 8 ++--- lib/posix/posix.nim | 3 ++ lib/pure/htmlparser.nim | 3 ++ lib/pure/json.nim | 3 ++ lib/pure/logging.nim | 11 ++++--- lib/pure/memfiles.nim | 4 +-- lib/pure/os.nim | 3 ++ lib/pure/parsecfg.nim | 4 +++ lib/pure/parsecsv.nim | 3 ++ lib/pure/pegs.nim | 2 ++ lib/pure/ropes.nim | 3 ++ lib/pure/streams.nim | 9 ++++-- lib/pure/terminal.nim | 3 ++ lib/pure/xmlparser.nim | 3 ++ lib/{system/io.nim => std/syncio.nim} | 45 +++++++++++++-------------- lib/system.nim | 9 ++++-- lib/system_overview.rst | 1 - lib/windows/winlean.nim | 3 ++ tests/cpp/t6986.nim | 3 ++ tests/test_nimscript.nims | 3 ++ tools/kochdocs.nim | 1 - 24 files changed, 92 insertions(+), 42 deletions(-) rename lib/{system/io.nim => std/syncio.nim} (96%) diff --git a/changelog.md b/changelog.md index bbbaf6e97..412e046eb 100644 --- a/changelog.md +++ b/changelog.md @@ -20,6 +20,8 @@ - `addr` is now available for all addressable locations, `unsafeAddr` is deprecated and becomes an alias for `addr`. +- io is about to move out of system; use `-d:nimPreviewSlimSystem` and import `std/syncio`. + ## Standard library additions and changes - `macros.parseExpr` and `macros.parseStmt` now accept an optional diff --git a/compiler/pathutils.nim b/compiler/pathutils.nim index d8f3613b0..8138a245a 100644 --- a/compiler/pathutils.nim +++ b/compiler/pathutils.nim @@ -12,6 +12,9 @@ import os, pathnorm +when defined(nimSlimSystem): + import std/syncio + type AbsoluteFile* = distinct string AbsoluteDir* = distinct string diff --git a/compiler/vmops.nim b/compiler/vmops.nim index 018e7b9c6..6b06cc68d 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -47,7 +47,7 @@ template systemop(op) {.dirty.} = registerCallback(c, "stdlib.system." & astToStr(op), `op Wrapper`) template ioop(op) {.dirty.} = - registerCallback(c, "stdlib.io." & astToStr(op), `op Wrapper`) + registerCallback(c, "stdlib.syncio." & astToStr(op), `op Wrapper`) template macrosop(op) {.dirty.} = registerCallback(c, "stdlib.macros." & astToStr(op), `op Wrapper`) diff --git a/doc/tut1.rst b/doc/tut1.rst index 3107aef45..405df57b1 100644 --- a/doc/tut1.rst +++ b/doc/tut1.rst @@ -84,8 +84,8 @@ done with spaces only, tabulators are not allowed. String literals are enclosed in double-quotes. The `var` statement declares a new variable named `name` of type `string` with the value that is -returned by the `readLine `_ procedure. Since the -compiler knows that `readLine `_ returns a string, +returned by the `readLine `_ procedure. Since the +compiler knows that `readLine `_ returns a string, you can leave out the type in the declaration (this is called `local type inference`:idx:). So this will work too: @@ -97,7 +97,7 @@ Note that this is basically the only form of type inference that exists in Nim: it is a good compromise between brevity and readability. The "hello world" program contains several identifiers that are already known -to the compiler: `echo`, `readLine `_, etc. +to the compiler: `echo`, `readLine `_, etc. These built-ins are declared in the system_ module which is implicitly imported by any other module. @@ -594,7 +594,7 @@ Procedures ========== To define new commands like `echo `_ -and `readLine `_ in the examples, the concept of a +and `readLine `_ in the examples, the concept of a *procedure* is needed. You might be used to them being called *methods* or *functions* in other languages, but Nim `differentiates these concepts `_. In diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 912e395a7..c10fc6ac9 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -37,6 +37,9 @@ when defined(nimHasStyleChecks): {.push styleChecks: off.} +when defined(nimSlimSystem): + import std/syncio + # TODO these constants don't seem to be fetched from a header file for unknown # platforms - where do they come from and why are they here? when false: diff --git a/lib/pure/htmlparser.nim b/lib/pure/htmlparser.nim index 21dff69ff..6b1300f11 100644 --- a/lib/pure/htmlparser.nim +++ b/lib/pure/htmlparser.nim @@ -51,6 +51,9 @@ import strutils, streams, parsexml, xmltree, unicode, strtabs +when defined(nimPreviewSlimSystem): + import std/syncio + type HtmlTag* = enum ## list of all supported HTML tags; order will always be ## alphabetically diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 922cd4e2f..bdc9fe5ab 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -164,6 +164,9 @@ import hashes, tables, strutils, lexbase, streams, macros, parsejson import options # xxx remove this dependency using same approach as https://github.com/nim-lang/Nim/pull/14563 import std/private/since +when defined(nimPreviewSlimSystem): + import std/syncio + export tables.`$` diff --git a/lib/pure/logging.nim b/lib/pure/logging.nim index b2ace79ab..6751a372a 100644 --- a/lib/pure/logging.nim +++ b/lib/pure/logging.nim @@ -48,7 +48,7 @@ ## .. warning:: ## For loggers that log to a console or to files, only error and fatal ## messages will cause their output buffers to be flushed immediately. -## Use the `flushFile proc `_ to flush the buffer +## Use the `flushFile proc `_ to flush the buffer ## manually if needed. ## ## Handlers @@ -146,6 +146,9 @@ import strutils, times when not defined(js): import os +when defined(nimPreviewSlimSystem): + import std/syncio + type Level* = enum ## \ ## Enumeration of logging levels. @@ -346,7 +349,7 @@ method log*(logger: ConsoleLogger, level: Level, args: varargs[string, `$`]) = ## ## **Note:** Only error and fatal messages will cause the output buffer ## to be flushed immediately. Use the `flushFile proc - ## `_ to flush the buffer manually if needed. + ## `_ to flush the buffer manually if needed. ## ## See also: ## * `log method<#log.e,FileLogger,Level,varargs[string,]>`_ @@ -422,7 +425,7 @@ when not defined(js): ## **Notes:** ## * Only error and fatal messages will cause the output buffer ## to be flushed immediately. Use the `flushFile proc - ## `_ to flush the buffer manually if needed. + ## `_ to flush the buffer manually if needed. ## * This method is not available for the JavaScript backend. ## ## See also: @@ -600,7 +603,7 @@ when not defined(js): ## **Notes:** ## * Only error and fatal messages will cause the output buffer ## to be flushed immediately. Use the `flushFile proc - ## `_ to flush the buffer manually if needed. + ## `_ to flush the buffer manually if needed. ## * This method is not available for the JavaScript backend. ## ## See also: diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim index 407a358fa..f65ca125e 100644 --- a/lib/pure/memfiles.nim +++ b/lib/pure/memfiles.nim @@ -431,7 +431,7 @@ iterator memSlices*(mfile: MemFile, delim = '\l', eat = '\r'): MemSlice {.inline iterator lines*(mfile: MemFile, buf: var string, delim = '\l', eat = '\r'): string {.inline.} = ## Replace contents of passed buffer with each new line, like - ## `readLine(File) `_. + ## `readLine(File) `_. ## `delim`, `eat`, and delimiting logic is exactly as for `memSlices ## <#memSlices.i,MemFile,char,char>`_, but Nim strings are returned. ## @@ -450,7 +450,7 @@ iterator lines*(mfile: MemFile, buf: var string, delim = '\l', iterator lines*(mfile: MemFile, delim = '\l', eat = '\r'): string {.inline.} = ## Return each line in a file as a Nim string, like - ## `lines(File) `_. + ## `lines(File) `_. ## `delim`, `eat`, and delimiting logic is exactly as for `memSlices ## <#memSlices.i,MemFile,char,char>`_, but Nim strings are returned. ## diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 07040d611..76164bc49 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -34,6 +34,9 @@ import std/private/since import strutils, pathnorm +when defined(nimPreviewSlimSystem): + import std/syncio + const weirdTarget = defined(nimscript) or defined(js) since (1, 1): diff --git a/lib/pure/parsecfg.nim b/lib/pure/parsecfg.nim index 0ee19912c..54584a253 100644 --- a/lib/pure/parsecfg.nim +++ b/lib/pure/parsecfg.nim @@ -175,8 +175,12 @@ import strutils, lexbase, streams, tables import std/private/decode_helpers import std/private/since +when defined(nimPreviewSlimSystem): + import std/syncio + include "system/inclrtl" + type CfgEventKind* = enum ## enumeration of all events that may occur when parsing cfgEof, ## end of file reached diff --git a/lib/pure/parsecsv.nim b/lib/pure/parsecsv.nim index 6db794682..a8d1cfaab 100644 --- a/lib/pure/parsecsv.nim +++ b/lib/pure/parsecsv.nim @@ -67,6 +67,9 @@ import lexbase, streams +when defined(nimPreviewSlimSystem): + import std/syncio + type CsvRow* = seq[string] ## A row in a CSV file. CsvParser* = object of BaseLexer ## The parser object. diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index bac8b1a0e..0eed1c388 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -16,6 +16,8 @@ ## include "system/inclrtl" +when defined(nimPreviewSlimSystem): + import std/syncio const useUnicode = true ## change this to deactivate proper UTF-8 support diff --git a/lib/pure/ropes.nim b/lib/pure/ropes.nim index 1300b4479..84b5b47c2 100644 --- a/lib/pure/ropes.nim +++ b/lib/pure/ropes.nim @@ -19,6 +19,9 @@ include system/inclrtl import streams +when defined(nimPreviewSlimSystem): + import std/syncio + {.push debugger: off.} # the user does not want to trace a part # of the standard library! diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim index 7dc81148f..7ad81685f 100644 --- a/lib/pure/streams.nim +++ b/lib/pure/streams.nim @@ -92,10 +92,13 @@ ## See also ## ======== ## * `asyncstreams module `_ -## * `io module `_ for `FileMode enum `_ +## * `io module `_ for `FileMode enum `_ import std/private/since +when defined(nimPreviewSlimSystem): + import std/syncio + proc newEIO(msg: string): owned(ref IOError) = new(result) result.msg = msg @@ -1331,7 +1334,7 @@ proc newFileStream*(f: File): owned FileStream = ## * `newStringStream proc <#newStringStream,string>`_ creates a new stream ## from string. ## * `newFileStream proc <#newFileStream,string,FileMode,int>`_ is the same - ## as using `open proc `_ + ## as using `open proc `_ ## on Examples. ## * `openFileStream proc <#openFileStream,string,FileMode,int>`_ creates a ## file stream from the file name and the mode. @@ -1370,7 +1373,7 @@ proc newFileStream*(filename: string, mode: FileMode = fmRead, ## Creates a new stream from the file named `filename` with the mode `mode`. ## ## If the file cannot be opened, `nil` is returned. See the `io module - ## `_ for a list of available `FileMode enums `_. + ## `_ for a list of available `FileMode enums `_. ## ## **Note:** ## * **This function returns nil in case of failure.** diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index c9aafc037..5755e142a 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -66,6 +66,9 @@ import colors when defined(windows): import winlean +when defined(nimPreviewSlimSystem): + import std/syncio + type PTerminal = ref object trueColorIsSupported: bool diff --git a/lib/pure/xmlparser.nim b/lib/pure/xmlparser.nim index 3d9c288ed..6785fa66e 100644 --- a/lib/pure/xmlparser.nim +++ b/lib/pure/xmlparser.nim @@ -11,6 +11,9 @@ import streams, parsexml, strtabs, xmltree +when defined(nimPreviewSlimSystem): + import std/syncio + type XmlError* = object of ValueError ## Exception that is raised ## for invalid XML. diff --git a/lib/system/io.nim b/lib/std/syncio.nim similarity index 96% rename from lib/system/io.nim rename to lib/std/syncio.nim index 04a43328d..eab96254a 100644 --- a/lib/system/io.nim +++ b/lib/std/syncio.nim @@ -1,18 +1,17 @@ # # # Nim's Runtime Library -# (c) Copyright 2019 Nim contributors +# (c) Copyright 2022 Nim contributors # # See the file "copying.txt", included in this # distribution, for details about the copyright. # -## This is a part of `system.nim`, you should not manually import it. +## This module implements various synchronized I/O operations. - -include inclrtl +include system/inclrtl import std/private/since -import formatfloat +import system/formatfloat # ----------------- IO Part ------------------------------------------------ type @@ -179,7 +178,7 @@ proc checkErr(f: File) = {.push stackTrace: off, profiler: off.} proc readBuffer*(f: File, buffer: pointer, len: Natural): int {. tags: [ReadIOEffect], benign.} = - ## reads `len` bytes into the buffer pointed to by `buffer`. Returns + ## Reads `len` bytes into the buffer pointed to by `buffer`. Returns ## the actual number of bytes that have been read which may be less than ## `len` (if not as many bytes are remaining), but not greater. result = cast[int](c_fread(buffer, 1, cast[csize_t](len), f)) @@ -188,13 +187,13 @@ proc readBuffer*(f: File, buffer: pointer, len: Natural): int {. proc readBytes*(f: File, a: var openArray[int8|uint8], start, len: Natural): int {. tags: [ReadIOEffect], benign.} = - ## reads `len` bytes into the buffer `a` starting at `a[start]`. Returns + ## Reads `len` bytes into the buffer `a` starting at `a[start]`. Returns ## the actual number of bytes that have been read which may be less than ## `len` (if not as many bytes are remaining), but not greater. result = readBuffer(f, addr(a[start]), len) proc readChars*(f: File, a: var openArray[char]): int {.tags: [ReadIOEffect], benign.} = - ## reads up to `a.len` bytes into the buffer `a`. Returns + ## Reads up to `a.len` bytes into the buffer `a`. Returns ## the actual number of bytes that have been read which may be less than ## `a.len` (if not as many bytes are remaining), but not greater. result = readBuffer(f, addr(a[0]), a.len) @@ -202,7 +201,7 @@ proc readChars*(f: File, a: var openArray[char]): int {.tags: [ReadIOEffect], be proc readChars*(f: File, a: var openArray[char], start, len: Natural): int {. tags: [ReadIOEffect], benign, deprecated: "use other `readChars` overload, possibly via: readChars(toOpenArray(buf, start, len-1))".} = - ## reads `len` bytes into the buffer `a` starting at `a[start]`. Returns + ## Reads `len` bytes into the buffer `a` starting at `a[start]`. Returns ## the actual number of bytes that have been read which may be less than ## `len` (if not as many bytes are remaining), but not greater. if (start + len) > len(a): @@ -216,7 +215,7 @@ proc write*(f: File, c: cstring) {.tags: [WriteIOEffect], benign.} = proc writeBuffer*(f: File, buffer: pointer, len: Natural): int {. tags: [WriteIOEffect], benign.} = - ## writes the bytes of buffer pointed to by the parameter `buffer` to the + ## Writes the bytes of buffer pointed to by the parameter `buffer` to the ## file `f`. Returns the number of actual written bytes, which may be less ## than `len` in case of an error. result = cast[int](c_fwrite(buffer, 1, cast[csize_t](len), f)) @@ -224,7 +223,7 @@ proc writeBuffer*(f: File, buffer: pointer, len: Natural): int {. proc writeBytes*(f: File, a: openArray[int8|uint8], start, len: Natural): int {. tags: [WriteIOEffect], benign.} = - ## writes the bytes of `a[start..start+len-1]` to the file `f`. Returns + ## Writes the bytes of `a[start..start+len-1]` to the file `f`. Returns ## the number of actual written bytes, which may be less than `len` in case ## of an error. var x = cast[ptr UncheckedArray[int8]](a) @@ -232,7 +231,7 @@ proc writeBytes*(f: File, a: openArray[int8|uint8], start, len: Natural): int {. proc writeChars*(f: File, a: openArray[char], start, len: Natural): int {. tags: [WriteIOEffect], benign.} = - ## writes the bytes of `a[start..start+len-1]` to the file `f`. Returns + ## Writes the bytes of `a[start..start+len-1]` to the file `f`. Returns ## the number of actual written bytes, which may be less than `len` in case ## of an error. var x = cast[ptr UncheckedArray[int8]](a) @@ -341,7 +340,7 @@ proc flushFile*(f: File) {.tags: [WriteIOEffect].} = discard c_fflush(f) proc getFileHandle*(f: File): FileHandle = - ## returns the file handle of the file `f`. This is only useful for + ## Returns the file handle of the file `f`. This is only useful for ## platform specific programming. ## Note that on Windows this doesn't return the Windows-specific handle, ## but the C library's notion of a handle, whatever that means. @@ -349,7 +348,7 @@ proc getFileHandle*(f: File): FileHandle = c_fileno(f) proc getOsFileHandle*(f: File): FileHandle = - ## returns the OS file handle of the file `f`. This is only useful for + ## Returns the OS file handle of the file `f`. This is only useful for ## platform specific programming. when defined(windows): result = FileHandle getOsfhandle(cint getFileHandle(f)) @@ -380,7 +379,7 @@ when defined(nimdoc) or (defined(posix) and not defined(nimscript)) or defined(w proc readLine*(f: File, line: var string): bool {.tags: [ReadIOEffect], benign.} = - ## reads a line of text from the file `f` into `line`. May throw an IO + ## Reads a line of text from the file `f` into `line`. May throw an IO ## exception. ## A line of text may be delimited by `LF` or `CRLF`. The newline ## character(s) are not part of the returned string. Returns `false` @@ -500,7 +499,7 @@ proc readLine*(f: File, line: var string): bool {.tags: [ReadIOEffect], line.setLen(pos+sp) proc readLine*(f: File): string {.tags: [ReadIOEffect], benign.} = - ## reads a line of text from the file `f`. May throw an IO exception. + ## Reads a line of text from the file `f`. May throw an IO exception. ## A line of text may be delimited by `LF` or `CRLF`. The newline ## character(s) are not part of the returned string. result = newStringOfCap(80) @@ -602,7 +601,7 @@ proc readAll*(file: File): string {.tags: [ReadIOEffect], benign.} = proc writeLine*[Ty](f: File, x: varargs[Ty, `$`]) {.inline, tags: [WriteIOEffect], benign.} = - ## writes the values `x` to `f` and then writes "\\n". + ## Writes the values `x` to `f` and then writes "\\n". ## May throw an IO exception. for i in items(x): write(f, i) @@ -723,7 +722,7 @@ proc open*(f: var File, filename: string, proc reopen*(f: File, filename: string, mode: FileMode = fmRead): bool {. tags: [], benign.} = - ## reopens the file `f` with given `filename` and `mode`. This + ## Reopens the file `f` with given `filename` and `mode`. This ## is often used to redirect the `stdin`, `stdout` or `stderr` ## file variables. ## @@ -765,19 +764,19 @@ proc open*(filename: string, sysFatal(IOError, "cannot open: " & filename) proc setFilePos*(f: File, pos: int64, relativeTo: FileSeekPos = fspSet) {.benign.} = - ## sets the position of the file pointer that is used for read/write + ## Sets the position of the file pointer that is used for read/write ## operations. The file's first byte has the index zero. if c_fseek(f, pos, cint(relativeTo)) != 0: raiseEIO("cannot set file position") proc getFilePos*(f: File): int64 {.benign.} = - ## retrieves the current position of the file pointer that is used to + ## Retrieves the current position of the file pointer that is used to ## read from the file `f`. The file's first byte has the index zero. result = c_ftell(f) if result < 0: raiseEIO("cannot retrieve file position") proc getFileSize*(f: File): int64 {.tags: [ReadIOEffect], benign.} = - ## retrieves the file size (in bytes) of `f`. + ## Retrieves the file size (in bytes) of `f`. let oldPos = getFilePos(f) discard c_fseek(f, 0, 2) # seek the end of the file result = getFilePos(f) @@ -906,7 +905,7 @@ proc writeFile*(filename: string, content: openArray[byte]) {.since: (1, 1).} = raise newException(IOError, "cannot open: " & filename) proc readLines*(filename: string, n: Natural): seq[string] = - ## read `n` lines from the file named `filename`. Raises an IO exception + ## Reads `n` lines from the file named `filename`. Raises an IO exception ## in case of an error. Raises EOF if file does not contain at least `n` lines. ## Available at compile time. A line of text may be delimited by `LF` or `CRLF`. ## The newline character(s) are not part of the returned strings. @@ -948,7 +947,7 @@ iterator lines*(filename: string): string {.tags: [ReadIOEffect].} = close(f) iterator lines*(f: File): string {.tags: [ReadIOEffect].} = - ## Iterate over any line in the file `f`. + ## Iterates over any line in the file `f`. ## ## The trailing newline character(s) are removed from the iterated lines. ## diff --git a/lib/system.nim b/lib/system.nim index 63cdc3ca8..56a6e2887 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2680,7 +2680,7 @@ proc slurp*(filename: string): string {.magic: "Slurp".} ## This is an alias for `staticRead <#staticRead,string>`_. proc staticRead*(filename: string): string {.magic: "Slurp".} - ## Compile-time `readFile `_ proc for easy + ## Compile-time `readFile `_ proc for easy ## `resource`:idx: embedding: ## ## The maximum file size limit that `staticRead` and `slurp` can read is @@ -3150,8 +3150,11 @@ when defined(genode): import system/widestrs export widestrs -import system/io -export io +when not defined(nimPreviewSlimSystem): + {.deprecated: """io is about to move out of system; use `-d:nimPreviewSlimSystem` and + import `std/syncio`.""".} + import std/syncio + export syncio when not defined(createNimHcr) and not defined(nimscript): include nimhcr diff --git a/lib/system_overview.rst b/lib/system_overview.rst index 0e4ffaf17..768fdcd6d 100644 --- a/lib/system_overview.rst +++ b/lib/system_overview.rst @@ -7,7 +7,6 @@ is in separate files: * `iterators `_ * `assertions `_ * `dollars `_ -* `io `_ * `widestrs `_ diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index 6e626e4ba..c02cd9775 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -17,6 +17,9 @@ when defined(nimHasStyleChecks): {.passc: "-DWIN32_LEAN_AND_MEAN".} +when defined(nimPreviewSlimSystem): + from std/syncio import FileHandle + const useWinUnicode* = not defined(useWinAnsi) diff --git a/tests/cpp/t6986.nim b/tests/cpp/t6986.nim index ffd277adb..16e455c3b 100644 --- a/tests/cpp/t6986.nim +++ b/tests/cpp/t6986.nim @@ -5,6 +5,9 @@ discard """ import sequtils, strutils +when defined(nimPreviewSlimSystem): + import std/syncio + let rules = toSeq(lines("input")) .mapIt(it.split(" => ").mapIt(it.replace("/", ""))) diff --git a/tests/test_nimscript.nims b/tests/test_nimscript.nims index 22a566307..b380570ff 100644 --- a/tests/test_nimscript.nims +++ b/tests/test_nimscript.nims @@ -5,6 +5,9 @@ from stdtest/specialpaths import buildDir +when defined(nimSlimSystem): + import std/syncio + import std/[ # Core: bitops, typetraits, lenientops, macros, volatile, diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 4fbea8c77..8fd9c84a5 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -187,7 +187,6 @@ proc getDocList(): seq[string] = # don't ignore these even though in lib/system (not include files) const goodSystem = """ -lib/system/io.nim lib/system/nimscript.nim lib/system/assertions.nim lib/system/iterators.nim From 772ed5eff29671b1e99ca17b63c646b341c88123 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 3 Feb 2022 11:15:46 +0800 Subject: [PATCH 1502/2472] correct typos (#19485) --- compiler/pathutils.nim | 2 +- lib/posix/posix.nim | 2 +- tests/test_nimscript.nims | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/pathutils.nim b/compiler/pathutils.nim index 8138a245a..3a501a417 100644 --- a/compiler/pathutils.nim +++ b/compiler/pathutils.nim @@ -12,7 +12,7 @@ import os, pathnorm -when defined(nimSlimSystem): +when defined(nimPreviewSlimSystem): import std/syncio type diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index c10fc6ac9..9c65d0732 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -37,7 +37,7 @@ when defined(nimHasStyleChecks): {.push styleChecks: off.} -when defined(nimSlimSystem): +when defined(nimPreviewSlimSystem): import std/syncio # TODO these constants don't seem to be fetched from a header file for unknown diff --git a/tests/test_nimscript.nims b/tests/test_nimscript.nims index b380570ff..5fc77f6e4 100644 --- a/tests/test_nimscript.nims +++ b/tests/test_nimscript.nims @@ -5,7 +5,7 @@ from stdtest/specialpaths import buildDir -when defined(nimSlimSystem): +when defined(nimPreviewSlimSystem): import std/syncio import std/[ From 33bfe5b12cf771072d93d7316945f3f5e6c727ff Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 4 Feb 2022 20:47:03 +0800 Subject: [PATCH 1503/2472] fix parseEnum cannot parse enum with const fields (#19466) fix #19463 --- lib/std/enumutils.nim | 9 +++++++-- tests/stdlib/tstrutils.nim | 11 +++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/lib/std/enumutils.nim b/lib/std/enumutils.nim index 81e602ad5..b7d2b9f89 100644 --- a/lib/std/enumutils.nim +++ b/lib/std/enumutils.nim @@ -14,7 +14,7 @@ from typetraits import OrdinalEnum, HoleyEnum macro genEnumCaseStmt*(typ: typedesc, argSym: typed, default: typed, userMin, userMax: static[int], normalizer: static[proc(s :string): string]): untyped = - # generates a case stmt, which assigns the correct enum field given + # Generates a case stmt, which assigns the correct enum field given # a normalized string comparison to the `argSym` input. # string normalization is done using passed normalizer. # NOTE: for an enum with fields Foo, Bar, ... we cannot generate @@ -49,7 +49,12 @@ macro genEnumCaseStmt*(typ: typedesc, argSym: typed, default: typed, of nnkIntLit: fStr = f[0].strVal fNum = f[1].intVal - else: error("Invalid tuple syntax!", f[1]) + else: + let fAst = f[0].getImpl + if fAst.kind == nnkStrLit: + fStr = fAst.strVal + else: + error("Invalid tuple syntax!", f[1]) else: error("Invalid node for enum type `" & $f.kind & "`!", f) # add field if string not already added if fNum >= userMin and fNum <= userMax: diff --git a/tests/stdlib/tstrutils.nim b/tests/stdlib/tstrutils.nim index 8000b6b07..26fc0c7b0 100644 --- a/tests/stdlib/tstrutils.nim +++ b/tests/stdlib/tstrutils.nim @@ -583,6 +583,17 @@ template main() = let g = parseEnum[Foo]("Bar", A) doAssert g == A + block: # bug #19463 + const CAMPAIGN_TABLE = "wikientries_campaign" + const CHARACTER_TABLE = "wikientries_character" + + type Tables = enum + a = CAMPAIGN_TABLE, + b = CHARACTER_TABLE, + + let myA = CAMPAIGN_TABLE + doAssert $parseEnum[Tables](myA) == "wikientries_campaign" + block: # check enum defined in block type Bar = enum From 7b09fd70ab8154eed3738dfd86f84c5642df3cf2 Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 5 Feb 2022 06:10:24 +0800 Subject: [PATCH 1504/2472] undefine C symbols in JS backend [backport:1.6] (#19437) fix #19330; fix #19059 --- compiler/cmdlinehelper.nim | 3 ++- compiler/commands.nim | 5 +++-- tests/js/tcsymbol.nim | 6 ++++++ 3 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 tests/js/tcsymbol.nim diff --git a/compiler/cmdlinehelper.nim b/compiler/cmdlinehelper.nim index 007dc2319..02162755c 100644 --- a/compiler/cmdlinehelper.nim +++ b/compiler/cmdlinehelper.nim @@ -64,7 +64,8 @@ proc loadConfigsAndProcessCmdLine*(self: NimProg, cache: IdentCache; conf: Confi if conf.cmd == cmdNimscript: return false # now process command line arguments again, because some options in the # command line can overwrite the config file's settings - extccomp.initVars(conf) + if conf.backend != backendJs: # bug #19059 + extccomp.initVars(conf) self.processCmdLine(passCmd2, "", conf) if conf.cmd == cmdNone: rawMessage(conf, errGenerated, "command missing") diff --git a/compiler/commands.nim b/compiler/commands.nim index a1fa55e74..fb2907bb1 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -899,8 +899,9 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; splitSwitch(conf, arg, key, val, pass, info) os.putEnv(key, val) of "cc": - expectArg(conf, switch, arg, pass, info) - setCC(conf, arg, info) + if conf.backend != backendJs: # bug #19330 + expectArg(conf, switch, arg, pass, info) + setCC(conf, arg, info) of "track": expectArg(conf, switch, arg, pass, info) track(conf, arg, info) diff --git a/tests/js/tcsymbol.nim b/tests/js/tcsymbol.nim new file mode 100644 index 000000000..07e52b9b6 --- /dev/null +++ b/tests/js/tcsymbol.nim @@ -0,0 +1,6 @@ +discard """ + matrix: "--cc:gcc; --cc:tcc" +""" + +doAssert not defined(gcc) +doAssert not defined(tcc) \ No newline at end of file From d0287748fefd9eae71cb3d66caef724473525b28 Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 6 Feb 2022 15:51:03 +0800 Subject: [PATCH 1505/2472] setjump => setjmp [backport: 1.2] (#19496) --- compiler/commands.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index fb2907bb1..6d3f23197 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -238,7 +238,7 @@ const errNoneBoehmRefcExpectedButXFound = "'arc', 'orc', 'markAndSweep', 'boehm', 'go', 'none', 'regions', or 'refc' expected, but '$1' found" errNoneSpeedOrSizeExpectedButXFound = "'none', 'speed' or 'size' expected, but '$1' found" errGuiConsoleOrLibExpectedButXFound = "'gui', 'console' or 'lib' expected, but '$1' found" - errInvalidExceptionSystem = "'goto', 'setjump', 'cpp' or 'quirky' expected, but '$1' found" + errInvalidExceptionSystem = "'goto', 'setjmp', 'cpp' or 'quirky' expected, but '$1' found" template warningOptionNoop(switch: string) = warningDeprecated(conf, info, "'$#' is deprecated, now a noop" % switch) From 6b302bb71b6d9aab708b91f8d68957652ec30480 Mon Sep 17 00:00:00 2001 From: Roj Date: Sun, 6 Feb 2022 18:28:54 -0800 Subject: [PATCH 1506/2472] Improve consistency (#19490) --- readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index 6bc7a2385..7ff056a20 100644 --- a/readme.md +++ b/readme.md @@ -28,7 +28,7 @@ the latest release, check out [Nim's website][nim-site] or [bleeding edge docs]( is the official Telegram channel. Not bridged to IRC. * [Stack Overflow][nim-stackoverflow] - a popular Q/A site for programming related topics that includes posts about Nim. -* [Github Wiki][nim-wiki] - Misc user-contributed content. +* [GitHub Wiki][nim-wiki] - Misc user-contributed content. ## Compiling @@ -73,7 +73,7 @@ the installation instructions on the website to do so: https://nim-lang.org/inst For package maintainers: see [packaging guidelines](https://nim-lang.github.io/Nim/packaging.html). -First, get Nim from github: +First, get Nim from GitHub: ``` git clone https://github.com/nim-lang/Nim.git From 28180e47a99cda88abee3e2d8e130c18100f3a4f Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 7 Feb 2022 14:04:04 +0800 Subject: [PATCH 1507/2472] disable nimlsp (#19499) --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 40a3aef36..456f42cd2 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -103,7 +103,7 @@ pkg "nimfp", "nim c -o:nfp -r src/fp.nim" pkg "nimgame2", "nim c -d:nimLegacyConvEnumEnum nimgame2/nimgame.nim" # XXX Doesn't work with deprecated 'randomize', will create a PR. pkg "nimgen", "nim c -o:nimgenn -r src/nimgen/runcfg.nim" -pkg "nimlsp" +pkg "nimlsp", allowFailure = true pkg "nimly", "nim c -r tests/test_readme_example.nim" pkg "nimongo", "nimble test_ci", allowFailure = true pkg "nimph", "nimble test", "https://github.com/disruptek/nimph", allowFailure = true From 4b0636fba018df407237811f9e687ed4fc249c10 Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Mon, 7 Feb 2022 23:05:41 +0000 Subject: [PATCH 1508/2472] Add compilers and hints to default nim.cfg (#18424) --- config/nim.cfg | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/config/nim.cfg b/config/nim.cfg index 59d7f48d2..7fe279f56 100644 --- a/config/nim.cfg +++ b/config/nim.cfg @@ -2,6 +2,7 @@ # (c) 2017 Andreas Rumpf # Feel free to edit the default values as you need. +# See https://nim-lang.org/docs/nimc.html # You may set environment variables with # @putenv "key" "val" @@ -17,11 +18,21 @@ hint[LineTooLong]=off #hint[XDeclaredButNotUsed]=off # Examples of how to setup a cross-compiler: +# Nim can target architectures and OSes different than the local host +# Syntax: ..gcc.exe = "" +# ..gcc.linkerexe = "" -# Cross-compiling for Raspberry Pi. -# (This compiler is available in gcc-arm-linux-gnueabihf package on Ubuntu) +# ARM e.g. Raspberry Pi 2: gcc-arm-linux-gnueabihf package on Debian/Ubuntu arm.linux.gcc.exe = "arm-linux-gnueabihf-gcc" arm.linux.gcc.linkerexe = "arm-linux-gnueabihf-gcc" +# ARM64/aarch64 e.g. Raspberry Pi 3: gcc-aarch64-linux-gnu package on Debian/Ubuntu +arm64.linux.gcc.exe = "aarch64-linux-gnu-gcc" +arm64.linux.gcc.linkerexe = "aarch64-linux-gnu-gcc" +# RISC-V: gcc-riscv64-linux-gnu package on Debian/Ubuntu +riscv32.linux.gcc.exe = "riscv64-linux-gnu-gcc" +riscv32.linux.gcc.linkerexe = "riscv64-linux-gnu-gcc" +riscv64.linux.gcc.exe = "riscv64-linux-gnu-gcc" +riscv64.linux.gcc.linkerexe = "arm-linux-gnueabihf-gcc" # For OpenWRT, you will also need to adjust PATH to point to your toolchain. mips.linux.gcc.exe = "mips-openwrt-linux-gcc" From 801c0f0369779a47da2ef6618abea11feb390973 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Tue, 8 Feb 2022 02:11:53 +0300 Subject: [PATCH 1509/2472] Fix bug 27 of #17340 (#19433) Fixes silent disappearance of Markdown (pseudo-)link when it's detected as unsafe protocol. Now it will be converted to plain text in spirit of [the specification](https://spec.commonmark.org/0.30/#links). For that sake the check for protocol is added to rst.nim also. --- lib/packages/docutils/rst.nim | 66 +++++++++++++++++++++++--------- lib/packages/docutils/rstgen.nim | 17 ++------ tests/stdlib/trst.nim | 27 ++++++++++--- tests/stdlib/trstgen.nim | 11 +++++- 4 files changed, 81 insertions(+), 40 deletions(-) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 93c2231dc..d4bfae48f 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -222,7 +222,7 @@ import os, strutils, rstast, dochelpers, std/enumutils, algorithm, lists, sequtils, - std/private/miscdollars, tables + std/private/miscdollars, tables, strscans from highlite import SourceLanguage, getSourceLanguage type @@ -1347,7 +1347,20 @@ proc match(p: RstParser, start: int, expr: string): bool = inc i result = true -proc fixupEmbeddedRef(n, a, b: PRstNode) = +proc safeProtocol*(linkStr: var string): string = + # Returns link's protocol and, if it's not safe, clears `linkStr` + result = "" + if scanf(linkStr, "$w:", result): + # if it has a protocol at all, ensure that it's not 'javascript:' or worse: + if cmpIgnoreCase(result, "http") == 0 or + cmpIgnoreCase(result, "https") == 0 or + cmpIgnoreCase(result, "ftp") == 0: + discard "it's fine" + else: + linkStr = "" + +proc fixupEmbeddedRef(p: var RstParser, n, a, b: PRstNode): bool = + # Returns `true` if the link belongs to an allowed protocol var sep = - 1 for i in countdown(n.len - 2, 0): if n.sons[i].text == "<": @@ -1355,7 +1368,15 @@ proc fixupEmbeddedRef(n, a, b: PRstNode) = break var incr = if sep > 0 and n.sons[sep - 1].text[0] == ' ': 2 else: 1 for i in countup(0, sep - incr): a.add(n.sons[i]) - for i in countup(sep + 1, n.len - 2): b.add(n.sons[i]) + var linkStr = "" + for i in countup(sep + 1, n.len - 2): linkStr.add(n.sons[i].addNodes) + if linkStr != "": + let protocol = safeProtocol(linkStr) + result = linkStr != "" + if not result: + rstMessage(p, mwBrokenLink, protocol, + p.tok[p.idx-3].line, p.tok[p.idx-3].col) + b.add newLeaf(linkStr) proc whichRole(p: RstParser, sym: string): RstNodeKind = result = whichRoleAux(sym) @@ -1407,14 +1428,17 @@ proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode = if p.tok[p.idx-2].symbol == "`" and p.tok[p.idx-3].symbol == ">": var a = newRstNode(rnInner) var b = newRstNode(rnInner) - fixupEmbeddedRef(n, a, b) - if a.len == 0: - newKind = rnStandaloneHyperlink - newSons = @[b] - else: - newKind = rnHyperlink - newSons = @[a, b] - setRef(p, rstnodeToRefname(a), b, implicitHyperlinkAlias) + if fixupEmbeddedRef(p, n, a, b): + if a.len == 0: # e.g. ``_ + newKind = rnStandaloneHyperlink + newSons = @[b] + else: # e.g. `link title `_ + newKind = rnHyperlink + newSons = @[a, b] + setRef(p, rstnodeToRefname(a), b, implicitHyperlinkAlias) + else: # include as plain text, not a link + newKind = rnInner + newSons = n.sons result = newRstNode(newKind, newSons) else: # some link that will be resolved in `resolveSubs` newKind = rnRef @@ -1623,7 +1647,6 @@ proc parseMarkdownCodeblock(p: var RstParser): PRstNode = result.add(lb) proc parseMarkdownLink(p: var RstParser; father: PRstNode): bool = - result = true var desc, link = "" var i = p.idx @@ -1642,14 +1665,21 @@ proc parseMarkdownLink(p: var RstParser; father: PRstNode): bool = parse("]", desc) if p.tok[i].symbol != "(": return false + let linkIdx = i + 1 parse(")", link) - let child = newRstNode(rnHyperlink) - child.add desc - child.add link # only commit if we detected no syntax error: - father.add child - p.idx = i - result = true + let protocol = safeProtocol(link) + if link == "": + result = false + rstMessage(p, mwBrokenLink, protocol, + p.tok[linkIdx].line, p.tok[linkIdx].col) + else: + let child = newRstNode(rnHyperlink) + child.add desc + child.add link + father.add child + p.idx = i + result = true proc getFootnoteType(label: PRstNode): (FootnoteType, int) = if label.sons.len >= 1 and label.sons[0].kind == rnLeaf and diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index d2180cb91..d662a667c 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -40,7 +40,7 @@ ## can be done by simply searching for [footnoteName]. import strutils, os, hashes, strtabs, rstast, rst, highlite, tables, sequtils, - algorithm, parseutils, std/strbasics, strscans + algorithm, parseutils, std/strbasics import ../../std/private/since @@ -826,17 +826,6 @@ proc renderOverline(d: PDoc, n: PRstNode, result: var string) = "\\rstov$4[$5]{$3}$2\n", [$n.level, rstnodeToRefname(n).idS, tmp, $chr(n.level - 1 + ord('A')), tocName]) - -proc safeProtocol(linkStr: var string) = - var protocol = "" - if scanf(linkStr, "$w:", protocol): - # if it has a protocol at all, ensure that it's not 'javascript:' or worse: - if cmpIgnoreCase(protocol, "http") == 0 or cmpIgnoreCase(protocol, "https") == 0 or - cmpIgnoreCase(protocol, "ftp") == 0: - discard "it's fine" - else: - linkStr = "" - proc renderTocEntry(d: PDoc, e: TocEntry, result: var string) = dispA(d.target, result, "

            • $2
              • \n", @@ -901,7 +890,7 @@ proc renderImage(d: PDoc, n: PRstNode, result: var string) = # support for `:target:` links for images: var target = esc(d.target, getFieldValue(n, "target").strip(), escMode=emUrl) - safeProtocol(target) + discard safeProtocol(target) if target.len > 0: # `htmlOut` needs to be of the following format for link to work for images: @@ -1205,7 +1194,7 @@ proc renderHyperlink(d: PDoc, text, link: PRstNode, result: var string, d.escMode = emUrl renderRstToOut(d, link, linkStr) d.escMode = mode - safeProtocol(linkStr) + discard safeProtocol(linkStr) var textStr = "" renderRstToOut(d, text, textStr) let nimDocStr = if nimdoc: " nimdoc" else: "" diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index 771e02477..8b98d1403 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -843,12 +843,7 @@ suite "Warnings": rnInner rnLeaf 'foo' rnInner - rnLeaf '#' - rnLeaf 'foo' - rnLeaf ',' - rnLeaf 'string' - rnLeaf ',' - rnLeaf 'string' + rnLeaf '#foo,string,string' rnParagraph anchor='foo' rnLeaf 'Paragraph' rnLeaf '.' @@ -1256,3 +1251,23 @@ suite "RST inline markup": rnLeaf 'my {link example' rnLeaf 'http://example.com/bracket_(symbol_[)' """) + + test "not a Markdown link": + # bug #17340 (27) `f` will be considered as a protocol and blocked as unsafe + var warnings = new seq[string] + check("[T](f: var Foo)".toAst(warnings = warnings) == + dedent""" + rnInner + rnLeaf '[' + rnLeaf 'T' + rnLeaf ']' + rnLeaf '(' + rnLeaf 'f' + rnLeaf ':' + rnLeaf ' ' + rnLeaf 'var' + rnLeaf ' ' + rnLeaf 'Foo' + rnLeaf ')' + """) + check(warnings[] == @["input(1, 5) Warning: broken link 'f'"]) diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index 995c0a151..2c28fe506 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -1593,8 +1593,15 @@ suite "invalid targets": test "invalid links": check("(([Nim](https://nim-lang.org/)))".toHtml == """((Nim))""") - check("(([Nim](javascript://nim-lang.org/)))".toHtml == - """((Nim))""") + # unknown protocol is treated just like plain text, not a link + var warnings = new seq[string] + check("(([Nim](javascript://nim-lang.org/)))".toHtml(warnings=warnings) == + """(([Nim](javascript://nim-lang.org/)))""") + check(warnings[] == @["input(1, 9) Warning: broken link 'javascript'"]) + warnings[].setLen 0 + check("`Nim `_".toHtml(warnings=warnings) == + """Nim <javascript://nim-lang.org/>""") + check(warnings[] == @["input(1, 33) Warning: broken link 'javascript'"]) suite "local file inclusion": test "cannot include files in sandboxed mode": From 27e548140b0f51a8dbc6f6094764e02f08ae509e Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 8 Feb 2022 15:34:44 +0800 Subject: [PATCH 1510/2472] don't use a temp for addr [backport: 1.6] (#19503) * don't use a temp for addr fix #19497 * Update compiler/ccgcalls.nim Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * add a test Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- compiler/ccgcalls.nim | 4 ++-- tests/arc/tarc_orc.nim | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 tests/arc/tarc_orc.nim diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index 197728ccf..ce5fbfdd7 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -376,8 +376,8 @@ proc genParams(p: BProc, ri: PNode, typ: PType): Rope = if not needTmp[i - 1]: needTmp[i - 1] = potentialAlias(n, potentialWrites) getPotentialWrites(ri[i], false, potentialWrites) - if ri[i].kind == nkHiddenAddr: - # Optimization: don't use a temp, if we would only take the adress anyway + if ri[i].kind in {nkHiddenAddr, nkAddr}: + # Optimization: don't use a temp, if we would only take the address anyway needTmp[i - 1] = false for i in 1.. Date: Fri, 11 Feb 2022 18:16:27 +0800 Subject: [PATCH 1511/2472] fix #16458; make useNimRtl compile for --gc:orc (#19512) * fix #16458; make useNimRtl compile for --gc:orc/arc * fix tests --- lib/system/mmdisp.nim | 5 +++-- lib/system/orc.nim | 4 ++-- lib/system/seqs_v2.nim | 3 ++- tests/arc/t16458.nim | 6 ++++++ 4 files changed, 13 insertions(+), 5 deletions(-) create mode 100644 tests/arc/t16458.nim diff --git a/lib/system/mmdisp.nim b/lib/system/mmdisp.nim index 3317ba627..e5038387f 100644 --- a/lib/system/mmdisp.nim +++ b/lib/system/mmdisp.nim @@ -82,8 +82,9 @@ else: # XXX due to bootstrapping reasons, we cannot use compileOption("gc", "stack") here include "system/gc_regions" elif defined(nimV2) or usesDestructors: - var allocator {.rtlThreadVar.}: MemRegion - instantiateForRegion(allocator) + when not defined(useNimRtl): + var allocator {.rtlThreadVar.}: MemRegion + instantiateForRegion(allocator) when defined(gcHooks): include "system/gc_hooks" elif defined(gcMarkAndSweep): diff --git a/lib/system/orc.nim b/lib/system/orc.nim index f9f5cd81f..4c23aea6c 100644 --- a/lib/system/orc.nim +++ b/lib/system/orc.nim @@ -119,7 +119,7 @@ template orcAssert(cond, msg) = when logOrc: proc strstr(s, sub: cstring): cstring {.header: "", importc.} -proc nimTraceRef(q: pointer; desc: PNimTypeV2; env: pointer) {.compilerRtl, inline.} = +proc nimTraceRef(q: pointer; desc: PNimTypeV2; env: pointer) {.compilerRtl, inl.} = let p = cast[ptr pointer](q) if p[] != nil: @@ -128,7 +128,7 @@ proc nimTraceRef(q: pointer; desc: PNimTypeV2; env: pointer) {.compilerRtl, inli var j = cast[ptr GcEnv](env) j.traceStack.add(p, desc) -proc nimTraceRefDyn(q: pointer; env: pointer) {.compilerRtl, inline.} = +proc nimTraceRefDyn(q: pointer; env: pointer) {.compilerRtl, inl.} = let p = cast[ptr pointer](q) if p[] != nil: var j = cast[ptr GcEnv](env) diff --git a/lib/system/seqs_v2.nim b/lib/system/seqs_v2.nim index 0c487b31d..1194f40ef 100644 --- a/lib/system/seqs_v2.nim +++ b/lib/system/seqs_v2.nim @@ -75,7 +75,8 @@ proc prepareSeqAdd(len: int; p: pointer; addlen, elemSize, elemAlign: int): poin proc shrink*[T](x: var seq[T]; newLen: Natural) {.tags: [], raises: [].} = when nimvm: - setLen(x, newLen) + {.cast(tags: []).}: + setLen(x, newLen) else: #sysAssert newLen <= x.len, "invalid newLen parameter for 'shrink'" when not supportsCopyMem(T): diff --git a/tests/arc/t16458.nim b/tests/arc/t16458.nim new file mode 100644 index 000000000..6ae114287 --- /dev/null +++ b/tests/arc/t16458.nim @@ -0,0 +1,6 @@ +discard """ + matrix: "--gc:orc --d:useNimRtl" + action: "compile" +""" + +echo 134 \ No newline at end of file From ed0dce7292480002be96d0ea5b7775c38767134d Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 12 Feb 2022 15:10:45 +0100 Subject: [PATCH 1512/2472] fixes #19404 by protecting the memory we borrow from. this replaces crashes with minor memory leaks which seems to be acceptable. In the longer run we need a better VM that didn't grow hacks over a decade. (#19515) Co-authored-by: flywind --- compiler/vm.nim | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/compiler/vm.nim b/compiler/vm.nim index 3343eb781..259648add 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -85,9 +85,9 @@ proc bailOut(c: PCtx; tos: PStackFrame) = when not defined(nimComputedGoto): {.pragma: computedGoto.} -proc ensureKind(n: var TFullReg, kind: TRegisterKind) = - if n.kind != kind: - n = TFullReg(kind: kind) +proc ensureKind(n: var TFullReg, k: TRegisterKind) {.inline.} = + if n.kind != k: + n = TFullReg(kind: k) template ensureKind(k: untyped) {.dirty.} = ensureKind(regs[ra], k) @@ -521,6 +521,11 @@ template maybeHandlePtr(node2: PNode, reg: TFullReg, isAssign2: bool): bool = when not defined(nimHasSinkInference): {.pragma: nosinks.} +template takeAddress(reg, source) = + reg.nodeAddr = addr source + when defined(gcDestructors): + GC_ref source + proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = var pc = start var tos = tos @@ -679,7 +684,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = let idx = regs[rc].intVal.int let src = if regs[rb].kind == rkNode: regs[rb].node else: regs[rb].nodeAddr[] if src.kind notin {nkEmpty..nkTripleStrLit} and idx <% src.len: - regs[ra].nodeAddr = addr src.sons[idx] + takeAddress regs[ra], src.sons[idx] else: stackTrace(c, tos, pc, formatErrorIndexBound(idx, src.safeLen-1)) of opcLdStrIdx: @@ -747,11 +752,11 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = of nkObjConstr: let n = src.sons[rc + 1] if n.kind == nkExprColonExpr: - regs[ra].nodeAddr = addr n.sons[1] + takeAddress regs[ra], n.sons[1] else: - regs[ra].nodeAddr = addr src.sons[rc + 1] + takeAddress regs[ra], src.sons[rc + 1] else: - regs[ra].nodeAddr = addr src.sons[rc] + takeAddress regs[ra], src.sons[rc] of opcWrObj: # a.b = c decodeBC(rkNode) @@ -778,7 +783,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = decodeB(rkNodeAddr) case regs[rb].kind of rkNode: - regs[ra].nodeAddr = addr(regs[rb].node) + takeAddress regs[ra], regs[rb].node of rkNodeAddr: # bug #14339 regs[ra].nodeAddr = regs[rb].nodeAddr else: From 551225d88808c6029d9246cf2bea5da349d1eb43 Mon Sep 17 00:00:00 2001 From: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Date: Mon, 14 Feb 2022 08:50:38 +0100 Subject: [PATCH 1513/2472] Remove backslash in glob pattern (#19524) --- lib/pure/os.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 76164bc49..7c0c64276 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -2131,7 +2131,7 @@ iterator walkPattern*(pattern: string): string {.tags: [ReadDirEffect], noWeirdT ## Iterate over all the files and directories that match the `pattern`. ## ## On POSIX this uses the `glob`:idx: call. - ## `pattern` is OS dependent, but at least the `"\*.ext"` + ## `pattern` is OS dependent, but at least the `"*.ext"` ## notation is supported. ## ## See also: @@ -2150,7 +2150,7 @@ iterator walkFiles*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTar ## Iterate over all the files that match the `pattern`. ## ## On POSIX this uses the `glob`:idx: call. - ## `pattern` is OS dependent, but at least the `"\*.ext"` + ## `pattern` is OS dependent, but at least the `"*.ext"` ## notation is supported. ## ## See also: @@ -2167,7 +2167,7 @@ iterator walkDirs*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTarg ## Iterate over all the directories that match the `pattern`. ## ## On POSIX this uses the `glob`:idx: call. - ## `pattern` is OS dependent, but at least the `"\*.ext"` + ## `pattern` is OS dependent, but at least the `"*.ext"` ## notation is supported. ## ## See also: From cda77c15209519ea04f889a826ac79c7cbb7e152 Mon Sep 17 00:00:00 2001 From: David Krause Date: Mon, 14 Feb 2022 08:56:35 +0100 Subject: [PATCH 1514/2472] use OrderedTable instead of OrderedTableRef for mimedb (#19522) * use OrderedTable instead of OrderedTableRef for mimedb Signed-off-by: David Krause * added changelog entry for mimedb change Signed-off-by: David Krause --- changelog.md | 2 ++ lib/pure/mimetypes.nim | 4 ++-- tests/stdlib/tmimetypes.nim | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/changelog.md b/changelog.md index 412e046eb..b21784bae 100644 --- a/changelog.md +++ b/changelog.md @@ -40,6 +40,8 @@ becomes an alias for `addr`. - Added `std/oserrors` for OS error reporting. Added `std/envvars` for environment variables handling. +- Changed mimedb to use an `OrderedTable` instead of `OrderedTableRef`, to use it in a const. + ## Language changes - Pragma macros on type definitions can now return `nnkTypeSection` nodes as well as `nnkTypeDef`, diff --git a/lib/pure/mimetypes.nim b/lib/pure/mimetypes.nim index 6ae63a0ae..d1566d897 100644 --- a/lib/pure/mimetypes.nim +++ b/lib/pure/mimetypes.nim @@ -31,7 +31,7 @@ from strutils import startsWith, toLowerAscii, strip type MimeDB* = object - mimes: OrderedTableRef[string, string] + mimes: OrderedTable[string, string] const mimes* = { "123": "application/vnd.lotus-1-2-3", @@ -1904,7 +1904,7 @@ func newMimetypes*(): MimeDB = ## Creates a new Mimetypes database. The database will contain the most ## common mimetypes. {.cast(noSideEffect).}: - result.mimes = mimes.newOrderedTable() + result.mimes = mimes.toOrderedTable() func getMimetype*(mimedb: MimeDB, ext: string, default = "text/plain"): string = ## Gets mimetype which corresponds to `ext`. Returns `default` if `ext` diff --git a/tests/stdlib/tmimetypes.nim b/tests/stdlib/tmimetypes.nim index 93c20d4a3..6435309e1 100644 --- a/tests/stdlib/tmimetypes.nim +++ b/tests/stdlib/tmimetypes.nim @@ -7,6 +7,8 @@ template main() = var m = newMimetypes() doAssert m.getMimetype("mp4") == "video/mp4" doAssert m.getExt("application/json") == "json" + m.register("foo", "baa") + doAssert m.getMimetype("foo") == "baa" # see also `runnableExamples`. # xxx we should have a way to avoid duplicating code between runnableExamples and tests From 42eeab5fd181ffca6d1197b704a15f056939cfa8 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Mon, 14 Feb 2022 00:09:21 -0800 Subject: [PATCH 1515/2472] Remove Deprecated oids.oidsToString (#19519) * Remove deprecated oids.oidToString * Remove deprecated oids.oidToString --- changelog.md | 2 ++ lib/pure/oids.nim | 5 ----- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/changelog.md b/changelog.md index b21784bae..ec13152d0 100644 --- a/changelog.md +++ b/changelog.md @@ -39,6 +39,8 @@ becomes an alias for `addr`. - Added `getIsoWeeksInYear` in `times` to return the number of weeks in an ISO week-based year. - Added `std/oserrors` for OS error reporting. Added `std/envvars` for environment variables handling. +- Removed deprecated `oids.oidToString`. + - Changed mimedb to use an `OrderedTable` instead of `OrderedTableRef`, to use it in a const. diff --git a/lib/pure/oids.nim b/lib/pure/oids.nim index da285a187..776c046b1 100644 --- a/lib/pure/oids.nim +++ b/lib/pure/oids.nim @@ -67,11 +67,6 @@ template toStringImpl[T: string | cstring](result: var T, oid: Oid) = when T is cstring: result[N] = '\0' -proc oidToString*(oid: Oid, str: cstring) {.deprecated: "unsafe; use `$`".} = - ## Converts an oid to a string which must have space allocated for 25 elements. - # work around a compiler bug: - var str = str - toStringImpl(str, oid) proc `$`*(oid: Oid): string = ## Converts an OID to a string. From 8e8231f9d6e586a32af7b80892244dc1033db871 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Mon, 14 Feb 2022 05:35:33 -0800 Subject: [PATCH 1516/2472] Remove deprecated math.c_frexp (#19518) * Remove Deprecated math proc * Remove Deprecated math proc * Remove Deprecated math proc --- changelog.md | 3 +++ lib/pure/math.nim | 5 ----- tests/stdlib/tmath.nim | 6 ------ 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/changelog.md b/changelog.md index ec13152d0..3fd66dba8 100644 --- a/changelog.md +++ b/changelog.md @@ -44,6 +44,9 @@ becomes an alias for `addr`. - Changed mimedb to use an `OrderedTable` instead of `OrderedTableRef`, to use it in a const. +- Removed deprecated `math.c_frexp`. + + ## Language changes - Pragma macros on type definitions can now return `nnkTypeSection` nodes as well as `nnkTypeDef`, diff --git a/lib/pure/math.nim b/lib/pure/math.nim index 66f1a2e2c..1c47258bc 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -69,11 +69,6 @@ when defined(c) or defined(cpp): proc c_signbit(x: SomeFloat): cint {.importc: "signbit", header: "".} - func c_frexp*(x: cfloat, exponent: var cint): cfloat {. - importc: "frexpf", header: "", deprecated: "Use `frexp` instead".} - func c_frexp*(x: cdouble, exponent: var cint): cdouble {. - importc: "frexp", header: "", deprecated: "Use `frexp` instead".} - # don't export `c_frexp` in the future and remove `c_frexp2`. func c_frexp2(x: cfloat, exponent: var cint): cfloat {. importc: "frexpf", header: "".} diff --git a/tests/stdlib/tmath.nim b/tests/stdlib/tmath.nim index 9bc5b994f..5e501c09b 100644 --- a/tests/stdlib/tmath.nim +++ b/tests/stdlib/tmath.nim @@ -437,12 +437,6 @@ template main() = doAssert lgamma(-0.0) == Inf doAssert lgamma(-1.0) == Inf - when nimvm: discard - else: - var exponent: cint - doAssert c_frexp(0.0, exponent) == 0.0 - doAssert c_frexp(-0.0, exponent) == -0.0 - doAssert classify(c_frexp(-0.0, exponent)) == fcNegZero static: main() main() From 2ef71c0a46b2209c906921f10f585306ce6199cc Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 14 Feb 2022 21:36:01 +0800 Subject: [PATCH 1517/2472] [testcase] genSym fails to make unique identifier for ref object types (#19506) close #15118 --- tests/macros/tmacros_various.nim | 60 ++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/tests/macros/tmacros_various.nim b/tests/macros/tmacros_various.nim index 00d74cb5c..d702db56a 100644 --- a/tests/macros/tmacros_various.nim +++ b/tests/macros/tmacros_various.nim @@ -23,6 +23,8 @@ x: some string ([("key", "val"), ("keyB", "2")], [("val", "key"), ("2", "keyB")]) ([("key", "val"), ("keyB", "2")], [("val", "key"), ("2", "keyB")]) 0 +0 +0 ''' """ @@ -270,3 +272,61 @@ block: # bug #13511 debugAst: add(foo(), component) + +block: # bug #15118 + macro flop(name: static string) = + let id = genSym(nskType, "env") + let r = + nnkStmtList.newTree( + nnkTypeSection.newTree( + nnkTypeDef.newTree( + id, + newEmptyNode(), + nnkRefTy.newTree( + nnkObjectTy.newTree( + newEmptyNode(), + newEmptyNode(), + nnkRecList.newTree( + nnkIdentDefs.newTree( + newIdentNode(name), + newIdentNode("int"), + newEmptyNode() + ) + ) + ) + ) + ) + ), + + # var f: T + + nnkVarSection.newTree( + nnkIdentDefs.newTree( + newIdentNode("f"), + id, + newEmptyNode() + ) + ), + + # echo f.a + nnkCommand.newTree( + newIdentNode("new"), + newIdentNode("f") + ), + + nnkCommand.newTree( + newIdentNode("echo"), + nnkDotExpr.newTree( + newIdentNode("f"), + newIdentNode(name) + ) + ) + ) + r + + + block: + flop("a") + + block: + flop("b") From b2c5d7b4ff070abe2145e998d090fb15b4df522f Mon Sep 17 00:00:00 2001 From: Sizhe Zhao Date: Tue, 15 Feb 2022 06:15:59 +0800 Subject: [PATCH 1518/2472] Documentation: Fix word usage (#19529) --- doc/lib.rst | 2 +- lib/pure/base64.nim | 4 ++-- lib/pure/hashes.nim | 2 +- lib/pure/md5.nim | 4 ++-- lib/std/sha1.nim | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/lib.rst b/doc/lib.rst index 881d857d3..f72012777 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -446,7 +446,7 @@ Hashing from the MongoDB interface and it thus binary compatible with a MongoDB OID. * `sha1 `_ - This module implements a sha1 encoder and decoder. + This module implements the SHA-1 checksum algorithm. Miscellaneous diff --git a/lib/pure/base64.nim b/lib/pure/base64.nim index 513ec5826..9b7af88ef 100644 --- a/lib/pure/base64.nim +++ b/lib/pure/base64.nim @@ -56,8 +56,8 @@ runnableExamples: ## ======== ## ## * `hashes module`_ for efficient computations of hash values for diverse Nim types -## * `md5 module`_ implements the MD5 checksum algorithm -## * `sha1 module`_ implements a sha1 encoder and decoder +## * `md5 module`_ for the MD5 checksum algorithm +## * `sha1 module`_ for the SHA-1 checksum algorithm template cbBase(a, b): untyped = [ 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', diff --git a/lib/pure/hashes.nim b/lib/pure/hashes.nim index d9f534670..47dacec7d 100644 --- a/lib/pure/hashes.nim +++ b/lib/pure/hashes.nim @@ -62,7 +62,7 @@ runnableExamples: ## ======== ## * `md5 module `_ for the MD5 checksum algorithm ## * `base64 module `_ for a Base64 encoder and decoder -## * `std/sha1 module `_ for a SHA-1 encoder and decoder +## * `std/sha1 module `_ for the SHA-1 checksum algorithm ## * `tables module `_ for hash tables import std/private/since diff --git a/lib/pure/md5.nim b/lib/pure/md5.nim index 06c5608a4..cd4d1e6b8 100644 --- a/lib/pure/md5.nim +++ b/lib/pure/md5.nim @@ -13,8 +13,8 @@ ## ## See also ## ======== -## * `base64 module`_ implements a Base64 encoder and decoder -## * `std/sha1 module `_ for a SHA-1 encoder and decoder +## * `base64 module`_ for a Base64 encoder and decoder +## * `std/sha1 module `_ for the SHA-1 checksum algorithm ## * `hashes module`_ for efficient computations of hash values ## for diverse Nim types diff --git a/lib/std/sha1.nim b/lib/std/sha1.nim index 120a81702..ed962707b 100644 --- a/lib/std/sha1.nim +++ b/lib/std/sha1.nim @@ -12,9 +12,9 @@ ## ## See also ## ======== -## * `base64 module`_ implements a Base64 encoder and decoder +## * `base64 module`_ for a Base64 encoder and decoder ## * `hashes module`_ for efficient computations of hash values for diverse Nim types -## * `md5 module`_ implements the MD5 checksum algorithm +## * `md5 module`_ for the MD5 checksum algorithm runnableExamples: let accessName = secureHash("John Doe") From 30e9f23ec04618616dcf2aaa7980794bad2135ed Mon Sep 17 00:00:00 2001 From: rockcavera Date: Thu, 17 Feb 2022 08:48:57 -0300 Subject: [PATCH 1519/2472] Update chcks.nim (#19540) --- lib/system/chcks.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/system/chcks.nim b/lib/system/chcks.nim index 8c7c95795..a404e9d40 100644 --- a/lib/system/chcks.nim +++ b/lib/system/chcks.nim @@ -29,7 +29,7 @@ proc raiseFieldError(f: string) {.compilerproc, noinline.} = ## remove after bootstrap > 1.5.1 sysFatal(FieldDefect, f) -when defined(gcdestructors): +when defined(nimV2): proc raiseFieldError2(f: string, discVal: int) {.compilerproc, noinline.} = ## raised when field is inaccessible given runtime value of discriminant sysFatal(FieldError, f & $discVal & "'") From f019e999d491ecf462421bd750aec0465d308094 Mon Sep 17 00:00:00 2001 From: tersec Date: Fri, 18 Feb 2022 15:06:11 +0000 Subject: [PATCH 1520/2472] keep casing of noinit and noreturn pragmas consistently documented (#19535) --- doc/manual.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual.rst b/doc/manual.rst index 2469a0525..e45133c38 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -2818,7 +2818,7 @@ The implicit initialization can be avoided for optimization reasons with the .. code-block:: nim var - a {.noInit.}: array[0..1023, char] + a {.noinit.}: array[0..1023, char] If a proc is annotated with the `noinit` pragma, this refers to its implicit `result` variable: @@ -6636,7 +6636,7 @@ but accessed at runtime: doAssert nameToProc[2][1]() == "baz" -noReturn pragma +noreturn pragma --------------- The `noreturn` pragma is used to mark a proc that never returns. From 516db3bac389ea554166ee1a6671b5005159a30a Mon Sep 17 00:00:00 2001 From: locriacyber <74560659+locriacyber@users.noreply.github.com> Date: Fri, 18 Feb 2022 22:24:42 +0000 Subject: [PATCH 1521/2472] Remove trailing space in `nim r` command; Amend one error message (#19487) --- compiler/nim.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/nim.nim b/compiler/nim.nim index b8256d576..86c7ab524 100644 --- a/compiler/nim.nim +++ b/compiler/nim.nim @@ -116,7 +116,8 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) = if cmdPrefix.len > 0: cmdPrefix.add " " # without the `cmdPrefix.len > 0` check, on windows you'd get a cryptic: # `The parameter is incorrect` - execExternalProgram(conf, cmdPrefix & output.quoteShell & ' ' & conf.arguments) + let cmd = cmdPrefix & output.quoteShell & ' ' & conf.arguments + execExternalProgram(conf, cmd.strip(leading=false,trailing=true)) of cmdDocLike, cmdRst2html, cmdRst2tex: # bugfix(cmdRst2tex was missing) if conf.arguments.len > 0: # reserved for future use From 0c915b5e47e554d657f6533f262cebbcf1554e8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Thu, 24 Feb 2022 08:31:40 +0100 Subject: [PATCH 1522/2472] compile pragma: cache the result sooner (#19554) extccomp.addExternalFileToCompile() relies on hashes to decide whether an external C file needs recompilation or not. Due to short-circuit evaluation of boolean expressions, the procedure that generates a corresponding hash file is not called the first time an external file is compiled, so an avoidable recompilation is triggered the next build. This patch fixes that by moving the proc call with a desired side effect from its boolean expression, so it's executed unconditionally. --- compiler/extccomp.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index e7e1bcd51..124cc2c34 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -649,8 +649,10 @@ proc externalFileChanged(conf: ConfigRef; cfile: Cfile): bool = close(f) proc addExternalFileToCompile*(conf: ConfigRef; c: var Cfile) = + # we want to generate the hash file unconditionally + let extFileChanged = externalFileChanged(conf, c) if optForceFullMake notin conf.globalOptions and fileExists(c.obj) and - not externalFileChanged(conf, c): + not extFileChanged: c.flags.incl CfileFlag.Cached else: # make sure Nim keeps recompiling the external file on reruns From 340b5a1676062aa9bc7f17a2e078eaba96821bf3 Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Fri, 25 Feb 2022 07:16:46 +1100 Subject: [PATCH 1523/2472] Document links repect --out in .idx file (#19543) * Link in .idx file now respects --out:file switch * Use clearer chained calls Co-authored-by: Clay Sweetser --- compiler/docgen.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 3e61032d8..aad3b9409 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -1499,7 +1499,10 @@ proc genOutFile(d: PDoc, groupedToc = false): string = # Extract the title. Non API modules generate an entry in the index table. if d.meta[metaTitle].len != 0: title = d.meta[metaTitle] - let external = presentationPath(d.conf, AbsoluteFile d.filename).changeFileExt(HtmlExt).string.nativeToUnixPath + let external = AbsoluteFile(d.destFile) + .relativeTo(d.conf.outDir, '/') + .changeFileExt(HtmlExt) + .string setIndexTerm(d[], external, "", title) else: # Modules get an automatic title for the HTML, but no entry in the index. From 9a49451124a781a2cd032b6e7616c67a7815fe57 Mon Sep 17 00:00:00 2001 From: PMunch Date: Fri, 25 Feb 2022 10:43:03 +0100 Subject: [PATCH 1524/2472] Remove volatiles when compiling with ARC/ORC (#19545) This removes volatiles on ARC/ORC targets in NimMain and PreMainInner. This avoids an issue where they couldn't be optimised out on microcontrollers leading to larger code. Since the stack bottom doesn't have to be initialised this way when using ARC or ORC (or None, which is also covered by this PR) these can be safely removed. --- compiler/cgen.nim | 68 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 18 deletions(-) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 10b4b55d0..b78ad10c4 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1363,16 +1363,25 @@ proc genMainProc(m: BModule) = # The use of a volatile function pointer to call Pre/NimMainInner # prevents inlining of the NimMainInner function and dependent # functions, which might otherwise merge their stack frames. - PreMainBody = "$N" & + + PreMainVolatileBody = + "\tvoid (*volatile inner)(void);$N" & + "\tinner = PreMainInner;$N" & + "$1" & + "\t(*inner)();$N" + + PreMainNonVolatileBody = + "$1" & + "\tPreMainInner();$N" + + PreMainBodyStart = "$N" & "N_LIB_PRIVATE void PreMainInner(void) {$N" & "$2" & "}$N$N" & PosixCmdLine & - "N_LIB_PRIVATE void PreMain(void) {$N" & - "\tvoid (*volatile inner)(void);$N" & - "\tinner = PreMainInner;$N" & - "$1" & - "\t(*inner)();$N" & + "N_LIB_PRIVATE void PreMain(void) {$N" + + PreMainBodyEnd = "}$N$N" MainProcs = @@ -1385,17 +1394,32 @@ proc genMainProc(m: BModule) = "$1" & "}$N$N" - NimMainProc = - "N_CDECL(void, $5NimMain)(void) {$N" & - "\tvoid (*volatile inner)(void);$N" & - "$4" & - "\tinner = NimMainInner;$N" & - "$2" & - "\t(*inner)();$N" & + NimMainVolatileBody = + "\tvoid (*volatile inner)(void);$N" & + "$4" & + "\tinner = NimMainInner;$N" & + "$2" & + "\t(*inner)();$N" + + NimMainNonVolatileBody = + "$4" & + "$2" & + "\tNimMainInner();$N" + + NimMainProcStart = + "N_CDECL(void, $5NimMain)(void) {$N" + + NimMainProcEnd = "}$N$N" + NimMainProc = NimMainProcStart & NimMainVolatileBody & NimMainProcEnd + + NimSlimMainProc = NimMainProcStart & NimMainNonVolatileBody & NimMainProcEnd + NimMainBody = NimMainInner & NimMainProc + NimSlimMainBody = NimMainInner & NimSlimMainProc + PosixCMain = "int main(int argc, char** args, char** env) {$N" & "\tcmdLine = args;$N" & @@ -1456,10 +1480,13 @@ proc genMainProc(m: BModule) = m.includeHeader("") let initStackBottomCall = - if m.config.target.targetOS == osStandalone or m.config.selectedGC == gcNone: "".rope + if m.config.target.targetOS == osStandalone or m.config.selectedGC in {gcNone, gcArc, gcOrc}: "".rope else: ropecg(m, "\t#initStackBottomWith((void *)&inner);$N", []) inc(m.labels) - appcg(m, m.s[cfsProcs], PreMainBody, [m.g.mainDatInit, m.g.otherModsInit]) + if m.config.selectedGC notin {gcNone, gcArc, gcOrc}: + appcg(m, m.s[cfsProcs], PreMainBodyStart & PreMainVolatileBody & PreMainBodyEnd, [m.g.mainDatInit, m.g.otherModsInit]) + else: + appcg(m, m.s[cfsProcs], PreMainBodyStart & PreMainNonVolatileBody & PreMainBodyEnd, [m.g.mainDatInit, m.g.otherModsInit]) if m.config.target.targetOS == osWindows and m.config.globalOptions * {optGenGuiApp, optGenDynLib} != {}: @@ -1484,9 +1511,14 @@ proc genMainProc(m: BModule) = appcg(m, m.s[cfsProcs], nimMain, [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix]) else: - const nimMain = NimMainBody - appcg(m, m.s[cfsProcs], nimMain, - [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix]) + if m.config.selectedGC notin {gcNone, gcArc, gcOrc}: + const nimMain = NimMainBody + appcg(m, m.s[cfsProcs], nimMain, + [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix]) + else: + const nimMain = NimSlimMainBody + appcg(m, m.s[cfsProcs], nimMain, + [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix]) if optNoMain notin m.config.globalOptions: if m.config.cppCustomNamespace.len > 0: From fe791c67b4c028d2e439e6f74e0a4225ba6cc2f1 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Fri, 25 Feb 2022 01:43:35 -0800 Subject: [PATCH 1525/2472] Remove Deprecated jsre func (#19551) * Remove Deprecated jsre func * Remove Deprecated jsre func * ReStart CI, because OSX is dumb and I have no permission to restart --- changelog.md | 2 +- lib/js/jsre.nim | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/changelog.md b/changelog.md index 3fd66dba8..cbad9b4e6 100644 --- a/changelog.md +++ b/changelog.md @@ -43,7 +43,7 @@ becomes an alias for `addr`. - Changed mimedb to use an `OrderedTable` instead of `OrderedTableRef`, to use it in a const. - +- Removed deprecated `jsre.test` and `jsre.toString`. - Removed deprecated `math.c_frexp`. diff --git a/lib/js/jsre.nim b/lib/js/jsre.nim index a23fccaa8..cd8fb2be7 100644 --- a/lib/js/jsre.nim +++ b/lib/js/jsre.nim @@ -47,10 +47,6 @@ func toCstring*(self: RegExp): cstring {.importjs: "#.toString()".} func `$`*(self: RegExp): string = $toCstring(self) -func test*(self: RegExp; pattern: cstring): bool {.importjs: "#.test(#)", deprecated: "Use contains instead".} - -func toString*(self: RegExp): cstring {.importjs: "#.toString()", deprecated: "Use toCstring instead".} - func contains*(pattern: cstring; self: RegExp): bool = ## Tests for a substring match in its string parameter. runnableExamples: From ef3f343ec2979d22a5d4e51f4328e4a4c21d68dc Mon Sep 17 00:00:00 2001 From: Tanguy Date: Fri, 25 Feb 2022 12:57:58 +0100 Subject: [PATCH 1526/2472] Allow std/macros.params to work with nnkProcTy (#19563) * Allow std/macros.params to work with nnkProcTy * Add tests for proc params & pragma --- lib/core/macros.nim | 15 +++++++++++---- tests/macros/tprochelpers.nim | 22 ++++++++++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 tests/macros/tprochelpers.nim diff --git a/lib/core/macros.nim b/lib/core/macros.nim index c0e6e5154..4d040cebc 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -1271,12 +1271,19 @@ proc `name=`*(someProc: NimNode; val: NimNode) = else: someProc[0] = val proc params*(someProc: NimNode): NimNode = - someProc.expectRoutine - result = someProc[3] + if someProc.kind == nnkProcTy: + someProc[0] + else: + someProc.expectRoutine + someProc[3] + proc `params=`* (someProc: NimNode; params: NimNode) = - someProc.expectRoutine expectKind(params, nnkFormalParams) - someProc[3] = params + if someProc.kind == nnkProcTy: + someProc[0] = params + else: + someProc.expectRoutine + someProc[3] = params proc pragma*(someProc: NimNode): NimNode = ## Get the pragma of a proc type. diff --git a/tests/macros/tprochelpers.nim b/tests/macros/tprochelpers.nim new file mode 100644 index 000000000..d95a2ced8 --- /dev/null +++ b/tests/macros/tprochelpers.nim @@ -0,0 +1,22 @@ +import std/macros +import stdtest/testutils + +macro test1(prc: untyped): untyped = + assertAll: + prc.params.len == 2 + prc.params[1].len == 4 + prc.pragma.len == 2 + + prc.params = nnkFormalParams.newTree( + ident("int") + ) + prc.pragma = newEmptyNode() + + assertAll: + prc.params.len == 1 + prc.pragma.len == 0 + prc + +proc test(a, b: int): int {.gcsafe, raises: [], test1.} = 5 + +type hello = proc(a, b: int): int {.gcsafe, raises: [], test1.} From 9c17a32e0ec45de83c21d84f98be7f351ccd1172 Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 26 Feb 2022 03:33:28 +0800 Subject: [PATCH 1527/2472] fix #19266; allow reassign discriminant field (#19567) * add inUncheckedAssignSection * add one more test --- compiler/ccgexprs.nim | 16 +++++++++++++++- compiler/ccgstmts.nim | 2 +- compiler/cgendata.nim | 1 + tests/objvariant/treassign.nim | 22 ++++++++++++++++++++++ 4 files changed, 39 insertions(+), 2 deletions(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 1f1035115..9848366a1 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -2932,7 +2932,21 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = nkFromStmt, nkTemplateDef, nkMacroDef, nkStaticStmt: discard of nkPragma: genPragma(p, n) - of nkPragmaBlock: expr(p, n.lastSon, d) + of nkPragmaBlock: + var inUncheckedAssignSection = 0 + let pragmaList = n[0] + for pi in pragmaList: + if whichPragma(pi) == wCast: + case whichPragma(pi[1]) + of wUncheckedAssign: + inUncheckedAssignSection = 1 + else: + discard + + inc p.inUncheckedAssignSection, inUncheckedAssignSection + expr(p, n.lastSon, d) + dec p.inUncheckedAssignSection, inUncheckedAssignSection + of nkProcDef, nkFuncDef, nkMethodDef, nkConverterDef: if n[genericParamsPos].kind == nkEmpty: var prc = n[namePos].sym diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index 58a6dd015..240fa55c8 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -1566,7 +1566,7 @@ proc asgnFieldDiscriminant(p: BProc, e: PNode) = initLocExpr(p, e[0], a) getTemp(p, a.t, tmp) expr(p, e[1], tmp) - if optTinyRtti notin p.config.globalOptions: + if optTinyRtti notin p.config.globalOptions and p.inUncheckedAssignSection == 0: let field = dotExpr[1].sym genDiscriminantCheck(p, a, tmp, dotExpr[0].typ, field) message(p.config, e.info, warnCaseTransition) diff --git a/compiler/cgendata.nim b/compiler/cgendata.nim index 88b8d4090..449076725 100644 --- a/compiler/cgendata.nim +++ b/compiler/cgendata.nim @@ -99,6 +99,7 @@ type withinTryWithExcept*: int # required for goto based exception handling withinBlockLeaveActions*: int # complex to explain sigConflicts*: CountTable[string] + inUncheckedAssignSection*: int TTypeSeq* = seq[PType] TypeCache* = Table[SigHash, Rope] diff --git a/tests/objvariant/treassign.nim b/tests/objvariant/treassign.nim index 9549cb29c..527204616 100644 --- a/tests/objvariant/treassign.nim +++ b/tests/objvariant/treassign.nim @@ -34,3 +34,25 @@ proc passToVar(x: var Token) = discard t.curr = TokenObject(kind: t.curr.kind, foo: "abc") t.curr.kind = Token.foo + + +block: + type + TokenKind = enum + strLit, intLit + Token = object + case kind*: TokenKind + of strLit: + s*: string + of intLit: + i*: int64 + + var t = Token(kind: strLit, s: "abc") + + {.cast(uncheckedAssign).}: + + # inside the 'cast' section it is allowed to assign to the 't.kind' field directly: + t.kind = intLit + + {.cast(uncheckedAssign).}: + t.kind = strLit \ No newline at end of file From f0bfc0bd3f619c0933df5b71a0a04d4181d99880 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Fri, 25 Feb 2022 11:34:16 -0800 Subject: [PATCH 1528/2472] Remove define for jsfetch (#19530) * Remove define nimExperimentalAsyncjsThen for std/asyncjs.then and std/jsfetch * Remove define nimExperimentalAsyncjsThen for std/asyncjs.then and std/jsfetch * Remove define nimExperimentalAsyncjsThen for std/asyncjs.then and std/jsfetch * Remove define nimExperimentalAsyncjsThen for std/asyncjs.then and std/jsfetch --- changelog.md | 2 +- lib/js/asyncjs.nim | 148 ++++++++++++++++++++++---------------------- lib/std/jsfetch.nim | 15 +++-- tools/kochdocs.nim | 2 +- 4 files changed, 83 insertions(+), 84 deletions(-) diff --git a/changelog.md b/changelog.md index cbad9b4e6..011f95ddb 100644 --- a/changelog.md +++ b/changelog.md @@ -40,7 +40,7 @@ becomes an alias for `addr`. - Added `std/oserrors` for OS error reporting. Added `std/envvars` for environment variables handling. - Removed deprecated `oids.oidToString`. - +- Remove define `nimExperimentalAsyncjsThen` for `std/asyncjs.then` and `std/jsfetch`. - Changed mimedb to use an `OrderedTable` instead of `OrderedTableRef`, to use it in a const. - Removed deprecated `jsre.test` and `jsre.toString`. diff --git a/lib/js/asyncjs.nim b/lib/js/asyncjs.nim index 861fe3fe2..8f91e1acd 100644 --- a/lib/js/asyncjs.nim +++ b/lib/js/asyncjs.nim @@ -64,6 +64,7 @@ when not defined(js) and not defined(nimsuggest): import std/jsffi import std/macros +import std/private/since type Future*[T] = ref object @@ -163,92 +164,91 @@ template maybeFuture(T): untyped = when T is Future: T else: Future[T] -when defined(nimExperimentalAsyncjsThen): - import std/private/since - since (1, 5, 1): - #[ - TODO: - * map `Promise.all()` - * proc toString*(a: Error): cstring {.importjs: "#.toString()".} - Note: - We probably can't have a `waitFor` in js in browser (single threaded), but maybe it would be possible - in in nodejs, see https://nodejs.org/api/child_process.html#child_process_child_process_execsync_command_options - and https://stackoverflow.com/questions/61377358/javascript-wait-for-async-call-to-finish-before-returning-from-function-witho - ]# +since (1, 5, 1): + #[ + TODO: + * map `Promise.all()` + * proc toString*(a: Error): cstring {.importjs: "#.toString()".} - type Error* {.importjs: "Error".} = ref object of JsRoot - ## https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error - message*: cstring - name*: cstring + Note: + We probably can't have a `waitFor` in js in browser (single threaded), but maybe it would be possible + in in nodejs, see https://nodejs.org/api/child_process.html#child_process_child_process_execsync_command_options + and https://stackoverflow.com/questions/61377358/javascript-wait-for-async-call-to-finish-before-returning-from-function-witho + ]# - type OnReject* = proc(reason: Error) + type Error* {.importjs: "Error".} = ref object of JsRoot + ## https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error + message*: cstring + name*: cstring - proc then*[T](future: Future[T], onSuccess: proc, onReject: OnReject = nil): auto = - ## See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/then - ## Returns a `Future` from the return type of `onSuccess(T.default)`. - runnableExamples("-d:nimExperimentalAsyncjsThen"): - from std/sugar import `=>` + type OnReject* = proc(reason: Error) - proc fn(n: int): Future[int] {.async.} = - if n >= 7: raise newException(ValueError, "foobar: " & $n) - else: result = n * 2 + proc then*[T](future: Future[T], onSuccess: proc, onReject: OnReject = nil): auto = + ## See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/then + ## Returns a `Future` from the return type of `onSuccess(T.default)`. + runnableExamples("-r:off"): + from std/sugar import `=>` - proc asyncFact(n: int): Future[int] {.async.} = - if n > 0: result = n * await asyncFact(n-1) - else: result = 1 + proc fn(n: int): Future[int] {.async.} = + if n >= 7: raise newException(ValueError, "foobar: " & $n) + else: result = n * 2 - proc main() {.async.} = - block: # then - assert asyncFact(3).await == 3*2 - assert asyncFact(3).then(asyncFact).await == 6*5*4*3*2 - let x1 = await fn(3) - assert x1 == 3 * 2 - let x2 = await fn(4) - .then((a: int) => a.float) - .then((a: float) => $a) - assert x2 == "8.0" + proc asyncFact(n: int): Future[int] {.async.} = + if n > 0: result = n * await asyncFact(n-1) + else: result = 1 - block: # then with `onReject` callback - var witness = 1 - await fn(6).then((a: int) => (witness = 2), (r: Error) => (witness = 3)) - assert witness == 2 - await fn(7).then((a: int) => (witness = 2), (r: Error) => (witness = 3)) - assert witness == 3 + proc main() {.async.} = + block: # then + assert asyncFact(3).await == 3*2 + assert asyncFact(3).then(asyncFact).await == 6*5*4*3*2 + let x1 = await fn(3) + assert x1 == 3 * 2 + let x2 = await fn(4) + .then((a: int) => a.float) + .then((a: float) => $a) + assert x2 == "8.0" - template impl(call): untyped = - # see D20210421T014713 - when typeof(block: call) is void: - var ret: Future[void] - else: - var ret = default(maybeFuture(typeof(call))) - typeof(ret) - when T is void: - type A = impl(onSuccess()) + block: # then with `onReject` callback + var witness = 1 + await fn(6).then((a: int) => (witness = 2), (r: Error) => (witness = 3)) + assert witness == 2 + await fn(7).then((a: int) => (witness = 2), (r: Error) => (witness = 3)) + assert witness == 3 + + template impl(call): untyped = + # see D20210421T014713 + when typeof(block: call) is void: + var ret: Future[void] else: - type A = impl(onSuccess(default(T))) - var ret: A - asm "`ret` = `future`.then(`onSuccess`, `onReject`)" - return ret + var ret = default(maybeFuture(typeof(call))) + typeof(ret) + when T is void: + type A = impl(onSuccess()) + else: + type A = impl(onSuccess(default(T))) + var ret: A + asm "`ret` = `future`.then(`onSuccess`, `onReject`)" + return ret - proc catch*[T](future: Future[T], onReject: OnReject): Future[void] = - ## See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/catch - runnableExamples("-d:nimExperimentalAsyncjsThen"): - from std/sugar import `=>` - from std/strutils import contains + proc catch*[T](future: Future[T], onReject: OnReject): Future[void] = + ## See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/catch + runnableExamples("-r:off"): + from std/sugar import `=>` + from std/strutils import contains - proc fn(n: int): Future[int] {.async.} = - if n >= 7: raise newException(ValueError, "foobar: " & $n) - else: result = n * 2 + proc fn(n: int): Future[int] {.async.} = + if n >= 7: raise newException(ValueError, "foobar: " & $n) + else: result = n * 2 - proc main() {.async.} = - var reason: Error - await fn(6).catch((r: Error) => (reason = r)) # note: `()` are needed, `=> reason = r` would not work - assert reason == nil - await fn(7).catch((r: Error) => (reason = r)) - assert reason != nil - assert "foobar: 7" in $reason.message + proc main() {.async.} = + var reason: Error + await fn(6).catch((r: Error) => (reason = r)) # note: `()` are needed, `=> reason = r` would not work + assert reason == nil + await fn(7).catch((r: Error) => (reason = r)) + assert reason != nil + assert "foobar: 7" in $reason.message - discard main() + discard main() - asm "`result` = `future`.catch(`onReject`)" + asm "`result` = `future`.catch(`onReject`)" diff --git a/lib/std/jsfetch.nim b/lib/std/jsfetch.nim index 808d699b0..034bb6332 100644 --- a/lib/std/jsfetch.nim +++ b/lib/std/jsfetch.nim @@ -190,12 +190,11 @@ runnableExamples("-r:off"): discard example() - when defined(nimExperimentalAsyncjsThen): - block: - proc example2 {.async.} = - await fetch("https://api.github.com/users/torvalds".cstring) - .then((response: Response) => response.json()) - .then((json: JsObject) => console.log(json)) - .catch((err: Error) => console.log("Request Failed", err)) + block: + proc example2 {.async.} = + await fetch("https://api.github.com/users/torvalds".cstring) + .then((response: Response) => response.json()) + .then((json: JsObject) => console.log(json)) + .catch((err: Error) => console.log("Request Failed", err)) - discard example2() + discard example2() diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 8fd9c84a5..764146cf5 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -11,7 +11,7 @@ const gaCode* = " --doc.googleAnalytics:UA-48159761-1" # errormax: subsequent errors are probably consequences of 1st one; a simple # bug could cause unlimited number of errors otherwise, hard to debug in CI. - docDefines = "-d:nimExperimentalAsyncjsThen -d:nimExperimentalLinenoiseExtra" + docDefines = "-d:nimExperimentalLinenoiseExtra" nimArgs = "--errormax:3 --hint:Conf:off --hint:Path:off --hint:Processing:off --hint:XDeclaredButNotUsed:off --warning:UnusedImport:off -d:boot --putenv:nimversion=$# $#" % [system.NimVersion, docDefines] gitUrl = "https://github.com/nim-lang/Nim" docHtmlOutput = "doc/html" From c2c9fbd7458db7adb515aad30cfe65b98fba47bf Mon Sep 17 00:00:00 2001 From: solarizedalias <60473183+solarizedalias@users.noreply.github.com> Date: Sun, 27 Feb 2022 16:22:52 +0900 Subject: [PATCH 1529/2472] make compileOption("panics") work (#19568) --- compiler/commands.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler/commands.nim b/compiler/commands.nim index 6d3f23197..8e1d96860 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -332,6 +332,7 @@ proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool result = contains(conf.options, optTrMacros) of "excessivestacktrace": result = contains(conf.globalOptions, optExcessiveStackTrace) of "nilseqs", "nilchecks", "taintmode": warningOptionNoop(switch) + of "panics": result = contains(conf.globalOptions, optPanics) else: invalidCmdLineOption(conf, passCmd1, switch, info) proc processPath(conf: ConfigRef; path: string, info: TLineInfo, From c870ff5946731dd1deff828abfe07c1900a5b364 Mon Sep 17 00:00:00 2001 From: Tomohiro Date: Sun, 27 Feb 2022 23:36:16 +0900 Subject: [PATCH 1530/2472] Add error message to pointer size check assert in lib/nimbase.h (#19549) --- lib/nimbase.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/nimbase.h b/lib/nimbase.h index a83bd3006..e4dec0dd2 100644 --- a/lib/nimbase.h +++ b/lib/nimbase.h @@ -547,7 +547,7 @@ static inline void GCGuard (void *ptr) { asm volatile ("" :: "X" (ptr)); } #endif // Test to see if Nim and the C compiler agree on the size of a pointer. -NIM_STATIC_ASSERT(sizeof(NI) == sizeof(void*) && NIM_INTBITS == sizeof(NI)*8, ""); +NIM_STATIC_ASSERT(sizeof(NI) == sizeof(void*) && NIM_INTBITS == sizeof(NI)*8, "Pointer size mismatch between Nim and C/C++ backend. You probably need to setup the backend compiler for target CPU."); #ifdef USE_NIM_NAMESPACE } From 207237cec229171eafbd3357dcf128bdd270332a Mon Sep 17 00:00:00 2001 From: Evan Typanski Date: Mon, 28 Feb 2022 07:18:41 -0500 Subject: [PATCH 1531/2472] Fix #18662 (#19534) --- compiler/jsgen.nim | 4 +++- compiler/ropes.nim | 7 ------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 0a7036b46..f4097efa4 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -2796,7 +2796,9 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode = var map: SourceMap (code, map) = genSourceMap($(code), outFile.string) writeFile(outFile.string & ".map", $(%map)) - discard writeRopeIfNotEqual(code, outFile) + if not equalsFile(code, outFile): + if not writeRope(code, outFile): + rawMessage(m.config, errCannotOpenFile, outFile.string) proc myOpen(graph: ModuleGraph; s: PSym; idgen: IdGenerator): PPassContext = diff --git a/compiler/ropes.nim b/compiler/ropes.nim index 1e7957337..cd696a545 100644 --- a/compiler/ropes.nim +++ b/compiler/ropes.nim @@ -329,10 +329,3 @@ proc equalsFile*(r: Rope, filename: AbsoluteFile): bool = if result: result = equalsFile(r, f) close(f) - -proc writeRopeIfNotEqual*(r: Rope, filename: AbsoluteFile): bool = - # returns true if overwritten - if not equalsFile(r, filename): - result = writeRope(r, filename) - else: - result = false From d6d36093b18dfa1e2b8b6325016f1def015250b4 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 1 Mar 2022 14:46:08 +0800 Subject: [PATCH 1532/2472] apply changes from #18017 and some fixes (#19571) * implements https://github.com/nim-lang/RFCs/issues/369 * deprecate unsafeAddr; extend addr addr is now available for all addressable locations, unsafeAddr is deprecated and become an alias for addr * follow @Vindaar's advice * change the signature of addr * unsafeAddr => addr (stdlib) * Update changelog.md * unsafeAddr => addr (tests) * Revert "unsafeAddr => addr (stdlib)" This reverts commit ab83c99c507048a8396e636bf22d55fdd84d7d1c. * doc changes; thanks to @konsumlamm Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * merge * remove * fix bug Co-authored-by: Araq Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- compiler/parampatterns.nim | 60 ++++++++++++++++++++------------------ compiler/semexprs.nim | 14 +++++---- compiler/semmagic.nim | 6 ++-- lib/system.nim | 3 -- 4 files changed, 42 insertions(+), 41 deletions(-) diff --git a/compiler/parampatterns.nim b/compiler/parampatterns.nim index bfbc72675..cceb236ae 100644 --- a/compiler/parampatterns.nim +++ b/compiler/parampatterns.nim @@ -179,6 +179,7 @@ type arLocalLValue, # is an l-value, but local var; must not escape # its stack frame! arDiscriminant, # is a discriminant + arAddressableConst, # an addressable const arLentValue, # lent value arStrange # it is a strange beast like 'typedesc[var T]' @@ -212,7 +213,7 @@ proc exprRoot*(n: PNode): PSym = else: break -proc isAssignable*(owner: PSym, n: PNode; isUnsafeAddr=false): TAssignableResult = +proc isAssignable*(owner: PSym, n: PNode): TAssignableResult = ## 'owner' can be nil! result = arNone case n.kind @@ -220,20 +221,20 @@ proc isAssignable*(owner: PSym, n: PNode; isUnsafeAddr=false): TAssignableResult if n.typ != nil and n.typ.kind in {tyVar}: result = arLValue of nkSym: - let kinds = if isUnsafeAddr: {skVar, skResult, skTemp, skParam, skLet, skForVar} - else: {skVar, skResult, skTemp} - if n.sym.kind == skParam and n.sym.typ.kind in {tyVar, tySink}: - result = arLValue - elif isUnsafeAddr and n.sym.kind == skParam: - result = arLValue - elif isUnsafeAddr and n.sym.kind == skConst and dontInlineConstant(n, n.sym.ast): - result = arLValue + const kinds = {skVar, skResult, skTemp, skParam, skLet, skForVar} + if n.sym.kind == skParam: + result = if n.sym.typ.kind in {tyVar, tySink}: arLValue else: arAddressableConst + elif n.sym.kind == skConst and dontInlineConstant(n, n.sym.ast): + result = arAddressableConst elif n.sym.kind in kinds: - if owner != nil and owner == n.sym.owner and - sfGlobal notin n.sym.flags: - result = arLocalLValue + if n.sym.kind in {skParam, skLet, skForVar}: + result = arAddressableConst else: - result = arLValue + if owner != nil and owner == n.sym.owner and + sfGlobal notin n.sym.flags: + result = arLocalLValue + else: + result = arLValue elif n.sym.kind == skType: let t = n.sym.typ.skipTypes({tyTypeDesc}) if t.kind in {tyVar}: result = arStrange @@ -241,10 +242,10 @@ proc isAssignable*(owner: PSym, n: PNode; isUnsafeAddr=false): TAssignableResult let t = skipTypes(n[0].typ, abstractInst-{tyTypeDesc}) if t.kind in {tyVar, tySink, tyPtr, tyRef}: result = arLValue - elif isUnsafeAddr and t.kind == tyLent: - result = arLValue + elif t.kind == tyLent: + result = arAddressableConst else: - result = isAssignable(owner, n[0], isUnsafeAddr) + result = isAssignable(owner, n[0]) if result != arNone and n[1].kind == nkSym and sfDiscriminant in n[1].sym.flags: result = arDiscriminant @@ -252,23 +253,23 @@ proc isAssignable*(owner: PSym, n: PNode; isUnsafeAddr=false): TAssignableResult let t = skipTypes(n[0].typ, abstractInst-{tyTypeDesc}) if t.kind in {tyVar, tySink, tyPtr, tyRef}: result = arLValue - elif isUnsafeAddr and t.kind == tyLent: - result = arLValue + elif t.kind == tyLent: + result = arAddressableConst else: - result = isAssignable(owner, n[0], isUnsafeAddr) + result = isAssignable(owner, n[0]) of nkHiddenStdConv, nkHiddenSubConv, nkConv: # Object and tuple conversions are still addressable, so we skip them # XXX why is 'tyOpenArray' allowed here? if skipTypes(n.typ, abstractPtrs-{tyTypeDesc}).kind in {tyOpenArray, tyTuple, tyObject}: - result = isAssignable(owner, n[1], isUnsafeAddr) + result = isAssignable(owner, n[1]) elif compareTypes(n.typ, n[1].typ, dcEqIgnoreDistinct): # types that are equal modulo distinction preserve l-value: - result = isAssignable(owner, n[1], isUnsafeAddr) + result = isAssignable(owner, n[1]) of nkHiddenDeref: let n0 = n[0] if n0.typ.kind == tyLent: - if isUnsafeAddr or (n0.kind == nkSym and n0.sym.kind == skResult): + if n0.kind == nkSym and n0.sym.kind == skResult: result = arLValue else: result = arLentValue @@ -277,18 +278,19 @@ proc isAssignable*(owner: PSym, n: PNode; isUnsafeAddr=false): TAssignableResult of nkDerefExpr, nkHiddenAddr: result = arLValue of nkObjUpConv, nkObjDownConv, nkCheckedFieldExpr: - result = isAssignable(owner, n[0], isUnsafeAddr) + result = isAssignable(owner, n[0]) of nkCallKinds: # builtin slice keeps lvalue-ness: if getMagic(n) in {mArrGet, mSlice}: - result = isAssignable(owner, n[1], isUnsafeAddr) - elif n.typ != nil and n.typ.kind in {tyVar}: - result = arLValue - elif isUnsafeAddr and n.typ != nil and n.typ.kind == tyLent: - result = arLValue + result = isAssignable(owner, n[1]) + elif n.typ != nil: + case n.typ.kind + of tyVar: result = arLValue + of tyLent: result = arLentValue + else: discard of nkStmtList, nkStmtListExpr: if n.typ != nil: - result = isAssignable(owner, n.lastSon, isUnsafeAddr) + result = isAssignable(owner, n.lastSon) of nkVarTy: # XXX: The fact that this is here is a bit of a hack. # The goal is to allow the use of checks such as "foo(var T)" diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index c593efe55..5f3d065b6 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -650,8 +650,8 @@ proc fixAbstractType(c: PContext, n: PNode) = changeType(c, it[1], s, check=true) n[i] = it[1] -proc isAssignable(c: PContext, n: PNode; isUnsafeAddr=false): TAssignableResult = - result = parampatterns.isAssignable(c.p.owner, n, isUnsafeAddr) +proc isAssignable(c: PContext, n: PNode): TAssignableResult = + result = parampatterns.isAssignable(c.p.owner, n) proc isUnresolvedSym(s: PSym): bool = result = s.kind == skGenericParam @@ -1658,9 +1658,11 @@ proc takeImplicitAddr(c: PContext, n: PNode; isLent: bool): PNode = # `proc fun(a: var int): var int = a` discard else: discard - let valid = isAssignable(c, n, isLent) + let valid = isAssignable(c, n) if valid != arLValue: - if valid == arLocalLValue: + if valid in {arAddressableConst, arLentValue} and isLent: + discard "ok" + elif valid == arLocalLValue: localError(c.config, n.info, errXStackEscape % renderTree(n, {renderNoComments})) else: localError(c.config, n.info, errExprHasNoAddress) @@ -1784,7 +1786,7 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode = if le == nil: localError(c.config, a.info, "expression has no type") elif (skipTypes(le, {tyGenericInst, tyAlias, tySink}).kind notin {tyVar} and - isAssignable(c, a) in {arNone, arLentValue}) or ( + isAssignable(c, a) in {arNone, arLentValue, arAddressableConst}) or ( skipTypes(le, abstractVar).kind in {tyOpenArray, tyVarargs} and views notin c.features): # Direct assignment to a discriminant is allowed! localError(c.config, a.info, errXCannotBeAssignedTo % @@ -2267,7 +2269,7 @@ proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode = markUsed(c, n.info, s) checkSonsLen(n, 2, c.config) result[0] = newSymNode(s, n[0].info) - result[1] = semAddrArg(c, n[1], s.name.s == "unsafeAddr") + result[1] = semAddrArg(c, n[1]) result.typ = makePtrType(c, result[1].typ) of mTypeOf: markUsed(c, n.info, s) diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim index c7fc75620..ed1826fd4 100644 --- a/compiler/semmagic.nim +++ b/compiler/semmagic.nim @@ -10,11 +10,11 @@ # This include file implements the semantic checking for magics. # included from sem.nim -proc semAddrArg(c: PContext; n: PNode; isUnsafeAddr = false): PNode = +proc semAddrArg(c: PContext; n: PNode): PNode = let x = semExprWithType(c, n) if x.kind == nkSym: x.sym.flags.incl(sfAddrTaken) - if isAssignable(c, x, true) notin {arLValue, arLocalLValue}: + if isAssignable(c, x) notin {arLValue, arLocalLValue, arAddressableConst, arLentValue}: localError(c.config, n.info, errExprHasNoAddress) result = x @@ -466,7 +466,7 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode, of mAddr: checkSonsLen(n, 2, c.config) result = n - result[1] = semAddrArg(c, n[1], n[0].sym.name.s == "unsafeAddr") + result[1] = semAddrArg(c, n[1]) result.typ = makePtrType(c, result[1].typ) of mTypeOf: result = semTypeOf(c, n) diff --git a/lib/system.nim b/lib/system.nim index 56a6e2887..572768de2 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -202,9 +202,6 @@ proc `addr`*[T](x: T): ptr T {.magic: "Addr", noSideEffect.} = ## ## Cannot be overloaded. ## - ## See also: - ## * `unsafeAddr <#unsafeAddr,T>`_ - ## ## .. code-block:: Nim ## var ## buf: seq[char] = @['a','b','c'] From 590d39785243afb6da3662cf77c842a5ff4e1412 Mon Sep 17 00:00:00 2001 From: huantian Date: Wed, 2 Mar 2022 05:53:59 +0000 Subject: [PATCH 1533/2472] Fix typo in Nim Tutorial pt3 (#19579) --- doc/tut3.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/tut3.rst b/doc/tut3.rst index 6b5a3b5ca..1c497f1f5 100644 --- a/doc/tut3.rst +++ b/doc/tut3.rst @@ -244,7 +244,7 @@ Building Your First Macro ------------------------- To give a starting point to writing macros we will show now how to -implement the `myDebug` macro mentioned earlier. The first thing to +implement the `myAssert` macro mentioned earlier. The first thing to do is to build a simple example of the macro usage, and then just print the argument. This way it is possible to get an idea of what a correct argument should look like. From 488e18f90d342b5a3b528c0c84891b31b7d1e6ca Mon Sep 17 00:00:00 2001 From: VlkrS <47375452+VlkrS@users.noreply.github.com> Date: Thu, 3 Mar 2022 14:41:35 +0100 Subject: [PATCH 1534/2472] Fix CPU detection for i386 (#19577) * Fix CPU detection for i386 Commit 787def271b1cabc6f898caa42f892125de9fa908 breaks CPU detection for i386 on OpenBSD and probably on other platforms. [ isOpenIndiana -eq "yes" ] always returns 0, so mycpu is always set to "amd64". * Update buildsh.nimf Fix accidental bashism --- tools/niminst/buildsh.nimf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/niminst/buildsh.nimf b/tools/niminst/buildsh.nimf index 5f1db9cd3..7a05ef342 100644 --- a/tools/niminst/buildsh.nimf +++ b/tools/niminst/buildsh.nimf @@ -164,7 +164,7 @@ esac case $ucpu in *i386* | *i486* | *i586* | *i686* | *bepc* | *i86pc* ) - if [ isOpenIndiana -eq "yes" ] ; then + if [ "$isOpenIndiana" = "yes" ] ; then mycpu="amd64" else mycpu="i386" From 645447293851749fcc3394cd387d7070d8a9c735 Mon Sep 17 00:00:00 2001 From: Hamid Bluri Date: Mon, 7 Mar 2022 12:51:42 +0330 Subject: [PATCH 1535/2472] fix not flushing stdout in MSYS (#19590) I did this pull request according to what xflywind said: https://github.com/nim-lang/Nim/pull/19584#issuecomment-1060085141 --- lib/impure/rdstdin.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/impure/rdstdin.nim b/lib/impure/rdstdin.nim index 32048b131..adc0e212d 100644 --- a/lib/impure/rdstdin.nim +++ b/lib/impure/rdstdin.nim @@ -27,6 +27,7 @@ when defined(windows): tags: [ReadIOEffect, WriteIOEffect].} = ## Reads a line from stdin. stdout.write(prompt) + stdout.flushFile() result = readLine(stdin) proc readLineFromStdin*(prompt: string, line: var string): bool {. From 8f9dd5b3492293e4e48647206c8858b58cd51666 Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 9 Mar 2022 01:12:31 +0800 Subject: [PATCH 1536/2472] register callback for marshal in VM (#19578) * register callback for marshal in VM * remove unrelated code * polish * more tests * more tests * add loadVM and toVM --- compiler/vm.nim | 17 +++------------- compiler/vmdef.nim | 5 ++--- compiler/vmgen.nim | 27 ------------------------- compiler/vmops.nim | 36 ++++++++++++++++++++++++++++++++- lib/pure/marshal.nim | 28 ++++++++++++++++++-------- tests/stdlib/tmarshal.nim | 42 +++++++++++++++++++++++++++++++++------ 6 files changed, 96 insertions(+), 59 deletions(-) diff --git a/compiler/vm.nim b/compiler/vm.nim index 259648add..fa1c71c85 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -15,7 +15,7 @@ import std/[strutils, tables, parseutils], msgs, vmdef, vmgen, nimsets, types, passes, parser, vmdeps, idents, trees, renderer, options, transf, - vmmarshal, gorgeimpl, lineinfos, btrees, macrocacheimpl, + gorgeimpl, lineinfos, btrees, macrocacheimpl, modulegraphs, sighashes, int128, vmprofiler import ast except getstr @@ -1224,7 +1224,8 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = c.callbacks[-prc.offset-2].value( VmArgs(ra: ra, rb: rb, rc: rc, slots: cast[ptr UncheckedArray[TFullReg]](addr regs[0]), currentException: c.currentExceptionA, - currentLineInfo: c.debug[pc])) + currentLineInfo: c.debug[pc]) + ) elif importcCond(c, prc): if compiletimeFFI notin c.config.features: globalError(c.config, c.debug[pc], "VM not allowed to do FFI, see `compiletimeFFI`") @@ -2100,18 +2101,6 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = while typ.kind == tyTypeDesc and typ.len > 0: typ = typ[0] createStr regs[ra] regs[ra].node.strVal = typ.typeToString(preferExported) - of opcMarshalLoad: - let ra = instr.regA - let rb = instr.regB - inc pc - let typ = c.types[c.code[pc].regBx - wordExcess] - putIntoReg(regs[ra], loadAny(regs[rb].node.strVal, typ, c.cache, c.config, c.idgen)) - of opcMarshalStore: - decodeB(rkNode) - inc pc - let typ = c.types[c.code[pc].regBx - wordExcess] - createStrKeepNode(regs[ra]) - storeAny(regs[ra].node.strVal, typ, regs[rb].regToNode, c.config) c.profiler.leave(c) diff --git a/compiler/vmdef.nim b/compiler/vmdef.nim index 2044b860a..ecdbeff89 100644 --- a/compiler/vmdef.nim +++ b/compiler/vmdef.nim @@ -180,7 +180,6 @@ type opcNBindSym, opcNDynBindSym, opcSetType, # dest.typ = types[Bx] opcTypeTrait, - opcMarshalLoad, opcMarshalStore, opcSymOwner, opcSymIsInstantiationOf @@ -307,8 +306,8 @@ proc registerCallback*(c: PCtx; name: string; callback: VmCallback): int {.disca const firstABxInstr* = opcTJmp largeInstrs* = { # instructions which use 2 int32s instead of 1: - opcSubStr, opcConv, opcCast, opcNewSeq, opcOf, - opcMarshalLoad, opcMarshalStore} + opcSubStr, opcConv, opcCast, opcNewSeq, opcOf + } slotSomeTemp* = slotTempUnknown relativeJumps* = {opcTJmp, opcFJmp, opcJmp, opcJmpBack} diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index c0c4bac19..1551fbb80 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -99,11 +99,6 @@ proc codeListing(c: PCtx, result: var string, start=0; last = -1) = let idx = x.regBx-wordExcess result.addf("\t$#\tr$#, $# ($#)", opc.toStr, x.regA, c.constants[idx].renderTree, $idx) - elif opc in {opcMarshalLoad, opcMarshalStore}: - let y = c.code[i+1] - result.addf("\t$#\tr$#, r$#, $#", opc.toStr, x.regA, x.regB, - c.types[y.regBx-wordExcess].typeToString) - inc i else: result.addf("\t$#\tr$#, $#", opc.toStr, x.regA, x.regBx-wordExcess) result.add("\t# ") @@ -1383,22 +1378,6 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = # mGCref, mGCunref, globalError(c.config, n.info, "cannot generate code for: " & $m) -proc genMarshalLoad(c: PCtx, n: PNode, dest: var TDest) = - ## Signature: proc to*[T](data: string): T - if dest < 0: dest = c.getTemp(n.typ) - var tmp = c.genx(n[1]) - c.gABC(n, opcMarshalLoad, dest, tmp) - c.gABx(n, opcMarshalLoad, 0, c.genType(n.typ)) - c.freeTemp(tmp) - -proc genMarshalStore(c: PCtx, n: PNode, dest: var TDest) = - ## Signature: proc `$$`*[T](x: T): string - if dest < 0: dest = c.getTemp(n.typ) - var tmp = c.genx(n[1]) - c.gABC(n, opcMarshalStore, dest, tmp) - c.gABx(n, opcMarshalStore, 0, c.genType(n[1].typ)) - c.freeTemp(tmp) - proc unneededIndirection(n: PNode): bool = n.typ.skipTypes(abstractInstOwned-{tyTypeDesc}).kind == tyRef @@ -2054,12 +2033,6 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) = elif s.kind == skMethod: localError(c.config, n.info, "cannot call method " & s.name.s & " at compile time") - elif matches(s, "stdlib.marshal.to"): - # XXX marshal load&store should not be opcodes, but use the - # general callback mechanisms. - genMarshalLoad(c, n, dest) - elif matches(s, "stdlib.marshal.$$"): - genMarshalStore(c, n, dest) else: genCall(c, n, dest) clearDest(c, n, dest) diff --git a/compiler/vmops.nim b/compiler/vmops.nim index 6b06cc68d..f13e4b1cc 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -32,7 +32,7 @@ from system/formatfloat import addFloatRoundtrip, addFloatSprintf # There are some useful procs in vmconv. -import vmconv +import vmconv, vmmarshal template mathop(op) {.dirty.} = registerCallback(c, "stdlib.math." & astToStr(op), `op Wrapper`) @@ -152,6 +152,7 @@ when defined(nimHasInvariant): proc stackTrace2(c: PCtx, msg: string, n: PNode) = stackTrace(c, PStackFrame(prc: c.prc.sym, comesFrom: 0, next: nil), c.exceptionInstr, msg, n.info) + proc registerAdditionalOps*(c: PCtx) = template wrapIterator(fqname: string, iter: untyped) = @@ -344,3 +345,36 @@ proc registerAdditionalOps*(c: PCtx) = addFloatSprintf(p.strVal, x) wrapIterator("stdlib.os.envPairsImplSeq"): envPairs() + + registerCallback c, "stdlib.marshal.toVM", proc(a: VmArgs) = + let typ = a.getNode(0).typ + case typ.kind + of tyInt..tyInt64, tyUInt..tyUInt64: + setResult(a, loadAny(a.getString(1), typ, c.cache, c.config, c.idgen).intVal) + of tyFloat..tyFloat128: + setResult(a, loadAny(a.getString(1), typ, c.cache, c.config, c.idgen).floatVal) + else: + setResult(a, loadAny(a.getString(1), typ, c.cache, c.config, c.idgen)) + + registerCallback c, "stdlib.marshal.loadVM", proc(a: VmArgs) = + let typ = a.getNode(0).typ + let p = a.getReg(1) + var res: string + + var node: PNode + case p.kind + of rkNone: + node = newNode(nkEmpty) + of rkInt: + node = newIntNode(nkIntLit, p.intVal) + of rkFloat: + node = newFloatNode(nkFloatLit, p.floatVal) + of rkNode: + node = p.node + of rkRegisterAddr: + node = p.regAddr.node + of rkNodeAddr: + node = p.nodeAddr[] + + storeAny(res, typ, node, c.config) + setResult(a, res) diff --git a/lib/pure/marshal.nim b/lib/pure/marshal.nim index 993e0f510..452af54d5 100644 --- a/lib/pure/marshal.nim +++ b/lib/pure/marshal.nim @@ -298,6 +298,9 @@ proc store*[T](s: Stream, data: T) = shallowCopy(d, data) storeAny(s, toAny(d), stored) +proc loadVM[T](typ: typedesc[T], x: T): string = + discard "the implementation is in the compiler/vmops" + proc `$$`*[T](x: T): string = ## Returns a string representation of `x` (serialization, marshalling). ## @@ -313,12 +316,18 @@ proc `$$`*[T](x: T): string = let y = $$x assert y == """{"id": 1, "bar": "baz"}""" - var stored = initIntSet() - var d: T - shallowCopy(d, x) - var s = newStringStream() - storeAny(s, toAny(d), stored) - result = s.data + when nimvm: + result = loadVM(T, x) + else: + var stored = initIntSet() + var d: T + shallowCopy(d, x) + var s = newStringStream() + storeAny(s, toAny(d), stored) + result = s.data + +proc toVM[T](typ: typedesc[T], data: string): T = + discard "the implementation is in the compiler/vmops" proc to*[T](data: string): T = ## Reads data and transforms it to a type `T` (deserialization, unmarshalling). @@ -335,5 +344,8 @@ proc to*[T](data: string): T = assert z.id == 1 assert z.bar == "baz" - var tab = initTable[BiggestInt, pointer]() - loadAny(newStringStream(data), toAny(result), tab) + when nimvm: + result = toVM(T, data) + else: + var tab = initTable[BiggestInt, pointer]() + loadAny(newStringStream(data), toAny(result), tab) diff --git a/tests/stdlib/tmarshal.nim b/tests/stdlib/tmarshal.nim index 508205c3a..e539b1c34 100644 --- a/tests/stdlib/tmarshal.nim +++ b/tests/stdlib/tmarshal.nim @@ -4,12 +4,16 @@ import std/marshal proc testit[T](x: T): string = $$to[T]($$x) -let test1: array[0..1, array[0..4, string]] = [ - ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"]] -doAssert testit(test1) == - """[["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"]]""" -let test2: tuple[name: string, s: int] = ("tuple test", 56) -doAssert testit(test2) == """{"Field0": "tuple test", "Field1": 56}""" +template check1 = + let test1: array[0..1, array[0..4, string]] = [ + ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"]] + doAssert testit(test1) == + """[["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"]]""" + let test2: tuple[name: string, s: int] = ("tuple test", 56) + doAssert testit(test2) == """{"Field0": "tuple test", "Field1": 56}""" + +static: check1() +check1() type TE = enum @@ -146,3 +150,29 @@ block: let a: ref A = new(B) doAssert $$a[] == "{}" # not "{f: 0}" + +template checkMarshal(data: typed) = + let orig = data + let m = $$orig + + let old = to[typeof(orig)](m) + doAssert data == old + +template main() = + type + Book = object + page: int + name: string + + let book = Book(page: 12, name: "persona") + + checkMarshal(486) + checkMarshal(3.14) + checkMarshal("azure sky") + checkMarshal(book) + checkMarshal([1, 2, 3]) + checkMarshal(@[1.5, 2.7, 3.9, 4.2]) + checkMarshal(@["dream", "is", "possible"]) + +static: main() +main() From 0d6795a771e46b07244f3a3f43502602acbebdc3 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 9 Mar 2022 07:22:10 +0100 Subject: [PATCH 1537/2472] fixes #19569 (#19595) [backport] * minor code refactorings * fixes #19569 --- compiler/cgen.nim | 2 +- compiler/sizealignoffsetimpl.nim | 14 ++++++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index b78ad10c4..2865473fb 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -881,7 +881,7 @@ proc containsResult(n: PNode): bool = if containsResult(n[i]): return true const harmless = {nkConstSection, nkTypeSection, nkEmpty, nkCommentStmt, nkTemplateDef, - nkMacroDef, nkMixinStmt, nkBindStmt} + + nkMacroDef, nkMixinStmt, nkBindStmt, nkFormalParams} + declarativeDefs proc easyResultAsgn(n: PNode): PNode = diff --git a/compiler/sizealignoffsetimpl.nim b/compiler/sizealignoffsetimpl.nim index c5cd3ba64..c2e97aa53 100644 --- a/compiler/sizealignoffsetimpl.nim +++ b/compiler/sizealignoffsetimpl.nim @@ -40,15 +40,15 @@ proc inc(arg: var OffsetAccum; value: int) = else: arg.offset += value -proc alignmentMax(a,b: int): int = +proc alignmentMax(a, b: int): int = if unlikely(a == szIllegalRecursion or b == szIllegalRecursion): raiseIllegalTypeRecursion() if a == szUnknownSize or b == szUnknownSize: szUnknownSize else: - max(a,b) + max(a, b) proc align(arg: var OffsetAccum; value: int) = - if unlikely(value == szIllegalRecursion): raiseIllegalTypeRecursion() + if unlikely(value == szIllegalRecursion): raiseIllegalTypeRecursion() if value == szUnknownSize or arg.maxAlign == szUnknownSize or arg.offset == szUnknownSize: arg.maxAlign = szUnknownSize arg.offset = szUnknownSize @@ -112,7 +112,7 @@ proc setOffsetsToUnknown(n: PNode) = for i in 0.. Date: Wed, 9 Mar 2022 11:42:09 +0100 Subject: [PATCH 1538/2472] fixes #19575 (#19596) [backport] * fixes #19575 * better bugfix --- compiler/closureiters.nim | 6 ++++-- tests/iter/titer_issues.nim | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/compiler/closureiters.nim b/compiler/closureiters.nim index a8da7485e..4d9aeb3f7 100644 --- a/compiler/closureiters.nim +++ b/compiler/closureiters.nim @@ -1351,13 +1351,15 @@ proc preprocess(c: var PreprocessContext; n: PNode): PNode = case n.kind of nkTryStmt: let f = n.lastSon + var didAddSomething = false if f.kind == nkFinally: c.finallys.add f.lastSon + didAddSomething = true for i in 0 ..< n.len: result[i] = preprocess(c, n[i]) - if f.kind == nkFinally: + if didAddSomething: discard c.finallys.pop() of nkWhileStmt, nkBlockStmt: @@ -1384,7 +1386,7 @@ proc preprocess(c: var PreprocessContext; n: PNode): PNode = result = newNodeI(nkStmtList, n.info) for i in countdown(c.finallys.high, fin): var vars = FreshVarsContext(tab: initTable[int, PSym](), config: c.config, info: n.info, idgen: c.idgen) - result.add freshVars(preprocess(c, c.finallys[i]), vars) + result.add freshVars(copyTree(c.finallys[i]), vars) c.idgen = vars.idgen result.add n of nkSkip: discard diff --git a/tests/iter/titer_issues.nim b/tests/iter/titer_issues.nim index 65f66ad26..15fe867c8 100644 --- a/tests/iter/titer_issues.nim +++ b/tests/iter/titer_issues.nim @@ -251,3 +251,26 @@ block: for x in ff(@[1, 2], @[1, 2, 3]): echo x + + +# bug #19575 + +iterator bb() {.closure.} = + while true: + try: discard + except: break + finally: break + +var a = bb + +iterator cc() {.closure.} = + while true: + try: discard + except: + if true: + break + finally: + if true: + break + +var a2 = cc From 6773ffa63d0b3ab8b8894e84ed417f4eaced9122 Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 9 Mar 2022 19:55:20 +0800 Subject: [PATCH 1539/2472] enable nimPreviewDotLikeOps (#19598) --- config/config.nims | 1 + tests/config.nims | 1 - tests/stdlib/texperimental.nim | 1 + 3 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 tests/stdlib/texperimental.nim diff --git a/config/config.nims b/config/config.nims index 88c344642..08d9d9f55 100644 --- a/config/config.nims +++ b/config/config.nims @@ -21,3 +21,4 @@ when defined(nimStrictMode): # future work: XDeclaredButNotUsed switch("define", "nimVersion:" & NimVersion) +switch("define", "nimPreviewDotLikeOps") diff --git a/tests/config.nims b/tests/config.nims index 894c4bea0..8c4305521 100644 --- a/tests/config.nims +++ b/tests/config.nims @@ -37,6 +37,5 @@ switch("define", "nimExperimentalLinenoiseExtra") # preview APIs are expected to be the new default in upcoming versions switch("define", "nimPreviewFloatRoundtrip") -switch("define", "nimPreviewDotLikeOps") switch("define", "nimPreviewJsonutilsHoleyEnum") switch("define", "nimPreviewHashRef") diff --git a/tests/stdlib/texperimental.nim b/tests/stdlib/texperimental.nim new file mode 100644 index 000000000..ba8c4eb80 --- /dev/null +++ b/tests/stdlib/texperimental.nim @@ -0,0 +1 @@ +doAssert defined(nimPreviewDotLikeOps) \ No newline at end of file From 4c76f9f1523a72f560138709642c2d51ea365b85 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 11 Mar 2022 00:01:27 +0800 Subject: [PATCH 1540/2472] support useVersion:1.6 (#19601) --- compiler/commands.nim | 4 +++ compiler/options.nim | 1 + doc/advopt.txt | 2 +- doc/manual.rst | 5 ---- tests/system/tuse_version.nim | 5 ++-- tests/system/tuse_version16.nim | 49 +++++++++++++++++++++++++++++++++ 6 files changed, 57 insertions(+), 9 deletions(-) create mode 100644 tests/system/tuse_version16.nim diff --git a/compiler/commands.nim b/compiler/commands.nim index 8e1d96860..b21b48801 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -1043,6 +1043,10 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; defineSymbol(conf.symbols, "NimMajor", "1") defineSymbol(conf.symbols, "NimMinor", "2") conf.globalOptions.incl optNimV12Emulation + of "1.6": + defineSymbol(conf.symbols, "NimMajor", "1") + defineSymbol(conf.symbols, "NimMinor", "6") + conf.globalOptions.incl optNimV16Emulation else: localError(conf, info, "unknown Nim version; currently supported values are: `1.0`, `1.2`") # always be compatible with 1.x.100: diff --git a/compiler/options.nim b/compiler/options.nim index 5cfe58119..9c0c9df72 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -96,6 +96,7 @@ type # please make sure we have under 32 options optPanics # turn panics (sysFatal) into a process termination optNimV1Emulation # emulate Nim v1.0 optNimV12Emulation # emulate Nim v1.2 + optNimV16Emulation # emulate Nim v1.6 optSourcemap optProfileVM # enable VM profiler optEnableDeepCopy # ORC specific: enable 'deepcopy' for all types. diff --git a/doc/advopt.txt b/doc/advopt.txt index e27c75ada..0a2349597 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -161,7 +161,7 @@ Advanced options: enable experimental language feature --legacy:$2 enable obsolete/legacy language feature - --useVersion:1.0|1.2 emulate Nim version X of the Nim compiler, for testing + --useVersion:1.0|1.2|1.6 emulate Nim version X of the Nim compiler, for testing --benchmarkVM:on|off turn benchmarking of VM code with cpuTime() on|off --profileVM:on|off turn compile time VM profiler on|off --sinkInference:on|off turn sink parameter inference on|off (default: on) diff --git a/doc/manual.rst b/doc/manual.rst index e45133c38..4c4b9c506 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -5797,11 +5797,6 @@ However, this means that the method call syntax is not available for tmp(12) -**Note**: The Nim compiler prior to version 1 was more lenient about this -requirement. Use the `--useVersion:0.19`:option: switch for a transition period. - - - Limitations of the method call syntax ------------------------------------- diff --git a/tests/system/tuse_version.nim b/tests/system/tuse_version.nim index 36dac46ec..116ef3596 100644 --- a/tests/system/tuse_version.nim +++ b/tests/system/tuse_version.nim @@ -1,6 +1,5 @@ discard """ - cmd: "nim c --useVersion:1.0 -r $file" - output: "1.0.100" + matrix: "--useVersion:1.0" """ {.warning[UnusedImport]: off.} @@ -47,4 +46,4 @@ import std/[ ] -echo NimVersion +doAssert NimVersion == "1.0.100" diff --git a/tests/system/tuse_version16.nim b/tests/system/tuse_version16.nim new file mode 100644 index 000000000..d5430db32 --- /dev/null +++ b/tests/system/tuse_version16.nim @@ -0,0 +1,49 @@ +discard """ + matrix: "--useVersion:1.6" +""" + +{.warning[UnusedImport]: off.} + +import std/[ + # Core: + bitops, typetraits, lenientops, macros, volatile, + + # Algorithms: + algorithm, sequtils, + + # Collections: + critbits, deques, heapqueue, intsets, lists, options, sets, + sharedlist, tables, + + # Strings: + editdistance, wordwrap, parseutils, ropes, + pegs, punycode, strformat, strmisc, strscans, strtabs, + strutils, unicode, unidecode, + + # Generic operator system services: + os, streams, + + # Math libraries: + complex, math, mersenne, random, rationals, stats, sums, + + # Internet protocols: + httpcore, mimetypes, uri, + + # Parsers: + htmlparser, json, lexbase, parsecfg, parsecsv, parsesql, parsexml, + + # XML processing: + xmltree, xmlparser, + + # Generators: + htmlgen, + + # Hashing: + base64, hashes, + + # Miscellaneous: + colors, sugar, varints, +] + + +doAssert NimVersion == "1.6.100" From 7a50d663467f0e64ea8390e8c86269b25f312816 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Fri, 18 Mar 2022 03:54:20 -0300 Subject: [PATCH 1541/2472] Removed deprecated httpcore func (#19550) * Remove Deprecated httpcore func * Remove Deprecated httpcore func * Fix a test with Deprecated func * Restart CI, Apple can code shit anymore I tell you --- changelog.md | 1 + lib/pure/httpcore.nim | 10 ---------- tests/stdlib/tasynchttpserver.nim | 8 ++++---- 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/changelog.md b/changelog.md index 011f95ddb..a9bf3842a 100644 --- a/changelog.md +++ b/changelog.md @@ -45,6 +45,7 @@ becomes an alias for `addr`. - Changed mimedb to use an `OrderedTable` instead of `OrderedTableRef`, to use it in a const. - Removed deprecated `jsre.test` and `jsre.toString`. - Removed deprecated `math.c_frexp`. +- Removed deprecated ``` httpcore.`==` ```. ## Language changes diff --git a/lib/pure/httpcore.nim b/lib/pure/httpcore.nim index fdd5926f7..7e995ac46 100644 --- a/lib/pure/httpcore.nim +++ b/lib/pure/httpcore.nim @@ -347,16 +347,6 @@ func `$`*(code: HttpCode): string = func `==`*(a, b: HttpCode): bool {.borrow.} -proc `==`*(rawCode: string, code: HttpCode): bool - {.deprecated: "Deprecated since v1.2; use rawCode == $code instead".} = - ## Compare the string form of the status code with a HttpCode - ## - ## **Note**: According to HTTP/1.1 specification, the reason phrase is - ## optional and should be ignored by the client, making this - ## proc only suitable for comparing the `HttpCode` against the - ## string form of itself. - return cmpIgnoreCase(rawCode, $code) == 0 - func is1xx*(code: HttpCode): bool {.inline, since: (1, 5).} = ## Determines whether `code` is a 1xx HTTP status code. runnableExamples: diff --git a/tests/stdlib/tasynchttpserver.nim b/tests/stdlib/tasynchttpserver.nim index 77ad7a071..d4176d55a 100644 --- a/tests/stdlib/tasynchttpserver.nim +++ b/tests/stdlib/tasynchttpserver.nim @@ -39,7 +39,7 @@ proc test200() {.async.} = return clientResponse proc test(response: AsyncResponse, body: string) {.async.} = - doAssert(response.status == Http200) + doAssert(response.status == $Http200) doAssert(body == "Hello World, 200") doAssert(response.headers.hasKey("Content-Length")) doAssert(response.headers["Content-Length"] == "16") @@ -60,7 +60,7 @@ proc test404() {.async.} = return clientResponse proc test(response: AsyncResponse, body: string) {.async.} = - doAssert(response.status == Http404) + doAssert(response.status == $Http404) doAssert(body == "Hello World, 404") doAssert(response.headers.hasKey("Content-Length")) doAssert(response.headers["Content-Length"] == "16") @@ -81,7 +81,7 @@ proc testCustomEmptyHeaders() {.async.} = return clientResponse proc test(response: AsyncResponse, body: string) {.async.} = - doAssert(response.status == Http200) + doAssert(response.status == $Http200) doAssert(body == "Hello World, 200") doAssert(response.headers.hasKey("Content-Length")) doAssert(response.headers["Content-Length"] == "16") @@ -104,7 +104,7 @@ proc testCustomContentLength() {.async.} = return clientResponse proc test(response: AsyncResponse, body: string) {.async.} = - doAssert(response.status == Http200) + doAssert(response.status == $Http200) doAssert(body == "") doAssert(response.headers.hasKey("Content-Length")) doAssert(response.headers["Content-Length"] == "0") From 7b811deeffc762f8370b4cf7e8a353fd516d6543 Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 18 Mar 2022 14:17:54 +0100 Subject: [PATCH 1542/2472] db_sqlite: added a space --- lib/impure/db_sqlite.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/impure/db_sqlite.nim b/lib/impure/db_sqlite.nim index b600576df..1638d38c6 100644 --- a/lib/impure/db_sqlite.nim +++ b/lib/impure/db_sqlite.nim @@ -700,7 +700,7 @@ proc insert*(db: DbConn, query: SqlQuery, pkName: string, args: varargs[string, `$`]): int64 {.tags: [WriteDbEffect], since: (1, 3).} = ## same as insertId - result = tryInsert(db, query,pkName, args) + result = tryInsert(db, query, pkName, args) if result < 0: dbError(db) proc execAffectedRows*(db: DbConn, query: SqlQuery, From 3e83d73f272afe8de85189da7d6d513916cc2efd Mon Sep 17 00:00:00 2001 From: Clyybber Date: Fri, 18 Mar 2022 16:41:45 +0100 Subject: [PATCH 1543/2472] compiler: Handle nkCheckedFieldExpr better in dfa (#19616) Simply recurse into their first child, which is always a nkDotExpr instead of treating them seperately. This fixes the rhs sym of a nkCheckedFieldExpr being checked twice in aliases. This double checking didn't cause any issues, but was unintentional and redundant. --- compiler/dfa.nim | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/compiler/dfa.nim b/compiler/dfa.nim index 0539f6699..d68e7be82 100644 --- a/compiler/dfa.nim +++ b/compiler/dfa.nim @@ -605,11 +605,11 @@ proc aliases*(obj, field: PNode): AliasKind = var n = n while true: case n.kind - of PathKinds0 - {nkDotExpr, nkCheckedFieldExpr, nkBracketExpr}: + of PathKinds0 - {nkDotExpr, nkBracketExpr}: n = n[0] of PathKinds1: n = n[1] - of nkDotExpr, nkCheckedFieldExpr, nkBracketExpr: + of nkDotExpr, nkBracketExpr: result.add n n = n[0] of nkSym: @@ -642,8 +642,6 @@ proc aliases*(obj, field: PNode): AliasKind = if currFieldPath.sym != currObjPath.sym: return no of nkDotExpr: if currFieldPath[1].sym != currObjPath[1].sym: return no - of nkCheckedFieldExpr: - if currFieldPath[0][1].sym != currObjPath[0][1].sym: return no of nkBracketExpr: if currFieldPath[1].kind in nkLiterals and currObjPath[1].kind in nkLiterals: if currFieldPath[1].intVal != currObjPath[1].intVal: From 731eabc9309997775c8be41f3e5eb5512460aad0 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 20 Mar 2022 14:02:44 +0100 Subject: [PATCH 1544/2472] fixes #19631 (#19618) Aliasing is hard and we have to watch out not to compile 'x = f(x.a)' into 'f(x.a, addr x)' --- compiler/ccgcalls.nim | 11 ++++++++--- tests/ccgbugs/tcgbug.nim | 31 +++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index ce5fbfdd7..05be51a45 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -21,7 +21,7 @@ proc canRaiseDisp(p: BProc; n: PNode): bool = # we have to be *very* conservative: result = canRaiseConservative(n) -proc preventNrvo(p: BProc; le, ri: PNode): bool = +proc preventNrvo(p: BProc; dest, le, ri: PNode): bool = proc locationEscapes(p: BProc; le: PNode; inTryStmt: bool): bool = var n = le while true: @@ -54,6 +54,11 @@ proc preventNrvo(p: BProc; le, ri: PNode): bool = if canRaise(ri[0]) and locationEscapes(p, le, p.nestedTryStmts.len > 0): message(p.config, le.info, warnObservableStores, $le) + # bug #19613 prevent dangerous aliasing too: + if dest != nil and dest != le: + for i in 1.. 1: pl.add(~", ") # beware of 'result = p(result)'. We may need to allocate a temporary: - if d.k in {locTemp, locNone} or not preventNrvo(p, le, ri): + if d.k in {locTemp, locNone} or not preventNrvo(p, d.lode, le, ri): # Great, we can use 'd': if d.k == locNone: getTemp(p, typ[0], d, needsInit=true) diff --git a/tests/ccgbugs/tcgbug.nim b/tests/ccgbugs/tcgbug.nim index db9c116be..0fe4b8852 100644 --- a/tests/ccgbugs/tcgbug.nim +++ b/tests/ccgbugs/tcgbug.nim @@ -91,3 +91,34 @@ proc test(c: Helper): string = c.formatted echo test(Helper(isKind: true, formatted: "ok")) + + +# bug #19613 + +type + Eth2Digest = object + data: array[42, byte] + + BlockId* = object + root*: Eth2Digest + + BlockSlotId* = object + bid*: BlockId + slot*: uint64 + +func init*(T: type BlockSlotId, bid: BlockId, slot: uint64): T = + #debugecho "init ", bid, " ", slot + BlockSlotId(bid: bid, slot: slot) + +proc bug19613 = + var x: BlockSlotId + x.bid.root.data[0] = 42 + + x = + if x.slot > 0: + BlockSlotId.init(x.bid, x.slot) + else: + BlockSlotId.init(x.bid, x.slot) + doAssert x.bid.root.data[0] == 42 + +bug19613() From c4a0d4c5e35f09430a1c3d465fc62eb1001b7f9f Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 22 Mar 2022 15:36:49 +0100 Subject: [PATCH 1545/2472] =?UTF-8?q?fixes=20#19615;=20emit=20better=20cod?= =?UTF-8?q?e=20for=20integer=20divisions=20when=20the=20divisor=E2=80=A6?= =?UTF-8?q?=20(#19626)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fixes #19615; emit better code for integer divisions when the divisor is known at compile-time * proper bugfix: unsigned numbers cannot be -1 --- compiler/ccgexprs.nim | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 9848366a1..1a3e217b2 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -584,13 +584,23 @@ proc binaryArithOverflow(p: BProc, e: PNode, d: var TLoc, m: TMagic) = else: # we handle div by zero here so that we know that the compilerproc's # result is only for overflows. + var needsOverflowCheck = true if m in {mDivI, mModI}: - linefmt(p, cpsStmts, "if ($1 == 0){ #raiseDivByZero(); $2}$n", - [rdLoc(b), raiseInstr(p)]) - - let res = binaryArithOverflowRaw(p, t, a, b, - if t.kind == tyInt64: prc64[m] else: prc[m]) - putIntoDest(p, d, e, "($#)($#)" % [getTypeDesc(p.module, e.typ), res]) + var canBeZero = true + if e[2].kind in {nkIntLit..nkUInt64Lit}: + canBeZero = e[2].intVal == 0 + if e[2].kind in {nkIntLit..nkInt64Lit}: + needsOverflowCheck = e[2].intVal == -1 + if canBeZero: + linefmt(p, cpsStmts, "if ($1 == 0){ #raiseDivByZero(); $2}$n", + [rdLoc(b), raiseInstr(p)]) + if needsOverflowCheck: + let res = binaryArithOverflowRaw(p, t, a, b, + if t.kind == tyInt64: prc64[m] else: prc[m]) + putIntoDest(p, d, e, "($#)($#)" % [getTypeDesc(p.module, e.typ), res]) + else: + let res = "($1)($2 $3 $4)" % [getTypeDesc(p.module, e.typ), rdLoc(a), rope(opr[m]), rdLoc(b)] + putIntoDest(p, d, e, res) proc unaryArithOverflow(p: BProc, e: PNode, d: var TLoc, m: TMagic) = var From 4c8934305c48bb52ed0e5a59b47229521cb333ce Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 22 Mar 2022 12:08:31 -0700 Subject: [PATCH 1546/2472] system: thread: stack dealloction on Zephyr (#19633) [backport:1.6] Try to free the stack allocation when a thread exits. Possibly works for FreeRTOS as well. --- lib/system/threads.nim | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/system/threads.nim b/lib/system/threads.nim index ac727be0a..1c1d1ca1c 100644 --- a/lib/system/threads.nim +++ b/lib/system/threads.nim @@ -47,7 +47,10 @@ when not declared(ThisIsSystem): {.error: "You must not import this module explicitly".} -when defined(zephyr) or defined(freertos): +const + hasAllocStack = defined(zephyr) # maybe freertos too? + +when hasAllocStack or defined(zephyr) or defined(freertos): const nimThreadStackSize {.intdefine.} = 8192 nimThreadStackGuard {.intdefine.} = 128 @@ -68,6 +71,12 @@ else: #const globalsSlot = ThreadVarSlot(0) #sysAssert checkSlot.int == globalsSlot.int +# Zephyr doesn't include this properly without some help +when defined(zephyr): + {.emit: """/*INCLUDESECTION*/ + #include + """.} + # create for the main thread. Note: do not insert this data into the list # of all threads; it's not to be stopped etc. when not defined(useNimRtl): @@ -98,6 +107,8 @@ type else: dataFn: proc (m: TArg) {.nimcall, gcsafe.} data: TArg + when hasAllocStack: + rawStack: pointer proc `=copy`*[TArg](x: var Thread[TArg], y: Thread[TArg]) {.error.} @@ -161,6 +172,8 @@ else: threadTrouble() finally: afterThreadRuns() + when hasAllocStack: + deallocShared(thrd.rawStack) proc threadProcWrapStackFrame[TArg](thrd: ptr Thread[TArg]) {.raises: [].} = when defined(boehmgc): @@ -330,11 +343,12 @@ else: when hasSharedHeap: t.core.stackSize = ThreadStackSize var a {.noinit.}: Pthread_attr doAssert pthread_attr_init(a) == 0 - when defined(zephyr): + when hasAllocStack: var rawstk = allocShared0(ThreadStackSize + StackGuardSize) stk = cast[pointer](cast[uint](rawstk) + StackGuardSize) let setstacksizeResult = pthread_attr_setstack(addr a, stk, ThreadStackSize) + t.rawStack = rawstk else: let setstacksizeResult = pthread_attr_setstacksize(a, ThreadStackSize) From a8b5ad845c4218b4f20595df097c593acee53d50 Mon Sep 17 00:00:00 2001 From: Dominic Ward Date: Wed, 23 Mar 2022 06:50:36 +0000 Subject: [PATCH 1547/2472] Fix process lines iterator (#19605) * Ensure lines when process done * eliminate post-EOF exit test * Recommend fixes for execCmdEx/execProcess --- lib/pure/osproc.nim | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 6a0ac9a8b..3da9737ec 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -451,7 +451,7 @@ proc execProcesses*(cmds: openArray[string], if afterRunEvent != nil: afterRunEvent(i, p) close(p) -iterator lines*(p: Process): string {.since: (1, 3), tags: [ReadIOEffect].} = +iterator lines*(p: Process, keepNewLines = false): string {.since: (1, 3), tags: [ReadIOEffect].} = ## Convenience iterator for working with `startProcess` to read data from a ## background process. ## @@ -474,11 +474,11 @@ iterator lines*(p: Process): string {.since: (1, 3), tags: [ReadIOEffect].} = ## p.close var outp = p.outputStream var line = newStringOfCap(120) - while true: - if outp.readLine(line): - yield line - else: - if p.peekExitCode != -1: break + while outp.readLine(line): + if keepNewLines: + line.add("\n") + yield line + discard waitForExit(p) proc readLines*(p: Process): (seq[string], int) {.since: (1, 3).} = ## Convenience function for working with `startProcess` to read data from a @@ -514,6 +514,7 @@ when not defined(useNimRtl): var outp = outputStream(p) result = "" var line = newStringOfCap(120) + # consider `p.lines(keepNewLines=true)` to circumvent `running` busy-wait while true: # FIXME: converts CR-LF to LF. if outp.readLine(line): @@ -1622,6 +1623,7 @@ proc execCmdEx*(command: string, options: set[ProcessOption] = { inputStream(p).write(input) close inputStream(p) + # consider `p.lines(keepNewLines=true)` to avoid exit code test result = ("", -1) var line = newStringOfCap(120) while true: From 7f6e800cafc7b73625893fb5280eb8b51a15b252 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 24 Mar 2022 03:34:53 +0800 Subject: [PATCH 1548/2472] move assertions out of system (#19599) --- compiler/aliases.nim | 3 +++ compiler/ast.nim | 3 +++ compiler/astalgo.nim | 3 +++ compiler/bitsets.nim | 3 +++ compiler/btrees.nim | 3 +++ compiler/ccgutils.nim | 3 +++ compiler/cgen.nim | 4 ++++ compiler/cgmeth.nim | 4 ++++ compiler/closureiters.nim | 3 +++ compiler/commands.nim | 3 +++ compiler/concepts.nim | 3 +++ compiler/dfa.nim | 3 +++ compiler/enumtostr.nim | 4 ++++ compiler/errorhandling.nim | 3 +++ compiler/extccomp.nim | 3 +++ compiler/gorgeimpl.nim | 3 +++ compiler/guards.nim | 3 +++ compiler/hlo.nim | 3 +++ compiler/ic/bitabs.nim | 3 +++ compiler/ic/cbackend.nim | 3 +++ compiler/ic/dce.nim | 4 ++++ compiler/ic/ic.nim | 3 +++ compiler/ic/integrity.nim | 4 ++++ compiler/ic/navigator.nim | 3 +++ compiler/ic/packed_ast.nim | 3 +++ compiler/ic/replayer.nim | 3 +++ compiler/ic/rodfiles.nim | 3 +++ compiler/idents.nim | 3 +++ compiler/importer.nim | 3 +++ compiler/injectdestructors.nim | 3 +++ compiler/int128.nim | 3 +++ compiler/isolation_check.nim | 3 +++ compiler/lambdalifting.nim | 3 +++ compiler/lexer.nim | 3 +++ compiler/liftdestructors.nim | 3 +++ compiler/llstream.nim | 3 +++ compiler/lookups.nim | 4 ++++ compiler/lowerings.nim | 3 +++ compiler/main.nim | 4 ++++ compiler/modulegraphs.nim | 3 +++ compiler/modules.nim | 3 +++ compiler/msgs.nim | 4 ++++ compiler/ndi.nim | 3 +++ compiler/nilcheck.nim | 3 +++ compiler/nim.nim | 4 ++++ compiler/nimblecmd.nim | 3 +++ compiler/nimconf.nim | 3 +++ compiler/nimlexbase.nim | 3 +++ compiler/nimpaths.nim | 4 ++++ compiler/nimsets.nim | 3 +++ compiler/options.nim | 5 +++++ compiler/parser.nim | 3 +++ compiler/passes.nim | 3 +++ compiler/pathutils.nim | 2 +- compiler/patterns.nim | 3 +++ compiler/platform.nim | 4 ++++ compiler/pragmas.nim | 3 +++ compiler/renderer.nim | 3 +++ compiler/reorder.nim | 3 +++ compiler/rodutils.nim | 3 +++ compiler/ropes.nim | 4 ++++ compiler/scriptconfig.nim | 3 +++ compiler/semdata.nim | 3 +++ compiler/semfold.nim | 3 +++ compiler/sempass2.nim | 3 +++ compiler/semtypinst.nim | 3 +++ compiler/sighashes.nim | 4 ++++ compiler/sigmatch.nim | 3 +++ compiler/syntaxes.nim | 3 +++ compiler/transf.nim | 3 +++ compiler/treetab.nim | 3 +++ compiler/typeallowed.nim | 3 +++ compiler/types.nim | 3 +++ compiler/varpartitions.nim | 3 +++ compiler/vmdeps.nim | 3 +++ compiler/vmgen.nim | 3 +++ compiler/vmhooks.nim | 3 +++ compiler/vmmarshal.nim | 3 +++ compiler/vmops.nim | 4 ++++ lib/core/macros.nim | 4 ++++ lib/pure/algorithm.nim | 4 ++++ lib/pure/browsers.nim | 3 +++ lib/pure/collections/hashcommon.nim | 4 ++++ lib/pure/collections/sequtils.nim | 4 ++++ lib/pure/hashes.nim | 4 ++++ lib/pure/json.nim | 2 +- lib/pure/lexbase.nim | 3 +++ lib/pure/math.nim | 4 ++++ lib/pure/options.nim | 4 ++++ lib/pure/os.nim | 2 +- lib/pure/osproc.nim | 4 ++++ lib/pure/parsejson.nim | 3 +++ lib/pure/streamwrapper.nim | 4 ++++ lib/pure/strformat.nim | 4 ++++ lib/pure/strtabs.nim | 4 ++++ lib/pure/strutils.nim | 3 +++ lib/pure/terminal.nim | 2 +- lib/pure/times.nim | 4 ++++ lib/pure/typetraits.nim | 4 ++++ lib/{system => std}/assertions.nim | 14 ++++++++++---- lib/std/enumutils.nim | 4 ++++ lib/std/jsonutils.nim | 3 +++ lib/std/packedsets.nim | 3 +++ lib/std/private/globs.nim | 4 ++++ lib/std/sha1.nim | 3 +++ lib/system.nim | 7 +++++-- lib/system/dragonbox.nim | 2 ++ lib/system/formatfloat.nim | 3 +++ lib/system/iterators.nim | 3 +++ lib/system/schubfach.nim | 3 +++ 110 files changed, 361 insertions(+), 10 deletions(-) rename lib/{system => std}/assertions.nim (94%) diff --git a/compiler/aliases.nim b/compiler/aliases.nim index 0006c9fe6..9ec72faa4 100644 --- a/compiler/aliases.nim +++ b/compiler/aliases.nim @@ -12,6 +12,9 @@ import ast, astalgo, types, trees, intsets +when defined(nimPreviewSlimSystem): + import std/assertions + type TAnalysisResult* = enum arNo, arMaybe, arYes diff --git a/compiler/ast.nim b/compiler/ast.nim index ad5ec12f5..d1e5ae2bf 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -13,6 +13,9 @@ import lineinfos, hashes, options, ropes, idents, int128, tables from strutils import toLowerAscii +when defined(nimPreviewSlimSystem): + import std/assertions + export int128 type diff --git a/compiler/astalgo.nim b/compiler/astalgo.nim index 94fa9da93..bc5b7d6e1 100644 --- a/compiler/astalgo.nim +++ b/compiler/astalgo.nim @@ -15,6 +15,9 @@ import ast, hashes, intsets, strutils, options, lineinfos, ropes, idents, rodutils, msgs +when defined(nimPreviewSlimSystem): + import std/assertions + proc hashNode*(p: RootRef): Hash proc treeToYaml*(conf: ConfigRef; n: PNode, indent: int = 0, maxRecDepth: int = - 1): Rope # Convert a tree into its YAML representation; this is used by the diff --git a/compiler/bitsets.nim b/compiler/bitsets.nim index e0d94409e..67598f9ca 100644 --- a/compiler/bitsets.nim +++ b/compiler/bitsets.nim @@ -10,6 +10,9 @@ # this unit handles Nim sets; it implements bit sets # the code here should be reused in the Nim standard library +when defined(nimPreviewSlimSystem): + import std/assertions + type ElemType = byte TBitSet* = seq[ElemType] # we use byte here to avoid issues with diff --git a/compiler/btrees.nim b/compiler/btrees.nim index d5e7e8099..1701f8d7b 100644 --- a/compiler/btrees.nim +++ b/compiler/btrees.nim @@ -10,6 +10,9 @@ ## BTree implementation with few features, but good enough for the ## Nim compiler's needs. +when defined(nimPreviewSlimSystem): + import std/assertions + const M = 512 # max children per B-tree node = M-1 # (must be even and greater than 2) diff --git a/compiler/ccgutils.nim b/compiler/ccgutils.nim index 06b75a20f..e19fccfa7 100644 --- a/compiler/ccgutils.nim +++ b/compiler/ccgutils.nim @@ -13,6 +13,9 @@ import ast, types, hashes, strutils, msgs, wordrecg, platform, trees, options, cgendata +when defined(nimPreviewSlimSystem): + import std/assertions + proc getPragmaStmt*(n: PNode, w: TSpecialWord): PNode = case n.kind of nkStmtList: diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 2865473fb..ad59b759f 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -17,6 +17,10 @@ import lowerings, tables, sets, ndi, lineinfos, pathutils, transf, injectdestructors, astmsgs + +when defined(nimPreviewSlimSystem): + import std/assertions + when not defined(leanCompiler): import spawn, semparallel diff --git a/compiler/cgmeth.nim b/compiler/cgmeth.nim index 484bc9d97..dab8826c1 100644 --- a/compiler/cgmeth.nim +++ b/compiler/cgmeth.nim @@ -13,6 +13,10 @@ import intsets, options, ast, msgs, idents, renderer, types, magicsys, sempass2, strutils, modulegraphs, lineinfos +when defined(nimPreviewSlimSystem): + import std/assertions + + proc genConv(n: PNode, d: PType, downcast: bool; conf: ConfigRef): PNode = var dest = skipTypes(d, abstractPtrs) var source = skipTypes(n.typ, abstractPtrs) diff --git a/compiler/closureiters.nim b/compiler/closureiters.nim index 4d9aeb3f7..6370d0dcb 100644 --- a/compiler/closureiters.nim +++ b/compiler/closureiters.nim @@ -133,6 +133,9 @@ import renderer, magicsys, lowerings, lambdalifting, modulegraphs, lineinfos, tables, options +when defined(nimPreviewSlimSystem): + import std/assertions + type Ctx = object g: ModuleGraph diff --git a/compiler/commands.nim b/compiler/commands.nim index b21b48801..4c1cec3d2 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -31,6 +31,9 @@ import from ast import setUseIc, eqTypeFlags, tfGcSafe, tfNoSideEffect +when defined(nimPreviewSlimSystem): + import std/assertions + # but some have deps to imported modules. Yay. bootSwitch(usedTinyC, hasTinyCBackend, "-d:tinyc") bootSwitch(usedFFI, hasFFI, "-d:nimHasLibFFI") diff --git a/compiler/concepts.nim b/compiler/concepts.nim index 885b69c60..89a2bdc10 100644 --- a/compiler/concepts.nim +++ b/compiler/concepts.nim @@ -15,6 +15,9 @@ import ast, astalgo, semdata, lookups, lineinfos, idents, msgs, renderer, types, from magicsys import addSonSkipIntLit +when defined(nimPreviewSlimSystem): + import std/assertions + const logBindings = false diff --git a/compiler/dfa.nim b/compiler/dfa.nim index d68e7be82..5b048ff6e 100644 --- a/compiler/dfa.nim +++ b/compiler/dfa.nim @@ -32,6 +32,9 @@ import ast, intsets, lineinfos, renderer import std/private/asciitables +when defined(nimPreviewSlimSystem): + import std/assertions + type InstrKind* = enum goto, fork, def, use diff --git a/compiler/enumtostr.nim b/compiler/enumtostr.nim index 9bfa7001a..4a28d066c 100644 --- a/compiler/enumtostr.nim +++ b/compiler/enumtostr.nim @@ -1,6 +1,10 @@ import ast, idents, lineinfos, modulegraphs, magicsys +when defined(nimPreviewSlimSystem): + import std/assertions + + proc genEnumToStrProc*(t: PType; info: TLineInfo; g: ModuleGraph; idgen: IdGenerator): PSym = result = newSym(skProc, getIdent(g.cache, "$"), nextSymId idgen, t.owner, info) diff --git a/compiler/errorhandling.nim b/compiler/errorhandling.nim index cda7ab3f4..a8361105e 100644 --- a/compiler/errorhandling.nim +++ b/compiler/errorhandling.nim @@ -12,6 +12,9 @@ import ast, renderer, options, strutils, types +when defined(nimPreviewSlimSystem): + import std/assertions + type ErrorKind* = enum ## expand as you need. RawTypeMismatchError diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 124cc2c34..09a1dfbb3 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -16,6 +16,9 @@ import ropes, platform, condsyms, options, msgs, lineinfos, pathutils import std/[os, strutils, osproc, sha1, streams, sequtils, times, strtabs, json, jsonutils, sugar] +when defined(nimPreviewSlimSystem): + import std/syncio + type TInfoCCProp* = enum # properties of the C compiler: hasSwitchRange, # CC allows ranges in switch statements (GNU C) diff --git a/compiler/gorgeimpl.nim b/compiler/gorgeimpl.nim index d4aeb6a77..f58ac5de9 100644 --- a/compiler/gorgeimpl.nim +++ b/compiler/gorgeimpl.nim @@ -12,6 +12,9 @@ import msgs, std / sha1, os, osproc, streams, options, lineinfos, pathutils +when defined(nimPreviewSlimSystem): + import std/syncio + proc readOutput(p: Process): (string, int) = result[0] = "" var output = p.outputStream diff --git a/compiler/guards.nim b/compiler/guards.nim index d1265d42c..a50593aca 100644 --- a/compiler/guards.nim +++ b/compiler/guards.nim @@ -12,6 +12,9 @@ import ast, astalgo, msgs, magicsys, nimsets, trees, types, renderer, idents, saturate, modulegraphs, options, lineinfos, int128 +when defined(nimPreviewSlimSystem): + import std/assertions + const someEq = {mEqI, mEqF64, mEqEnum, mEqCh, mEqB, mEqRef, mEqProc, mEqStr, mEqSet, mEqCString} diff --git a/compiler/hlo.nim b/compiler/hlo.nim index af54cabbb..8620c4c4d 100644 --- a/compiler/hlo.nim +++ b/compiler/hlo.nim @@ -9,6 +9,9 @@ # This include implements the high level optimization pass. +when defined(nimPreviewSlimSystem): + import std/assertions + proc hlo(c: PContext, n: PNode): PNode proc evalPattern(c: PContext, n, orig: PNode): PNode = diff --git a/compiler/ic/bitabs.nim b/compiler/ic/bitabs.nim index 0bce30b5d..ae673b574 100644 --- a/compiler/ic/bitabs.nim +++ b/compiler/ic/bitabs.nim @@ -3,6 +3,9 @@ import hashes, rodfiles +when defined(nimPreviewSlimSystem): + import std/assertions + type LitId* = distinct uint32 diff --git a/compiler/ic/cbackend.nim b/compiler/ic/cbackend.nim index f5811cb3b..e7ab000e6 100644 --- a/compiler/ic/cbackend.nim +++ b/compiler/ic/cbackend.nim @@ -20,6 +20,9 @@ import std/packedsets, algorithm, tables +when defined(nimPreviewSlimSystem): + import std/assertions + import ".."/[ast, options, lineinfos, modulegraphs, cgendata, cgen, pathutils, extccomp, msgs] diff --git a/compiler/ic/dce.nim b/compiler/ic/dce.nim index 0a436a5d1..bc61a38de 100644 --- a/compiler/ic/dce.nim +++ b/compiler/ic/dce.nim @@ -10,6 +10,10 @@ ## Dead code elimination (=DCE) for IC. import std/[intsets, tables] + +when defined(nimPreviewSlimSystem): + import std/assertions + import ".." / [ast, options, lineinfos, types] import packed_ast, ic, bitabs diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim index 7387d165b..193e5f517 100644 --- a/compiler/ic/ic.nim +++ b/compiler/ic/ic.nim @@ -14,6 +14,9 @@ import ".." / [ast, idents, lineinfos, msgs, ropes, options, #import ".." / [renderer, astalgo] from os import removeFile, isAbsolute +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + type PackedConfig* = object backend: TBackend diff --git a/compiler/ic/integrity.nim b/compiler/ic/integrity.nim index ed367ef61..d341fd653 100644 --- a/compiler/ic/integrity.nim +++ b/compiler/ic/integrity.nim @@ -11,6 +11,10 @@ ## The set must cover a complete Nim project. import sets + +when defined(nimPreviewSlimSystem): + import std/assertions + import ".." / [ast, modulegraphs] import packed_ast, bitabs, ic diff --git a/compiler/ic/navigator.nim b/compiler/ic/navigator.nim index a1a14885d..cbba591c5 100644 --- a/compiler/ic/navigator.nim +++ b/compiler/ic/navigator.nim @@ -16,6 +16,9 @@ import sets from os import nil from std/private/miscdollars import toLocation +when defined(nimPreviewSlimSystem): + import std/assertions + import ".." / [ast, modulegraphs, msgs, options] import packed_ast, bitabs, ic diff --git a/compiler/ic/packed_ast.nim b/compiler/ic/packed_ast.nim index 17beda2c1..c78fe56f5 100644 --- a/compiler/ic/packed_ast.nim +++ b/compiler/ic/packed_ast.nim @@ -16,6 +16,9 @@ import hashes, tables, strtabs import bitabs import ".." / [ast, options] +when defined(nimPreviewSlimSystem): + import std/assertions + type SymId* = distinct int32 ModuleId* = distinct int32 diff --git a/compiler/ic/replayer.nim b/compiler/ic/replayer.nim index 61aa0e697..0188eaee3 100644 --- a/compiler/ic/replayer.nim +++ b/compiler/ic/replayer.nim @@ -16,6 +16,9 @@ import ".." / [ast, modulegraphs, trees, extccomp, btrees, import tables +when defined(nimPreviewSlimSystem): + import std/assertions + import packed_ast, ic, bitabs proc replayStateChanges*(module: PSym; g: ModuleGraph) = diff --git a/compiler/ic/rodfiles.nim b/compiler/ic/rodfiles.nim index a52bd18b3..d811b9b99 100644 --- a/compiler/ic/rodfiles.nim +++ b/compiler/ic/rodfiles.nim @@ -16,6 +16,9 @@ from typetraits import supportsCopyMem +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + ## Overview ## ======== ## `RodFile` represents a Rod File (versioned binary format), and the diff --git a/compiler/idents.nim b/compiler/idents.nim index d2a84fd36..f36ce09f3 100644 --- a/compiler/idents.nim +++ b/compiler/idents.nim @@ -14,6 +14,9 @@ import hashes, wordrecg +when defined(nimPreviewSlimSystem): + import std/assertions + type PIdent* = ref TIdent TIdent*{.acyclic.} = object diff --git a/compiler/importer.nim b/compiler/importer.nim index acca2c644..719b75f0f 100644 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -15,6 +15,9 @@ import modulegraphs, wordrecg, tables from strutils import `%` +when defined(nimPreviewSlimSystem): + import std/assertions + proc readExceptSet*(c: PContext, n: PNode): IntSet = assert n.kind in {nkImportExceptStmt, nkExportExceptStmt} result = initIntSet() diff --git a/compiler/injectdestructors.nim b/compiler/injectdestructors.nim index 985f27878..6500c5bc7 100644 --- a/compiler/injectdestructors.nim +++ b/compiler/injectdestructors.nim @@ -19,6 +19,9 @@ import lineinfos, parampatterns, sighashes, liftdestructors, optimizer, varpartitions +when defined(nimPreviewSlimSystem): + import std/assertions + from trees import exprStructuralEquivalent, getRoot type diff --git a/compiler/int128.nim b/compiler/int128.nim index 6ba7c1961..afa07094b 100644 --- a/compiler/int128.nim +++ b/compiler/int128.nim @@ -5,6 +5,9 @@ from math import trunc +when defined(nimPreviewSlimSystem): + import std/assertions + type Int128* = object udata: array[4, uint32] diff --git a/compiler/isolation_check.nim b/compiler/isolation_check.nim index 68a212794..2674605dc 100644 --- a/compiler/isolation_check.nim +++ b/compiler/isolation_check.nim @@ -13,6 +13,9 @@ import ast, types, renderer, intsets +when defined(nimPreviewSlimSystem): + import std/assertions + proc canAlias(arg, ret: PType; marker: var IntSet): bool proc canAliasN(arg: PType; n: PNode; marker: var IntSet): bool = diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index a622f6de6..6838fd80d 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -14,6 +14,9 @@ import idents, renderer, types, magicsys, lowerings, tables, modulegraphs, lineinfos, transf, liftdestructors, typeallowed +when defined(nimPreviewSlimSystem): + import std/assertions + discard """ The basic approach is that captured vars need to be put on the heap and that the calling chain needs to be explicitly modelled. Things to consider: diff --git a/compiler/lexer.nim b/compiler/lexer.nim index 506b0e924..e795d52c0 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -19,6 +19,9 @@ import hashes, options, msgs, strutils, platform, idents, nimlexbase, llstream, wordrecg, lineinfos, pathutils, parseutils +when defined(nimPreviewSlimSystem): + import std/assertions + const MaxLineLength* = 80 # lines longer than this lead to a warning numChars*: set[char] = {'0'..'9', 'a'..'z', 'A'..'Z'} diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim index 51b4ddfb0..68c93a179 100644 --- a/compiler/liftdestructors.nim +++ b/compiler/liftdestructors.nim @@ -15,6 +15,9 @@ import modulegraphs, lineinfos, idents, ast, renderer, semdata, from trees import isCaseObj +when defined(nimPreviewSlimSystem): + import std/assertions + type TLiftCtx = object g: ModuleGraph diff --git a/compiler/llstream.nim b/compiler/llstream.nim index 865a98ee0..004d990fa 100644 --- a/compiler/llstream.nim +++ b/compiler/llstream.nim @@ -12,6 +12,9 @@ import pathutils +when defined(nimPreviewSlimSystem): + import std/syncio + # support `useGnuReadline`, `useLinenoise` for backwards compatibility const hasRstdin = (defined(nimUseLinenoise) or defined(useLinenoise) or defined(useGnuReadline)) and not defined(windows) diff --git a/compiler/lookups.nim b/compiler/lookups.nim index fc30408e5..d61e15915 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -9,6 +9,10 @@ # This module implements lookup helpers. import std/[algorithm, strutils] + +when defined(nimPreviewSlimSystem): + import std/assertions + import intsets, ast, astalgo, idents, semdata, types, msgs, options, renderer, nimfix/prettybase, lineinfos, modulegraphs, astmsgs diff --git a/compiler/lowerings.nim b/compiler/lowerings.nim index 37405d8d9..20003b946 100644 --- a/compiler/lowerings.nim +++ b/compiler/lowerings.nim @@ -15,6 +15,9 @@ const import ast, astalgo, types, idents, magicsys, msgs, options, modulegraphs, lineinfos +when defined(nimPreviewSlimSystem): + import std/assertions + proc newDeref*(n: PNode): PNode {.inline.} = result = newNodeIT(nkHiddenDeref, n.info, n.typ[0]) result.add n diff --git a/compiler/main.nim b/compiler/main.nim index e4ec4d729..3cddbffb3 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -22,6 +22,10 @@ import modules, modulegraphs, lineinfos, pathutils, vmprofiler + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + import ic / [cbackend, integrity, navigator] from ic / ic import rodViewer diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index 28e0ddb82..c44908dc3 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -15,6 +15,9 @@ import intsets, tables, hashes, md5_old import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils import ic / [packed_ast, ic] +when defined(nimPreviewSlimSystem): + import std/assertions + type SigHash* = distinct MD5Digest diff --git a/compiler/modules.nim b/compiler/modules.nim index 6fba606b2..dd5db63fa 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -14,6 +14,9 @@ import idents, lexer, passes, syntaxes, llstream, modulegraphs, lineinfos, pathutils, tables +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + import ic / replayer proc resetSystemArtifacts*(g: ModuleGraph) = diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 56531eb68..9d111b2e2 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -12,6 +12,10 @@ import std/private/miscdollars, options, ropes, lineinfos, pathutils, strutils2 +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + + type InstantiationInfo* = typeof(instantiationInfo()) template instLoc*(): InstantiationInfo = instantiationInfo(-2, fullPaths = true) diff --git a/compiler/ndi.nim b/compiler/ndi.nim index 5af87237f..a9d9cfe79 100644 --- a/compiler/ndi.nim +++ b/compiler/ndi.nim @@ -12,6 +12,9 @@ import ast, msgs, ropes, options, pathutils +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + type NdiFile* = object enabled: bool diff --git a/compiler/nilcheck.nim b/compiler/nilcheck.nim index 9c2d091f7..49ceb8942 100644 --- a/compiler/nilcheck.nim +++ b/compiler/nilcheck.nim @@ -10,6 +10,9 @@ import ast, renderer, intsets, tables, msgs, options, lineinfos, strformat, idents, treetab, hashes import sequtils, strutils, sets +when defined(nimPreviewSlimSystem): + import std/assertions + # IMPORTANT: notes not up to date, i'll update this comment again # # notes: diff --git a/compiler/nim.nim b/compiler/nim.nim index 86c7ab524..bfb07ba20 100644 --- a/compiler/nim.nim +++ b/compiler/nim.nim @@ -8,6 +8,10 @@ # import std/[os, strutils, parseopt] + +when defined(nimPreviewSlimSystem): + import std/assertions + when defined(windows) and not defined(nimKochBootstrap): # remove workaround pending bootstrap >= 1.5.1 # refs https://github.com/nim-lang/Nim/issues/18334#issuecomment-867114536 diff --git a/compiler/nimblecmd.nim b/compiler/nimblecmd.nim index 9cd2941ba..b6b08ccd4 100644 --- a/compiler/nimblecmd.nim +++ b/compiler/nimblecmd.nim @@ -12,6 +12,9 @@ import parseutils, strutils, os, options, msgs, sequtils, lineinfos, pathutils, std/sha1, tables +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + proc addPath*(conf: ConfigRef; path: AbsoluteDir, info: TLineInfo) = if not conf.searchPaths.contains(path): conf.searchPaths.insert(path, 0) diff --git a/compiler/nimconf.nim b/compiler/nimconf.nim index 1cf22e20a..319a3de6d 100644 --- a/compiler/nimconf.nim +++ b/compiler/nimconf.nim @@ -13,6 +13,9 @@ import llstream, commands, os, strutils, msgs, lexer, ast, options, idents, wordrecg, strtabs, lineinfos, pathutils, scriptconfig +when defined(nimPreviewSlimSystem): + import std/syncio + # ---------------- configuration file parser ----------------------------- # we use Nim's lexer here to save space and work diff --git a/compiler/nimlexbase.nim b/compiler/nimlexbase.nim index b8cb9f78a..3bc9af9c9 100644 --- a/compiler/nimlexbase.nim +++ b/compiler/nimlexbase.nim @@ -15,6 +15,9 @@ import llstream, strutils +when defined(nimPreviewSlimSystem): + import std/assertions + const Lrz* = ' ' Apo* = '\'' diff --git a/compiler/nimpaths.nim b/compiler/nimpaths.nim index a93b488fd..216071c5c 100644 --- a/compiler/nimpaths.nim +++ b/compiler/nimpaths.nim @@ -19,6 +19,10 @@ Unstable API import os, strutils +when defined(nimPreviewSlimSystem): + import std/assertions + + const docCss* = "$nimr/doc/nimdoc.css" docHackNim* = "$nimr/tools/dochack/dochack.nim" diff --git a/compiler/nimsets.nim b/compiler/nimsets.nim index 8683604af..49c80065a 100644 --- a/compiler/nimsets.nim +++ b/compiler/nimsets.nim @@ -12,6 +12,9 @@ import ast, astalgo, lineinfos, bitsets, types, options +when defined(nimPreviewSlimSystem): + import std/assertions + proc inSet*(s: PNode, elem: PNode): bool = assert s.kind == nkCurly if s.kind != nkCurly: diff --git a/compiler/options.nim b/compiler/options.nim index 9c0c9df72..69eafb67f 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -14,6 +14,11 @@ import from terminal import isatty from times import utc, fromUnix, local, getTime, format, DateTime from std/private/globs import nativeToUnixPath + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + + const hasTinyCBackend* = defined(tinyc) useEffectSystem* = true diff --git a/compiler/parser.nim b/compiler/parser.nim index 0fb60440c..7973f7d37 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -36,6 +36,9 @@ import when defined(nimpretty): import layouter +when defined(nimPreviewSlimSystem): + import std/assertions + type Parser* = object # A Parser object represents a file that # is being parsed diff --git a/compiler/passes.nim b/compiler/passes.nim index 3debce1f6..ce52b10c5 100644 --- a/compiler/passes.nim +++ b/compiler/passes.nim @@ -16,6 +16,9 @@ import syntaxes, modulegraphs, reorder, lineinfos, pathutils +when defined(nimPreviewSlimSystem): + import std/syncio + type TPassData* = tuple[input: PNode, closeOutput: PNode] diff --git a/compiler/pathutils.nim b/compiler/pathutils.nim index 3a501a417..1ef0143d5 100644 --- a/compiler/pathutils.nim +++ b/compiler/pathutils.nim @@ -13,7 +13,7 @@ import os, pathnorm when defined(nimPreviewSlimSystem): - import std/syncio + import std/[syncio, assertions] type AbsoluteFile* = distinct string diff --git a/compiler/patterns.nim b/compiler/patterns.nim index 4b39de3ba..87e9c825c 100644 --- a/compiler/patterns.nim +++ b/compiler/patterns.nim @@ -13,6 +13,9 @@ import ast, types, semdata, sigmatch, idents, aliases, parampatterns, trees +when defined(nimPreviewSlimSystem): + import std/assertions + type TPatternContext = object owner: PSym diff --git a/compiler/platform.nim b/compiler/platform.nim index 4e6054d5c..4b4316bc2 100644 --- a/compiler/platform.nim +++ b/compiler/platform.nim @@ -16,6 +16,10 @@ import strutils +when defined(nimPreviewSlimSystem): + import std/assertions + + type TSystemOS* = enum # Also add OS in initialization section and alias # conditionals to condsyms (end of module). diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 702a9e082..01399d4ff 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -14,6 +14,9 @@ import wordrecg, ropes, options, strutils, extccomp, math, magicsys, trees, types, lookups, lineinfos, pathutils, linter +when defined(nimPreviewSlimSystem): + import std/assertions + from ic / ic import addCompilerProc const diff --git a/compiler/renderer.nim b/compiler/renderer.nim index 22a2d4cbd..3503c4bf6 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -17,6 +17,9 @@ when defined(nimHasUsed): import lexer, options, idents, strutils, ast, msgs, lineinfos +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + type TRenderFlag* = enum renderNone, renderNoBody, renderNoComments, renderDocComments, diff --git a/compiler/reorder.nim b/compiler/reorder.nim index 65c1fb9ee..daeb48248 100644 --- a/compiler/reorder.nim +++ b/compiler/reorder.nim @@ -4,6 +4,9 @@ import msgs, modulegraphs, syntaxes, options, modulepaths, lineinfos +when defined(nimPreviewSlimSystem): + import std/assertions + type DepN = ref object pnode: PNode diff --git a/compiler/rodutils.nim b/compiler/rodutils.nim index a4f7a5146..95b7b2d9e 100644 --- a/compiler/rodutils.nim +++ b/compiler/rodutils.nim @@ -10,6 +10,9 @@ ## Serialization utilities for the compiler. import strutils, math +when defined(nimPreviewSlimSystem): + import std/assertions + # bcc on windows doesn't have C99 functions when defined(windows) and defined(bcc): {.emit: """#if defined(_MSC_VER) && _MSC_VER < 1900 diff --git a/compiler/ropes.nim b/compiler/ropes.nim index cd696a545..a44d84ddc 100644 --- a/compiler/ropes.nim +++ b/compiler/ropes.nim @@ -60,6 +60,10 @@ import from pathutils import AbsoluteFile +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] + + type FormatStr* = string # later we may change it to CString for better # performance of the code generator (assignments diff --git a/compiler/scriptconfig.nim b/compiler/scriptconfig.nim index 89510f6b2..adc228d1e 100644 --- a/compiler/scriptconfig.nim +++ b/compiler/scriptconfig.nim @@ -16,6 +16,9 @@ import os, times, osproc, wordrecg, strtabs, modulegraphs, pathutils +when defined(nimPreviewSlimSystem): + import std/syncio + # we support 'cmpIgnoreStyle' natively for efficiency: from strutils import cmpIgnoreStyle, contains diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 422d1223a..d7a4d2c1c 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -11,6 +11,9 @@ import tables +when defined(nimPreviewSlimSystem): + import std/assertions + import intsets, options, ast, astalgo, msgs, idents, renderer, magicsys, vmdef, modulegraphs, lineinfos, sets, pathutils diff --git a/compiler/semfold.nim b/compiler/semfold.nim index d0473b5ce..d2bbc63be 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -17,6 +17,9 @@ import from system/memory import nimCStrLen +when defined(nimPreviewSlimSystem): + import std/assertions + proc errorType*(g: ModuleGraph): PType = ## creates a type representing an error state result = newType(tyError, nextTypeId(g.idgen), g.owners[^1]) diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index d6cc4180b..9bc811a03 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -12,6 +12,9 @@ import wordrecg, strutils, options, guards, lineinfos, semfold, semdata, modulegraphs, varpartitions, typeallowed, nilcheck, errorhandling, tables +when defined(nimPreviewSlimSystem): + import std/assertions + when defined(useDfa): import dfa diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim index 5d300fc58..504b83b4c 100644 --- a/compiler/semtypinst.nim +++ b/compiler/semtypinst.nim @@ -14,6 +14,9 @@ import ast, astalgo, msgs, types, magicsys, semdata, renderer, options, from concepts import makeTypeDesc +when defined(nimPreviewSlimSystem): + import std/assertions + const tfInstClearedFlags = {tfHasMeta, tfUnresolved} proc checkPartialConstructedType(conf: ConfigRef; info: TLineInfo, t: PType) = diff --git a/compiler/sighashes.nim b/compiler/sighashes.nim index 3d36d2533..1835d9d0f 100644 --- a/compiler/sighashes.nim +++ b/compiler/sighashes.nim @@ -13,6 +13,10 @@ import ast, tables, ropes, md5_old, modulegraphs from hashes import Hash import types +when defined(nimPreviewSlimSystem): + import std/assertions + + proc `&=`(c: var MD5Context, s: string) = md5Update(c, s, s.len) proc `&=`(c: var MD5Context, ch: char) = # XXX suspicious code here; relies on ch being zero terminated? diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 14f4c5e19..f195c4e45 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -15,6 +15,9 @@ import magicsys, idents, lexer, options, parampatterns, strutils, trees, linter, lineinfos, lowerings, modulegraphs, concepts +when defined(nimPreviewSlimSystem): + import std/assertions + type MismatchKind* = enum kUnknown, kAlreadyGiven, kUnknownNamedParam, kTypeMismatch, kVarNeeded, diff --git a/compiler/syntaxes.nim b/compiler/syntaxes.nim index 03a9702a3..e47b0483d 100644 --- a/compiler/syntaxes.nim +++ b/compiler/syntaxes.nim @@ -13,6 +13,9 @@ import strutils, llstream, ast, idents, lexer, options, msgs, parser, filters, filter_tmpl, renderer, lineinfos, pathutils +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + export Parser, parseAll, parseTopLevelStmt, closeParser type diff --git a/compiler/transf.nim b/compiler/transf.nim index edb8d3573..bdd7c680c 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -24,6 +24,9 @@ import lowerings, liftlocals, modulegraphs, lineinfos +when defined(nimPreviewSlimSystem): + import std/assertions + proc transformBody*(g: ModuleGraph; idgen: IdGenerator, prc: PSym, cache: bool): PNode import closureiters, lambdalifting diff --git a/compiler/treetab.nim b/compiler/treetab.nim index 26afd102c..d8dd8d33e 100644 --- a/compiler/treetab.nim +++ b/compiler/treetab.nim @@ -12,6 +12,9 @@ import hashes, ast, astalgo, types +when defined(nimPreviewSlimSystem): + import std/assertions + proc hashTree*(n: PNode): Hash = if n.isNil: return diff --git a/compiler/typeallowed.nim b/compiler/typeallowed.nim index 3d6ea0edb..57dd039ad 100644 --- a/compiler/typeallowed.nim +++ b/compiler/typeallowed.nim @@ -13,6 +13,9 @@ import intsets, ast, renderer, options, semdata, types +when defined(nimPreviewSlimSystem): + import std/assertions + type TTypeAllowedFlag* = enum taField, diff --git a/compiler/types.nim b/compiler/types.nim index 007a61356..4d2801633 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -13,6 +13,9 @@ import intsets, ast, astalgo, trees, msgs, strutils, platform, renderer, options, lineinfos, int128, modulegraphs, astmsgs +when defined(nimPreviewSlimSystem): + import std/assertions + type TPreferedDesc* = enum preferName, # default diff --git a/compiler/varpartitions.nim b/compiler/varpartitions.nim index 721de900e..d04a7b40c 100644 --- a/compiler/varpartitions.nim +++ b/compiler/varpartitions.nim @@ -32,6 +32,9 @@ import ast, types, lineinfos, options, msgs, renderer, typeallowed, modulegraphs from trees import getMagic, isNoSideEffectPragma, stupidStmtListExpr from isolation_check import canAlias +when defined(nimPreviewSlimSystem): + import std/assertions + type AbstractTime = distinct int diff --git a/compiler/vmdeps.nim b/compiler/vmdeps.nim index 1afda14b0..118a3031e 100644 --- a/compiler/vmdeps.nim +++ b/compiler/vmdeps.nim @@ -10,6 +10,9 @@ import ast, types, msgs, os, options, idents, lineinfos from pathutils import AbsoluteFile +when defined(nimPreviewSlimSystem): + import std/syncio + proc opSlurp*(file: string, info: TLineInfo, module: PSym; conf: ConfigRef): string = try: var filename = parentDir(toFullPath(conf, info)) / file diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 1551fbb80..72bcef4db 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -29,6 +29,9 @@ import tables +when defined(nimPreviewSlimSystem): + import std/assertions + import strutils, ast, types, msgs, renderer, vmdef, intsets, magicsys, options, lowerings, lineinfos, transf, astmsgs diff --git a/compiler/vmhooks.nim b/compiler/vmhooks.nim index 1ede87e5e..84ecf586f 100644 --- a/compiler/vmhooks.nim +++ b/compiler/vmhooks.nim @@ -9,6 +9,9 @@ import pathutils +when defined(nimPreviewSlimSystem): + import std/assertions + template setX(k, field) {.dirty.} = a.slots[a.ra].ensureKind(k) a.slots[a.ra].field = v diff --git a/compiler/vmmarshal.nim b/compiler/vmmarshal.nim index d28f0325b..83c441283 100644 --- a/compiler/vmmarshal.nim +++ b/compiler/vmmarshal.nim @@ -12,6 +12,9 @@ import streams, json, intsets, tables, ast, astalgo, idents, types, msgs, options, lineinfos +when defined(nimPreviewSlimSystem): + import std/assertions + proc ptrToInt(x: PNode): int {.inline.} = result = cast[int](x) # don't skip alignment diff --git a/compiler/vmops.nim b/compiler/vmops.nim index f13e4b1cc..640826cc8 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -31,6 +31,10 @@ from std/osproc import nil from system/formatfloat import addFloatRoundtrip, addFloatSprintf +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + + # There are some useful procs in vmconv. import vmconv, vmmarshal diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 4d040cebc..f20ca9385 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -10,6 +10,10 @@ include "system/inclrtl" import std/private/since +when defined(nimPreviewSlimSystem): + import std/assertions + + ## This module contains the interface to the compiler's abstract syntax ## tree (`AST`:idx:). Macros operate on this tree. ## diff --git a/lib/pure/algorithm.nim b/lib/pure/algorithm.nim index 1ddcc9843..c43545f78 100644 --- a/lib/pure/algorithm.nim +++ b/lib/pure/algorithm.nim @@ -44,6 +44,10 @@ runnableExamples: import std/private/since +when defined(nimPreviewSlimSystem): + import std/assertions + + type SortOrder* = enum Descending, Ascending diff --git a/lib/pure/browsers.nim b/lib/pure/browsers.nim index 08f5208d2..c36e31b11 100644 --- a/lib/pure/browsers.nim +++ b/lib/pure/browsers.nim @@ -16,6 +16,9 @@ import std/private/since import strutils +when defined(nimPreviewSlimSystem): + import std/assertions + when defined(windows): import winlean from os import absolutePath diff --git a/lib/pure/collections/hashcommon.nim b/lib/pure/collections/hashcommon.nim index a169418ce..deff8fa21 100644 --- a/lib/pure/collections/hashcommon.nim +++ b/lib/pure/collections/hashcommon.nim @@ -10,6 +10,10 @@ # An `include` file which contains common code for # hash sets and tables. +when defined(nimPreviewSlimSystem): + import std/assertions + + const growthFactor = 2 diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index 64a7be7a9..5e9b492c2 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -84,6 +84,10 @@ import std/private/since import macros +when defined(nimPreviewSlimSystem): + import std/assertions + + when defined(nimHasEffectsOf): {.experimental: "strictEffects".} else: diff --git a/lib/pure/hashes.nim b/lib/pure/hashes.nim index 47dacec7d..e88210757 100644 --- a/lib/pure/hashes.nim +++ b/lib/pure/hashes.nim @@ -67,6 +67,10 @@ runnableExamples: import std/private/since +when defined(nimPreviewSlimSystem): + import std/assertions + + type Hash* = int ## A hash value. Hash tables using these values should ## always have a size of a power of two so they can use the `and` diff --git a/lib/pure/json.nim b/lib/pure/json.nim index bdc9fe5ab..dd9232ea6 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -165,7 +165,7 @@ import options # xxx remove this dependency using same approach as https://githu import std/private/since when defined(nimPreviewSlimSystem): - import std/syncio + import std/[syncio, assertions] export tables.`$` diff --git a/lib/pure/lexbase.nim b/lib/pure/lexbase.nim index bbc0a38ae..336a57ec1 100644 --- a/lib/pure/lexbase.nim +++ b/lib/pure/lexbase.nim @@ -14,6 +14,9 @@ import strutils, streams +when defined(nimPreviewSlimSystem): + import std/assertions + const EndOfFile* = '\0' ## end of file marker NewLines* = {'\c', '\L'} diff --git a/lib/pure/math.nim b/lib/pure/math.nim index 1c47258bc..15324f882 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -60,6 +60,10 @@ import std/private/since import bitops, fenv +when defined(nimPreviewSlimSystem): + import std/assertions + + when defined(c) or defined(cpp): proc c_isnan(x: float): bool {.importc: "isnan", header: "".} # a generic like `x: SomeFloat` might work too if this is implemented via a C macro. diff --git a/lib/pure/options.nim b/lib/pure/options.nim index 850bfa555..562ed6361 100644 --- a/lib/pure/options.nim +++ b/lib/pure/options.nim @@ -71,6 +71,10 @@ supports pattern matching on `Option`s, with the `Some()` and import typetraits +when defined(nimPreviewSlimSystem): + import std/assertions + + when (NimMajor, NimMinor) >= (1, 1): type SomePointer = ref | ptr | pointer | proc diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 7c0c64276..fa379a228 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -35,7 +35,7 @@ import std/private/since import strutils, pathnorm when defined(nimPreviewSlimSystem): - import std/syncio + import std/[syncio, assertions] const weirdTarget = defined(nimscript) or defined(js) diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 3da9737ec..dca5099a4 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -31,6 +31,10 @@ else: when defined(linux) and defined(useClone): import linux +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + + type ProcessOption* = enum ## Options that can be passed to `startProcess proc ## <#startProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_. diff --git a/lib/pure/parsejson.nim b/lib/pure/parsejson.nim index 196d8c360..c92eac26e 100644 --- a/lib/pure/parsejson.nim +++ b/lib/pure/parsejson.nim @@ -14,6 +14,9 @@ import strutils, lexbase, streams, unicode import std/private/decode_helpers +when defined(nimPreviewSlimSystem): + import std/assertions + type JsonEventKind* = enum ## enumeration of all events that may occur when parsing jsonError, ## an error occurred during parsing diff --git a/lib/pure/streamwrapper.nim b/lib/pure/streamwrapper.nim index 7a501760b..a6c1901d2 100644 --- a/lib/pure/streamwrapper.nim +++ b/lib/pure/streamwrapper.nim @@ -13,6 +13,10 @@ import deques, streams +when defined(nimPreviewSlimSystem): + import std/assertions + + type PipeOutStream*[T] = ref object of T # When stream peek operation is called, it reads from base stream diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim index 40a33951c..ce3439600 100644 --- a/lib/pure/strformat.nim +++ b/lib/pure/strformat.nim @@ -316,6 +316,10 @@ single letter DSLs. import macros, parseutils, unicode import strutils except format +when defined(nimPreviewSlimSystem): + import std/assertions + + proc mkDigit(v: int, typ: char): string {.inline.} = assert(v < 26) if v < 10: diff --git a/lib/pure/strtabs.nim b/lib/pure/strtabs.nim index 3b90fea50..aa2886cfa 100644 --- a/lib/pure/strtabs.nim +++ b/lib/pure/strtabs.nim @@ -53,6 +53,10 @@ import std/private/since import hashes, strutils +when defined(nimPreviewSlimSystem): + import std/assertions + + when defined(js) or defined(nimscript) or defined(Standalone): {.pragma: rtlFunc.} else: diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 5e505ec3a..bf7bd6aa8 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -82,6 +82,9 @@ include "system/inclrtl" import std/private/since from std/private/strimpl import cmpIgnoreStyleImpl, cmpIgnoreCaseImpl, startsWithImpl, endsWithImpl +when defined(nimPreviewSlimSystem): + import std/assertions + const Whitespace* = {' ', '\t', '\v', '\r', '\l', '\f'} diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index 5755e142a..571c9b13c 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -67,7 +67,7 @@ when defined(windows): import winlean when defined(nimPreviewSlimSystem): - import std/syncio + import std/[syncio, assertions] type PTerminal = ref object diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 7851bf158..b70c5cedc 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -201,6 +201,10 @@ import strutils, math, options import std/private/since include "system/inclrtl" +when defined(nimPreviewSlimSystem): + import std/assertions + + when defined(js): import jscore diff --git a/lib/pure/typetraits.nim b/lib/pure/typetraits.nim index 8dc1b8cf2..3fc1c7c5c 100644 --- a/lib/pure/typetraits.nim +++ b/lib/pure/typetraits.nim @@ -15,6 +15,10 @@ import std/private/since export system.`$` # for backward compatibility +when defined(nimPreviewSlimSystem): + import std/assertions + + type HoleyEnum* = (not Ordinal) and enum ## Enum with holes. type OrdinalEnum* = Ordinal and enum ## Enum without holes. diff --git a/lib/system/assertions.nim b/lib/std/assertions.nim similarity index 94% rename from lib/system/assertions.nim rename to lib/std/assertions.nim index 6f64a55b7..3d2112b1a 100644 --- a/lib/system/assertions.nim +++ b/lib/std/assertions.nim @@ -1,7 +1,13 @@ -## This module provides various assertion utilities. -## -## **Note:** This module is reexported by `system` and thus does not need to be -## imported directly (with `system/assertions`). +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements assertion handling. when not declared(sysFatal): include "system/fatal" diff --git a/lib/std/enumutils.nim b/lib/std/enumutils.nim index b7d2b9f89..9d4ff1bcf 100644 --- a/lib/std/enumutils.nim +++ b/lib/std/enumutils.nim @@ -10,6 +10,10 @@ import macros from typetraits import OrdinalEnum, HoleyEnum +when defined(nimPreviewSlimSystem): + import std/assertions + + # xxx `genEnumCaseStmt` needs tests and runnableExamples macro genEnumCaseStmt*(typ: typedesc, argSym: typed, default: typed, diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim index 19384b5d1..722ea49b5 100644 --- a/lib/std/jsonutils.nim +++ b/lib/std/jsonutils.nim @@ -34,6 +34,9 @@ import macros from enumutils import symbolName from typetraits import OrdinalEnum +when defined(nimPreviewSlimSystem): + import std/assertions + when not defined(nimFixedForwardGeneric): # xxx remove pending csources_v1 update >= 1.2.0 proc to[T](node: JsonNode, t: typedesc[T]): T = diff --git a/lib/std/packedsets.nim b/lib/std/packedsets.nim index b2ee917eb..1e2892658 100644 --- a/lib/std/packedsets.nim +++ b/lib/std/packedsets.nim @@ -24,6 +24,9 @@ import std/private/since import hashes +when defined(nimPreviewSlimSystem): + import std/assertions + type BitScalar = uint diff --git a/lib/std/private/globs.nim b/lib/std/private/globs.nim index 190316f93..28a810372 100644 --- a/lib/std/private/globs.nim +++ b/lib/std/private/globs.nim @@ -8,6 +8,10 @@ import os when defined(windows): from strutils import replace +when defined(nimPreviewSlimSystem): + import std/assertions + + when defined(nimHasEffectsOf): {.experimental: "strictEffects".} else: diff --git a/lib/std/sha1.nim b/lib/std/sha1.nim index ed962707b..50175024c 100644 --- a/lib/std/sha1.nim +++ b/lib/std/sha1.nim @@ -29,6 +29,9 @@ runnableExamples("-r:off"): import strutils from endians import bigEndian32, bigEndian64 +when defined(nimPreviewSlimSystem): + import std/syncio + const Sha1DigestSize = 20 type diff --git a/lib/system.nim b/lib/system.nim index 572768de2..4080fee06 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1830,8 +1830,11 @@ when not defined(nimscript): when defined(nimV2): include system/arc -import system/assertions -export assertions +when not defined(nimPreviewSlimSystem): + {.deprecated: """assertions is about to move out of system; use `-d:nimPreviewSlimSystem` and + import `std/assertions`.""".} + import std/assertions + export assertions import system/iterators export iterators diff --git a/lib/system/dragonbox.nim b/lib/system/dragonbox.nim index 336c982c1..34ae9e210 100644 --- a/lib/system/dragonbox.nim +++ b/lib/system/dragonbox.nim @@ -24,6 +24,8 @@ import std/private/digitsutils +when defined(nimPreviewSlimSystem): + import std/assertions const dtoaMinBufferLength*: cint = 64 diff --git a/lib/system/formatfloat.nim b/lib/system/formatfloat.nim index 3bcd3257b..aada3e1bf 100644 --- a/lib/system/formatfloat.nim +++ b/lib/system/formatfloat.nim @@ -7,6 +7,9 @@ # distribution, for details about the copyright. # +when defined(nimPreviewSlimSystem): + import std/assertions + proc c_memcpy(a, b: pointer, size: csize_t): pointer {.importc: "memcpy", header: "", discardable.} proc addCstringN(result: var string, buf: cstring; buflen: int) = diff --git a/lib/system/iterators.nim b/lib/system/iterators.nim index f23f7aa11..6d33fc0bc 100644 --- a/lib/system/iterators.nim +++ b/lib/system/iterators.nim @@ -1,3 +1,6 @@ +when defined(nimPreviewSlimSystem): + import std/assertions + when defined(nimHasLentIterators) and not defined(nimNoLentIterators): template lent2(T): untyped = lent T else: diff --git a/lib/system/schubfach.nim b/lib/system/schubfach.nim index 06813f632..7d6861e35 100644 --- a/lib/system/schubfach.nim +++ b/lib/system/schubfach.nim @@ -12,6 +12,9 @@ import std/private/digitsutils +when defined(nimPreviewSlimSystem): + import std/assertions + template sf_Assert(x: untyped): untyped = assert(x) From e93eaac223045543e753643765b6982a109b41a1 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 24 Mar 2022 03:42:08 +0800 Subject: [PATCH 1549/2472] build external deps with userversion:1.6 (#19612) --- koch.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/koch.nim b/koch.nim index 295b1584b..a3cfd5930 100644 --- a/koch.nim +++ b/koch.nim @@ -142,7 +142,7 @@ proc csource(args: string) = proc bundleC2nim(args: string) = cloneDependency(distDir, "https://github.com/nim-lang/c2nim.git") nimCompile("dist/c2nim/c2nim", - options = "--noNimblePath --path:. " & args) + options = "--noNimblePath --useVersion:1.6 --path:. " & args) proc bundleNimbleExe(latest: bool, args: string) = let commit = if latest: "HEAD" else: NimbleStableCommit @@ -150,7 +150,7 @@ proc bundleNimbleExe(latest: bool, args: string) = commit = commit, allowBundled = true) # installer.ini expects it under $nim/bin nimCompile("dist/nimble/src/nimble.nim", - options = "-d:release --noNimblePath " & args) + options = "-d:release --useVersion:1.6 --noNimblePath " & args) proc bundleNimsuggest(args: string) = nimCompileFold("Compile nimsuggest", "nimsuggest/nimsuggest.nim", From d14e8e8dccc87671acb832b4aea8e736ce60c13e Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Wed, 23 Mar 2022 16:42:54 -0300 Subject: [PATCH 1550/2472] Remove Deprecated Nimscript proc (#19629) --- lib/system/nimscript.nim | 8 -------- 1 file changed, 8 deletions(-) diff --git a/lib/system/nimscript.nim b/lib/system/nimscript.nim index b4554d778..0ef5fc584 100644 --- a/lib/system/nimscript.nim +++ b/lib/system/nimscript.nim @@ -136,14 +136,6 @@ proc dirExists*(dir: string): bool {. ## Checks if the directory `dir` exists. builtin -template existsFile*(args: varargs[untyped]): untyped {.deprecated: "use fileExists".} = - # xxx: warning won't be shown for nimsscript because of current logic handling - # `foreignPackageNotes` - fileExists(args) - -template existsDir*(args: varargs[untyped]): untyped {.deprecated: "use dirExists".} = - dirExists(args) - proc selfExe*(): string = ## Returns the currently running nim or nimble executable. # TODO: consider making this as deprecated alias of `getCurrentCompilerExe` From d7a966c4ebdb13a1cc414ce98bbff87ffa922261 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Wed, 23 Mar 2022 16:43:28 -0300 Subject: [PATCH 1551/2472] Remove Deprecated domextension (#19630) * Remove Deprecated domextension * Remove Deprecated domextension --- changelog.md | 1 + lib/js/dom_extensions.nim | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) delete mode 100644 lib/js/dom_extensions.nim diff --git a/changelog.md b/changelog.md index a9bf3842a..73cb5e2b5 100644 --- a/changelog.md +++ b/changelog.md @@ -46,6 +46,7 @@ becomes an alias for `addr`. - Removed deprecated `jsre.test` and `jsre.toString`. - Removed deprecated `math.c_frexp`. - Removed deprecated ``` httpcore.`==` ```. +- Removed deprecated `std/dom_extensions`. ## Language changes diff --git a/lib/js/dom_extensions.nim b/lib/js/dom_extensions.nim deleted file mode 100644 index c15a03195..000000000 --- a/lib/js/dom_extensions.nim +++ /dev/null @@ -1,3 +0,0 @@ -import std/dom -export elementsFromPoint -{.deprecated: "use `std/dom` instead".} From b0bd4320a0cd9a90acf44bca0dddf10fa5022969 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Wed, 23 Mar 2022 16:43:58 -0300 Subject: [PATCH 1552/2472] Remove deprecated typo poDemon (#19631) * Remove Deprecated poDemon * Remove Deprecated poDemon --- changelog.md | 2 ++ lib/pure/osproc.nim | 5 ----- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/changelog.md b/changelog.md index 73cb5e2b5..92d2612e8 100644 --- a/changelog.md +++ b/changelog.md @@ -48,6 +48,8 @@ becomes an alias for `addr`. - Removed deprecated ``` httpcore.`==` ```. - Removed deprecated `std/dom_extensions`. +- Remove deprecated `osproc.poDemon`, symbol with typo. + ## Language changes diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index dca5099a4..512db92a4 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -70,11 +70,6 @@ type Process* = ref ProcessObj ## Represents an operating system process. -const poDemon* {.deprecated.} = poDaemon ## Nim versions before 0.20 - ## used the wrong spelling ("demon"). - ## Now `ProcessOption` uses the correct spelling ("daemon"), - ## and this is needed just for backward compatibility. - proc execProcess*(command: string, workingDir: string = "", args: openArray[string] = [], env: StringTableRef = nil, From 2c01c9c4c8a20772ebbb91f3333feb5dbcc94e9e Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 24 Mar 2022 03:57:13 +0800 Subject: [PATCH 1553/2472] output byref types into --header file [backport: 1.6] (#19505) * output byref types into --header file fix #19445 * fix comments * set targets --- compiler/ccgtypes.nim | 9 +++++++-- tests/ccgbugs/m19445.c | 3 +++ tests/ccgbugs/t19445.nim | 13 +++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 tests/ccgbugs/m19445.c create mode 100644 tests/ccgbugs/t19445.nim diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index f010b25c9..c5b4d3857 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -471,8 +471,13 @@ proc genProcParams(m: BModule, t: PType, rettype, params: var Rope, var arr = t[0] if params != nil: params.add(", ") if mapReturnType(m.config, t[0]) != ctArray: - params.add(getTypeDescWeak(m, arr, check, skResult)) - params.add("*") + if isHeaderFile in m.flags: + # still generates types for `--header` + params.add(getTypeDescAux(m, arr, check, skResult)) + params.add("*") + else: + params.add(getTypeDescWeak(m, arr, check, skResult)) + params.add("*") else: params.add(getTypeDescAux(m, arr, check, skResult)) params.addf(" Result", []) diff --git a/tests/ccgbugs/m19445.c b/tests/ccgbugs/m19445.c new file mode 100644 index 000000000..74c23d4b4 --- /dev/null +++ b/tests/ccgbugs/m19445.c @@ -0,0 +1,3 @@ +#include "m19445.h" + +const Foo f = {10, 20, 30, 40}; \ No newline at end of file diff --git a/tests/ccgbugs/t19445.nim b/tests/ccgbugs/t19445.nim new file mode 100644 index 000000000..b6e8d028c --- /dev/null +++ b/tests/ccgbugs/t19445.nim @@ -0,0 +1,13 @@ +discard """ + matrix: "--nimcache:tests/ccgbugs/nimcache19445 --cincludes:nimcache19445 --header:m19445" + targets: "c" +""" + +# bug #19445 +type + Foo* {.exportc.} = object + a*, b*, c*, d*: int + +proc dummy(): Foo {.exportc.} = discard + +{.compile:"m19445.c".} \ No newline at end of file From a262a87bbebdaf41ea248f7caa780f8ea27e20e1 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 24 Mar 2022 04:07:05 +0800 Subject: [PATCH 1554/2472] [add testcase] arc problems with recursive types (#19456) * [add testcase] arc problems with recursive types close #9650 * do test * expand * Update tests/arc/t9650.nim --- tests/arc/t9650.nim | 87 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 tests/arc/t9650.nim diff --git a/tests/arc/t9650.nim b/tests/arc/t9650.nim new file mode 100644 index 000000000..a8182db68 --- /dev/null +++ b/tests/arc/t9650.nim @@ -0,0 +1,87 @@ +discard """ + matrix: "--gc:arc" +""" + +import typetraits + +# bug #9650 +type + SharedPtr*[T] = object + val: ptr tuple[atomicCounter: int, value: T] + + Node*[T] = object + value: T + next: SharedPtr[Node[T]] + + ForwardList*[T] = object + first: SharedPtr[Node[T]] + +proc `=destroy`*[T](p: var SharedPtr[T]) = + if p.val != nil: + let c = atomicDec(p.val[].atomicCounter) + if c == 0: + when not supportsCopyMem(T): + `=destroy`(p.val[]) + dealloc(p.val) + p.val = nil + +proc `=`*[T](dest: var SharedPtr[T], src: SharedPtr[T]) = + if dest.val != src.val: + if dest.val != nil: + `=destroy`(dest) + if src.val != nil: + discard atomicInc(src.val[].atomicCounter) + dest.val = src.val + +proc `=sink`*[T](dest: var SharedPtr[T], src: SharedPtr[T]) = + if dest.val != nil and dest.val != src.val: + `=destroy`(dest) + dest.val = src.val + + +proc newSharedPtr*[T](val: sink T): SharedPtr[T] = + result.val = cast[type(result.val)](alloc(sizeof(result.val[]))) + reset(result.val[]) + result.val.atomicCounter = 1 + result.val.value = val + +proc isNil*[T](p: SharedPtr[T]): bool = + p.val == nil + +template `->`*[T](p: SharedPtr[T], name: untyped): untyped = + p.val.value.name + +proc createNode[T](val: T): SharedPtr[ Node[T] ]= + result = newSharedPtr(Node[T](value: val)) + +proc push_front*[T](list: var ForwardList[T], val: T) = + var newElem = createNode(val) + newElem->next = list.first + list.first = newElem + +proc pop_front*[T](list: var ForwardList[T]) = + let head = list.first + list.first = head->next + +proc toString*[T](list: ForwardList[T]): string = + result = "[" + var head = list.first + while not head.isNil: + result &= $(head->value) & ", " + head = head->next + result &= ']' + +block: + var x: ForwardList[int] + x.push_front(1) + x.push_front(2) + x.push_front(3) + + doAssert toString(x) == "[3, 2, 1, ]" + + x.pop_front() + x.pop_front() + doAssert toString(x) == "[1, ]" + + x.pop_front() + doAssert toString(x) == "[]" From 7c3c61f2f1e898e5c70dfd619d94d09e339836e4 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 24 Mar 2022 04:09:44 +0800 Subject: [PATCH 1555/2472] fix nim check nimscript [backport: 1.6] (#19444) fix #19440; fix #3858 --- compiler/commands.nim | 2 ++ compiler/main.nim | 6 +++++- tests/newconfig/tconfigcheck.nims | 11 +++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 tests/newconfig/tconfigcheck.nims diff --git a/compiler/commands.nim b/compiler/commands.nim index 4c1cec3d2..98537e9ef 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -1112,6 +1112,8 @@ proc processArgument*(pass: TCmdLinePass; p: OptParser; else: if pass == passCmd1: config.commandArgs.add p.key if argsCount == 1: + if p.key.endsWith(".nims"): + incl(config.globalOptions, optWasNimscript) # support UNIX style filenames everywhere for portable build scripts: if config.projectName.len == 0: config.projectName = unixToNativePath(p.key) diff --git a/compiler/main.nim b/compiler/main.nim index 3cddbffb3..e27a4f6d1 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -62,6 +62,9 @@ proc commandCheck(graph: ModuleGraph) = let conf = graph.config conf.setErrorMaxHighMaybe defineSymbol(conf.symbols, "nimcheck") + if optWasNimscript in conf.globalOptions: + defineSymbol(conf.symbols, "nimscript") + defineSymbol(conf.symbols, "nimconfig") semanticPasses(graph) # use an empty backend for semantic checking only compileProject(graph) @@ -367,7 +370,8 @@ proc mainCommand*(graph: ModuleGraph) = msgWriteln(conf, "-- end of list --", {msgStdout, msgSkipHook}) for it in conf.searchPaths: msgWriteln(conf, it.string) - of cmdCheck: commandCheck(graph) + of cmdCheck: + commandCheck(graph) of cmdParse: wantMainModule(conf) discard parseFile(conf.projectMainIdx, cache, conf) diff --git a/tests/newconfig/tconfigcheck.nims b/tests/newconfig/tconfigcheck.nims new file mode 100644 index 000000000..2e12a0bda --- /dev/null +++ b/tests/newconfig/tconfigcheck.nims @@ -0,0 +1,11 @@ +discard """ + cmd: "nim check $file" +""" + +mode = ScriptMode.Verbose +proc build() = + echo "building nim... " + exec "sleep 10" + echo getCurrentDir() + +echo "hello" From 400e0260b854f34a91af47bbe0a1b1c9d112e51b Mon Sep 17 00:00:00 2001 From: treeform Date: Wed, 23 Mar 2022 23:35:29 -0700 Subject: [PATCH 1556/2472] Add more info for {.bycopy.} (#18815) * Add more info for {.bycopy.} See confusion here: https://github.com/nim-lang/Nim/issues/18807 I hope this will help people googling to find this. * Update doc/manual.rst Co-authored-by: Andreas Rumpf --- doc/manual.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manual.rst b/doc/manual.rst index 4c4b9c506..8206d8342 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -7912,6 +7912,7 @@ instructs the compiler to pass the type by value to procs: Vector {.bycopy.} = object x, y, z: float +The Nim compiler automatically determines whether a parameter is passed by value or by reference based on the parameter type's size. If a parameter must be passed by value or by reference, (such as when interfacing with a C library) use the bycopy or byref pragmas. Byref pragma ------------ From afbcba909b37b8c06250b141ddb9da4bf5bb9922 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 25 Mar 2022 17:46:42 +0800 Subject: [PATCH 1557/2472] remove unnecessary framePtr code (#19645) --- compiler/jsgen.nim | 9 ++++++--- lib/system/jssys.nim | 3 ++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index f4097efa4..d7955b2f9 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -773,7 +773,8 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) = var tmpFramePtr = rope"F" if optStackTrace notin p.options: tmpFramePtr = p.getTemp(true) - line(p, tmpFramePtr & " = framePtr;\L") + if hasFrameInfo(p): + line(p, tmpFramePtr & " = framePtr;\L") lineF(p, "try {$n", []) var a: TCompRes gen(p, n[0], a) @@ -782,7 +783,8 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) = if catchBranchesExist: p.body.addf("--excHandler;$n} catch (EXCEPTION) {$n var prevJSError = lastJSError;$n" & " lastJSError = EXCEPTION;$n --excHandler;$n", []) - line(p, "framePtr = $1;$n" % [tmpFramePtr]) + if hasFrameInfo(p): + line(p, "framePtr = $1;$n" % [tmpFramePtr]) while i < n.len and n[i].kind == nkExceptBranch: if n[i].len == 1: # general except section: @@ -841,7 +843,8 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) = line(p, "}\L") lineF(p, "lastJSError = prevJSError;$n") line(p, "} finally {\L") - line(p, "framePtr = $1;$n" % [tmpFramePtr]) + if hasFrameInfo(p): + line(p, "framePtr = $1;$n" % [tmpFramePtr]) if i < n.len and n[i].kind == nkFinally: genStmt(p, n[i][0]) line(p, "}\L") diff --git a/lib/system/jssys.nim b/lib/system/jssys.nim index 6608a2927..00a4a8ab6 100644 --- a/lib/system/jssys.nim +++ b/lib/system/jssys.nim @@ -130,7 +130,8 @@ proc unhandledException(e: ref Exception) {. when NimStackTrace: add(buf, rawWriteStackTrace()) let cbuf = cstring(buf) - framePtr = nil + when NimStackTrace: + framePtr = nil {.emit: """ if (typeof(Error) !== "undefined") { throw new Error(`cbuf`); From 12a0f88a52bae74e31d6d612f0e2aae3f90340a9 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 26 Mar 2022 06:37:04 +0100 Subject: [PATCH 1558/2472] mitigates #19364 [backport]; we make this bug more unlikely to appear by producing better code to begin with; real fix will come later (#19647) --- compiler/lowerings.nim | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/compiler/lowerings.nim b/compiler/lowerings.nim index 20003b946..382e48a16 100644 --- a/compiler/lowerings.nim +++ b/compiler/lowerings.nim @@ -74,14 +74,20 @@ proc lowerTupleUnpacking*(g: ModuleGraph; n: PNode; idgen: IdGenerator; owner: P let value = n.lastSon result = newNodeI(nkStmtList, n.info) - var temp = newSym(skTemp, getIdent(g.cache, genPrefix), nextSymId(idgen), - owner, value.info, g.config.options) - temp.typ = skipTypes(value.typ, abstractInst) - incl(temp.flags, sfFromGeneric) + var tempAsNode: PNode + let avoidTemp = value.kind == nkSym + if avoidTemp: + tempAsNode = value + else: + var temp = newSym(skTemp, getIdent(g.cache, genPrefix), nextSymId(idgen), + owner, value.info, g.config.options) + temp.typ = skipTypes(value.typ, abstractInst) + incl(temp.flags, sfFromGeneric) + tempAsNode = newSymNode(temp) var v = newNodeI(nkVarSection, value.info) - let tempAsNode = newSymNode(temp) - v.addVar(tempAsNode, value) + if not avoidTemp: + v.addVar(tempAsNode, value) result.add(v) for i in 0.. Date: Sat, 26 Mar 2022 06:47:04 +0100 Subject: [PATCH 1559/2472] fixes #19603; some pragmas were really only supported as top level statements. Now this is enforced properly. (#19646) --- compiler/pragmas.nim | 15 +++++++++------ compiler/semexprs.nim | 8 +++++++- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 01399d4ff..81c6a42d3 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -47,18 +47,21 @@ const wDiscardable, wGensym, wInject, wRaises, wEffectsOf, wTags, wLocks, wGcSafe, wRequires, wEnsures} exprPragmas* = {wLine, wLocks, wNoRewrite, wGcSafe, wNoSideEffect} - stmtPragmas* = {wChecks, wObjChecks, wFieldChecks, wRangeChecks, - wBoundChecks, wOverflowChecks, wNilChecks, wStaticBoundchecks, - wStyleChecks, wAssertions, - wWarnings, wHints, - wLineDir, wStackTrace, wLineTrace, wOptimization, wHint, wWarning, wError, + stmtPragmas* = { + wHint, wWarning, wError, wFatal, wDefine, wUndef, wCompile, wLink, wLinksys, wPure, wPush, wPop, wPassl, wPassc, wLocalPassc, wDeadCodeElimUnused, # deprecated, always on wDeprecated, - wFloatChecks, wInfChecks, wNanChecks, wPragma, wEmit, wUnroll, + wPragma, wEmit, wUnroll, wLinearScanEnd, wPatterns, wTrMacros, wEffects, wNoForward, wReorder, wComputedGoto, wExperimental, wThis, wUsed, wInvariant, wAssume, wAssert} + stmtPragmasTopLevel* = {wChecks, wObjChecks, wFieldChecks, wRangeChecks, + wBoundChecks, wOverflowChecks, wNilChecks, wStaticBoundchecks, + wStyleChecks, wAssertions, + wWarnings, wHints, + wLineDir, wStackTrace, wLineTrace, wOptimization, + wFloatChecks, wInfChecks, wNanChecks} lambdaPragmas* = {FirstCallConv..LastCallConv, wNoSideEffect, wSideEffect, wNoreturn, wNosinks, wDynlib, wHeader, wThread, wAsmNoStackFrame, diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 5f3d065b6..f224bd0b3 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -2769,6 +2769,12 @@ proc enumFieldSymChoice(c: PContext, n: PNode, s: PSym): PNode = onUse(info, a) a = nextOverloadIter(o, c, n) +proc semPragmaStmt(c: PContext; n: PNode) = + if c.p.owner.kind == skModule: + pragma(c, c.p.owner, n, stmtPragmas+stmtPragmasTopLevel, true) + else: + pragma(c, c.p.owner, n, stmtPragmas, true) + proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = when defined(nimCompilerStacktraceHints): setFrameMsg c.config$n.info & " " & $n.kind @@ -3017,7 +3023,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = of nkUsingStmt: result = semUsing(c, n) of nkAsmStmt: result = semAsm(c, n) of nkYieldStmt: result = semYield(c, n) - of nkPragma: pragma(c, c.p.owner, n, stmtPragmas, true) + of nkPragma: semPragmaStmt(c, n) of nkIteratorDef: result = semIterator(c, n) of nkProcDef: result = semProc(c, n) of nkFuncDef: result = semFunc(c, n) From 4dfe420c2f1791c722ff949072e1a74f3e7c1499 Mon Sep 17 00:00:00 2001 From: tandy1000 Date: Sat, 26 Mar 2022 05:57:39 +0000 Subject: [PATCH 1560/2472] Add support for `Window.matchMedia` (#19648) --- lib/js/dom.nim | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/js/dom.nim b/lib/js/dom.nim index aca79faca..c1a677258 100644 --- a/lib/js/dom.nim +++ b/lib/js/dom.nim @@ -1322,6 +1322,11 @@ type `block`*: cstring inline*: cstring + MediaQueryList* = ref MediaQueryListObj + MediaQueryListObj {.importc.} = object of EventTargetObj + matches*: bool + media*: cstring + since (1, 3): type DomParser* = ref object @@ -1529,6 +1534,7 @@ proc setTimeout*(w: Window, function: proc (), pause: int): Interval proc stop*(w: Window) proc requestAnimationFrame*(w: Window, function: proc (time: float)): int proc cancelAnimationFrame*(w: Window, id: int) +proc matchMedia*(w: Window, mediaQueryString: cstring): MediaQueryList # Node "methods" proc appendData*(n: Node, data: cstring) From 84ac0035190e81cb991544460786bd5618386bec Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 26 Mar 2022 15:57:29 +0800 Subject: [PATCH 1561/2472] remove tmpFramePtr when optStackTrace is absent (#19649) --- compiler/jsgen.nim | 4 ---- 1 file changed, 4 deletions(-) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index d7955b2f9..b9a73daf9 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -771,10 +771,6 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) = if catchBranchesExist: p.body.add("++excHandler;\L") var tmpFramePtr = rope"F" - if optStackTrace notin p.options: - tmpFramePtr = p.getTemp(true) - if hasFrameInfo(p): - line(p, tmpFramePtr & " = framePtr;\L") lineF(p, "try {$n", []) var a: TCompRes gen(p, n[0], a) From 86e93eb2938329db921d5edb718fa7f3ecee83b1 Mon Sep 17 00:00:00 2001 From: Omar Flores <47699779+yoyojambo@users.noreply.github.com> Date: Sat, 26 Mar 2022 02:56:10 -0600 Subject: [PATCH 1562/2472] Fixed description as option in advopt.txt (#19635) * Fixed formatting error in warningAsError. There was only a single space character between the warning and its description, so it shows up as part of the name (in bold) and with no description. Copied the way hotCodeReloading was formatted, with the description in a new line. Also changed descriptions from 'ditto' to what the description it references says. * Fixed typo in advopt.txt Corrected 'hints' to 'warnings' in updated advopt.txt. --- doc/advopt.txt | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/doc/advopt.txt b/doc/advopt.txt index 0a2349597..244fe035b 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -55,9 +55,13 @@ Advanced options: (only `all:off` is supported). --hintAsError:X:on|off turn specific hint X into an error on|off -w:on|off|list, --warnings:on|off|list - same as `--hints` but for warnings. - --warning:X:on|off ditto - --warningAsError:X:on|off ditto + `on|off` enables or disables warnings. + `list` reports which warnings are selected. + --warning:X:on|off turn specific warning X on|off. `warning:X` means `warning:X:on`, + as with similar flags. `all` is the set of all warning + (only `all:off` is supported). + --warningAsError:X:on|off + turn specific warning X into an error on|off --styleCheck:off|hint|error produce hints or errors for Nim identifiers that do not adhere to Nim's official style guide From 8cdd8867c0237c28f7a43e6f871507eda207a15d Mon Sep 17 00:00:00 2001 From: John Titor Date: Sat, 26 Mar 2022 10:43:26 +0000 Subject: [PATCH 1563/2472] Fix dial ignoring buffered parameter (#19650) [backport] --- lib/pure/net.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index c1d4c6228..9922bbfa4 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -1963,7 +1963,7 @@ proc dial*(address: string, port: Port, closeUnusedFds(ord(domain)) if success: - result = newSocket(lastFd, domain, sockType, protocol) + result = newSocket(lastFd, domain, sockType, protocol, buffered) elif lastError != 0.OSErrorCode: raiseOSError(lastError) else: From 82319ef00dc9ce6f65b1664024a10ee5d9465f38 Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 26 Mar 2022 18:50:15 +0800 Subject: [PATCH 1564/2472] fix #8219; nim check/dump shouldn't run single nimscript project [backport: 1.6] (#19641) * fix #8219; nim check/dump shouldn't run single nimscript project [backport: 1.6] --- compiler/nimconf.nim | 3 ++- tests/misc/trunner.nim | 5 +++++ tests/newconfig/{tconfigcheck.nims => mconfigcheck.nims} | 6 ++---- 3 files changed, 9 insertions(+), 5 deletions(-) rename tests/newconfig/{tconfigcheck.nims => mconfigcheck.nims} (74%) diff --git a/compiler/nimconf.nim b/compiler/nimconf.nim index 319a3de6d..fceedb2c4 100644 --- a/compiler/nimconf.nim +++ b/compiler/nimconf.nim @@ -301,7 +301,7 @@ proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen: if conf.cmd == cmdNimscript: showHintConf() conf.configFiles.setLen 0 - if conf.cmd != cmdIdeTools: + if conf.cmd notin {cmdIdeTools, cmdCheck, cmdDump}: if conf.cmd == cmdNimscript: runNimScriptIfExists(conf.projectFull, isMain = true) else: @@ -311,5 +311,6 @@ proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen: runNimScriptIfExists(scriptFile, isMain = true) else: # 'nimsuggest foo.nims' means to just auto-complete the NimScript file + # `nim check foo.nims' means to check the syntax of the NimScript file discard showHintConf() diff --git a/tests/misc/trunner.nim b/tests/misc/trunner.nim index f874d38d9..5d12c38b6 100644 --- a/tests/misc/trunner.nim +++ b/tests/misc/trunner.nim @@ -249,6 +249,11 @@ tests/newconfig/bar/mfoo.nims""".splitLines expected.add &"Hint: used config file '{b}' [Conf]\n" doAssert outp.endsWith expected, outp & "\n" & expected + block: # bug #8219 + let file = "tests/newconfig/mconfigcheck.nims" + let cmd = fmt"{nim} check --hints:off {file}" + check execCmdEx(cmd) == ("", 0) + block: # mfoo2.customext let filename = testsDir / "newconfig/foo2/mfoo2.customext" let cmd = fmt"{nim} e --hint:conf {filename}" diff --git a/tests/newconfig/tconfigcheck.nims b/tests/newconfig/mconfigcheck.nims similarity index 74% rename from tests/newconfig/tconfigcheck.nims rename to tests/newconfig/mconfigcheck.nims index 2e12a0bda..8df6715f6 100644 --- a/tests/newconfig/tconfigcheck.nims +++ b/tests/newconfig/mconfigcheck.nims @@ -1,11 +1,9 @@ -discard """ - cmd: "nim check $file" -""" - mode = ScriptMode.Verbose proc build() = echo "building nim... " exec "sleep 10" + exec "nonexistant command" echo getCurrentDir() echo "hello" +build() From 7d32425a577053bda356ca71c81d6701d8ad6372 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 26 Mar 2022 13:29:04 +0100 Subject: [PATCH 1565/2472] fixes #14318 (#14335) --- compiler/pragmas.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 81c6a42d3..458c7547a 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -656,7 +656,7 @@ proc pragmaRaisesOrTags(c: PContext, n: PNode) = x.typ = makeTypeFromExpr(c, x) else: var t = skipTypes(c.semTypeNode(c, x, nil), skipPtrs) - if t.kind != tyObject and not t.isMetaType: + if t.kind notin {tyObject, tyOr}: localError(c.config, x.info, errGenerated, "invalid type for raises/tags list") x.typ = t From eae29e8eafded41ce966c131bff5e66964354805 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 28 Mar 2022 03:37:59 -0700 Subject: [PATCH 1566/2472] Embedded Network patches - eventfd & socket getters (#19632) --- lib/pure/ioselects/ioselectors_poll.nim | 35 ++++++++++++++++++------- lib/pure/net.nim | 10 +++++++ 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/lib/pure/ioselects/ioselectors_poll.nim b/lib/pure/ioselects/ioselectors_poll.nim index 0d8fef78a..12812ac80 100644 --- a/lib/pure/ioselects/ioselectors_poll.nim +++ b/lib/pure/ioselects/ioselectors_poll.nim @@ -14,6 +14,12 @@ import posix, times # Maximum number of events that can be returned const MAX_POLL_EVENTS = 64 +const hasEventFds = defined(zephyr) or defined(nimPollHasEventFds) + +when hasEventFds: + proc eventfd(count: cuint, flags: cint): cint + {.cdecl, importc: "eventfd", header: "".} + when hasThreadSupport: type SelectorImpl[T] = object @@ -184,14 +190,22 @@ proc unregister*[T](s: Selector[T], ev: SelectEvent) = s.pollRemove(fdi.cint) proc newSelectEvent*(): SelectEvent = - var fds: array[2, cint] - if posix.pipe(fds) != 0: - raiseIOSelectorsError(osLastError()) - setNonBlocking(fds[0]) - setNonBlocking(fds[1]) - result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) - result.rfd = fds[0] - result.wfd = fds[1] + when not hasEventFds: + var fds: array[2, cint] + if posix.pipe(fds) != 0: + raiseIOSelectorsError(osLastError()) + setNonBlocking(fds[0]) + setNonBlocking(fds[1]) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rfd = fds[0] + result.wfd = fds[1] + else: + let fdci = eventfd(0, posix.O_NONBLOCK) + if fdci == -1: + raiseIOSelectorsError(osLastError()) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rfd = fdci + result.wfd = fdci proc trigger*(ev: SelectEvent) = var data: uint64 = 1 @@ -200,7 +214,10 @@ proc trigger*(ev: SelectEvent) = proc close*(ev: SelectEvent) = let res1 = posix.close(ev.rfd) - let res2 = posix.close(ev.wfd) + let res2 = + when hasEventFds: 0 + else: posix.close(ev.wfd) + deallocShared(cast[pointer](ev)) if res1 != 0 or res2 != 0: raiseIOSelectorsError(osLastError()) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 9922bbfa4..c59a4e563 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -1798,6 +1798,16 @@ proc isSsl*(socket: Socket): bool = proc getFd*(socket: Socket): SocketHandle = return socket.fd ## Returns the socket's file descriptor +when defined(zephyr) or defined(nimNetSocketExtras): # Remove in future + proc getDomain*(socket: Socket): Domain = return socket.domain + ## Returns the socket's domain + + proc getType*(socket: Socket): SockType = return socket.sockType + ## Returns the socket's type + + proc getProtocol*(socket: Socket): Protocol = return socket.protocol + ## Returns the socket's protocol + when defined(nimHasStyleChecks): {.push styleChecks: off.} From 49844415ce829cb627d0b02dd6b057a1574f44d6 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Thu, 31 Mar 2022 16:06:13 -0300 Subject: [PATCH 1567/2472] Deprecate selfExe (#19660) * Deprecate selfExe Nimscript * Deprecate selfExe Nimscript --- changelog.md | 2 ++ lib/system/nimscript.nim | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/changelog.md b/changelog.md index 92d2612e8..8b9d0bc29 100644 --- a/changelog.md +++ b/changelog.md @@ -50,6 +50,8 @@ becomes an alias for `addr`. - Remove deprecated `osproc.poDemon`, symbol with typo. +- Deprecated `selfExe` for Nimscript. + ## Language changes diff --git a/lib/system/nimscript.nim b/lib/system/nimscript.nim index 0ef5fc584..5d1916c7f 100644 --- a/lib/system/nimscript.nim +++ b/lib/system/nimscript.nim @@ -136,9 +136,8 @@ proc dirExists*(dir: string): bool {. ## Checks if the directory `dir` exists. builtin -proc selfExe*(): string = +proc selfExe*(): string {.deprecated: "Deprecated since v1.7; Use getCurrentCompilerExe".} = ## Returns the currently running nim or nimble executable. - # TODO: consider making this as deprecated alias of `getCurrentCompilerExe` builtin proc toExe*(filename: string): string = From 1275763284d540c0e8ad53b3c27e52743ee08f2b Mon Sep 17 00:00:00 2001 From: rockcavera Date: Fri, 1 Apr 2022 03:01:51 -0300 Subject: [PATCH 1568/2472] fix 19655 - fixing url fragment (#19667) --- lib/pure/parseopt.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/parseopt.nim b/lib/pure/parseopt.nim index b2d024a39..aff9312f9 100644 --- a/lib/pure/parseopt.nim +++ b/lib/pure/parseopt.nim @@ -16,7 +16,7 @@ ## ## The following syntax is supported when arguments for the `shortNoVal` and ## `longNoVal` parameters, which are -## `described later<#shortnoval-and-longnoval>`_, are not provided: +## `described later<#nimshortnoval-and-nimlongnoval>`_, are not provided: ## ## 1. Short options: `-abcd`, `-e:5`, `-e=5` ## 2. Long options: `--foo:bar`, `--foo=bar`, `--foo` From a7024f49afe5623ce90a800986dea7447e249d3b Mon Sep 17 00:00:00 2001 From: Nan Xiao Date: Sat, 2 Apr 2022 01:29:15 +0800 Subject: [PATCH 1569/2472] fix 19655 - fixing more url fragments (#19669) --- lib/pure/parseopt.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/parseopt.nim b/lib/pure/parseopt.nim index aff9312f9..71deaa535 100644 --- a/lib/pure/parseopt.nim +++ b/lib/pure/parseopt.nim @@ -203,7 +203,7 @@ proc initOptParser*(cmdline = "", shortNoVal: set[char] = {}, ## ## `shortNoVal` and `longNoVal` are used to specify which options ## do not take values. See the `documentation about these - ## parameters<#shortnoval-and-longnoval>`_ for more information on + ## parameters<#nimshortnoval-and-nimlongnoval>`_ for more information on ## how this affects parsing. ## ## See also: @@ -465,7 +465,7 @@ iterator getopt*(cmdline: seq[string] = @[], ## ## `shortNoVal` and `longNoVal` are used to specify which options ## do not take values. See the `documentation about these - ## parameters<#shortnoval-and-longnoval>`_ for more information on + ## parameters<#nimshortnoval-and-nimlongnoval>`_ for more information on ## how this affects parsing. ## ## There is no need to check for `cmdEnd` while iterating. From c3f03cfa5dfa2ca47f8e4cf99bbcdbf5a7d16eda Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 2 Apr 2022 01:30:02 +0800 Subject: [PATCH 1570/2472] add somes links to docs (#19668) --- lib/core/macros.nim | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index f20ca9385..4d0e64868 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -933,21 +933,21 @@ proc treeTraverse(n: NimNode; res: var string; level = 0; isLisp = false, indent proc treeRepr*(n: NimNode): string {.benign.} = ## Convert the AST `n` to a human-readable tree-like string. ## - ## See also `repr`, `lispRepr`, and `astGenRepr`. + ## See also `repr`, `lispRepr`_, and `astGenRepr`_. result = "" n.treeTraverse(result, isLisp = false, indented = true) proc lispRepr*(n: NimNode; indented = false): string {.benign.} = ## Convert the AST `n` to a human-readable lisp-like string. ## - ## See also `repr`, `treeRepr`, and `astGenRepr`. + ## See also `repr`, `treeRepr`_, and `astGenRepr`_. result = "" n.treeTraverse(result, isLisp = true, indented = indented) proc astGenRepr*(n: NimNode): string {.benign.} = ## Convert the AST `n` to the code required to generate that AST. ## - ## See also `repr`, `treeRepr`, and `lispRepr`. + ## See also `repr`_, `treeRepr`_, and `lispRepr`_. const NodeKinds = {nnkEmpty, nnkIdent, nnkSym, nnkNone, nnkCommentStmt} @@ -1602,7 +1602,7 @@ macro hasCustomPragma*(n: typed, cp: typed{nkSym}): untyped = ## Expands to `true` if expression `n` which is expected to be `nnkDotExpr` ## (if checking a field), a proc or a type has custom pragma `cp`. ## - ## See also `getCustomPragmaVal`. + ## See also `getCustomPragmaVal`_. ## ## .. code-block:: nim ## template myAttr() {.pragma.} @@ -1626,7 +1626,7 @@ macro getCustomPragmaVal*(n: typed, cp: typed{nkSym}): untyped = ## Expands to value of custom pragma `cp` of expression `n` which is expected ## to be `nnkDotExpr`, a proc or a type. ## - ## See also `hasCustomPragma` + ## See also `hasCustomPragma`_. ## ## .. code-block:: nim ## template serializationKey(key: string) {.pragma.} From 83dabb69ae0f6c0bb269594a5b73af964b809bc7 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 4 Apr 2022 18:05:23 +0800 Subject: [PATCH 1571/2472] Fix bug in freshVarForClosureIter. Fixes #18474 (#19675) [backport] * Fix bug in freshVarForClosureIter. Fixes #18474. freshVarForClosureIter was returning non-fresh symbols sometimes. Fixed by making addField return the generated PSym. * remove discardable Co-authored-by: Nick Smallbone --- compiler/lambdalifting.nim | 13 ++++--------- compiler/lowerings.nim | 3 ++- compiler/spawn.nim | 22 +++++++++++----------- tests/iter/tclosureiters.nim | 18 ++++++++++++++++++ 4 files changed, 35 insertions(+), 21 deletions(-) diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 6838fd80d..4b7ca89cd 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -275,16 +275,11 @@ proc liftIterSym*(g: ModuleGraph; n: PNode; idgen: IdGenerator; owner: PSym): PN proc freshVarForClosureIter*(g: ModuleGraph; s: PSym; idgen: IdGenerator; owner: PSym): PNode = let envParam = getHiddenParam(g, owner) let obj = envParam.typ.skipTypes({tyOwned, tyRef, tyPtr}) - addField(obj, s, g.cache, idgen) + let field = addField(obj, s, g.cache, idgen) var access = newSymNode(envParam) assert obj.kind == tyObject - let field = getFieldFromObj(obj, s) - if field != nil: - result = rawIndirectAccess(access, field, s.info) - else: - localError(g.config, s.info, "internal error: cannot generate fresh variable") - result = access + result = rawIndirectAccess(access, field, s.info) # ------------------ new stuff ------------------------------------------- @@ -452,7 +447,7 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = if s.name.id == getIdent(c.graph.cache, ":state").id: obj.n[0].sym.itemId = ItemId(module: s.itemId.module, item: -s.itemId.item) else: - addField(obj, s, c.graph.cache, c.idgen) + discard addField(obj, s, c.graph.cache, c.idgen) # direct or indirect dependency: elif (innerProc and s.typ.callConv == ccClosure) or interestingVar(s): discard """ @@ -474,7 +469,7 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = if interestingVar(s) and not c.capturedVars.containsOrIncl(s.id): let obj = c.getEnvTypeForOwner(ow, n.info).skipTypes({tyOwned, tyRef, tyPtr}) #getHiddenParam(owner).typ.skipTypes({tyOwned, tyRef, tyPtr}) - addField(obj, s, c.graph.cache, c.idgen) + discard addField(obj, s, c.graph.cache, c.idgen) # create required upFields: var w = owner.skipGenericOwner if isInnerProc(w) or owner.isIterator: diff --git a/compiler/lowerings.nim b/compiler/lowerings.nim index 382e48a16..de2f678d7 100644 --- a/compiler/lowerings.nim +++ b/compiler/lowerings.nim @@ -230,7 +230,7 @@ proc lookupInRecord(n: PNode, id: ItemId): PSym = if n.sym.itemId.module == id.module and n.sym.itemId.item == -abs(id.item): result = n.sym else: discard -proc addField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator) = +proc addField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator): PSym = # because of 'gensym' support, we have to mangle the name with its ID. # This is hacky but the clean solution is much more complex than it looks. var field = newSym(skField, getIdent(cache, s.name.s & $obj.n.len), @@ -244,6 +244,7 @@ proc addField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator) = field.flags = s.flags * {sfCursor} obj.n.add newSymNode(field) fieldCheck() + result = field proc addUniqueField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator): PSym {.discardable.} = result = lookupInRecord(obj.n, s.itemId) diff --git a/compiler/spawn.nim b/compiler/spawn.nim index e57686cb4..581f722d5 100644 --- a/compiler/spawn.nim +++ b/compiler/spawn.nim @@ -221,7 +221,7 @@ proc setupArgsForConcurrency(g: ModuleGraph; n: PNode; objType: PType; let fieldname = if i < formals.len: formals[i].sym.name else: tmpName var field = newSym(skField, fieldname, nextSymId idgen, objType.owner, n.info, g.config.options) field.typ = argType - objType.addField(field, g.cache, idgen) + discard objType.addField(field, g.cache, idgen) result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n[i]) let temp = addLocalVar(g, varSection, varInit, idgen, owner, argType, @@ -260,17 +260,17 @@ proc setupArgsForParallelism(g: ModuleGraph; n: PNode; objType: PType; slice[0].typ = getSysType(g, n.info, tyInt) # fake type var fieldB = newSym(skField, tmpName, nextSymId idgen, objType.owner, n.info, g.config.options) fieldB.typ = getSysType(g, n.info, tyInt) - objType.addField(fieldB, g.cache, idgen) + discard objType.addField(fieldB, g.cache, idgen) if getMagic(n) == mSlice: let a = genAddrOf(n[1], idgen) field.typ = a.typ - objType.addField(field, g.cache, idgen) + discard objType.addField(field, g.cache, idgen) result.add newFastAsgnStmt(newDotExpr(scratchObj, field), a) var fieldA = newSym(skField, tmpName, nextSymId idgen, objType.owner, n.info, g.config.options) fieldA.typ = getSysType(g, n.info, tyInt) - objType.addField(fieldA, g.cache, idgen) + discard objType.addField(fieldA, g.cache, idgen) result.add newFastAsgnStmt(newDotExpr(scratchObj, fieldA), n[2]) result.add newFastAsgnStmt(newDotExpr(scratchObj, fieldB), n[3]) @@ -281,7 +281,7 @@ proc setupArgsForParallelism(g: ModuleGraph; n: PNode; objType: PType; else: let a = genAddrOf(n, idgen) field.typ = a.typ - objType.addField(field, g.cache, idgen) + discard objType.addField(field, g.cache, idgen) result.add newFastAsgnStmt(newDotExpr(scratchObj, field), a) result.add newFastAsgnStmt(newDotExpr(scratchObj, fieldB), genHigh(g, n)) @@ -299,7 +299,7 @@ proc setupArgsForParallelism(g: ModuleGraph; n: PNode; objType: PType; # it is more efficient to pass a pointer instead: let a = genAddrOf(n, idgen) field.typ = a.typ - objType.addField(field, g.cache, idgen) + discard objType.addField(field, g.cache, idgen) result.add newFastAsgnStmt(newDotExpr(scratchObj, field), a) let threadLocal = addLocalVar(g, varSection, nil, idgen, owner, field.typ, indirectAccess(castExpr, field, n.info), @@ -308,7 +308,7 @@ proc setupArgsForParallelism(g: ModuleGraph; n: PNode; objType: PType; else: # boring case field.typ = argType - objType.addField(field, g.cache, idgen) + discard objType.addField(field, g.cache, idgen) result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n) let threadLocal = addLocalVar(g, varSection, varInit, idgen, owner, field.typ, @@ -377,7 +377,7 @@ proc wrapProcForSpawn*(g: ModuleGraph; idgen: IdGenerator; owner: PSym; spawnExp var argType = n[0].typ.skipTypes(abstractInst) var field = newSym(skField, getIdent(g.cache, "fn"), nextSymId idgen, owner, n.info, g.config.options) field.typ = argType - objType.addField(field, g.cache, idgen) + discard objType.addField(field, g.cache, idgen) result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n[0]) fn = indirectAccess(castExpr, field, n.info) elif fn.kind == nkSym and fn.sym.kind == skIterator: @@ -399,7 +399,7 @@ proc wrapProcForSpawn*(g: ModuleGraph; idgen: IdGenerator; owner: PSym; spawnExp typ.rawAddSon(magicsys.getCompilerProc(g, "Barrier").typ) var field = newSym(skField, getIdent(g.cache, "barrier"), nextSymId idgen, owner, n.info, g.config.options) field.typ = typ - objType.addField(field, g.cache, idgen) + discard objType.addField(field, g.cache, idgen) result.add newFastAsgnStmt(newDotExpr(scratchObj, field), barrier) barrierAsExpr = indirectAccess(castExpr, field, n.info) @@ -407,7 +407,7 @@ proc wrapProcForSpawn*(g: ModuleGraph; idgen: IdGenerator; owner: PSym; spawnExp if spawnKind == srFlowVar: var field = newSym(skField, getIdent(g.cache, "fv"), nextSymId idgen, owner, n.info, g.config.options) field.typ = retType - objType.addField(field, g.cache, idgen) + discard objType.addField(field, g.cache, idgen) fvField = newDotExpr(scratchObj, field) fvAsExpr = indirectAccess(castExpr, field, n.info) # create flowVar: @@ -419,7 +419,7 @@ proc wrapProcForSpawn*(g: ModuleGraph; idgen: IdGenerator; owner: PSym; spawnExp var field = newSym(skField, getIdent(g.cache, "fv"), nextSymId idgen, owner, n.info, g.config.options) field.typ = newType(tyPtr, nextTypeId idgen, objType.owner) field.typ.rawAddSon(retType) - objType.addField(field, g.cache, idgen) + discard objType.addField(field, g.cache, idgen) fvAsExpr = indirectAccess(castExpr, field, n.info) result.add newFastAsgnStmt(newDotExpr(scratchObj, field), genAddrOf(dest, idgen)) diff --git a/tests/iter/tclosureiters.nim b/tests/iter/tclosureiters.nim index afeaabc7d..85611373c 100644 --- a/tests/iter/tclosureiters.nim +++ b/tests/iter/tclosureiters.nim @@ -21,6 +21,15 @@ discard """ 2 70 0 +(1, 1) +(1, 2) +(1, 3) +(2, 1) +(2, 2) +(2, 3) +(3, 1) +(3, 2) +(3, 3) ''' """ @@ -152,3 +161,12 @@ var love = iterator: int {.closure.} = for i in love(): echo i + +# bug #18474 +iterator pairs(): (int, int) {.closure.} = + for i in 1..3: + for j in 1..3: + yield (i, j) + +for pair in pairs(): + echo pair From 5a995ffc53ac7c1a51ab62440a58af2f4e43963b Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 6 Apr 2022 22:17:09 +0800 Subject: [PATCH 1572/2472] fix #18986; Import/except doesn't work on devel [backport: 1.6] (#19687) * fix #18986; Import/except doesn't work on devel [backport: 1.6] * add testcase --- compiler/importer.nim | 2 +- tests/converter/m18986.nim | 3 +++ tests/converter/t18986.nim | 10 ++++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 tests/converter/m18986.nim create mode 100644 tests/converter/t18986.nim diff --git a/compiler/importer.nim b/compiler/importer.nim index 719b75f0f..c4e37c269 100644 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -202,7 +202,7 @@ template addUnnamedIt(c: PContext, fromMod: PSym; filter: untyped) {.dirty.} = proc importAllSymbolsExcept(c: PContext, fromMod: PSym, exceptSet: IntSet) = c.addImport ImportedModule(m: fromMod, mode: importExcept, exceptSet: exceptSet) - addUnnamedIt(c, fromMod, it.sym.id notin exceptSet) + addUnnamedIt(c, fromMod, it.sym.name.id notin exceptSet) proc importAllSymbols*(c: PContext, fromMod: PSym) = c.addImport ImportedModule(m: fromMod, mode: importAll) diff --git a/tests/converter/m18986.nim b/tests/converter/m18986.nim new file mode 100644 index 000000000..0ebf343ae --- /dev/null +++ b/tests/converter/m18986.nim @@ -0,0 +1,3 @@ +import std/macros + +converter Lit*(x: uint): NimNode = newLit(x) diff --git a/tests/converter/t18986.nim b/tests/converter/t18986.nim new file mode 100644 index 000000000..ef300fa49 --- /dev/null +++ b/tests/converter/t18986.nim @@ -0,0 +1,10 @@ +discard """ + output: "Found a 0" +""" + +import m18986 except Lit +import std/macros + +# bug #18986 +var x = 0.uint +echo "Found a ", x From c322faaf384bfcde4dd5a6e6eb9355d10c2e9597 Mon Sep 17 00:00:00 2001 From: Miran Date: Wed, 6 Apr 2022 16:18:10 +0200 Subject: [PATCH 1573/2472] [backport] fix broken SSL tests (#19684) * [backport] fix broken SSL tests * remove a flaky one --- tests/untestable/thttpclient_ssl_remotenetwork.nim | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/untestable/thttpclient_ssl_remotenetwork.nim b/tests/untestable/thttpclient_ssl_remotenetwork.nim index d8137e516..cbb86dc13 100644 --- a/tests/untestable/thttpclient_ssl_remotenetwork.nim +++ b/tests/untestable/thttpclient_ssl_remotenetwork.nim @@ -32,7 +32,7 @@ when enableRemoteNetworking and (defined(nimTestsEnableFlaky) or not defined(win good, bad, dubious, good_broken, bad_broken, dubious_broken CertTest = tuple[url:string, category:Category, desc: string] - const certificate_tests: array[0..55, CertTest] = [ + const certificate_tests: array[0..54, CertTest] = [ ("https://wrong.host.badssl.com/", bad, "wrong.host"), ("https://captive-portal.badssl.com/", bad, "captive-portal"), ("https://expired.badssl.com/", bad, "expired"), @@ -41,13 +41,12 @@ when enableRemoteNetworking and (defined(nimTestsEnableFlaky) or not defined(win ("https://untrusted-root.badssl.com/", bad, "untrusted-root"), ("https://revoked.badssl.com/", bad_broken, "revoked"), ("https://pinning-test.badssl.com/", bad_broken, "pinning-test"), - ("https://no-common-name.badssl.com/", dubious_broken, "no-common-name"), - ("https://no-subject.badssl.com/", dubious_broken, "no-subject"), - ("https://incomplete-chain.badssl.com/", dubious_broken, "incomplete-chain"), + ("https://no-common-name.badssl.com/", bad, "no-common-name"), + ("https://no-subject.badssl.com/", bad, "no-subject"), ("https://sha1-intermediate.badssl.com/", bad, "sha1-intermediate"), ("https://sha256.badssl.com/", good, "sha256"), - ("https://sha384.badssl.com/", good, "sha384"), - ("https://sha512.badssl.com/", good, "sha512"), + ("https://sha384.badssl.com/", bad, "sha384"), + ("https://sha512.badssl.com/", bad, "sha512"), ("https://1000-sans.badssl.com/", bad, "1000-sans"), ("https://10000-sans.badssl.com/", good_broken, "10000-sans"), ("https://ecc256.badssl.com/", good, "ecc256"), From abe3b0eb644c60be4423bd3595382a126c4746db Mon Sep 17 00:00:00 2001 From: locriacyber <74560659+locriacyber@users.noreply.github.com> Date: Wed, 6 Apr 2022 16:08:27 +0000 Subject: [PATCH 1574/2472] Load Google Fonts last (#19683) --- config/nimdoc.cfg | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index c1074f344..0ae768bfd 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -233,10 +233,6 @@ doc.file = """ - - - - $title @@ -282,8 +278,6 @@ function main() { } } } - -window.addEventListener('DOMContentLoaded', main); @@ -301,7 +295,15 @@ window.addEventListener('DOMContentLoaded', main); + + + $analytics + + + + + """ From e649ddd09ba0b1195163f456dbf6b1308affec37 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 7 Apr 2022 02:27:51 +0800 Subject: [PATCH 1575/2472] Revert "Load Google Fonts last (#19683)" (#19693) This reverts commit abe3b0eb644c60be4423bd3595382a126c4746db. --- config/nimdoc.cfg | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index 0ae768bfd..c1074f344 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -233,6 +233,10 @@ doc.file = """ + + + + $title @@ -278,6 +282,8 @@ function main() { } } } + +window.addEventListener('DOMContentLoaded', main); @@ -295,15 +301,7 @@ function main() { - - - $analytics - - - - - """ From cb6ce80cb8dcf21733b437606ada1dfff9884316 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Wed, 6 Apr 2022 20:28:37 +0200 Subject: [PATCH 1576/2472] std/tasks: fix spelling (#19691) [backport] why aren't these not being caught by style check options? --styleCheck:usages finds it. Co-authored-by: flywind --- lib/std/tasks.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/std/tasks.nim b/lib/std/tasks.nim index e2ea5377f..dbe2e265b 100644 --- a/lib/std/tasks.nim +++ b/lib/std/tasks.nim @@ -87,7 +87,7 @@ template checkIsolate(scratchAssignList: seq[NimNode], procParam, scratchDotExpr let isolatedTemp = genSym(nskTemp, "isoTemp") scratchAssignList.add newVarStmt(isolatedTemp, newCall(newidentNode("isolate"), procParam)) scratchAssignList.add newAssignment(scratchDotExpr, - newcall(newIdentNode("extract"), isolatedTemp)) + newCall(newIdentNode("extract"), isolatedTemp)) template addAllNode(assignParam: NimNode, procParam: NimNode) = let scratchDotExpr = newDotExpr(scratchIdent, formalParams[i][0]) @@ -154,7 +154,7 @@ macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkC error("'toTask'ed function cannot have a 'typed' or 'untyped' parameter") let seqType = nnkBracketExpr.newTree(newIdentNode("seq"), param[1]) - seqCallNode = newcall("@", e[i]) + seqCallNode = newCall("@", e[i]) addAllNode(seqType, seqCallNode) else: addAllNode(param, e[i]) From 065f568470af944cfc43ecfed8df6ac8021b7cbe Mon Sep 17 00:00:00 2001 From: Nan Xiao Date: Thu, 7 Apr 2022 02:28:58 +0800 Subject: [PATCH 1577/2472] No need to export pos from OptParser (#19688) Co-authored-by: flywind --- lib/pure/parseopt.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/parseopt.nim b/lib/pure/parseopt.nim index 71deaa535..fef354733 100644 --- a/lib/pure/parseopt.nim +++ b/lib/pure/parseopt.nim @@ -164,7 +164,7 @@ type ## ## To initialize it, use the ## `initOptParser proc<#initOptParser,string,set[char],seq[string]>`_. - pos*: int + pos: int inShortState: bool allowWhitespaceAfterColon: bool shortNoVal: set[char] From e1929deac7dd41595e998ce68ee703a113ce8746 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 7 Apr 2022 07:11:25 +0800 Subject: [PATCH 1578/2472] remove useless compilation condition (#19690) ref https://github.com/nim-lang/Nim/pull/18546 --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 456f42cd2..6b0b15fd8 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -100,7 +100,7 @@ pkg "nimcrypto", "nim r --path:. tests/testall.nim" # `--path:.` workaround need pkg "NimData", "nim c -o:nimdataa src/nimdata.nim" pkg "nimes", "nim c src/nimes.nim" pkg "nimfp", "nim c -o:nfp -r src/fp.nim" -pkg "nimgame2", "nim c -d:nimLegacyConvEnumEnum nimgame2/nimgame.nim" +pkg "nimgame2", "nim c nimgame2/nimgame.nim" # XXX Doesn't work with deprecated 'randomize', will create a PR. pkg "nimgen", "nim c -o:nimgenn -r src/nimgen/runcfg.nim" pkg "nimlsp", allowFailure = true From 0978276ed9882a25782f91795b355a0dca7b0d3b Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 7 Apr 2022 13:07:39 +0800 Subject: [PATCH 1579/2472] use two spaces indentation (#19696) --- lib/core/macros.nim | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 4d0e64868..ba19712cf 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -1505,20 +1505,20 @@ macro expandMacros*(body: typed): untyped = result = body proc extractTypeImpl(n: NimNode): NimNode = - ## attempts to extract the type definition of the given symbol - case n.kind - of nnkSym: # can extract an impl - result = n.getImpl.extractTypeImpl() - of nnkObjectTy, nnkRefTy, nnkPtrTy: result = n - of nnkBracketExpr: - if n.typeKind == ntyTypeDesc: - result = n[1].extractTypeImpl() - else: - doAssert n.typeKind == ntyGenericInst - result = n[0].getImpl() - of nnkTypeDef: - result = n[2] - else: error("Invalid node to retrieve type implementation of: " & $n.kind) + ## attempts to extract the type definition of the given symbol + case n.kind + of nnkSym: # can extract an impl + result = n.getImpl.extractTypeImpl() + of nnkObjectTy, nnkRefTy, nnkPtrTy: result = n + of nnkBracketExpr: + if n.typeKind == ntyTypeDesc: + result = n[1].extractTypeImpl() + else: + doAssert n.typeKind == ntyGenericInst + result = n[0].getImpl() + of nnkTypeDef: + result = n[2] + else: error("Invalid node to retrieve type implementation of: " & $n.kind) proc customPragmaNode(n: NimNode): NimNode = expectKind(n, {nnkSym, nnkDotExpr, nnkBracketExpr, nnkTypeOfExpr, nnkCheckedFieldExpr}) From 810d5e91e435fffb7898ac26b5e4cc5fbb1678d7 Mon Sep 17 00:00:00 2001 From: Ivan Yonchovski Date: Thu, 7 Apr 2022 14:39:27 +0300 Subject: [PATCH 1580/2472] [nimsuggest] return the type when on symbol in let/var (#19639) - make sure `suggestSym` is called after `PSym.typ` is set. --- compiler/semstmts.nim | 9 ++++++--- nimsuggest/tests/tdef_let.nim | 7 +++++++ 2 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 nimsuggest/tests/tdef_let.nim diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index c2b385c80..3513fd767 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -306,7 +306,7 @@ proc identWithin(n: PNode, s: PIdent): bool = if identWithin(n[i], s): return true result = n.kind == nkSym and n.sym.name.id == s.id -proc semIdentDef(c: PContext, n: PNode, kind: TSymKind): PSym = +proc semIdentDef(c: PContext, n: PNode, kind: TSymKind, reportToNimsuggest = true): PSym = if isTopLevel(c): result = semIdentWithPragma(c, kind, n, {sfExported}) incl(result.flags, sfGlobal) @@ -330,7 +330,8 @@ proc semIdentDef(c: PContext, n: PNode, kind: TSymKind): PSym = discard result = n.info let info = getLineInfo(n) - suggestSym(c.graph, info, result, c.graph.usageSym) + if reportToNimsuggest: + suggestSym(c.graph, info, result, c.graph.usageSym) proc checkNilable(c: PContext; v: PSym) = if {sfGlobal, sfImportc} * v.flags == {sfGlobal} and v.typ.requiresInit: @@ -660,7 +661,7 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = if a.kind != nkVarTuple: typ else: tup[j]) addToVarSection(c, result, n, a) continue - var v = semIdentDef(c, a[j], symkind) + var v = semIdentDef(c, a[j], symkind, false) styleCheckDef(c.config, v) onDef(a[j].info, v) if sfGenSym notin v.flags: @@ -725,6 +726,8 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = if v.flags * {sfGlobal, sfThread} == {sfGlobal}: message(c.config, v.info, hintGlobalVar) + suggestSym(c.graph, v.info, v, c.graph.usageSym) + proc semConst(c: PContext, n: PNode): PNode = result = copyNode(n) inc c.inStaticContext diff --git a/nimsuggest/tests/tdef_let.nim b/nimsuggest/tests/tdef_let.nim new file mode 100644 index 000000000..3e9456d2f --- /dev/null +++ b/nimsuggest/tests/tdef_let.nim @@ -0,0 +1,7 @@ +discard """ +$nimsuggest --tester $file +>def $1 +def;;skLet;;tdef_let.intVar;;int;;$file;;7;;4;;"";;100 +""" + +let int#[!]#Var = 10 From c8aeea9d6205198306c3cd802136915f909ba5dc Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 8 Apr 2022 02:56:34 +0800 Subject: [PATCH 1581/2472] improve the error messages for std/tasks [backport: 1.6] (#19695) --- lib/std/tasks.nim | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/std/tasks.nim b/lib/std/tasks.nim index dbe2e265b..3ceb4c8ba 100644 --- a/lib/std/tasks.nim +++ b/lib/std/tasks.nim @@ -111,10 +111,10 @@ macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkC when compileOption("threads"): if not isGcSafe(e[0]): - error("'toTask' takes a GC safe call expression") + error("'toTask' takes a GC safe call expression", e) if hasClosure(e[0]): - error("closure call is not allowed") + error("closure call is not allowed", e) if e.len > 1: let scratchIdent = genSym(kind = nskTemp, ident = "scratch") @@ -141,17 +141,17 @@ macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkC param = param[0] if param.typeKind in {ntyExpr, ntyStmt}: - error("'toTask'ed function cannot have a 'typed' or 'untyped' parameter") + error("'toTask'ed function cannot have a 'typed' or 'untyped' parameter", e) case param.kind of nnkVarTy: - error("'toTask'ed function cannot have a 'var' parameter") + error("'toTask'ed function cannot have a 'var' parameter", e) of nnkBracketExpr: if param[0].typeKind == ntyTypeDesc: callNode.add nnkExprEqExpr.newTree(formalParams[i][0], e[i]) elif param[0].typeKind in {ntyVarargs, ntyOpenArray}: if param[1].typeKind in {ntyExpr, ntyStmt}: - error("'toTask'ed function cannot have a 'typed' or 'untyped' parameter") + error("'toTask'ed function cannot have a 'typed' or 'untyped' parameter", e) let seqType = nnkBracketExpr.newTree(newIdentNode("seq"), param[1]) seqCallNode = newCall("@", e[i]) @@ -167,7 +167,7 @@ macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkC of nnkCharLit..nnkNilLit: callNode.add nnkExprEqExpr.newTree(formalParams[i][0], e[i]) else: - error("not supported type kinds") + error("'toTask'ed function cannot have a parameter of " & $param.kind & " kind", e) let scratchObjType = genSym(kind = nskType, ident = "ScratchObj") let scratchObj = nnkTypeSection.newTree( From 1807de38e52f45c2fb88dac9b99b47729b12ebae Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 8 Apr 2022 02:57:50 +0800 Subject: [PATCH 1582/2472] add testcase for #16462 (#19692) --- tests/enum/m16462_1.nim | 3 +++ tests/enum/m16462_2.nim | 3 +++ tests/enum/t16462.nim | 5 +++++ 3 files changed, 11 insertions(+) create mode 100644 tests/enum/m16462_1.nim create mode 100644 tests/enum/m16462_2.nim create mode 100644 tests/enum/t16462.nim diff --git a/tests/enum/m16462_1.nim b/tests/enum/m16462_1.nim new file mode 100644 index 000000000..631c63256 --- /dev/null +++ b/tests/enum/m16462_1.nim @@ -0,0 +1,3 @@ +type + Scancode* {.pure.} = enum + SCANCODE_LEFT = 80 \ No newline at end of file diff --git a/tests/enum/m16462_2.nim b/tests/enum/m16462_2.nim new file mode 100644 index 000000000..631c63256 --- /dev/null +++ b/tests/enum/m16462_2.nim @@ -0,0 +1,3 @@ +type + Scancode* {.pure.} = enum + SCANCODE_LEFT = 80 \ No newline at end of file diff --git a/tests/enum/t16462.nim b/tests/enum/t16462.nim new file mode 100644 index 000000000..9f38286bb --- /dev/null +++ b/tests/enum/t16462.nim @@ -0,0 +1,5 @@ +import m16462_1 except Scancode +import m16462_2 + +# bug #16462 +let a = SCANCODE_LEFT \ No newline at end of file From e78ef57c93d66da483e0482ce0907dfe16dc8d27 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 7 Apr 2022 14:38:01 -0700 Subject: [PATCH 1583/2472] typetraits: add toSigned, toUnsigned (#18445) * typetraits: add toSigned, toUnsigned * improve and add tests Co-authored-by: Andreas Rumpf Co-authored-by: flywind --- lib/pure/bitops.nim | 26 ++++++++-------------- lib/pure/typetraits.nim | 38 +++++++++++++++++++++++++++----- lib/std/private/bitops_utils.nim | 15 ++++++++----- lib/system/countbits_impl.nim | 6 ++--- tests/metatype/ttypetraits.nim | 15 +++++++++++++ tests/stdlib/tbitops_utils.nim | 14 ++++++++++++ tests/stdlib/ttypetraits.nim | 4 ++-- 7 files changed, 85 insertions(+), 33 deletions(-) create mode 100644 tests/stdlib/tbitops_utils.nim diff --git a/lib/pure/bitops.nim b/lib/pure/bitops.nim index 1b3838621..a518c25d2 100644 --- a/lib/pure/bitops.nim +++ b/lib/pure/bitops.nim @@ -27,7 +27,7 @@ import macros import std/private/since -from std/private/bitops_utils import forwardImpl, toUnsigned +from std/private/bitops_utils import forwardImpl, castToUnsigned func bitnot*[T: SomeInteger](x: T): T {.magic: "BitnotI".} ## Computes the `bitwise complement` of the integer `x`. @@ -72,7 +72,7 @@ func bitsliced*[T: SomeInteger](v: T; slice: Slice[int]): T {.inline, since: (1, let upmost = sizeof(T) * 8 - 1 - uv = when v is SomeUnsignedInt: v else: v.toUnsigned + uv = v.castToUnsigned (uv shl (upmost - slice.b) shr (upmost - slice.b + slice.a)).T proc bitslice*[T: SomeInteger](v: var T; slice: Slice[int]) {.inline, since: (1, 3).} = @@ -84,7 +84,7 @@ proc bitslice*[T: SomeInteger](v: var T; slice: Slice[int]) {.inline, since: (1, let upmost = sizeof(T) * 8 - 1 - uv = when v is SomeUnsignedInt: v else: v.toUnsigned + uv = v.castToUnsigned v = (uv shl (upmost - slice.b) shr (upmost - slice.b + slice.a)).T func toMask*[T: SomeInteger](slice: Slice[int]): T {.inline, since: (1, 3).} = @@ -95,10 +95,7 @@ func toMask*[T: SomeInteger](slice: Slice[int]): T {.inline, since: (1, 3).} = let upmost = sizeof(T) * 8 - 1 - bitmask = when T is SomeUnsignedInt: - bitnot(0.T) - else: - bitnot(0.T).toUnsigned + bitmask = bitnot(0.T).castToUnsigned (bitmask shl (upmost - slice.b + slice.a) shr (upmost - slice.b)).T proc masked*[T: SomeInteger](v, mask :T): T {.inline, since: (1, 3).} = @@ -505,8 +502,7 @@ func parityBits*(x: SomeInteger): int {.inline.} = # Can be used a base if creating ASM version. # https://stackoverflow.com/questions/21617970/how-to-check-if-value-has-even-parity-of-bits-or-odd - when x is SomeSignedInt: - let x = x.toUnsigned + let x = x.castToUnsigned when nimvm: result = forwardImpl(parityImpl, x) else: @@ -529,8 +525,7 @@ func firstSetBit*(x: SomeInteger): int {.inline.} = doAssert firstSetBit(0b0000_1111'u8) == 1 # GCC builtin 'builtin_ffs' already handle zero input. - when x is SomeSignedInt: - let x = x.toUnsigned + let x = x.castToUnsigned when nimvm: when noUndefined: if x == 0: @@ -572,8 +567,7 @@ func fastLog2*(x: SomeInteger): int {.inline.} = doAssert fastLog2(0b0000_1000'u8) == 3 doAssert fastLog2(0b0000_1111'u8) == 3 - when x is SomeSignedInt: - let x = x.toUnsigned + let x = x.castToUnsigned when noUndefined: if x == 0: return -1 @@ -615,8 +609,7 @@ func countLeadingZeroBits*(x: SomeInteger): int {.inline.} = doAssert countLeadingZeroBits(0b0000_1000'u8) == 4 doAssert countLeadingZeroBits(0b0000_1111'u8) == 4 - when x is SomeSignedInt: - let x = x.toUnsigned + let x = x.castToUnsigned when noUndefined: if x == 0: return 0 @@ -644,8 +637,7 @@ func countTrailingZeroBits*(x: SomeInteger): int {.inline.} = doAssert countTrailingZeroBits(0b0000_1000'u8) == 3 doAssert countTrailingZeroBits(0b0000_1111'u8) == 0 - when x is SomeSignedInt: - let x = x.toUnsigned + let x = x.castToUnsigned when noUndefined: if x == 0: return 0 diff --git a/lib/pure/typetraits.nim b/lib/pure/typetraits.nim index 3fc1c7c5c..c20f9e645 100644 --- a/lib/pure/typetraits.nim +++ b/lib/pure/typetraits.nim @@ -35,7 +35,7 @@ runnableExamples: assert C[float] is HoleyEnum proc name*(t: typedesc): string {.magic: "TypeTrait".} = - ## Returns the name of the given type. + ## Returns the name of `t`. ## ## Alias for `system.\`$\`(t) `_ since Nim v0.20. runnableExamples: @@ -43,7 +43,7 @@ proc name*(t: typedesc): string {.magic: "TypeTrait".} = doAssert name(seq[string]) == "seq[string]" proc arity*(t: typedesc): int {.magic: "TypeTrait".} = - ## Returns the arity of the given type. This is the number of "type" + ## Returns the arity of `t`. This is the number of "type" ## components or the number of generic parameters a given type `t` has. runnableExamples: doAssert arity(int) == 0 @@ -92,8 +92,7 @@ proc stripGenericParams*(t: typedesc): typedesc {.magic: "TypeTrait".} = doAssert stripGenericParams(int) is int proc supportsCopyMem*(t: typedesc): bool {.magic: "TypeTrait".} - ## This trait returns true if the type `t` is safe to use for - ## `copyMem`:idx:. + ## Returns true if `t` is safe to use for `copyMem`:idx:. ## ## Other languages name a type like these `blob`:idx:. @@ -289,9 +288,38 @@ since (1, 1): proc hasClosureImpl(n: NimNode): bool = discard "see compiler/vmops.nim" proc hasClosure*(fn: NimNode): bool {.since: (1, 5, 1).} = - ## Return true if the func/proc/etc `fn` has `closure`. + ## Returns true if the func/proc/etc `fn` has `closure`. ## `fn` has to be a resolved symbol of kind `nnkSym`. This ## implies that the macro that calls this proc should accept `typed` ## arguments and not `untyped` arguments. expectKind fn, nnkSym result = hasClosureImpl(fn) + +template toUnsigned*(T: typedesc[SomeInteger and not range]): untyped = + ## Returns an unsigned type with same bit size as `T`. + runnableExamples: + assert int8.toUnsigned is uint8 + assert uint.toUnsigned is uint + assert int.toUnsigned is uint + # range types are currently unsupported: + assert not compiles(toUnsigned(range[0..7])) + when T is int8: uint8 + elif T is int16: uint16 + elif T is int32: uint32 + elif T is int64: uint64 + elif T is int: uint + else: T + +template toSigned*(T: typedesc[SomeInteger and not range]): untyped = + ## Returns a signed type with same bit size as `T`. + runnableExamples: + assert int8.toSigned is int8 + assert uint16.toSigned is int16 + # range types are currently unsupported: + assert not compiles(toSigned(range[0..7])) + when T is uint8: int8 + elif T is uint16: int16 + elif T is uint32: int32 + elif T is uint64: int64 + elif T is uint: int + else: T diff --git a/lib/std/private/bitops_utils.nim b/lib/std/private/bitops_utils.nim index d54977b4e..0b9484416 100644 --- a/lib/std/private/bitops_utils.nim +++ b/lib/std/private/bitops_utils.nim @@ -10,8 +10,13 @@ template forwardImpl*(impl, arg) {.dirty.} = else: impl(x.uint64) -template toUnsigned*(x: int8): uint8 = cast[uint8](x) -template toUnsigned*(x: int16): uint16 = cast[uint16](x) -template toUnsigned*(x: int32): uint32 = cast[uint32](x) -template toUnsigned*(x: int64): uint64 = cast[uint64](x) -template toUnsigned*(x: int): uint = cast[uint](x) +# this could also be implemented via: +# import std/typetraits +# template castToUnsigned*(x: SomeInteger): auto = cast[toUnsigned(typeof(x))](x) + +template castToUnsigned*(x: int8): uint8 = cast[uint8](x) +template castToUnsigned*(x: int16): uint16 = cast[uint16](x) +template castToUnsigned*(x: int32): uint32 = cast[uint32](x) +template castToUnsigned*(x: int64): uint64 = cast[uint64](x) +template castToUnsigned*(x: int): uint = cast[uint](x) +template castToUnsigned*[T: SomeUnsignedInt](x: T): T = x diff --git a/lib/system/countbits_impl.nim b/lib/system/countbits_impl.nim index 020423e01..34969cb32 100644 --- a/lib/system/countbits_impl.nim +++ b/lib/system/countbits_impl.nim @@ -9,8 +9,7 @@ ## Contains the used algorithms for counting bits. -from std/private/bitops_utils import forwardImpl, toUnsigned - +from std/private/bitops_utils import forwardImpl, castToUnsigned const useBuiltins* = not defined(noIntrinsicsBitOpts) const noUndefined* = defined(noUndefinedBitOpts) @@ -65,8 +64,7 @@ func countSetBitsImpl*(x: SomeInteger): int {.inline.} = ## Counts the set bits in an integer (also called `Hamming weight`:idx:). # TODO: figure out if ICC support _popcnt32/_popcnt64 on platform without POPCNT. # like GCC and MSVC - when x is SomeSignedInt: - let x = x.toUnsigned + let x = x.castToUnsigned when nimvm: result = forwardImpl(countBitsImpl, x) else: diff --git a/tests/metatype/ttypetraits.nim b/tests/metatype/ttypetraits.nim index 3ff5c5ea6..bfaa23057 100644 --- a/tests/metatype/ttypetraits.nim +++ b/tests/metatype/ttypetraits.nim @@ -1,6 +1,21 @@ import typetraits import macros +block: # toUnsigned, toSigned + var a1: toSigned(int16) + doAssert a1 is int16 + var a2: toSigned(uint16) + doAssert $a2.typeof == "int16" + doAssert toSigned(uint32) is int32 + doAssert uint64.toSigned is int64 + doAssert int64.toSigned is int64 + doAssert int64.toUnsigned is uint64 + doAssert int.toUnsigned is uint + doAssert $uint.toUnsigned == "uint" + # disallowed for now + doAssert not compiles(toUnsigned(range[0..7])) + doAssert not compiles(toSigned(range[0..7])) + block: # isNamedTuple type Foo1 = (a:1,).type type Foo2 = (Field0:1,).type diff --git a/tests/stdlib/tbitops_utils.nim b/tests/stdlib/tbitops_utils.nim new file mode 100644 index 000000000..b571baeae --- /dev/null +++ b/tests/stdlib/tbitops_utils.nim @@ -0,0 +1,14 @@ +import std/private/bitops_utils + +template chk(a, b) = + let a2 = castToUnsigned(a) + doAssert a2 == b + doAssert type(a2) is type(b) + doAssert type(b) is type(a2) + +chk 1'i8, 1'u8 +chk -1'i8, 255'u8 +chk 1'u8, 1'u8 +chk 1'u, 1'u +chk -1, cast[uint](-1) +chk -1'i64, cast[uint64](-1) diff --git a/tests/stdlib/ttypetraits.nim b/tests/stdlib/ttypetraits.nim index de8259ab0..799bcf6e2 100644 --- a/tests/stdlib/ttypetraits.nim +++ b/tests/stdlib/ttypetraits.nim @@ -2,10 +2,10 @@ discard """ targets: "c cpp js" """ +# xxx merge with tests/metatype/ttypetraits.nim + import std/typetraits - - macro testClosure(fn: typed, flag: static bool) = if flag: doAssert hasClosure(fn) From 00775f6880733695d187d84bb742e8c9d6c65d6d Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 8 Apr 2022 14:04:46 +0800 Subject: [PATCH 1584/2472] fix stylecheck bug with nre (#19356) * stylecheck usages part two: stdlib cleanup typeinfo.nim: importCompilerProc => importcompilerproc nre.nim: newLineFlags => newlineFlags system.nim: JSRoot => JsRoot ref #19319 * prefer importCompilerProc * fix stylecheck error with asyncdispatch it is a partial regression since #12842 * add tests * don't use echo in tests * fix stylecheck bug with nre * Update compiler/linter.nim * no need to check dotexpr again * neither did let/var/const --- compiler/semexprs.nim | 9 ++++++--- tests/stylecheck/taccept.nim | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index f224bd0b3..702b47929 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1188,7 +1188,8 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode = let s = getGenSym(c, sym) case s.kind of skConst: - markUsed(c, n.info, s) + if n.kind != nkDotExpr: # dotExpr is already checked by builtinFieldAccess + markUsed(c, n.info, s) onUse(n.info, s) let typ = skipTypes(s.typ, abstractInst-{tyTypeDesc}) case typ.kind @@ -1249,7 +1250,8 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode = if s.magic == mNimvm: localError(c.config, n.info, "illegal context for 'nimvm' magic") - markUsed(c, n.info, s) + if n.kind != nkDotExpr: # dotExpr is already checked by builtinFieldAccess + markUsed(c, n.info, s) onUse(n.info, s) result = newSymNode(s, n.info) # We cannot check for access to outer vars for example because it's still @@ -1270,7 +1272,8 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode = n.typ = s.typ return n of skType: - markUsed(c, n.info, s) + if n.kind != nkDotExpr: # dotExpr is already checked by builtinFieldAccess + markUsed(c, n.info, s) onUse(n.info, s) if s.typ.kind == tyStatic and s.typ.base.kind != tyNone and s.typ.n != nil: return s.typ.n diff --git a/tests/stylecheck/taccept.nim b/tests/stylecheck/taccept.nim index 6fb55f835..afe02a65c 100644 --- a/tests/stylecheck/taccept.nim +++ b/tests/stylecheck/taccept.nim @@ -2,7 +2,7 @@ discard """ matrix: "--styleCheck:error --styleCheck:usages" """ -import asyncdispatch +import std/[asyncdispatch, nre] type Name = object From ded8b0e5410518ffed24b46565eb6e42f18a63f1 Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 9 Apr 2022 23:50:57 +0800 Subject: [PATCH 1585/2472] rewrite docs JS in Nim (#19701) * rewrite docs JS in Nim * fixup * fix nimdoc/rsttester --- config/nimdoc.cfg | 43 ------------------- nimdoc/rst2html/expected/rst_examples.html | 43 ------------------- .../expected/index.html | 43 ------------------- .../expected/theindex.html | 43 ------------------- .../expected/subdir/subdir_b/utils.html | 43 ------------------- nimdoc/testproject/expected/testproject.html | 43 ------------------- nimdoc/testproject/expected/theindex.html | 43 ------------------- tools/dochack/dochack.nim | 36 +++++++++++++++- 8 files changed, 35 insertions(+), 302 deletions(-) diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index c1074f344..4efa1f637 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -243,49 +243,6 @@ doc.file = """ - -
              diff --git a/nimdoc/rst2html/expected/rst_examples.html b/nimdoc/rst2html/expected/rst_examples.html index 1df91ff1f..2b5218d9f 100644 --- a/nimdoc/rst2html/expected/rst_examples.html +++ b/nimdoc/rst2html/expected/rst_examples.html @@ -22,49 +22,6 @@ - -
              diff --git a/nimdoc/test_out_index_dot_html/expected/index.html b/nimdoc/test_out_index_dot_html/expected/index.html index 69dd2eb2e..c6a116bd6 100644 --- a/nimdoc/test_out_index_dot_html/expected/index.html +++ b/nimdoc/test_out_index_dot_html/expected/index.html @@ -22,49 +22,6 @@ - -
              diff --git a/nimdoc/test_out_index_dot_html/expected/theindex.html b/nimdoc/test_out_index_dot_html/expected/theindex.html index 34ddf8f6a..8ee62a330 100644 --- a/nimdoc/test_out_index_dot_html/expected/theindex.html +++ b/nimdoc/test_out_index_dot_html/expected/theindex.html @@ -22,49 +22,6 @@ - -
              diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html index 574ac5b9b..f94da7f40 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html @@ -22,49 +22,6 @@ - -
              diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html index e45492dac..cba9391af 100644 --- a/nimdoc/testproject/expected/testproject.html +++ b/nimdoc/testproject/expected/testproject.html @@ -22,49 +22,6 @@ - -
              diff --git a/nimdoc/testproject/expected/theindex.html b/nimdoc/testproject/expected/theindex.html index 88d4b86aa..47fae2491 100644 --- a/nimdoc/testproject/expected/theindex.html +++ b/nimdoc/testproject/expected/theindex.html @@ -22,49 +22,6 @@ - -
              diff --git a/tools/dochack/dochack.nim b/tools/dochack/dochack.nim index 3a663808d..c7b8232ef 100644 --- a/tools/dochack/dochack.nim +++ b/tools/dochack/dochack.nim @@ -1,6 +1,39 @@ import dom import fuzzysearch + +proc switchTheme(event: Event) = + if event.target.checked: + document.documentElement.setAttribute("data-theme", "dark") + window.localStorage.setItem("theme", "dark") + else: + document.documentElement.setAttribute("data-theme", "light") + window.localStorage.setItem("theme", "light") + + +proc nimThemeSwitch(event: Event) {.exportC.} = + var pragmaDots = document.getElementsByClassName("pragmadots") + for i in 0.. 0: + document.documentElement.setAttribute("data-theme", currentTheme); + + if currentTheme == "dark" and toggleSwitch != nil: + toggleSwitch.checked = true + proc textContent(e: Element): cstring {. importcpp: "#.textContent", nodecl.} @@ -396,5 +429,6 @@ proc copyToClipboard*() {.exportc.} = """ .} - + copyToClipboard() +window.addEventListener("DOMContentLoaded", nimThemeSwitch) \ No newline at end of file From 152dab09838adc5a4ab8dc220c5017c868563467 Mon Sep 17 00:00:00 2001 From: Leon Date: Sun, 10 Apr 2022 01:51:44 +1000 Subject: [PATCH 1586/2472] fix: Fix introductory macro examples (#19706) Co-authored-by: adigitoleo --- doc/tut3.rst | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/doc/tut3.rst b/doc/tut3.rst index 1c497f1f5..c2c956107 100644 --- a/doc/tut3.rst +++ b/doc/tut3.rst @@ -188,21 +188,35 @@ Backticks are used to insert code from `NimNode` symbols into the generated expression. .. code-block:: nim - macro a(i) = quote do: let `i` = 0 + :test: "nim c $1" + import std/macros + macro a(i) = quote do: + let `i` = 0 + a b + doAssert b == 0 A custom prefix operator can be defined whenever backticks are needed. .. code-block:: nim - macro a(i) = quote("@") do: assert @i == 0 + :test: "nim c $1" + import std/macros + macro a(i) = quote("@") do: + assert @i == 0 + let b = 0 a b The injected symbol needs accent quoted when it resolves to a symbol. .. code-block:: nim - macro a(i) = quote("@") do: let `@i` == 0 + :test: "nim c $1" + import std/macros + macro a(i) = quote("@") do: + let `@i` = 0 + a b + doAssert b == 0 Make sure to inject only symbols of type `NimNode` into the generated syntax tree. You can use `newLit` to convert arbitrary values into From a77ffdb7ea4ec66983d3481cc486c821a5df6b1b Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 9 Apr 2022 23:52:11 +0800 Subject: [PATCH 1587/2472] fix #17286 nim check -b:js works (#19704) * fix #17286 nim check -b:js works * fix --- compiler/main.nim | 2 ++ tests/misc/t17286.nim | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 tests/misc/t17286.nim diff --git a/compiler/main.nim b/compiler/main.nim index e27a4f6d1..a4425e510 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -65,6 +65,8 @@ proc commandCheck(graph: ModuleGraph) = if optWasNimscript in conf.globalOptions: defineSymbol(conf.symbols, "nimscript") defineSymbol(conf.symbols, "nimconfig") + elif conf.backend == backendJs: + setTarget(conf.target, osJS, cpuJS) semanticPasses(graph) # use an empty backend for semantic checking only compileProject(graph) diff --git a/tests/misc/t17286.nim b/tests/misc/t17286.nim new file mode 100644 index 000000000..3a54e624e --- /dev/null +++ b/tests/misc/t17286.nim @@ -0,0 +1,16 @@ +discard """ + cmd: "nim check -b:js $file" + action: "compile" +""" + +# bug #17286 + +import std/compilesettings + +static: + doAssert querySetting(backend) == "js" + doAssert defined(js) + doAssert not defined(c) + +import random +randomize() \ No newline at end of file From 26acc97864ede8e19dddd52aa04e0139da356e3e Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Mon, 11 Apr 2022 06:27:12 -0600 Subject: [PATCH 1588/2472] StringStreams no longer errors when intialized with literals on arc/orc (#19708) --- lib/pure/streams.nim | 2 ++ tests/stdlib/tstreams.nim | 2 ++ 2 files changed, 4 insertions(+) diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim index 7ad81685f..84451f989 100644 --- a/lib/pure/streams.nim +++ b/lib/pure/streams.nim @@ -1274,6 +1274,8 @@ else: # after 1.3 or JS not defined new(result) result.data = s + when defined(gcOrc) or defined(gcArc): + prepareMutation(result.data) # Allows us to mutate using `addr` logic like `copyMem`, otherwise it errors. result.pos = 0 result.closeImpl = ssClose result.atEndImpl = ssAtEnd diff --git a/tests/stdlib/tstreams.nim b/tests/stdlib/tstreams.nim index c2ceed624..d70d4995a 100644 --- a/tests/stdlib/tstreams.nim +++ b/tests/stdlib/tstreams.nim @@ -74,3 +74,5 @@ block: doAssert(ss.peekLine(str)) doAssert(str == "uick brown fox jumped over the lazy dog.") doAssert(ss.getPosition == 5) # haven't moved + ss.setPosition(0) # Ensure we dont error with writing over literals on arc/orc #19707 + ss.write("hello") From 26bcf18f918b2bcc7b6b2e230ae74d7726dca6d0 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 12 Apr 2022 15:48:48 +0800 Subject: [PATCH 1589/2472] fix #19680; check if stderr is static (#19709) --- compiler/vmgen.nim | 2 +- tests/misc/tvarious1.nim | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 72bcef4db..dadcc3865 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1479,7 +1479,7 @@ proc checkCanEval(c: PCtx; n: PNode) = # proc foo() = var x ... let s = n.sym if {sfCompileTime, sfGlobal} <= s.flags: return - if s.importcCondVar: return + if compiletimeFFI in c.config.features and s.importcCondVar: return if s.kind in {skVar, skTemp, skLet, skParam, skResult} and not s.isOwnedBy(c.prc.sym) and s.owner != c.module and c.mode != emRepl: # little hack ahead for bug #12612: assume gensym'ed variables diff --git a/tests/misc/tvarious1.nim b/tests/misc/tvarious1.nim index 9c0b541db..9c912ee8e 100644 --- a/tests/misc/tvarious1.nim +++ b/tests/misc/tvarious1.nim @@ -46,3 +46,13 @@ echo value var ys = @[4.1, 5.6, 7.2, 1.7, 9.3, 4.4, 3.2] #var x = int(ys.high / 2) #echo ys[x] # Works echo ys[int(ys.high / 2)] # Doesn't work + + +# bug #19680 +var here = "" +when stderr is static: + doAssert false +else: + here = "works" + +doAssert here == "works" From 98cebad7debddfb147ee22bc6f3d81221582c4d6 Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 13 Apr 2022 17:53:02 +0800 Subject: [PATCH 1590/2472] enable style:usages for stdlib tests [backport: 1.6] (#19715) * enable style:usages for stdlib tests * freeAddrInfo * more tests * importc * bufSize * fix more * => parseSql and renderSql --- lib/impure/db_mysql.nim | 20 +++--- lib/posix/posix.nim | 2 +- lib/pure/asyncdispatch.nim | 6 +- lib/pure/asyncnet.nim | 6 +- lib/pure/cookies.nim | 2 +- lib/pure/ioselects/ioselectors_epoll.nim | 2 +- lib/pure/ioselects/ioselectors_select.nim | 16 ++--- lib/pure/json.nim | 12 ++-- lib/pure/nativesockets.nim | 4 +- lib/pure/net.nim | 14 ++-- lib/pure/parsesql.nim | 12 ++-- lib/pure/pegs.nim | 4 +- lib/pure/segfaults.nim | 2 +- lib/std/strbasics.nim | 2 +- lib/std/tasks.nim | 2 +- lib/windows/registry.nim | 18 ++--- lib/windows/winlean.nim | 2 +- tests/stdlib/config.nims | 2 + tests/stdlib/nre/misc.nim | 4 +- tests/stdlib/talgorithm.nim | 4 +- tests/stdlib/tasynchttpserver.nim | 8 +-- tests/stdlib/tbase64.nim | 6 +- tests/stdlib/tbitops.nim | 4 +- tests/stdlib/thttpclient.nim | 2 +- tests/stdlib/tjsonmacro.nim | 4 +- tests/stdlib/tjsonutils.nim | 2 +- tests/stdlib/tnet.nim | 2 +- tests/stdlib/tnetdial.nim | 2 +- tests/stdlib/tosproc.nim | 2 +- tests/stdlib/tparsesql.nim | 86 +++++++++++------------ tests/stdlib/trst.nim | 10 +-- tests/stdlib/trstgen.nim | 8 +-- tests/stdlib/tsha1.nim | 2 +- tests/stdlib/tssl.nim | 4 +- tests/stdlib/tstrbasics.nim | 2 +- tests/stdlib/tstreams.nim | 2 +- tests/stdlib/tstrset.nim | 2 +- tests/stdlib/ttimes.nim | 2 +- tests/stdlib/ttypeinfo.nim | 2 +- tests/stdlib/ttypeinfo.nims | 1 + tests/stdlib/tunicode.nim | 4 +- tests/stdlib/tvarints.nim | 2 +- 42 files changed, 149 insertions(+), 146 deletions(-) create mode 100644 tests/stdlib/config.nims create mode 100644 tests/stdlib/ttypeinfo.nims diff --git a/lib/impure/db_mysql.nim b/lib/impure/db_mysql.nim index 242ea1b0d..df878e25a 100644 --- a/lib/impure/db_mysql.nim +++ b/lib/impure/db_mysql.nim @@ -150,17 +150,17 @@ proc tryExec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): bool {. tags: [ReadDbEffect, WriteDbEffect].} = ## tries to execute the query and returns true if successful, false otherwise. var q = dbFormat(query, args) - return mysql.realQuery(PMySQL db, q, q.len) == 0'i32 + return mysql.real_query(PMySQL db, q, q.len) == 0'i32 proc rawExec(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) = var q = dbFormat(query, args) - if mysql.realQuery(PMySQL db, q, q.len) != 0'i32: dbError(db) + if mysql.real_query(PMySQL db, q, q.len) != 0'i32: dbError(db) proc exec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) {. tags: [ReadDbEffect, WriteDbEffect].} = ## executes the query and raises EDB if not successful. var q = dbFormat(query, args) - if mysql.realQuery(PMySQL db, q, q.len) != 0'i32: dbError(db) + if mysql.real_query(PMySQL db, q, q.len) != 0'i32: dbError(db) proc newRow(L: int): Row = newSeq(result, L) @@ -168,7 +168,7 @@ proc newRow(L: int): Row = proc properFreeResult(sqlres: mysql.PRES, row: cstringArray) = if row != nil: - while mysql.fetchRow(sqlres) != nil: discard + while mysql.fetch_row(sqlres) != nil: discard mysql.freeResult(sqlres) iterator fastRows*(db: DbConn, query: SqlQuery, @@ -190,7 +190,7 @@ iterator fastRows*(db: DbConn, query: SqlQuery, backup: Row newSeq(result, L) while true: - row = mysql.fetchRow(sqlres) + row = mysql.fetch_row(sqlres) if row == nil: break for i in 0..L-1: setLen(result[i], 0) @@ -209,7 +209,7 @@ iterator instantRows*(db: DbConn, query: SqlQuery, let L = int(mysql.numFields(sqlres)) var row: cstringArray while true: - row = mysql.fetchRow(sqlres) + row = mysql.fetch_row(sqlres) if row == nil: break yield InstantRow(row: row, len: L) properFreeResult(sqlres, row) @@ -290,7 +290,7 @@ iterator instantRows*(db: DbConn; columns: var DbColumns; query: SqlQuery; setColumnInfo(columns, sqlres, L) var row: cstringArray while true: - row = mysql.fetchRow(sqlres) + row = mysql.fetch_row(sqlres) if row == nil: break yield InstantRow(row: row, len: L) properFreeResult(sqlres, row) @@ -317,7 +317,7 @@ proc getRow*(db: DbConn, query: SqlQuery, if sqlres != nil: var L = int(mysql.numFields(sqlres)) result = newRow(L) - var row = mysql.fetchRow(sqlres) + var row = mysql.fetch_row(sqlres) if row != nil: for i in 0..L-1: setLen(result[i], 0) @@ -335,7 +335,7 @@ proc getAllRows*(db: DbConn, query: SqlQuery, var row: cstringArray var j = 0 while true: - row = mysql.fetchRow(sqlres) + row = mysql.fetch_row(sqlres) if row == nil: break setLen(result, j+1) newSeq(result[j], L) @@ -361,7 +361,7 @@ proc tryInsertId*(db: DbConn, query: SqlQuery, ## executes the query (typically "INSERT") and returns the ## generated ID for the row or -1 in case of an error. var q = dbFormat(query, args) - if mysql.realQuery(PMySQL db, q, q.len) != 0'i32: + if mysql.real_query(PMySQL db, q, q.len) != 0'i32: result = -1'i64 else: result = mysql.insertId(PMySQL db) diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 9c65d0732..146ba886f 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -1016,7 +1016,7 @@ proc endhostent*() {.importc, header: "".} proc endnetent*() {.importc, header: "".} proc endprotoent*() {.importc, header: "".} proc endservent*() {.importc, header: "".} -proc freeaddrinfo*(a1: ptr AddrInfo) {.importc, header: "".} +proc freeAddrInfo*(a1: ptr AddrInfo) {.importc: "freeaddrinfo", header: "".} proc gai_strerror*(a1: cint): cstring {.importc:"(char *)$1", header: "".} diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 5e2a8d98e..b67b0c308 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -1791,7 +1791,7 @@ template asyncAddrInfoLoop(addrInfo: ptr AddrInfo, fd: untyped, curAddrInfo = curAddrInfo.ai_next if curAddrInfo == nil: - freeaddrinfo(addrInfo) + freeAddrInfo(addrInfo) when shouldCreateFd: closeUnusedFds() if lastException != nil: @@ -1807,7 +1807,7 @@ template asyncAddrInfoLoop(addrInfo: ptr AddrInfo, fd: untyped, try: curFd = createAsyncNativeSocket(domain, sockType, protocol) except: - freeaddrinfo(addrInfo) + freeAddrInfo(addrInfo) closeUnusedFds() raise getCurrentException() when defined(windows): @@ -1817,7 +1817,7 @@ template asyncAddrInfoLoop(addrInfo: ptr AddrInfo, fd: untyped, doConnect(curFd, curAddrInfo).callback = tryNextAddrInfo curAddrInfo = curAddrInfo.ai_next else: - freeaddrinfo(addrInfo) + freeAddrInfo(addrInfo) when shouldCreateFd: closeUnusedFds(ord(domain)) retFuture.complete(curFd) diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim index abb4dd075..61c2fc75a 100644 --- a/lib/pure/asyncnet.nim +++ b/lib/pure/asyncnet.nim @@ -648,9 +648,9 @@ proc bindAddr*(socket: AsyncSocket, port = Port(0), address = "") {. var aiList = getAddrInfo(realaddr, port, socket.domain) if bindAddr(socket.fd, aiList.ai_addr, aiList.ai_addrlen.SockLen) < 0'i32: - freeaddrinfo(aiList) + freeAddrInfo(aiList) raiseOSError(osLastError()) - freeaddrinfo(aiList) + freeAddrInfo(aiList) proc hasDataBuffered*(s: AsyncSocket): bool {.since: (1, 5).} = ## Determines whether an AsyncSocket has data buffered. @@ -859,7 +859,7 @@ proc sendTo*(socket: AsyncSocket, address: string, port: Port, data: string, it = it.ai_next - freeaddrinfo(aiList) + freeAddrInfo(aiList) if not success: if lastException != nil: diff --git a/lib/pure/cookies.nim b/lib/pure/cookies.nim index 7bf34f75b..0306558d6 100644 --- a/lib/pure/cookies.nim +++ b/lib/pure/cookies.nim @@ -75,4 +75,4 @@ proc setCookie*(key, value: string, expires: DateTime|Time, ## `Set-Cookie: key=value; Domain=...; ...` result = setCookie(key, value, domain, path, format(expires.utc, "ddd',' dd MMM yyyy HH:mm:ss 'GMT'"), - noname, secure, httpOnly, maxAge, sameSite) + noName, secure, httpOnly, maxAge, sameSite) diff --git a/lib/pure/ioselects/ioselectors_epoll.nim b/lib/pure/ioselects/ioselectors_epoll.nim index 05d9d19a8..8526eb8a3 100644 --- a/lib/pure/ioselects/ioselectors_epoll.nim +++ b/lib/pure/ioselects/ioselectors_epoll.nim @@ -526,4 +526,4 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, body2 proc getFd*[T](s: Selector[T]): int = - return s.epollFd.int + return s.epollFD.int diff --git a/lib/pure/ioselects/ioselectors_select.nim b/lib/pure/ioselects/ioselectors_select.nim index 88cbaf28c..4c38ba604 100644 --- a/lib/pure/ioselects/ioselectors_select.nim +++ b/lib/pure/ioselects/ioselectors_select.nim @@ -26,27 +26,27 @@ else: #include #include """ type - Fdset {.importc: "fd_set", header: platformHeaders, pure, final.} = object + FdSet {.importc: "fd_set", header: platformHeaders, pure, final.} = object var FD_SETSIZE {.importc: "FD_SETSIZE", header: platformHeaders.}: cint -proc IOFD_SET(fd: SocketHandle, fdset: ptr Fdset) +proc IOFD_SET(fd: SocketHandle, fdset: ptr FdSet) {.cdecl, importc: "FD_SET", header: platformHeaders, inline.} -proc IOFD_CLR(fd: SocketHandle, fdset: ptr Fdset) +proc IOFD_CLR(fd: SocketHandle, fdset: ptr FdSet) {.cdecl, importc: "FD_CLR", header: platformHeaders, inline.} -proc IOFD_ZERO(fdset: ptr Fdset) +proc IOFD_ZERO(fdset: ptr FdSet) {.cdecl, importc: "FD_ZERO", header: platformHeaders, inline.} when defined(windows): - proc IOFD_ISSET(fd: SocketHandle, fdset: ptr Fdset): cint + proc IOFD_ISSET(fd: SocketHandle, fdset: ptr FdSet): cint {.stdcall, importc: "FD_ISSET", header: platformHeaders, inline.} - proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr Fdset, + proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr FdSet, timeout: ptr Timeval): cint {.stdcall, importc: "select", header: platformHeaders.} else: - proc IOFD_ISSET(fd: SocketHandle, fdset: ptr Fdset): cint + proc IOFD_ISSET(fd: SocketHandle, fdset: ptr FdSet): cint {.cdecl, importc: "FD_ISSET", header: platformHeaders, inline.} - proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr Fdset, + proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr FdSet, timeout: ptr Timeval): cint {.cdecl, importc: "select", header: platformHeaders.} diff --git a/lib/pure/json.nim b/lib/pure/json.nim index dd9232ea6..8e4db93b0 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -958,12 +958,12 @@ proc parseJson*(s: Stream, filename: string = ""; rawIntegers = false, rawFloats when defined(js): from math import `mod` - from std/jsffi import JSObject, `[]`, to + from std/jsffi import JsObject, `[]`, to from std/private/jsutils import getProtoName, isInteger, isSafeInteger - proc parseNativeJson(x: cstring): JSObject {.importjs: "JSON.parse(#)".} + proc parseNativeJson(x: cstring): JsObject {.importjs: "JSON.parse(#)".} - proc getVarType(x: JSObject, isRawNumber: var bool): JsonNodeKind = + proc getVarType(x: JsObject, isRawNumber: var bool): JsonNodeKind = result = JNull case $getProtoName(x) # TODO: Implicit returns fail here. of "[object Array]": return JArray @@ -982,12 +982,12 @@ when defined(js): of "[object String]": return JString else: assert false - proc len(x: JSObject): int = + proc len(x: JsObject): int = asm """ `result` = `x`.length; """ - proc convertObject(x: JSObject): JsonNode = + proc convertObject(x: JsObject): JsonNode = var isRawNumber = false case getVarType(x, isRawNumber) of JArray: @@ -1000,7 +1000,7 @@ when defined(js): if (`x`.hasOwnProperty(property)) { """ var nimProperty: cstring - var nimValue: JSObject + var nimValue: JsObject asm "`nimProperty` = property; `nimValue` = `x`[property];" result[$nimProperty] = nimValue.convertObject() asm "}}" diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index eb0514a8f..5453226a0 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -633,8 +633,8 @@ when useNimNetLite: INET_ADDRSTRLEN = 16 INET6_ADDRSTRLEN = 46 # it's actually 46 in both cases - proc sockAddrToStr(sa: ptr Sockaddr): string {.noinit.} = - let af_family = sa.sa_family + proc sockAddrToStr(sa: ptr SockAddr): string {.noinit.} = + let af_family = sa.sa_family var nl, v4Slice: cint var si_addr: ptr InAddr diff --git a/lib/pure/net.nim b/lib/pure/net.nim index c59a4e563..370b83e54 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -984,11 +984,11 @@ proc bindAddr*(socket: Socket, port = Port(0), address = "") {. var aiList = getAddrInfo(realaddr, port, socket.domain) if bindAddr(socket.fd, aiList.ai_addr, aiList.ai_addrlen.SockLen) < 0'i32: - freeaddrinfo(aiList) + freeAddrInfo(aiList) var address2: string address2.addQuoted address raiseOSError(osLastError(), "address: $# port: $#" % [address2, $port]) - freeaddrinfo(aiList) + freeAddrInfo(aiList) proc acceptAddr*(server: Socket, client: var owned(Socket), address: var string, flags = {SocketFlag.SafeDisconn}, @@ -1745,7 +1745,7 @@ proc sendTo*(socket: Socket, address: string, port: Port, data: pointer, it = it.ai_next let osError = osLastError() - freeaddrinfo(aiList) + freeAddrInfo(aiList) if not success: raiseOSError(osError) @@ -1960,7 +1960,7 @@ proc dial*(address: string, port: Port, # network system problem (e.g. not enough FDs), and not an unreachable # address. let err = osLastError() - freeaddrinfo(aiList) + freeAddrInfo(aiList) closeUnusedFds() raiseOSError(err) fdPerDomain[ord(domain)] = lastFd @@ -1969,7 +1969,7 @@ proc dial*(address: string, port: Port, break lastError = osLastError() it = it.ai_next - freeaddrinfo(aiList) + freeAddrInfo(aiList) closeUnusedFds(ord(domain)) if success: @@ -1999,7 +1999,7 @@ proc connect*(socket: Socket, address: string, else: lastError = osLastError() it = it.ai_next - freeaddrinfo(aiList) + freeAddrInfo(aiList) if not success: raiseOSError(lastError) when defineSsl: @@ -2051,7 +2051,7 @@ proc connectAsync(socket: Socket, name: string, port = Port(0), it = it.ai_next - freeaddrinfo(aiList) + freeAddrInfo(aiList) if not success: raiseOSError(lastError) proc connect*(socket: Socket, address: string, port = Port(0), diff --git a/lib/pure/parsesql.nim b/lib/pure/parsesql.nim index 9d74d08d0..61196aead 100644 --- a/lib/pure/parsesql.nim +++ b/lib/pure/parsesql.nim @@ -1451,7 +1451,7 @@ proc ra(n: SqlNode, s: var SqlWriter) = s.addKeyw("enum") rs(n, s) -proc renderSQL*(n: SqlNode, upperCase = false): string = +proc renderSql*(n: SqlNode, upperCase = false): string = ## Converts an SQL abstract syntax tree to its string representation. var s: SqlWriter s.buffer = "" @@ -1460,8 +1460,8 @@ proc renderSQL*(n: SqlNode, upperCase = false): string = return s.buffer proc `$`*(n: SqlNode): string = - ## an alias for `renderSQL`. - renderSQL(n) + ## an alias for `renderSql`. + renderSql(n) proc treeReprAux(s: SqlNode, level: int, result: var string) = result.add('\n') @@ -1493,7 +1493,7 @@ proc open(p: var SqlParser, input: Stream, filename: string) = p.tok.literal = "" getTok(p) -proc parseSQL*(input: Stream, filename: string): SqlNode = +proc parseSql*(input: Stream, filename: string): SqlNode = ## parses the SQL from `input` into an AST and returns the AST. ## `filename` is only used for error messages. ## Syntax errors raise an `SqlParseError` exception. @@ -1504,8 +1504,8 @@ proc parseSQL*(input: Stream, filename: string): SqlNode = finally: close(p) -proc parseSQL*(input: string, filename = ""): SqlNode = +proc parseSql*(input: string, filename = ""): SqlNode = ## parses the SQL from `input` into an AST and returns the AST. ## `filename` is only used for error messages. ## Syntax errors raise an `SqlParseError` exception. - parseSQL(newStringStream(input), "") + parseSql(newStringStream(input), "") diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index 0eed1c388..78f03d308 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -1023,7 +1023,7 @@ template eventParser*(pegAst, handlers: untyped): (proc(s: string): int) = ## Symbols declared in an *enter* handler can be made visible in the ## corresponding *leave* handler by annotating them with an *inject* pragma. proc rawParse(s: string, p: Peg, start: int, c: var Captures): int - {.genSym.} = + {.gensym.} = # binding from *macros* bind strVal @@ -1058,7 +1058,7 @@ template eventParser*(pegAst, handlers: untyped): (proc(s: string): int) = matchOrParse(parseIt) parseIt(s, p, start, c) - proc parser(s: string): int {.genSym.} = + proc parser(s: string): int {.gensym.} = # the proc to be returned var ms: array[MaxSubpatterns, (int, int)] diff --git a/lib/pure/segfaults.nim b/lib/pure/segfaults.nim index e0da8b81d..b0eac2299 100644 --- a/lib/pure/segfaults.nim +++ b/lib/pure/segfaults.nim @@ -30,7 +30,7 @@ when defined(windows): const EXCEPTION_ACCESS_VIOLATION = DWORD(0xc0000005'i32) - EXCEPTION_CONTINUE_SEARCH = Long(0) + EXCEPTION_CONTINUE_SEARCH = LONG(0) type PEXCEPTION_RECORD = ptr object diff --git a/lib/std/strbasics.nim b/lib/std/strbasics.nim index 0d4c96f44..6f6db5c37 100644 --- a/lib/std/strbasics.nim +++ b/lib/std/strbasics.nim @@ -46,7 +46,7 @@ func setSlice*(s: var string, slice: Slice[int]) = import std/sugar var a = "Hello, Nim!" - doassert a.dup(setSlice(7 .. 9)) == "Nim" + doAssert a.dup(setSlice(7 .. 9)) == "Nim" doAssert a.dup(setSlice(0 .. 0)) == "H" doAssert a.dup(setSlice(0 .. 1)) == "He" doAssert a.dup(setSlice(0 .. 10)) == a diff --git a/lib/std/tasks.nim b/lib/std/tasks.nim index 3ceb4c8ba..7b931ab14 100644 --- a/lib/std/tasks.nim +++ b/lib/std/tasks.nim @@ -85,7 +85,7 @@ template checkIsolate(scratchAssignList: seq[NimNode], procParam, scratchDotExpr # var isoTempB = isolate(literal) # scratch.b = extract(isolateB) let isolatedTemp = genSym(nskTemp, "isoTemp") - scratchAssignList.add newVarStmt(isolatedTemp, newCall(newidentNode("isolate"), procParam)) + scratchAssignList.add newVarStmt(isolatedTemp, newCall(newIdentNode("isolate"), procParam)) scratchAssignList.add newAssignment(scratchDotExpr, newCall(newIdentNode("extract"), isolatedTemp)) diff --git a/lib/windows/registry.nim b/lib/windows/registry.nim index ab7d0b1fe..44e4e4420 100644 --- a/lib/windows/registry.nim +++ b/lib/windows/registry.nim @@ -46,26 +46,26 @@ template call(f) = proc getUnicodeValue*(path, key: string; handle: HKEY): string = let hh = newWideCString path let kk = newWideCString key - var bufsize: int32 + var bufSize: int32 # try a couple of different flag settings: var flags: int32 = RRF_RT_ANY - let err = regGetValue(handle, hh, kk, flags, nil, nil, addr bufsize) + let err = regGetValue(handle, hh, kk, flags, nil, nil, addr bufSize) if err != 0: var newHandle: HKEY call regOpenKeyEx(handle, hh, 0, KEY_READ or KEY_WOW64_64KEY, newHandle) - call regGetValue(newHandle, nil, kk, flags, nil, nil, addr bufsize) + call regGetValue(newHandle, nil, kk, flags, nil, nil, addr bufSize) if bufSize > 0: - var res = newWideCString(bufsize) + var res = newWideCString(bufSize) call regGetValue(newHandle, nil, kk, flags, nil, addr res[0], - addr bufsize) - result = res $ bufsize + addr bufSize) + result = res $ bufSize call regCloseKey(newHandle) else: if bufSize > 0: - var res = newWideCString(bufsize) + var res = newWideCString(bufSize) call regGetValue(handle, hh, kk, flags, nil, addr res[0], - addr bufsize) - result = res $ bufsize + addr bufSize) + result = res $ bufSize proc regSetValue(key: HKEY, lpSubKey, lpValueName: WideCString, dwType: int32; lpData: WideCString; cbData: int32): int32 {. diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index c02cd9775..cd59b68a4 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -670,7 +670,7 @@ proc getaddrinfo*(nodename, servname: cstring, hints: ptr AddrInfo, res: var ptr AddrInfo): cint {. stdcall, importc: "getaddrinfo", dynlib: ws2dll.} -proc freeaddrinfo*(ai: ptr AddrInfo) {. +proc freeAddrInfo*(ai: ptr AddrInfo) {. stdcall, importc: "freeaddrinfo", dynlib: ws2dll.} proc inet_ntoa*(i: InAddr): cstring {. diff --git a/tests/stdlib/config.nims b/tests/stdlib/config.nims new file mode 100644 index 000000000..0c0c5bd88 --- /dev/null +++ b/tests/stdlib/config.nims @@ -0,0 +1,2 @@ +switch("styleCheck", "usages") +switch("styleCheck", "error") \ No newline at end of file diff --git a/tests/stdlib/nre/misc.nim b/tests/stdlib/nre/misc.nim index dbb0ecdf9..b7df08ee9 100644 --- a/tests/stdlib/nre/misc.nim +++ b/tests/stdlib/nre/misc.nim @@ -6,8 +6,8 @@ block: # Misc tests check("перевірка".replace(re"(*U)\w", "") == "") block: # empty or non-empty match - check("abc".findall(re"|.").join(":") == ":a::b::c:") - check("abc".findall(re".|").join(":") == "a:b:c:") + check("abc".findAll(re"|.").join(":") == ":a::b::c:") + check("abc".findAll(re".|").join(":") == "a:b:c:") check("abc".replace(re"|.", "x") == "xxxxxxx") check("abc".replace(re".|", "x") == "xxxx") diff --git a/tests/stdlib/talgorithm.nim b/tests/stdlib/talgorithm.nim index 5bb5a6bdf..61d2bc62f 100644 --- a/tests/stdlib/talgorithm.nim +++ b/tests/stdlib/talgorithm.nim @@ -101,10 +101,10 @@ block: doAssert binarySearch(noData, 7) == -1 let oneData = @[1] doAssert binarySearch(oneData, 1) == 0 - doAssert binarySearch(onedata, 7) == -1 + doAssert binarySearch(oneData, 7) == -1 let someData = @[1, 3, 4, 7] doAssert binarySearch(someData, 1) == 0 - doAssert binarySearch(somedata, 7) == 3 + doAssert binarySearch(someData, 7) == 3 doAssert binarySearch(someData, -1) == -1 doAssert binarySearch(someData, 5) == -1 doAssert binarySearch(someData, 13) == -1 diff --git a/tests/stdlib/tasynchttpserver.nim b/tests/stdlib/tasynchttpserver.nim index d4176d55a..aed21099d 100644 --- a/tests/stdlib/tasynchttpserver.nim +++ b/tests/stdlib/tasynchttpserver.nim @@ -111,9 +111,9 @@ proc testCustomContentLength() {.async.} = runTest(handler, request, test) -waitfor(test200()) -waitfor(test404()) -waitfor(testCustomEmptyHeaders()) -waitfor(testCustomContentLength()) +waitFor(test200()) +waitFor(test404()) +waitFor(testCustomEmptyHeaders()) +waitFor(testCustomContentLength()) echo "OK" diff --git a/tests/stdlib/tbase64.nim b/tests/stdlib/tbase64.nim index 491ec5db0..8ef5adf57 100644 --- a/tests/stdlib/tbase64.nim +++ b/tests/stdlib/tbase64.nim @@ -27,13 +27,13 @@ template main() = const tests = ["", "abc", "xyz", "man", "leasure.", "sure.", "easure.", "asure.", longText, testInputExpandsTo76, testInputExpands] - doAssert encodeMIME("foobarbaz", lineLen=4) == "Zm9v\r\nYmFy\r\nYmF6" + doAssert encodeMime("foobarbaz", lineLen=4) == "Zm9v\r\nYmFy\r\nYmF6" doAssert decode("Zm9v\r\nYmFy\r\nYmF6") == "foobarbaz" for t in items(tests): doAssert decode(encode(t)) == t - doAssert decode(encodeMIME(t, lineLen=40)) == t - doAssert decode(encodeMIME(t, lineLen=76)) == t + doAssert decode(encodeMime(t, lineLen=40)) == t + doAssert decode(encodeMime(t, lineLen=76)) == t doAssertRaises(ValueError): discard decode("SGVsbG\x008gV29ybGQ=") diff --git a/tests/stdlib/tbitops.nim b/tests/stdlib/tbitops.nim index 6b5e0aeb9..f807993d6 100644 --- a/tests/stdlib/tbitops.nim +++ b/tests/stdlib/tbitops.nim @@ -263,8 +263,8 @@ proc main() = doAssert v == 0b1000_0010 v.flipBit(1) doAssert v == 0b1000_0000 - doAssert v.testbit(7) - doAssert not v.testbit(6) + doAssert v.testBit(7) + doAssert not v.testBit(6) block: # multi bit operations var v: uint8 diff --git a/tests/stdlib/thttpclient.nim b/tests/stdlib/thttpclient.nim index e81590d95..8563a0c26 100644 --- a/tests/stdlib/thttpclient.nim +++ b/tests/stdlib/thttpclient.nim @@ -21,7 +21,7 @@ proc makeIPv6HttpServer(hostname: string, port: Port, let fd = createNativeSocket(AF_INET6) setSockOptInt(fd, SOL_SOCKET, SO_REUSEADDR, 1) var aiList = getAddrInfo(hostname, port, AF_INET6) - if bindAddr(fd, aiList.ai_addr, aiList.ai_addrlen.Socklen) < 0'i32: + if bindAddr(fd, aiList.ai_addr, aiList.ai_addrlen.SockLen) < 0'i32: freeAddrInfo(aiList) raiseOSError(osLastError()) freeAddrInfo(aiList) diff --git a/tests/stdlib/tjsonmacro.nim b/tests/stdlib/tjsonmacro.nim index 779c0ce65..9b59c7dc3 100644 --- a/tests/stdlib/tjsonmacro.nim +++ b/tests/stdlib/tjsonmacro.nim @@ -259,7 +259,7 @@ proc testJson() = colors: array[2, BirdColor] var red = BirdColor(name: "red", rgb: [1.0, 0.0, 0.0]) - var blue = Birdcolor(name: "blue", rgb: [0.0, 0.0, 1.0]) + var blue = BirdColor(name: "blue", rgb: [0.0, 0.0, 1.0]) var b = Bird(age: 3, height: 1.734, name: "bardo", colors: [red, blue]) let jnode = %b let data = jnode.to(Bird) @@ -497,7 +497,7 @@ proc testJson() = doAssert array[3, float](t.arr) == [1.0,2.0,7.0] doAssert MyRef(t.person).name == "boney" - doAssert MyObj(t.distFruit).color == 11 + doAssert MyObj(t.distfruit).color == 11 doAssert t.dog.name == "honey" doAssert t.fruit.color == 10 doAssert seq[string](t.emails) == @["abc", "123"] diff --git a/tests/stdlib/tjsonutils.nim b/tests/stdlib/tjsonutils.nim index 205160471..d5809ee73 100644 --- a/tests/stdlib/tjsonutils.nim +++ b/tests/stdlib/tjsonutils.nim @@ -320,7 +320,7 @@ template fn() = b: int var a = A() - fromJson(a, """{"is_a": true, "a":1, "extra_key": 1}""".parse_json, Joptions(allowExtraKeys: true)) + fromJson(a, """{"is_a": true, "a":1, "extra_key": 1}""".parseJson, Joptions(allowExtraKeys: true)) doAssert $a[] == "(is_a: true, a: 1)" block testAllowMissingKeys: diff --git a/tests/stdlib/tnet.nim b/tests/stdlib/tnet.nim index 671e0f7bc..4ec62d88f 100644 --- a/tests/stdlib/tnet.nim +++ b/tests/stdlib/tnet.nim @@ -93,7 +93,7 @@ block: # "IpAddress/Sockaddr conversion" doAssert($ipaddrstr == $ipaddr_1) var sockaddr: Sockaddr_storage - var socklen: Socklen + var socklen: SockLen var ipaddr_2: IpAddress var port_2: Port diff --git a/tests/stdlib/tnetdial.nim b/tests/stdlib/tnetdial.nim index 3de00ff99..b836fb78d 100644 --- a/tests/stdlib/tnetdial.nim +++ b/tests/stdlib/tnetdial.nim @@ -14,7 +14,7 @@ proc initIPv6Server(hostname: string, port: Port): AsyncFD = let fd = createNativeSocket(AF_INET6) setSockOptInt(fd, SOL_SOCKET, SO_REUSEADDR, 1) var aiList = getAddrInfo(hostname, port, AF_INET6) - if bindAddr(fd, aiList.ai_addr, aiList.ai_addrlen.Socklen) < 0'i32: + if bindAddr(fd, aiList.ai_addr, aiList.ai_addrlen.SockLen) < 0'i32: freeAddrInfo(aiList) raiseOSError(osLastError()) freeAddrInfo(aiList) diff --git a/tests/stdlib/tosproc.nim b/tests/stdlib/tosproc.nim index d767aac80..c54e0d112 100644 --- a/tests/stdlib/tosproc.nim +++ b/tests/stdlib/tosproc.nim @@ -12,7 +12,7 @@ see also: tests/osproc/*.nim; consider merging those into a single test here when defined(case_testfile): # compiled test file for child process from posix import exitnow - proc c_exit2(code: c_int): void {.importc: "_exit", header: "".} + proc c_exit2(code: cint): void {.importc: "_exit", header: "".} import os var a = 0 proc fun(b = 0) = diff --git a/tests/stdlib/tparsesql.nim b/tests/stdlib/tparsesql.nim index b440dc90f..8ef67f5dc 100644 --- a/tests/stdlib/tparsesql.nim +++ b/tests/stdlib/tparsesql.nim @@ -14,8 +14,8 @@ nkStmtList nkIntegerLit 10 nkNumericLit 5.5""" -doAssert $parseSQL("SELECT foo FROM table;") == "select foo from table;" -doAssert $parseSQL(""" +doAssert $parseSql("SELECT foo FROM table;") == "select foo from table;" +doAssert $parseSql(""" SELECT CustomerName, ContactName, @@ -31,26 +31,26 @@ SELECT Country FROM table;""") == "select CustomerName, ContactName, Address, City, PostalCode, Country, CustomerName, ContactName, Address, City, PostalCode, Country from table;" -doAssert $parseSQL("SELECT foo FROM table limit 10") == "select foo from table limit 10;" -doAssert $parseSQL("SELECT foo, bar, baz FROM table limit 10") == "select foo, bar, baz from table limit 10;" -doAssert $parseSQL("SELECT foo AS bar FROM table") == "select foo as bar from table;" -doAssert $parseSQL("SELECT foo AS foo_prime, bar AS bar_prime, baz AS baz_prime FROM table") == "select foo as foo_prime, bar as bar_prime, baz as baz_prime from table;" -doAssert $parseSQL("SELECT * FROM table") == "select * from table;" -doAssert $parseSQL("SELECT count(*) FROM table") == "select count(*) from table;" -doAssert $parseSQL("SELECT count(*) as 'Total' FROM table") == "select count(*) as 'Total' from table;" -doAssert $parseSQL("SELECT count(*) as 'Total', sum(a) as 'Aggr' FROM table") == "select count(*) as 'Total', sum(a) as 'Aggr' from table;" +doAssert $parseSql("SELECT foo FROM table limit 10") == "select foo from table limit 10;" +doAssert $parseSql("SELECT foo, bar, baz FROM table limit 10") == "select foo, bar, baz from table limit 10;" +doAssert $parseSql("SELECT foo AS bar FROM table") == "select foo as bar from table;" +doAssert $parseSql("SELECT foo AS foo_prime, bar AS bar_prime, baz AS baz_prime FROM table") == "select foo as foo_prime, bar as bar_prime, baz as baz_prime from table;" +doAssert $parseSql("SELECT * FROM table") == "select * from table;" +doAssert $parseSql("SELECT count(*) FROM table") == "select count(*) from table;" +doAssert $parseSql("SELECT count(*) as 'Total' FROM table") == "select count(*) as 'Total' from table;" +doAssert $parseSql("SELECT count(*) as 'Total', sum(a) as 'Aggr' FROM table") == "select count(*) as 'Total', sum(a) as 'Aggr' from table;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT * FROM table WHERE a = b and c = d """) == "select * from table where a = b and c = d;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT * FROM table WHERE not b """) == "select * from table where not b;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT * FROM @@ -59,63 +59,63 @@ WHERE a and not b """) == "select * from table where a and not b;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT * FROM table ORDER BY 1 """) == "select * from table order by 1;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT * FROM table GROUP BY 1 ORDER BY 1 """) == "select * from table group by 1 order by 1;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT * FROM table ORDER BY 1 LIMIT 100 """) == "select * from table order by 1 limit 100;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT * FROM table WHERE a = b and c = d or n is null and not b + 1 = 3 """) == "select * from table where a = b and c = d or n is null and not b + 1 = 3;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT * FROM table WHERE (a = b and c = d) or (n is null and not b + 1 = 3) """) == "select * from table where(a = b and c = d) or (n is null and not b + 1 = 3);" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT * FROM table HAVING a = b and c = d """) == "select * from table having a = b and c = d;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT a, b FROM table GROUP BY a """) == "select a, b from table group by a;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT a, b FROM table GROUP BY 1, 2 """) == "select a, b from table group by 1, 2;" -doAssert $parseSQL("SELECT t.a FROM t as t") == "select t.a from t as t;" +doAssert $parseSql("SELECT t.a FROM t as t") == "select t.a from t as t;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT a, b FROM ( SELECT * FROM t ) """) == "select a, b from(select * from t);" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT a, b FROM ( SELECT * FROM t ) as foo """) == "select a, b from(select * from t) as foo;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT a, b FROM ( SELECT * FROM ( SELECT * FROM ( @@ -127,49 +127,49 @@ SELECT a, b FROM ( ) as inner5 """) == "select a, b from(select * from(select * from(select * from(select * from innerTable as inner1) as inner2) as inner3) as inner4) as inner5;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT a, b FROM (SELECT * FROM a), (SELECT * FROM b), (SELECT * FROM c) """) == "select a, b from(select * from a),(select * from b),(select * from c);" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT * FROM Products WHERE Price BETWEEN 10 AND 20; """) == "select * from Products where Price between 10 and 20;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT id FROM a JOIN b ON a.id == b.id """) == "select id from a join b on a.id == b.id;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT id FROM a JOIN (SELECT id from c) as b ON a.id == b.id """) == "select id from a join(select id from c) as b on a.id == b.id;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT id FROM a INNER JOIN b ON a.id == b.id """) == "select id from a inner join b on a.id == b.id;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT id FROM a OUTER JOIN b ON a.id == b.id """) == "select id from a outer join b on a.id == b.id;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT id FROM a CROSS JOIN b ON a.id == b.id """) == "select id from a cross join b on a.id == b.id;" -doAssert $parseSQL(""" +doAssert $parseSql(""" CREATE TYPE happiness AS ENUM ('happy', 'very happy', 'ecstatic'); CREATE TABLE holidays ( num_weeks int, @@ -179,16 +179,16 @@ CREATE INDEX table1_attr1 ON table1(attr1); SELECT * FROM myTab WHERE col1 = 'happy'; """) == "create type happiness as enum ('happy' , 'very happy' , 'ecstatic' ); create table holidays(num_weeks int , happiness happiness );; create index table1_attr1 on table1(attr1 );; select * from myTab where col1 = 'happy';" -doAssert $parseSQL(""" +doAssert $parseSql(""" INSERT INTO Customers (CustomerName, ContactName, Address, City, PostalCode, Country) VALUES ('Cardinal', 'Tom B. Erichsen', 'Skagen 21', 'Stavanger', '4006', 'Norway'); """) == "insert into Customers (CustomerName , ContactName , Address , City , PostalCode , Country ) values ('Cardinal' , 'Tom B. Erichsen' , 'Skagen 21' , 'Stavanger' , '4006' , 'Norway' );" -doAssert $parseSQL(""" +doAssert $parseSql(""" INSERT INTO TableName DEFAULT VALUES """) == "insert into TableName default values;" -doAssert $parseSQL(""" +doAssert $parseSql(""" UPDATE Customers SET ContactName = 'Alfred Schmidt', City= 'Frankfurt' WHERE CustomerID = 1; @@ -209,9 +209,9 @@ nkStmtList nkStringLit Frankfurt nkNone""" -doAssert $parseSQL("DELETE FROM table_name;") == "delete from table_name;" +doAssert $parseSql("DELETE FROM table_name;") == "delete from table_name;" -doAssert treeRepr(parseSQL("DELETE FROM table_name;") +doAssert treeRepr(parseSql("DELETE FROM table_name;") ) == """ nkStmtList @@ -219,14 +219,14 @@ nkStmtList nkIdent table_name nkNone""" -doAssert $parseSQL("DELETE * FROM table_name;") == "delete from table_name;" +doAssert $parseSql("DELETE * FROM table_name;") == "delete from table_name;" -doAssert $parseSQL(""" +doAssert $parseSql(""" --Select all: SELECT * FROM Customers; """) == "select * from Customers;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT * FROM Customers WHERE (CustomerName LIKE 'L%' OR CustomerName LIKE 'R%' /*OR CustomerName LIKE 'S%' OR CustomerName LIKE 'T%'*/ OR CustomerName LIKE 'W%') @@ -235,9 +235,9 @@ ORDER BY CustomerName; """) == "select * from Customers where(CustomerName like 'L%' or CustomerName like 'R%' or CustomerName like 'W%') and Country = 'USA' order by CustomerName;" # parse quoted keywords as identifires -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT `SELECT`, `FROM` as `GROUP` FROM `WHERE`; """) == """select "SELECT", "FROM" as "GROUP" from "WHERE";""" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT "SELECT", "FROM" as "GROUP" FROM "WHERE"; """) == """select "SELECT", "FROM" as "GROUP" from "WHERE";""" diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index 8b98d1403..9787c13eb 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -861,7 +861,7 @@ suite "RST include directive": test "Include whole": "other.rst".writeFile("**test1**") let input = ".. include:: other.rst" - doAssert "test1" == rstTohtml(input, {roSandboxDisabled}, defaultConfig()) + doAssert "test1" == rstToHtml(input, {roSandboxDisabled}, defaultConfig()) removeFile("other.rst") test "Include starting from": @@ -875,7 +875,7 @@ OtherStart .. include:: other.rst :start-after: OtherStart """ - check "Visible" == rstTohtml(input, {roSandboxDisabled}, defaultConfig()) + check "Visible" == rstToHtml(input, {roSandboxDisabled}, defaultConfig()) removeFile("other.rst") test "Include everything before": @@ -889,7 +889,7 @@ And this should **NOT** be visible in `docs.html` .. include:: other.rst :end-before: OtherEnd """ - doAssert "Visible" == rstTohtml(input, {roSandboxDisabled}, defaultConfig()) + doAssert "Visible" == rstToHtml(input, {roSandboxDisabled}, defaultConfig()) removeFile("other.rst") @@ -907,7 +907,7 @@ And this should **NOT** be visible in `docs.html` :start-after: OtherStart :end-before: OtherEnd """ - check "Visible" == rstTohtml(input, {roSandboxDisabled}, defaultConfig()) + check "Visible" == rstToHtml(input, {roSandboxDisabled}, defaultConfig()) removeFile("other.rst") @@ -927,7 +927,7 @@ And this should **NOT** be visible in `docs.html` :start-after: OtherStart :end-before: OtherEnd """ - doAssert "Visible" == rstTohtml(input, {roSandboxDisabled}, defaultConfig()) + doAssert "Visible" == rstToHtml(input, {roSandboxDisabled}, defaultConfig()) removeFile("other.rst") suite "RST escaping": diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index 2c28fe506..d91b5615e 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -417,7 +417,7 @@ Some chapter """ var error9Bad = new string - let output9Bad = input9bad.toHtml(error=error9Bad) + let output9Bad = input9Bad.toHtml(error=error9Bad) check(error9Bad[] == "input(15, 1) Error: new section expected (section " & "level inconsistent: underline ~~~~~ unexpectedly found, while " & "the following intermediate section level(s) are missing on " & @@ -464,7 +464,7 @@ Some chapter rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", filenames = files) rstGenera.renderRstToOut(rst, output) doAssert rstGenera.meta[metaTitle] == "Title0" - doAssert rstGenera.meta[metaSubTitle] == "SubTitle0" + doAssert rstGenera.meta[metaSubtitle] == "SubTitle0" doAssert "

              Level1

              " in output doAssert "

              Level2

              " in output doAssert "

              Level3

              " in output @@ -491,7 +491,7 @@ Some chapter rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", filenames=files) rstGenera.renderRstToOut(rst, output) doAssert rstGenera.meta[metaTitle] == "" - doAssert rstGenera.meta[metaSubTitle] == "" + doAssert rstGenera.meta[metaSubtitle] == "" doAssert "

              Title0

              " in output doAssert "

              SubTitle0

              " in output @@ -521,7 +521,7 @@ Some chapter rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", filenames=files) rstGenera.renderRstToOut(rst, output) doAssert rstGenera.meta[metaTitle] == "Title0" - doAssert rstGenera.meta[metaSubTitle] == "" + doAssert rstGenera.meta[metaSubtitle] == "" doAssert output == "\n

              MySection1a

              " & # RST "\n

              MySection1b

              " & # Markdown diff --git a/tests/stdlib/tsha1.nim b/tests/stdlib/tsha1.nim index a20183039..6dca1f197 100644 --- a/tests/stdlib/tsha1.nim +++ b/tests/stdlib/tsha1.nim @@ -15,7 +15,7 @@ checkVector("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", proc testIsValidSha1Hash = doAssert not isValidSha1Hash("") doAssert not isValidSha1Hash("042D4BE2B90ED0672E717D71850ABDB0A2D19CD11") - doAssert not isValidSha1hash("042G4BE2B90ED0672E717D71850ABDB0A2D19CD1") + doAssert not isValidSha1Hash("042G4BE2B90ED0672E717D71850ABDB0A2D19CD1") doAssert isValidSha1Hash("042D4BE2B90ED0672E717D71850ABDB0A2D19CD1") doAssert isValidSha1Hash("042d4be2b90ed0672e717d71850abdb0a2d19cd1") doAssert isValidSha1Hash("042d4be2b90ed0672e717D71850ABDB0A2D19CD1") diff --git a/tests/stdlib/tssl.nim b/tests/stdlib/tssl.nim index 7625f3694..379c1b1e5 100644 --- a/tests/stdlib/tssl.nim +++ b/tests/stdlib/tssl.nim @@ -37,8 +37,8 @@ proc notifiedShutdown(port: Port) {.thread.} = proc main() = when defined(posix): var - ignoreAction = SigAction(sa_handler: SIG_IGN) - oldSigPipeHandler: SigAction + ignoreAction = Sigaction(sa_handler: SIG_IGN) + oldSigPipeHandler: Sigaction if sigemptyset(ignoreAction.sa_mask) == -1: raiseOSError(osLastError(), "Couldn't create an empty signal set") if sigaction(SIGPIPE, ignoreAction, oldSigPipeHandler) == -1: diff --git a/tests/stdlib/tstrbasics.nim b/tests/stdlib/tstrbasics.nim index b340ad509..bf562d9ec 100644 --- a/tests/stdlib/tstrbasics.nim +++ b/tests/stdlib/tstrbasics.nim @@ -73,7 +73,7 @@ proc main() = block: # setSlice var a = "Hello, Nim!" - doassert a.dup(setSlice(7 .. 9)) == "Nim" + doAssert a.dup(setSlice(7 .. 9)) == "Nim" doAssert a.dup(setSlice(0 .. 0)) == "H" doAssert a.dup(setSlice(0 .. 1)) == "He" doAssert a.dup(setSlice(0 .. 10)) == a diff --git a/tests/stdlib/tstreams.nim b/tests/stdlib/tstreams.nim index d70d4995a..d199037e3 100644 --- a/tests/stdlib/tstreams.nim +++ b/tests/stdlib/tstreams.nim @@ -40,7 +40,7 @@ block tstreams2: block tstreams3: try: var fs = openFileStream("shouldneverexist.txt") - except IoError: + except IOError: echo "threw exception" static: diff --git a/tests/stdlib/tstrset.nim b/tests/stdlib/tstrset.nim index 1b253f862..71a5520c2 100644 --- a/tests/stdlib/tstrset.nim +++ b/tests/stdlib/tstrset.nim @@ -50,7 +50,7 @@ proc search(r: PRadixNode, s: string): PRadixNode = proc contains*(r: PRadixNode, s: string): bool = return search(r, s) != nil -proc testOrincl*(r: var PRadixNode, s: string): bool = +proc testOrIncl*(r: var PRadixNode, s: string): bool = nil proc incl*(r: var PRadixNode, s: string) = discard testOrIncl(r, s) diff --git a/tests/stdlib/ttimes.nim b/tests/stdlib/ttimes.nim index 2cf945786..fd440eb20 100644 --- a/tests/stdlib/ttimes.nim +++ b/tests/stdlib/ttimes.nim @@ -20,7 +20,7 @@ proc staticTz(hours, minutes, seconds: int = 0): Timezone {.noSideEffect.} = result.utcOffset = offset result.time = time - newTimezone("", zonedTimeFromTime, zonedTImeFromAdjTime) + newTimezone("", zonedTimeFromTime, zonedTimeFromAdjTime) template parseTest(s, f, sExpected: string, ydExpected: int) = let diff --git a/tests/stdlib/ttypeinfo.nim b/tests/stdlib/ttypeinfo.nim index 61c661a58..1cddea781 100644 --- a/tests/stdlib/ttypeinfo.nim +++ b/tests/stdlib/ttypeinfo.nim @@ -50,7 +50,7 @@ doAssert($x4[].kind() == "akString") block: # gimme a new scope dammit - var myarr: array[0..4, array[0..4, string]] = [ + var myArr: array[0..4, array[0..4, string]] = [ ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"]] diff --git a/tests/stdlib/ttypeinfo.nims b/tests/stdlib/ttypeinfo.nims new file mode 100644 index 000000000..d31d0b26f --- /dev/null +++ b/tests/stdlib/ttypeinfo.nims @@ -0,0 +1 @@ +--styleCheck:off \ No newline at end of file diff --git a/tests/stdlib/tunicode.nim b/tests/stdlib/tunicode.nim index 33405cf38..3a8206c8f 100644 --- a/tests/stdlib/tunicode.nim +++ b/tests/stdlib/tunicode.nim @@ -218,5 +218,5 @@ block: # bug #17768 let s1 = "abcdef" let s2 = "abcdéf" - doAssert s1.runeSubstr(0, -1) == "abcde" - doAssert s2.runeSubstr(0, -1) == "abcdé" + doAssert s1.runeSubStr(0, -1) == "abcde" + doAssert s2.runeSubStr(0, -1) == "abcdé" diff --git a/tests/stdlib/tvarints.nim b/tests/stdlib/tvarints.nim index 772121f9f..3bba4f457 100644 --- a/tests/stdlib/tvarints.nim +++ b/tests/stdlib/tvarints.nim @@ -48,7 +48,7 @@ block: chk 0.1 chk Inf chk NegInf - chk Nan + chk NaN chk 3.1415926535897932384626433 block: From ef7d7f24594b1ea560fbf25d3fa0d6c51122725f Mon Sep 17 00:00:00 2001 From: huantian Date: Wed, 13 Apr 2022 14:03:46 -0700 Subject: [PATCH 1591/2472] Better error message and tests for bad await (#19622) * Better error message and tests for bad await * Use compiles to check if await is valid * temp: disable windows noasync test * Better error report, simplify test Co-authored-by: flywind --- lib/pure/asyncmacro.nim | 16 +++++++++++++--- tests/async/tasync_noasync.nim | 35 +++++++++++++++++++++++++++++++--- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/lib/pure/asyncmacro.nim b/lib/pure/asyncmacro.nim index ce538913f..63c8e6e5c 100644 --- a/lib/pure/asyncmacro.nim +++ b/lib/pure/asyncmacro.nim @@ -126,9 +126,19 @@ template await*(f: typed): untyped {.used.} = error "await expects Future[T], got " & $typeof(f) template await*[T](f: Future[T]): auto {.used.} = - var internalTmpFuture: FutureBase = f - yield internalTmpFuture - (cast[typeof(f)](internalTmpFuture)).read() + template yieldFuture = yield FutureBase() + + when compiles(yieldFuture): + var internalTmpFuture: FutureBase = f + yield internalTmpFuture + (cast[typeof(f)](internalTmpFuture)).read() + else: + macro errorAsync(futureError: Future[T]) = + error( + "Can only 'await' inside a proc marked as 'async'. Use " & + "'waitFor' when calling an 'async' proc in a non-async scope instead", + futureError) + errorAsync(f) proc asyncSingleProc(prc: NimNode): NimNode = ## This macro transforms a single procedure into a closure iterator. diff --git a/tests/async/tasync_noasync.nim b/tests/async/tasync_noasync.nim index 54f4f597f..0927148bf 100644 --- a/tests/async/tasync_noasync.nim +++ b/tests/async/tasync_noasync.nim @@ -1,13 +1,42 @@ discard """ - errormsg: "'yield' only allowed in an iterator" - cmd: "nim c $file" - file: "asyncmacro.nim" + cmd: "nim check --hints:off --warnings:off $file" + action: "reject" + nimout: ''' +tasync_noasync.nim(21, 10) Error: Can only 'await' inside a proc marked as 'async'. Use 'waitFor' when calling an 'async' proc in a non-async scope instead +tasync_noasync.nim(25, 12) Error: Can only 'await' inside a proc marked as 'async'. Use 'waitFor' when calling an 'async' proc in a non-async scope instead +tasync_noasync.nim(28, 10) Error: Can only 'await' inside a proc marked as 'async'. Use 'waitFor' when calling an 'async' proc in a non-async scope instead +tasync_noasync.nim(31, 10) Error: Can only 'await' inside a proc marked as 'async'. Use 'waitFor' when calling an 'async' proc in a non-async scope instead +tasync_noasync.nim(35, 10) Error: Can only 'await' inside a proc marked as 'async'. Use 'waitFor' when calling an 'async' proc in a non-async scope instead +tasync_noasync.nim(38, 10) Error: Can only 'await' inside a proc marked as 'async'. Use 'waitFor' when calling an 'async' proc in a non-async scope instead +tasync_noasync.nim(40, 8) Error: Can only 'await' inside a proc marked as 'async'. Use 'waitFor' when calling an 'async' proc in a non-async scope instead +''' """ import async proc a {.async.} = discard +# Bad await usage +proc nonAsyncProc = + await a() + +proc nestedNonAsyncProc {.async.} = + proc nested = + await a() + +iterator customIterator: int = + await a() + +macro awaitInMacro = + await a() + +type DummyRef = ref object of RootObj +method awaitInMethod(_: DummyRef) {.base.} = + await a() + +proc improperMultisync {.multisync.} = + await a() + await a() # if we overload a fallback handler to get From dc4cc2dca53e3772efb3654a4ddbbe8350d1db43 Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Thu, 14 Apr 2022 03:30:59 -0600 Subject: [PATCH 1592/2472] Fix string stream crashing when created on nimscript due to last fix (#19717) --- lib/pure/streams.nim | 2 +- tests/stdlib/tstreams.nim | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim index 84451f989..2ac6a82f1 100644 --- a/lib/pure/streams.nim +++ b/lib/pure/streams.nim @@ -1274,7 +1274,7 @@ else: # after 1.3 or JS not defined new(result) result.data = s - when defined(gcOrc) or defined(gcArc): + when declared(prepareMutation): prepareMutation(result.data) # Allows us to mutate using `addr` logic like `copyMem`, otherwise it errors. result.pos = 0 result.closeImpl = ssClose diff --git a/tests/stdlib/tstreams.nim b/tests/stdlib/tstreams.nim index d199037e3..6fda30f51 100644 --- a/tests/stdlib/tstreams.nim +++ b/tests/stdlib/tstreams.nim @@ -76,3 +76,7 @@ block: doAssert(ss.getPosition == 5) # haven't moved ss.setPosition(0) # Ensure we dont error with writing over literals on arc/orc #19707 ss.write("hello") + +static: # Ensure streams it doesnt break with nimscript on arc/orc #19716 + let s = newStringStream("a") + discard s.data From bb839029d5065bcd8367661802bf04e1e7eebb58 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Thu, 21 Apr 2022 22:06:09 +0300 Subject: [PATCH 1593/2472] Move common Latex code into class nimdoc.cls (#19734) --- compiler/docgen.nim | 6 +- compiler/nimpaths.nim | 2 + config/nimdoc.tex.cfg | 188 +---------------------------------------- doc/nimdoc.cls | 190 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 198 insertions(+), 188 deletions(-) create mode 100644 doc/nimdoc.cls diff --git a/compiler/docgen.nim b/compiler/docgen.nim index aad3b9409..ecf98d0b6 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -1576,7 +1576,11 @@ proc writeOutput*(d: PDoc, useWarning = false, groupedToc = false) = outfile.string) if not d.wroteSupportFiles: # nimdoc.css + dochack.js let nimr = $d.conf.getPrefixDir() - copyFile(docCss.interp(nimr = nimr), $d.conf.outDir / nimdocOutCss) + case d.target + of outHtml: + copyFile(docCss.interp(nimr = nimr), $d.conf.outDir / nimdocOutCss) + of outLatex: + copyFile(docCls.interp(nimr = nimr), $d.conf.outDir / nimdocOutCls) if optGenIndex in d.conf.globalOptions: let docHackJs2 = getDocHacksJs(nimr, nim = getAppFilename()) copyFile(docHackJs2, $d.conf.outDir / docHackJs2.lastPathPart) diff --git a/compiler/nimpaths.nim b/compiler/nimpaths.nim index 216071c5c..c6e188289 100644 --- a/compiler/nimpaths.nim +++ b/compiler/nimpaths.nim @@ -25,11 +25,13 @@ when defined(nimPreviewSlimSystem): const docCss* = "$nimr/doc/nimdoc.css" + docCls* = "$nimr/doc/nimdoc.cls" docHackNim* = "$nimr/tools/dochack/dochack.nim" docHackJs* = docHackNim.changeFileExt("js") docHackJsFname* = docHackJs.lastPathPart theindexFname* = "theindex.html" nimdocOutCss* = "nimdoc.out.css" + nimdocOutCls* = "nimdoc.cls" # `out` to make it easier to use with gitignore in user's repos htmldocsDirname* = "htmldocs" dotdotMangle* = "_._" ## refs #13223 diff --git a/config/nimdoc.tex.cfg b/config/nimdoc.tex.cfg index 3912d1279..4fb1aec90 100644 --- a/config/nimdoc.tex.cfg +++ b/config/nimdoc.tex.cfg @@ -69,198 +69,12 @@ doc.file = """ % makeindex file % xelatex file.tex % -\documentclass[a4paper,11pt]{article} -\usepackage[a4paper,xetex,left=3cm,right=3cm,top=1.5cm,bottom=2cm]{geometry} - -% for 2-sided printing with larger inner "binding" margin -%\usepackage[a4paper,xetex,twoside,left=4cm,right=2cm,top=1.5cm,bottom=2cm]{geometry} -% for e-readers with 1.77:1 aspect ratio (e.g. 1920x1080) -%\usepackage[xetex,paperheight=27.6cm,paperwidth=15.5cm,left=3mm,right=3mm,top=3mm,bottom=3mm]{geometry} -% for e-readers with 1.45:1 aspect ratio (e.g. 1200x825) -%\usepackage[xetex,paperheight=22.5cm,paperwidth=15.5cm,left=3mm,right=3mm,top=3mm,bottom=3mm]{geometry} -% for e-readers with 1.33:1 aspect ratio (e.g. 1872x1404) -%\usepackage[xetex,paperheight=20.7cm,paperwidth=15.5cm,left=3mm,right=3mm,top=3mm,bottom=3mm]{geometry} - -\usepackage{fontspec} -% logic to select default font with some fall-back fonts. -\IfFontExistsTF{Times New Roman}{% - \setmainfont{Times New Roman} % the default font - \typeout{========================================= nim: using Times New Roman} -}{ - \IfFontExistsTF{FreeSerif}{% - \setmainfont{FreeSerif} % fallback #1 - official GNU font, resembles Times - \typeout{========================================= nim: using FreeSerif} - }{ - \IfFontExistsTF{DejaVuSerif}{% - \setmainfont{DejaVuSerif} % fallback #2 - very widespread free font - \typeout{========================================= nim: using DejaVuSerif} - }{ - \typeout{!!!!!!!!!!!!!!!!!!! Fonts not found !!!!!!!!!!!!!!!!!!!!!!!} - } - } -} - -% default monospace font for code: -\usepackage{GoMono} -\usepackage{relsize} -% make this monospace font 2 steps smaller to hold 80-character line -\newcommand{\rstverbblockfont}{\smaller[2]} -\newcommand{\rstverbinlinefont}{\smaller} - -\usepackage{parskip} % paragraphs delimited by vertical space, no indent -\usepackage{graphicx} - -\usepackage{makeidx} -\newcommand{\nimindexterm}[2]{#2\index{#2}\label{#1}} -\makeindex - -\usepackage{dingbat} % for \carriagereturn, etc -\usepackage{fvextra} % for code blocks (works better than original fancyvrb) -\fvset{ - breaklines, - breakafter={=}:|\_\{\}[](){,}.;+-*/'", - breaksymbolleft=\color{red}{\ensuremath{\hookrightarrow}}, - breaksymbolright=\color{red}{\small\carriagereturn} -} -\fvinlineset{% - breaklines, - breakafter={=}:|\_\{\}[](){,}.;+-*/'", - % that does not work at all when we underline inline code by ulem :-( - commandchars=\\\{\} -} - -\usepackage{scrextend} % for the `addmargin` environment - -\usepackage{xcolor} -\usepackage[urlbordercolor=blue,linkbordercolor=cyan, - pdfborderstyle={/S/U/W 1}]{hyperref} -\usepackage{enumitem} % for option list, enumList, and rstfootnote - -\usepackage[most]{tcolorbox} % boxes around admonitions, code blocks, doc.item - -\newtcolorbox{rstadmonition}[1][]{blanker, breakable, - left=3mm, right=0mm, top=1mm, bottom=1mm, - before upper=\indent, parbox=false, #1} - -\newtcolorbox{rstquote}[1][]{blanker, breakable, - left=3mm, right=3mm, top=1mm, bottom=1mm, - parbox=false, - borderline west={0.3em}{0pt}{lightgray}, - borderline north={0.05em}{0pt}{lightgray}, - borderline east={0.05em}{0pt}{lightgray}, - borderline south={0.05em}{0pt}{lightgray}} - -\definecolor{rstframecolor}{rgb}{0.85, 0.8, 0.6} - -\newtcolorbox{rstprebox}[1][]{blanker, breakable, - left=3mm, right=3mm, top=1mm, bottom=1mm, - borderline ={0.1em}{0pt}{rstframecolor}, - before upper=\indent, parbox=false, #1} - -\newenvironment{rstpre}{% -\VerbatimEnvironment\begingroup\begin{rstprebox}% -\begin{Verbatim}[fontsize=\rstverbblockfont , commandchars=\\\{\}]}% -{\end{Verbatim}\end{rstprebox}\endgroup} - -\newtcolorbox{rstdocitembox}[1][]{blanker, breakable, - left=3mm, right=3mm, top=1mm, bottom=1mm, - borderline ={1pt}{0pt}{cyan}, - before upper=\indent, parbox=false, #1} - -% Inline code formatting: grey underline, -% use \Verb from fvextras e.g. to display -- correctly as double - -\usepackage[normalem]{ulem} -\newcommand\rstuline{\bgroup\markoverwith{\textcolor{rstframecolor}{\rule[-0.8ex]{2pt}{1.0pt}}}\ULon} - -\newcommand{\rstcode}[1]{% -{\rstverbinlinefont\Verb{\rstuline{#1}}}% -} - -\newcommand{\rstcodeitem}[1]{\Verb{#1}} - -\newenvironment{rstdocitem}{% -\VerbatimEnvironment\begingroup\begin{rstdocitembox}% -\begin{Verbatim}[fontsize=\rstverbblockfont , commandchars=\\\{\}]}% -{\end{Verbatim}\end{rstdocitembox}\endgroup} - - -\newenvironment{rstfootnote}{\begin{description}[labelindent=1em,leftmargin=1em,labelwidth=2.6em]}{\end{description}} -\ifdim\linewidth<30em - \def\rstoptleftmargin{0.4\linewidth} - \def\rstoptlabelwidth{0.35\linewidth} -\else - \def\rstoptleftmargin{12em} - \def\rstoptlabelwidth{10.5em} -\fi -\newenvironment{rstoptlist}{% -\begin{description}[font=\sffamily\bfseries,style=nextline,leftmargin=\rstoptleftmargin,labelwidth=\rstoptlabelwidth]}{\end{description}} - -\usepackage{tabulary} % tables with adjustable cell width and no overflow -% make tabulary prevent overflows (https://tex.stackexchange.com/a/195088) -\tymin=60pt -\tymax=\maxdimen -% to pack tabulary into a new environment, special syntax is needed :-( -\newenvironment{rsttab}[1]{\tabulary{\linewidth}{#1}}{\endtabulary} - -\newcommand{\rstsub}[1]{\raisebox{-0.5ex}{\scriptsize{#1}}} -\newcommand{\rstsup}[1]{\raisebox{0.5ex}{\scriptsize{#1}}} - -\newcommand{\rsthA}[2][]{\section[#1]{#2}} -\newcommand{\rsthB}[2][]{\subsection[#1]{#2}} -\newcommand{\rsthC}[2][]{\subsubsection[#1]{#2}} -\newcommand{\rsthD}[2][]{\paragraph[#1]{#2}} -\newcommand{\rsthE}[2][]{\paragraph[#1]{#2}} - -\newcommand{\rstovA}[2][]{\section*[#1]{#2}} -\newcommand{\rstovB}[2][]{\subsection*[#1]{#2}} -\newcommand{\rstovC}[2][]{\subsubsection*[#1]{#2}} -\newcommand{\rstovD}[2][]{\paragraph*[#1]{#2}} -\newcommand{\rstovE}[2][]{\paragraph*[#1]{#2}} - -% Syntax highlighting: -\newcommand{\spanDecNumber}[1]{\textbf{\textcolor{darkgray}{#1}}} -\newcommand{\spanBinNumber}[1]{\textbf{\textcolor{darkgray}{#1}}} -\newcommand{\spanHexNumber}[1]{\textbf{\textcolor{darkgray}{#1}}} -\newcommand{\spanOctNumber}[1]{\textbf{\textcolor{darkgray}{#1}}} -\newcommand{\spanFloatNumber}[1]{\textbf{\textcolor{darkgray}{#1}}} -\newcommand{\spanIdentifier}[1]{#1} -\newcommand{\spanKeyword}[1]{\textbf{#1}} -\newcommand{\spanStringLit}[1]{\textbf{\textcolor{darkgray}{#1}}} -\newcommand{\spanLongStringLit}[1]{\textbf{\textcolor{darkgray}{#1}}} -\newcommand{\spanCharLit}[1]{#1} -\newcommand{\spanEscapeSequence}[1]{#1} -\newcommand{\spanOperator}[1]{\textbf{#1}} -\newcommand{\spanPunctuation}[1]{#1} -\newcommand{\spanComment}[1]{\emph{#1}} -\newcommand{\spanLongComment}[1]{\emph{#1}} -\newcommand{\spanRegularExpression}[1]{#1} -\newcommand{\spanTagStart}[1]{#1} -\newcommand{\spanTagEnd}[1]{#1} -\newcommand{\spanKey}[1]{#1} -\newcommand{\spanValue}[1]{#1} -\newcommand{\spanRawData}[1]{\textbf{\textcolor{darkgray}{#1}}} -\newcommand{\spanAssembler}[1]{#1} -\newcommand{\spanPreprocessor}[1]{#1} -\newcommand{\spanDirective}[1]{#1} -\newcommand{\spanCommand}[1]{#1} -\newcommand{\spanRule}[1]{#1} -\newcommand{\spanHyperlink}[1]{#1} -\newcommand{\spanLabel}[1]{#1} -\newcommand{\spanReference}[1]{#1} -\newcommand{\spanOther}[1]{#1} -\newcommand{\spantok}[1]{\fbox{#1}} -\newcommand{\spanPrompt}[1]{\textcolor{red}{\textbf{#1}}} -\newcommand{\spanProgramOutput}[1]{\textcolor{darkgray}{\textbf{#1}}} -\newcommand{\spanprogram}[1]{\textbf{\underline{#1}}} -\newcommand{\spanoption}[1]{\textbf{\textcolor{darkgray}{#1}}} +\documentclass{nimdoc} \begin{document} \title{$title $version $subtitle} \author{$author} -% Never allow text overflow to margin: -\setlength\emergencystretch{\hsize}\hbadness=10000 - \maketitle $content diff --git a/doc/nimdoc.cls b/doc/nimdoc.cls new file mode 100644 index 000000000..271dc5dc9 --- /dev/null +++ b/doc/nimdoc.cls @@ -0,0 +1,190 @@ +\ProvidesClass{nimdoc}[2022/04/17, 2018/01/01 LaTeX2e nonstandard class] + +\LoadClass[a4paper,11pt]{article} + +\usepackage[a4paper,xetex,left=3cm,right=3cm,top=1.5cm,bottom=2cm]{geometry} + +% for 2-sided printing with larger inner "binding" margin +%\usepackage[a4paper,xetex,twoside,left=4cm,right=2cm,top=1.5cm,bottom=2cm]{geometry} +% for e-readers with 1.77:1 aspect ratio (e.g. 1920x1080) +%\usepackage[xetex,paperheight=27.6cm,paperwidth=15.5cm,left=3mm,right=3mm,top=3mm,bottom=3mm]{geometry} +% for e-readers with 1.45:1 aspect ratio (e.g. 1200x825) +%\usepackage[xetex,paperheight=22.5cm,paperwidth=15.5cm,left=3mm,right=3mm,top=3mm,bottom=3mm]{geometry} +% for e-readers with 1.33:1 aspect ratio (e.g. 1872x1404) +%\usepackage[xetex,paperheight=20.7cm,paperwidth=15.5cm,left=3mm,right=3mm,top=3mm,bottom=3mm]{geometry} + +\usepackage{fontspec} +% logic to select default font with some fall-back fonts. +\IfFontExistsTF{Times New Roman}{% + \setmainfont{Times New Roman} % the default font + \typeout{========================================= nim: using Times New Roman} +}{ + \IfFontExistsTF{FreeSerif}{% + \setmainfont{FreeSerif} % fallback #1 - official GNU font, resembles Times + \typeout{========================================= nim: using FreeSerif} + }{ + \IfFontExistsTF{DejaVuSerif}{% + \setmainfont{DejaVuSerif} % fallback #2 - very widespread free font + \typeout{========================================= nim: using DejaVuSerif} + }{ + \typeout{!!!!!!!!!!!!!!!!!!! Fonts not found !!!!!!!!!!!!!!!!!!!!!!!} + } + } +} + +% default monospace font for code: +\usepackage{GoMono} +\usepackage{relsize} +% make this monospace font 2 steps smaller to hold 80-character line +\newcommand{\rstverbblockfont}{\smaller[2]} +\newcommand{\rstverbinlinefont}{\smaller} + +\usepackage{parskip} % paragraphs delimited by vertical space, no indent +\usepackage{graphicx} + +\usepackage{makeidx} +\newcommand{\nimindexterm}[2]{#2\index{#2}\label{#1}} +\makeindex + +\usepackage{dingbat} % for \carriagereturn, etc +\usepackage{fvextra} % for code blocks (works better than original fancyvrb) +\fvset{ + breaklines, + breakafter={=}:|\_\{\}[](){,}.;+-*/'", + breaksymbolleft=\color{red}{\ensuremath{\hookrightarrow}}, + breaksymbolright=\color{red}{\small\carriagereturn} +} +\fvinlineset{% + breaklines, + breakafter={=}:|\_\{\}[](){,}.;+-*/'", + % that does not work at all when we underline inline code by ulem :-( + commandchars=\\\{\} +} + +\usepackage{scrextend} % for the `addmargin` environment + +\usepackage{xcolor} +\usepackage[urlbordercolor=blue,linkbordercolor=cyan, + pdfborderstyle={/S/U/W 1}]{hyperref} +\usepackage{enumitem} % for option list, enumList, and rstfootnote + +\usepackage[most]{tcolorbox} % boxes around admonitions, code blocks, doc.item + +\newtcolorbox{rstadmonition}[1][]{blanker, breakable, + left=3mm, right=0mm, top=1mm, bottom=1mm, + before upper=\indent, parbox=false, #1} + +\newtcolorbox{rstquote}[1][]{blanker, breakable, + left=3mm, right=3mm, top=1mm, bottom=1mm, + parbox=false, + borderline west={0.3em}{0pt}{lightgray}, + borderline north={0.05em}{0pt}{lightgray}, + borderline east={0.05em}{0pt}{lightgray}, + borderline south={0.05em}{0pt}{lightgray}} + +\definecolor{rstframecolor}{rgb}{0.85, 0.8, 0.6} + +\newtcolorbox{rstprebox}[1][]{blanker, breakable, + left=3mm, right=3mm, top=1mm, bottom=1mm, + borderline ={0.1em}{0pt}{rstframecolor}, + before upper=\indent, parbox=false, #1} + +\newenvironment{rstpre}{% +\VerbatimEnvironment\begingroup\begin{rstprebox}% +\begin{Verbatim}[fontsize=\rstverbblockfont , commandchars=\\\{\}]}% +{\end{Verbatim}\end{rstprebox}\endgroup} + +\newtcolorbox{rstdocitembox}[1][]{blanker, breakable, + left=3mm, right=3mm, top=1mm, bottom=1mm, + borderline ={1pt}{0pt}{cyan}, + before upper=\indent, parbox=false, #1} + +% Inline code formatting: grey underline, +% use \Verb from fvextras e.g. to display -- correctly as double - +\usepackage[normalem]{ulem} +\newcommand\rstuline{\bgroup\markoverwith{\textcolor{rstframecolor}{\rule[-0.8ex]{2pt}{1.0pt}}}\ULon} + +\newcommand{\rstcode}[1]{% +{\rstverbinlinefont\Verb{\rstuline{#1}}}% +} + +\newcommand{\rstcodeitem}[1]{\Verb{#1}} + +\newenvironment{rstdocitem}{% +\VerbatimEnvironment\begingroup\begin{rstdocitembox}% +\begin{Verbatim}[fontsize=\rstverbblockfont , commandchars=\\\{\}]}% +{\end{Verbatim}\end{rstdocitembox}\endgroup} + + +\newenvironment{rstfootnote}{\begin{description}[labelindent=1em,leftmargin=1em,labelwidth=2.6em]}{\end{description}} +\ifdim\linewidth<30em + \def\rstoptleftmargin{0.4\linewidth} + \def\rstoptlabelwidth{0.35\linewidth} +\else + \def\rstoptleftmargin{12em} + \def\rstoptlabelwidth{10.5em} +\fi +\newenvironment{rstoptlist}{% +\begin{description}[font=\sffamily\bfseries,style=nextline,leftmargin=\rstoptleftmargin,labelwidth=\rstoptlabelwidth]}{\end{description}} + +\usepackage{tabulary} % tables with adjustable cell width and no overflow +% make tabulary prevent overflows (https://tex.stackexchange.com/a/195088) +\tymin=60pt +\tymax=\maxdimen +% to pack tabulary into a new environment, special syntax is needed :-( +\newenvironment{rsttab}[1]{\tabulary{\linewidth}{#1}}{\endtabulary} + +\newcommand{\rstsub}[1]{\raisebox{-0.5ex}{\scriptsize{#1}}} +\newcommand{\rstsup}[1]{\raisebox{0.5ex}{\scriptsize{#1}}} + +\newcommand{\rsthA}[2][]{\section[#1]{#2}} +\newcommand{\rsthB}[2][]{\subsection[#1]{#2}} +\newcommand{\rsthC}[2][]{\subsubsection[#1]{#2}} +\newcommand{\rsthD}[2][]{\paragraph[#1]{#2}} +\newcommand{\rsthE}[2][]{\paragraph[#1]{#2}} + +\newcommand{\rstovA}[2][]{\section*[#1]{#2}} +\newcommand{\rstovB}[2][]{\subsection*[#1]{#2}} +\newcommand{\rstovC}[2][]{\subsubsection*[#1]{#2}} +\newcommand{\rstovD}[2][]{\paragraph*[#1]{#2}} +\newcommand{\rstovE}[2][]{\paragraph*[#1]{#2}} + +% Syntax highlighting: +\newcommand{\spanDecNumber}[1]{\textbf{\textcolor{darkgray}{#1}}} +\newcommand{\spanBinNumber}[1]{\textbf{\textcolor{darkgray}{#1}}} +\newcommand{\spanHexNumber}[1]{\textbf{\textcolor{darkgray}{#1}}} +\newcommand{\spanOctNumber}[1]{\textbf{\textcolor{darkgray}{#1}}} +\newcommand{\spanFloatNumber}[1]{\textbf{\textcolor{darkgray}{#1}}} +\newcommand{\spanIdentifier}[1]{#1} +\newcommand{\spanKeyword}[1]{\textbf{#1}} +\newcommand{\spanStringLit}[1]{\textbf{\textcolor{darkgray}{#1}}} +\newcommand{\spanLongStringLit}[1]{\textbf{\textcolor{darkgray}{#1}}} +\newcommand{\spanCharLit}[1]{#1} +\newcommand{\spanEscapeSequence}[1]{#1} +\newcommand{\spanOperator}[1]{\textbf{#1}} +\newcommand{\spanPunctuation}[1]{#1} +\newcommand{\spanComment}[1]{\emph{#1}} +\newcommand{\spanLongComment}[1]{\emph{#1}} +\newcommand{\spanRegularExpression}[1]{#1} +\newcommand{\spanTagStart}[1]{#1} +\newcommand{\spanTagEnd}[1]{#1} +\newcommand{\spanKey}[1]{#1} +\newcommand{\spanValue}[1]{#1} +\newcommand{\spanRawData}[1]{\textbf{\textcolor{darkgray}{#1}}} +\newcommand{\spanAssembler}[1]{#1} +\newcommand{\spanPreprocessor}[1]{#1} +\newcommand{\spanDirective}[1]{#1} +\newcommand{\spanCommand}[1]{#1} +\newcommand{\spanRule}[1]{#1} +\newcommand{\spanHyperlink}[1]{#1} +\newcommand{\spanLabel}[1]{#1} +\newcommand{\spanReference}[1]{#1} +\newcommand{\spanOther}[1]{#1} +\newcommand{\spantok}[1]{\fbox{#1}} +\newcommand{\spanPrompt}[1]{\textcolor{red}{\textbf{#1}}} +\newcommand{\spanProgramOutput}[1]{\textcolor{darkgray}{\textbf{#1}}} +\newcommand{\spanprogram}[1]{\textbf{\underline{#1}}} +\newcommand{\spanoption}[1]{\textbf{\textcolor{darkgray}{#1}}} + +% Never allow text overflow to margin: +\setlength\emergencystretch{\hsize}\hbadness=10000 From 02e8aa9660ae78a4d6a67656709f9379f30c5197 Mon Sep 17 00:00:00 2001 From: huantian Date: Sun, 24 Apr 2022 02:47:39 -0700 Subject: [PATCH 1594/2472] Fix doc: list of async backends (#19741) --- lib/pure/asyncdispatch.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index b67b0c308..b1e1c19fd 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -187,9 +187,9 @@ ## ## Known `async` backends include: ## -## * `none` - ``-d:asyncBackend=none`` - disable ``async`` support completely -## * `asyncdispatch -``-d:asyncBackend=asyncdispatch`` -## * `chronos ` - ``-d:asyncBackend=chronos`` +## * `-d:asyncBackend=none`: disable `async` support completely +## * `-d:asyncBackend=asyncdispatch`: https://nim-lang.org/docs/asyncdispatch.html +## * `-d:asyncBackend=chronos`: https://github.com/status-im/nim-chronos/ ## ## ``none`` can be used when a library supports both a synchronous and ## asynchronous API, to disable the latter. From b10f0e7bca43761316f6424786a771af33254e19 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 25 Apr 2022 02:20:18 +0800 Subject: [PATCH 1595/2472] fix db_sqlite.tryInsertID does raise exceptions in 1.6.0 #19743 (#19744) --- lib/impure/db_sqlite.nim | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/lib/impure/db_sqlite.nim b/lib/impure/db_sqlite.nim index 1638d38c6..db40dddd8 100644 --- a/lib/impure/db_sqlite.nim +++ b/lib/impure/db_sqlite.nim @@ -635,7 +635,7 @@ proc getValue*(db: DbConn, stmtName: SqlPrepared): string proc tryInsertID*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): int64 - {.tags: [WriteDbEffect], raises: [DbError].} = + {.tags: [WriteDbEffect], raises: [].} = ## Executes the query (typically "INSERT") and returns the ## generated ID for the row or -1 in case of an error. ## @@ -650,16 +650,19 @@ proc tryInsertID*(db: DbConn, query: SqlQuery, ## 1, "item#1") == -1 ## db.close() assert(not db.isNil, "Database not connected.") - var q = dbFormat(query, args) - var stmt: sqlite3.PStmt result = -1 - if prepare_v2(db, q, q.len.cint, stmt, nil) == SQLITE_OK: - if step(stmt) == SQLITE_DONE: - result = last_insert_rowid(db) - if finalize(stmt) != SQLITE_OK: - result = -1 - else: - discard finalize(stmt) + try: + var q = dbFormat(query, args) + var stmt: sqlite3.PStmt + if prepare_v2(db, q, q.len.cint, stmt, nil) == SQLITE_OK: + if step(stmt) == SQLITE_DONE: + result = last_insert_rowid(db) + if finalize(stmt) != SQLITE_OK: + result = -1 + else: + discard finalize(stmt) + except DbError: + discard proc insertID*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): int64 {.tags: [WriteDbEffect].} = @@ -692,7 +695,7 @@ proc insertID*(db: DbConn, query: SqlQuery, proc tryInsert*(db: DbConn, query: SqlQuery, pkName: string, args: varargs[string, `$`]): int64 - {.tags: [WriteDbEffect], raises: [DbError], since: (1, 3).} = + {.tags: [WriteDbEffect], raises: [], since: (1, 3).} = ## same as tryInsertID tryInsertID(db, query, args) From 8e6136dd59c737dab7c244e69f52864bd3fa65c9 Mon Sep 17 00:00:00 2001 From: tersec Date: Mon, 25 Apr 2022 09:06:53 +0000 Subject: [PATCH 1596/2472] document localPassC consistently with compiler; fix ANSI capitalization (#19721) --- doc/manual.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/manual.rst b/doc/manual.rst index 8206d8342..adc1d9d16 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -1370,7 +1370,7 @@ cstring type The `cstring` type meaning `compatible string` is the native representation of a string for the compilation backend. For the C backend the `cstring` type represents a pointer to a zero-terminated char array -compatible with the type `char*` in Ansi C. Its primary purpose lies in easy +compatible with the type `char*` in ANSI C. Its primary purpose lies in easy interfacing with C. The index operation `s[i]` means the i-th *char* of `s`; however no bounds checking for `cstring` is performed making the index operation unsafe. @@ -7218,16 +7218,16 @@ during semantic analysis: {.passc: gorge("pkg-config --cflags sdl").} -LocalPassc pragma +localPassC pragma ----------------- -The `localPassc` pragma can be used to pass additional parameters to the C +The `localPassC` pragma can be used to pass additional parameters to the C compiler, but only for the C/C++ file that is produced from the Nim module the pragma resides in: .. code-block:: Nim # Module A.nim # Produces: A.nim.cpp - {.localPassc: "-Wall -Werror".} # Passed when compiling A.nim.cpp + {.localPassC: "-Wall -Werror".} # Passed when compiling A.nim.cpp PassL pragma From efaa6777a464b94d88cb948390397b490ccee5b8 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 25 Apr 2022 17:07:55 +0800 Subject: [PATCH 1597/2472] fix #19435; don't create TypeBoundOps for tyOpenArray, tyVarargs [backport: 1.6] (#19723) * fix #19435; openArray wronyly registers typebounds * add testcase * don't create TypeBoundOps for tyOpenArray, tyVarargs --- compiler/sempass2.nim | 2 +- tests/arc/t19435.nim | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 tests/arc/t19435.nim diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index 9bc811a03..0ee806f36 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -1054,7 +1054,7 @@ proc track(tracked: PEffects, n: PNode) = addAsgnFact(tracked.guards, n[0], n[1]) notNilCheck(tracked, n[1], n[0].typ) when false: cstringCheck(tracked, n) - if tracked.owner.kind != skMacro: + if tracked.owner.kind != skMacro and n[0].typ.kind notin {tyOpenArray, tyVarargs}: createTypeBoundOps(tracked, n[0].typ, n.info) if n[0].kind != nkSym or not isLocalVar(tracked, n[0].sym): checkForSink(tracked.config, tracked.c.idgen, tracked.owner, n[1]) diff --git a/tests/arc/t19435.nim b/tests/arc/t19435.nim new file mode 100644 index 000000000..519216bad --- /dev/null +++ b/tests/arc/t19435.nim @@ -0,0 +1,29 @@ +discard """ + matrix: "--gc:arc" +""" + +# bug #19435 +{.experimental: "views".} + +type + Bar = object + placeholder: int + Foo = object + placeholder: int + c: seq[Bar] # remove this line to make things right + +func children*(s: var seq[Foo]): openArray[Foo] = + s.toOpenArray(0, s.len-1) + +proc test = + var foos = @[Foo(), Foo()] + + assert foos.children.len == 2 + var flag = true + for a in foos.children: + flag = false + + if flag: + doAssert false + +test() \ No newline at end of file From 2f32b450d39030c287df4d56be9d06473772f5cd Mon Sep 17 00:00:00 2001 From: Danil Yarantsev Date: Mon, 25 Apr 2022 12:15:03 +0300 Subject: [PATCH 1598/2472] Really fix StringStream with ARC at compile-time, improve streams test (#19739) * Fix compile-time StringStream with ARC * make readDataStr work with ARC, improve test --- lib/pure/streams.nim | 10 ++++++++-- tests/stdlib/tstreams.nim | 9 +++++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim index 2ac6a82f1..f58273ee8 100644 --- a/lib/pure/streams.nim +++ b/lib/pure/streams.nim @@ -243,6 +243,9 @@ proc readDataStr*(s: Stream, buffer: var string, slice: Slice[int]): int = result = s.readDataStrImpl(s, buffer, slice) else: # fallback + when declared(prepareMutation): + # buffer might potentially be a CoW literal with ARC + prepareMutation(buffer) result = s.readData(addr buffer[slice.a], slice.b + 1 - slice.a) template jsOrVmBlock(caseJsOrVm, caseElse: untyped): untyped = @@ -1274,8 +1277,11 @@ else: # after 1.3 or JS not defined new(result) result.data = s - when declared(prepareMutation): - prepareMutation(result.data) # Allows us to mutate using `addr` logic like `copyMem`, otherwise it errors. + when nimvm: + discard + else: + when declared(prepareMutation): + prepareMutation(result.data) # Allows us to mutate using `addr` logic like `copyMem`, otherwise it errors. result.pos = 0 result.closeImpl = ssClose result.atEndImpl = ssAtEnd diff --git a/tests/stdlib/tstreams.nim b/tests/stdlib/tstreams.nim index 6fda30f51..d9857926e 100644 --- a/tests/stdlib/tstreams.nim +++ b/tests/stdlib/tstreams.nim @@ -1,4 +1,5 @@ discard """ + matrix: "--gc:refc; --gc:arc" input: "Arne" output: ''' Hello! What is your name? @@ -74,9 +75,13 @@ block: doAssert(ss.peekLine(str)) doAssert(str == "uick brown fox jumped over the lazy dog.") doAssert(ss.getPosition == 5) # haven't moved - ss.setPosition(0) # Ensure we dont error with writing over literals on arc/orc #19707 + # bug #19707 - Ensure we dont error with writing over literals on arc/orc + ss.setPosition(0) ss.write("hello") + ss.setPosition(0) + doAssert(ss.peekStr(5) == "hello") +# bug #19716 static: # Ensure streams it doesnt break with nimscript on arc/orc #19716 let s = newStringStream("a") - discard s.data + doAssert s.data == "a" From 465fd06d50d0dd2e65762b718ea9a726d94c110b Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 25 Apr 2022 17:18:26 +0800 Subject: [PATCH 1599/2472] add warnings for gc:option (#19722) --- compiler/commands.nim | 91 +++++++++++++++++++++++-------------------- 1 file changed, 49 insertions(+), 42 deletions(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index 98537e9ef..bab98b376 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -527,6 +527,50 @@ proc unregisterArcOrc(conf: ConfigRef) = excl conf.globalOptions, optSeqDestructors excl conf.globalOptions, optTinyRtti +proc processMemoryManagementOption(switch, arg: string, pass: TCmdLinePass, + info: TLineInfo; conf: ConfigRef) = + if conf.backend == backendJs: return # for: bug #16033 + expectArg(conf, switch, arg, pass, info) + if pass in {passCmd2, passPP}: + case arg.normalize + of "boehm": + unregisterArcOrc(conf) + conf.selectedGC = gcBoehm + defineSymbol(conf.symbols, "boehmgc") + incl conf.globalOptions, optTlsEmulation # Boehm GC doesn't scan the real TLS + of "refc": + unregisterArcOrc(conf) + conf.selectedGC = gcRefc + of "markandsweep": + unregisterArcOrc(conf) + conf.selectedGC = gcMarkAndSweep + defineSymbol(conf.symbols, "gcmarkandsweep") + of "destructors", "arc": + registerArcOrc(pass, conf, false) + of "orc": + registerArcOrc(pass, conf, true) + of "hooks": + conf.selectedGC = gcHooks + defineSymbol(conf.symbols, "gchooks") + incl conf.globalOptions, optSeqDestructors + processOnOffSwitchG(conf, {optSeqDestructors}, arg, pass, info) + if pass in {passCmd2, passPP}: + defineSymbol(conf.symbols, "nimSeqsV2") + of "go": + unregisterArcOrc(conf) + conf.selectedGC = gcGo + defineSymbol(conf.symbols, "gogc") + of "none": + unregisterArcOrc(conf) + conf.selectedGC = gcNone + defineSymbol(conf.symbols, "nogc") + of "stack", "regions": + unregisterArcOrc(conf) + conf.selectedGC = gcRegions + defineSymbol(conf.symbols, "gcregions") + of "v2": warningOptionNoop(arg) + else: localError(conf, info, errNoneBoehmRefcExpectedButXFound % arg) + proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; conf: ConfigRef) = var @@ -626,48 +670,11 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; processOnOffSwitchG(conf, {optForceFullMake}, arg, pass, info) of "project": processOnOffSwitchG(conf, {optWholeProject, optGenIndex}, arg, pass, info) - of "gc", "mm": - if conf.backend == backendJs: return # for: bug #16033 - expectArg(conf, switch, arg, pass, info) - if pass in {passCmd2, passPP}: - case arg.normalize - of "boehm": - unregisterArcOrc(conf) - conf.selectedGC = gcBoehm - defineSymbol(conf.symbols, "boehmgc") - incl conf.globalOptions, optTlsEmulation # Boehm GC doesn't scan the real TLS - of "refc": - unregisterArcOrc(conf) - conf.selectedGC = gcRefc - of "markandsweep": - unregisterArcOrc(conf) - conf.selectedGC = gcMarkAndSweep - defineSymbol(conf.symbols, "gcmarkandsweep") - of "destructors", "arc": - registerArcOrc(pass, conf, false) - of "orc": - registerArcOrc(pass, conf, true) - of "hooks": - conf.selectedGC = gcHooks - defineSymbol(conf.symbols, "gchooks") - incl conf.globalOptions, optSeqDestructors - processOnOffSwitchG(conf, {optSeqDestructors}, arg, pass, info) - if pass in {passCmd2, passPP}: - defineSymbol(conf.symbols, "nimSeqsV2") - of "go": - unregisterArcOrc(conf) - conf.selectedGC = gcGo - defineSymbol(conf.symbols, "gogc") - of "none": - unregisterArcOrc(conf) - conf.selectedGC = gcNone - defineSymbol(conf.symbols, "nogc") - of "stack", "regions": - unregisterArcOrc(conf) - conf.selectedGC = gcRegions - defineSymbol(conf.symbols, "gcregions") - of "v2": warningOptionNoop(arg) - else: localError(conf, info, errNoneBoehmRefcExpectedButXFound % arg) + of "gc": + warningDeprecated(conf, info, "`gc:option` is deprecated; use `mm:option` instead") + processMemoryManagementOption(switch, arg, pass, info, conf) + of "mm": + processMemoryManagementOption(switch, arg, pass, info, conf) of "warnings", "w": if processOnOffSwitchOrList(conf, {optWarns}, arg, pass, info): listWarnings(conf) of "warning": processSpecificNote(arg, wWarning, pass, info, switch, conf) From 15ae9323e8f1e045bdd68e16231528f8c08fb611 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 25 Apr 2022 17:19:21 +0800 Subject: [PATCH 1600/2472] fix NimNode comment repr() regression [backport: 1.2] (#19726) --- compiler/renderer.nim | 2 ++ tests/macros/tmacros1.nim | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/compiler/renderer.nim b/compiler/renderer.nim index 3503c4bf6..f847aa094 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -165,6 +165,7 @@ proc putNL(g: var TSrcGen) = proc optNL(g: var TSrcGen, indent: int) = g.pendingNL = indent g.lineLen = indent + g.col = g.indent when defined(nimpretty): g.pendingNewlineCount = 0 proc optNL(g: var TSrcGen) = @@ -173,6 +174,7 @@ proc optNL(g: var TSrcGen) = proc optNL(g: var TSrcGen; a, b: PNode) = g.pendingNL = g.indent g.lineLen = g.indent + g.col = g.indent when defined(nimpretty): g.pendingNewlineCount = lineDiff(a, b) proc indentNL(g: var TSrcGen) = diff --git a/tests/macros/tmacros1.nim b/tests/macros/tmacros1.nim index 53af4287c..c588ff7e6 100644 --- a/tests/macros/tmacros1.nim +++ b/tests/macros/tmacros1.nim @@ -60,3 +60,22 @@ macro foo(t: static Tuple): untyped = doAssert t.b == 12345 foo((a: "foo", b: 12345)) + + +# bug #16307 + +macro bug(x: untyped): string = + newLit repr(x) + +let res = bug: + block: + ## one + ## two + ## three + +doAssert res == """ + +block: + ## one + ## two + ## three""" From ee0a47029443e6d717ed4a85825b7dcf971dbdf8 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 25 Apr 2022 11:59:37 +0200 Subject: [PATCH 1601/2472] Revert "fix db_sqlite.tryInsertID does raise exceptions in 1.6.0 #19743 (#19744)" (#19745) This reverts commit b10f0e7bca43761316f6424786a771af33254e19. --- lib/impure/db_sqlite.nim | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/lib/impure/db_sqlite.nim b/lib/impure/db_sqlite.nim index db40dddd8..1638d38c6 100644 --- a/lib/impure/db_sqlite.nim +++ b/lib/impure/db_sqlite.nim @@ -635,7 +635,7 @@ proc getValue*(db: DbConn, stmtName: SqlPrepared): string proc tryInsertID*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): int64 - {.tags: [WriteDbEffect], raises: [].} = + {.tags: [WriteDbEffect], raises: [DbError].} = ## Executes the query (typically "INSERT") and returns the ## generated ID for the row or -1 in case of an error. ## @@ -650,19 +650,16 @@ proc tryInsertID*(db: DbConn, query: SqlQuery, ## 1, "item#1") == -1 ## db.close() assert(not db.isNil, "Database not connected.") + var q = dbFormat(query, args) + var stmt: sqlite3.PStmt result = -1 - try: - var q = dbFormat(query, args) - var stmt: sqlite3.PStmt - if prepare_v2(db, q, q.len.cint, stmt, nil) == SQLITE_OK: - if step(stmt) == SQLITE_DONE: - result = last_insert_rowid(db) - if finalize(stmt) != SQLITE_OK: - result = -1 - else: - discard finalize(stmt) - except DbError: - discard + if prepare_v2(db, q, q.len.cint, stmt, nil) == SQLITE_OK: + if step(stmt) == SQLITE_DONE: + result = last_insert_rowid(db) + if finalize(stmt) != SQLITE_OK: + result = -1 + else: + discard finalize(stmt) proc insertID*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): int64 {.tags: [WriteDbEffect].} = @@ -695,7 +692,7 @@ proc insertID*(db: DbConn, query: SqlQuery, proc tryInsert*(db: DbConn, query: SqlQuery, pkName: string, args: varargs[string, `$`]): int64 - {.tags: [WriteDbEffect], raises: [], since: (1, 3).} = + {.tags: [WriteDbEffect], raises: [DbError], since: (1, 3).} = ## same as tryInsertID tryInsertID(db, query, args) From 42ac50e988a4734624b204740613c795a0e789a3 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 25 Apr 2022 18:00:23 +0800 Subject: [PATCH 1602/2472] fixes #19662; Indent level errored for first line (#19718) --- compiler/parser.nim | 4 ++++ compiler/passes.nim | 1 + compiler/syntaxes.nim | 2 +- tests/parser/t19662.nim | 6 ++++++ 4 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 tests/parser/t19662.nim diff --git a/compiler/parser.nim b/compiler/parser.nim index 7973f7d37..52466d9fa 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -2393,6 +2393,10 @@ proc parseAll(p: var Parser): PNode = if p.tok.indent != 0: parMessage(p, errInvalidIndentation) +proc checkFirstLineIndentation*(p: var Parser) = + if p.tok.indent != 0 and p.tok.strongSpaceA > 0: + parMessage(p, errInvalidIndentation) + proc parseTopLevelStmt(p: var Parser): PNode = ## Implements an iterator which, when called repeatedly, returns the next ## top-level statement or emptyNode if end of stream. diff --git a/compiler/passes.nim b/compiler/passes.nim index ce52b10c5..7fb2842f5 100644 --- a/compiler/passes.nim +++ b/compiler/passes.nim @@ -150,6 +150,7 @@ proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator; processImplicits graph, graph.config.implicitImports, nkImportStmt, a, module processImplicits graph, graph.config.implicitIncludes, nkIncludeStmt, a, module + checkFirstLineIndentation(p) while true: if graph.stopCompile(): break var n = parseTopLevelStmt(p) diff --git a/compiler/syntaxes.nim b/compiler/syntaxes.nim index e47b0483d..c00fe8b67 100644 --- a/compiler/syntaxes.nim +++ b/compiler/syntaxes.nim @@ -16,7 +16,7 @@ import when defined(nimPreviewSlimSystem): import std/[syncio, assertions] -export Parser, parseAll, parseTopLevelStmt, closeParser +export Parser, parseAll, parseTopLevelStmt, checkFirstLineIndentation, closeParser type FilterKind = enum diff --git a/tests/parser/t19662.nim b/tests/parser/t19662.nim new file mode 100644 index 000000000..7a1864ac6 --- /dev/null +++ b/tests/parser/t19662.nim @@ -0,0 +1,6 @@ + var i: int # bug #19662 +echo i + +discard """ + errormsg: "invalid indentation" +""" \ No newline at end of file From 82680a12a7a6acfbb6f5fdd22c042e409081b812 Mon Sep 17 00:00:00 2001 From: ehmry Date: Mon, 25 Apr 2022 15:16:11 -0500 Subject: [PATCH 1603/2472] macros: make hasCustomPragma more permissive (#19747) Make hasCustomPragma return false rather than fail for invalid parameters. --- lib/core/macros.nim | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index ba19712cf..b370aaa7b 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -1521,7 +1521,7 @@ proc extractTypeImpl(n: NimNode): NimNode = else: error("Invalid node to retrieve type implementation of: " & $n.kind) proc customPragmaNode(n: NimNode): NimNode = - expectKind(n, {nnkSym, nnkDotExpr, nnkBracketExpr, nnkTypeOfExpr, nnkCheckedFieldExpr}) + expectKind(n, {nnkSym, nnkDotExpr, nnkBracketExpr, nnkTypeOfExpr, nnkType, nnkCheckedFieldExpr}) let typ = n.getTypeInst() @@ -1532,7 +1532,9 @@ proc customPragmaNode(n: NimNode): NimNode = if kind(typ[1]) == nnkBracketExpr: typ[1][0] else: typ[1] ) - if impl[0].kind == nnkPragmaExpr: + if impl.kind == nnkNilLit: + return impl + elif impl[0].kind == nnkPragmaExpr: return impl[0][1] else: return impl[0] # handle types which don't have macro at all @@ -1560,7 +1562,7 @@ proc customPragmaNode(n: NimNode): NimNode = while typDef != nil: typDef.expectKind(nnkTypeDef) let typ = typDef[2].extractTypeImpl() - typ.expectKind({nnkRefTy, nnkPtrTy, nnkObjectTy}) + if typ.kind notin {nnkRefTy, nnkPtrTy, nnkObjectTy}: break let isRef = typ.kind in {nnkRefTy, nnkPtrTy} if isRef and typ[0].kind in {nnkSym, nnkBracketExpr}: # defines ref type for another object(e.g. X = ref X) typDef = getImpl(typ[0]) From ef4ac5a0d23fcffac9963670b73fa69ad0d0aa73 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 26 Apr 2022 08:58:15 +0200 Subject: [PATCH 1604/2472] use signed comparisons for the index checking in the hope it improves the code generation (#19712) --- compiler/ccgexprs.nim | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 1a3e217b2..5a58b4bcd 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -988,12 +988,12 @@ proc genBoundsCheck(p: BProc; arr, a, b: TLoc) = if reifiedOpenArray(arr.lode): linefmt(p, cpsStmts, "if ($2-$1 != -1 && " & - "((NU)($1) >= (NU)($3.Field1) || (NU)($2) >= (NU)($3.Field1))){ #raiseIndexError(); $4}$n", + "($1 < 0 || $1 >= $3.Field1 || $2 < 0 || $2 >= $3.Field1)){ #raiseIndexError(); $4}$n", [rdLoc(a), rdLoc(b), rdLoc(arr), raiseInstr(p)]) else: linefmt(p, cpsStmts, - "if ($2-$1 != -1 && " & - "((NU)($1) >= (NU)($3Len_0) || (NU)($2) >= (NU)($3Len_0))){ #raiseIndexError(); $4}$n", + "if ($2-$1 != -1 && ($1 < 0 || $1 >= $3Len_0 || $2 < 0 || $2 >= $3Len_0))" & + "{ #raiseIndexError(); $4}$n", [rdLoc(a), rdLoc(b), rdLoc(arr), raiseInstr(p)]) of tyArray: let first = intLiteral(firstOrd(p.config, ty)) @@ -1004,7 +1004,7 @@ proc genBoundsCheck(p: BProc; arr, a, b: TLoc) = of tySequence, tyString: linefmt(p, cpsStmts, "if ($2-$1 != -1 && " & - "((NU)($1) >= (NU)$3 || (NU)($2) >= (NU)$3)){ #raiseIndexError(); $4}$n", + "($1 < 0 || $1 >= $3 || $2 < 0 || $2 >= $3)){ #raiseIndexError(); $4}$n", [rdLoc(a), rdLoc(b), lenExpr(p, arr), raiseInstr(p)]) else: discard @@ -1015,14 +1015,14 @@ proc genOpenArrayElem(p: BProc, n, x, y: PNode, d: var TLoc) = if not reifiedOpenArray(x): # emit range check: if optBoundsCheck in p.options: - linefmt(p, cpsStmts, "if ((NU)($1) >= (NU)($2Len_0)){ #raiseIndexError2($1,$2Len_0-1); $3}$n", + linefmt(p, cpsStmts, "if ($1 < 0 || $1 >= $2Len_0){ #raiseIndexError2($1,$2Len_0-1); $3}$n", [rdCharLoc(b), rdLoc(a), raiseInstr(p)]) # BUGFIX: ``>=`` and not ``>``! inheritLocation(d, a) putIntoDest(p, d, n, ropecg(p.module, "$1[$2]", [rdLoc(a), rdCharLoc(b)]), a.storage) else: if optBoundsCheck in p.options: - linefmt(p, cpsStmts, "if ((NU)($1) >= (NU)($2.Field1)){ #raiseIndexError2($1,$2.Field1-1); $3}$n", + linefmt(p, cpsStmts, "if ($1 < 0 || $1 >= $2.Field1){ #raiseIndexError2($1,$2.Field1-1); $3}$n", [rdCharLoc(b), rdLoc(a), raiseInstr(p)]) # BUGFIX: ``>=`` and not ``>``! inheritLocation(d, a) putIntoDest(p, d, n, @@ -1037,7 +1037,7 @@ proc genSeqElem(p: BProc, n, x, y: PNode, d: var TLoc) = ty = skipTypes(ty.lastSon, abstractVarRange) # emit range check: if optBoundsCheck in p.options: linefmt(p, cpsStmts, - "if ((NU)($1) >= (NU)$2){ #raiseIndexError2($1,$2-1); $3}$n", + "if ($1 < 0 || $1 >= $2){ #raiseIndexError2($1,$2-1); $3}$n", [rdCharLoc(b), lenExpr(p, a), raiseInstr(p)]) if d.k == locNone: d.storage = OnHeap if skipTypes(a.t, abstractVar).kind in {tyRef, tyPtr}: From 8bfc396a4dfa722239818f399a119452a53fe07f Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 27 Apr 2022 02:14:39 +0800 Subject: [PATCH 1605/2472] fixes #18612; apply cache and memcmp for methods in arc/orc (#19749) * try using endsWith * use memcmp * add cache * cleanup * better * minor * fix * improve test coverage for methods with ARC --- compiler/ccgexprs.nim | 7 +++++-- lib/system/arc.nim | 30 +++++++++++++++++++++++++++--- tests/metatype/ttypedesc3.nim | 3 ++- tests/method/tgeneric_methods.nim | 1 + tests/method/tmethod_issues.nim | 1 + tests/method/tmethod_various.nim | 1 + tests/method/tsingle_methods.nim | 2 +- tests/misc/parsecomb.nim | 4 ++++ tests/pragmas/tlocks.nim | 3 +++ 9 files changed, 45 insertions(+), 7 deletions(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 5a58b4bcd..8e7a21c67 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1615,8 +1615,11 @@ proc genNewFinalize(p: BProc, e: PNode) = proc genOfHelper(p: BProc; dest: PType; a: Rope; info: TLineInfo): Rope = if optTinyRtti in p.config.globalOptions: - result = ropecg(p.module, "#isObj($1.m_type, $2)", - [a, genTypeInfo2Name(p.module, dest)]) + let ti = genTypeInfo2Name(p.module, dest) + inc p.module.labels + let cache = "Nim_OfCheck_CACHE" & p.module.labels.rope + p.module.s[cfsVars].addf("static TNimTypeV2* $#[2];$n", [cache]) + result = ropecg(p.module, "#isObjWithCache($#.m_type, $#, $#)", [a, ti, cache]) else: # unfortunately 'genTypeInfoV1' sets tfObjHasKids as a side effect, so we # have to call it here first: diff --git a/lib/system/arc.nim b/lib/system/arc.nim index d66f4b997..17142b277 100644 --- a/lib/system/arc.nim +++ b/lib/system/arc.nim @@ -227,10 +227,34 @@ template tearDownForeignThreadGc* = ## With `--gc:arc` a nop. discard -proc isObj(obj: PNimTypeV2, subclass: cstring): bool {.compilerRtl, inl.} = - proc strstr(s, sub: cstring): cstring {.header: "", importc.} +type ObjCheckCache = array[0..1, PNimTypeV2] - result = strstr(obj.name, subclass) != nil +proc memcmp(str1, str2: cstring, n: csize_t): cint {.importc, header: "".} + +func endsWith(s, suffix: cstring): bool {.inline.} = + let + sLen = s.len + suffixLen = suffix.len + + if suffixLen <= sLen: + result = memcmp(cstring(addr s[sLen - suffixLen]), suffix, csize_t(suffixLen)) == 0 + +proc isObj(obj: PNimTypeV2, subclass: cstring): bool {.compilerRtl, inl.} = + result = endsWith(obj.name, subclass) + +proc isObjSlowPath(obj: PNimTypeV2, subclass: cstring, cache: var ObjCheckCache): bool {.compilerRtl, inline.} = + if endsWith(obj.name, subclass): + cache[1] = obj + result = true + else: + cache[0] = obj + result = false + +proc isObjWithCache(obj: PNimTypeV2, subclass: cstring, cache: var ObjCheckCache): bool {.compilerRtl.} = + if cache[0] == obj: result = false + elif cache[1] == obj: result = true + else: + result = isObjSlowPath(obj, subclass, cache) proc chckObj(obj: PNimTypeV2, subclass: cstring) {.compilerRtl.} = # checks if obj is of type subclass: diff --git a/tests/metatype/ttypedesc3.nim b/tests/metatype/ttypedesc3.nim index 98a59f613..d3a58853d 100644 --- a/tests/metatype/ttypedesc3.nim +++ b/tests/metatype/ttypedesc3.nim @@ -1,5 +1,6 @@ discard """ -output: ''' + matrix: "--mm:arc; --mm:refc" + output: ''' proc Base proc Child method Base diff --git a/tests/method/tgeneric_methods.nim b/tests/method/tgeneric_methods.nim index 0e2aeeede..ab92c66d8 100644 --- a/tests/method/tgeneric_methods.nim +++ b/tests/method/tgeneric_methods.nim @@ -1,4 +1,5 @@ discard """ + matrix: "--mm:arc; --mm:refc" output: '''wow2 X 1 X 3''' diff --git a/tests/method/tmethod_issues.nim b/tests/method/tmethod_issues.nim index df4c3771a..13467f2b3 100644 --- a/tests/method/tmethod_issues.nim +++ b/tests/method/tmethod_issues.nim @@ -1,4 +1,5 @@ discard """ + matrix: "--mm:arc; --mm:refc" output: ''' wof! wof! diff --git a/tests/method/tmethod_various.nim b/tests/method/tmethod_various.nim index fd022717b..f9d067a72 100644 --- a/tests/method/tmethod_various.nim +++ b/tests/method/tmethod_various.nim @@ -1,4 +1,5 @@ discard """ + matrix: "--mm:arc; --mm:refc" output: ''' do nothing HELLO WORLD! diff --git a/tests/method/tsingle_methods.nim b/tests/method/tsingle_methods.nim index 40269559a..b564e7d87 100644 --- a/tests/method/tsingle_methods.nim +++ b/tests/method/tsingle_methods.nim @@ -1,5 +1,5 @@ discard """ - cmd: "nim c --multimethods:off $file" + matrix: "--mm:arc --multimethods:off; --mm:refc --multimethods:off" output: '''base base base diff --git a/tests/misc/parsecomb.nim b/tests/misc/parsecomb.nim index 4ff2f65d2..4f149cbf6 100644 --- a/tests/misc/parsecomb.nim +++ b/tests/misc/parsecomb.nim @@ -1,3 +1,7 @@ +discard """ + matrix: "--mm:arc; --mm:refc" +""" + type Input[T] = object toks: seq[T] index: int diff --git a/tests/pragmas/tlocks.nim b/tests/pragmas/tlocks.nim index ba66a2dca..6c2a9f9e9 100644 --- a/tests/pragmas/tlocks.nim +++ b/tests/pragmas/tlocks.nim @@ -1,3 +1,6 @@ +discard """ + matrix: "--mm:arc; --mm:refc" +""" type SomeBase* = ref object of RootObj type SomeDerived* = ref object of SomeBase From e4a2c2d474f725b01acfa3a6169686f5e16f2ce9 Mon Sep 17 00:00:00 2001 From: Ivan Yonchovski Date: Fri, 29 Apr 2022 12:16:07 +0300 Subject: [PATCH 1606/2472] Make sure that field usage preserves the original line info (#19751) Currently `struct.field` will generate a node with `info` that points to the symbol definition instead of having the actual node location. --- compiler/semexprs.nim | 2 ++ nimsuggest/tests/tuse_structure.nim | 15 +++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 nimsuggest/tests/tuse_structure.nim diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 702b47929..b8742ff65 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1436,8 +1436,10 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode = # is the access to a public field or in the same module or in a friend? markUsed(c, n[1].info, f) onUse(n[1].info, f) + let info = n[1].info n[0] = makeDeref(n[0]) n[1] = newSymNode(f) # we now have the correct field + n[1].info = info # preserve the original info n.typ = f.typ if check == nil: result = n diff --git a/nimsuggest/tests/tuse_structure.nim b/nimsuggest/tests/tuse_structure.nim new file mode 100644 index 000000000..f65ab9060 --- /dev/null +++ b/nimsuggest/tests/tuse_structure.nim @@ -0,0 +1,15 @@ +# tests for use and structures + +type + Foo* = ref object of RootObj + bar*: string + +proc test(f: Foo) = + echo f.#[!]#bar + +discard """ +$nimsuggest --tester $file +>use $1 +def skField tuse_structure.Foo.bar string $file 5 4 "" 100 +use skField tuse_structure.Foo.bar string $file 8 9 "" 100 +""" From 4680ab61c06782d142492d1fcdebf8e942373c09 Mon Sep 17 00:00:00 2001 From: nc-x Date: Sat, 30 Apr 2022 19:28:58 +0530 Subject: [PATCH 1607/2472] Fix fixAbstractType for user defined typeclasses, fixes #19730 & #18409 (#19732) --- compiler/semexprs.nim | 2 +- tests/concepts/t18409.nim | 37 +++++++++++++++++++++++++++++++++++++ tests/concepts/t19730.nim | 20 ++++++++++++++++++++ 3 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 tests/concepts/t18409.nim create mode 100644 tests/concepts/t19730.nim diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index b8742ff65..ed28d8145 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -645,7 +645,7 @@ proc fixAbstractType(c: PContext, n: PNode) = skipTypes(it.typ, abstractVar).kind notin {tyOpenArray, tyVarargs}: if skipTypes(it[1].typ, abstractVar).kind in {tyNil, tyTuple, tySet} or it[1].isArrayConstr: - var s = skipTypes(it.typ, abstractVar) + var s = skipTypes(it.typ, abstractVar + tyUserTypeClasses) if s.kind != tyUntyped: changeType(c, it[1], s, check=true) n[i] = it[1] diff --git a/tests/concepts/t18409.nim b/tests/concepts/t18409.nim new file mode 100644 index 000000000..0edba2d31 --- /dev/null +++ b/tests/concepts/t18409.nim @@ -0,0 +1,37 @@ +discard """ + action: "compile" +""" + +# A vector space over a field F concept. +type VectorSpace*[F] = concept x, y, type V + vector_add(x, y) is V + scalar_mul(x, F) is V + dimension(V) is Natural + +# Real numbers (here floats) form a vector space. +func vector_add*(v: float, w: float): float = v + w +func scalar_mul*(v: float, s: float): float = v * s +func dimension*(x: typedesc[float]): Natural = 1 + +# 2-tuples of real numbers form a vector space. +func vector_add*(v, w: (float, float)): (float, float) = + (vector_add(v[0], w[0]), vector_add(v[1], w[1])) + +func scalar_mul*(v: (float, float), s: float): (float, float) = + (scalar_mul(v[0], s), scalar_mul(v[1], s)) + +func dimension*(x: typedesc[(float, float)]): Natural = 2 + +# Check concept requirements. +assert float is VectorSpace +assert (float, float) is VectorSpace + +# Commutivity axiom for vector spaces over the same field. +func axiom_commutivity*[F](u, v: VectorSpace[F]): bool = + vector_add(u, v) == vector_add(v, u) + +# This is okay. +assert axiom_commutivity(2.2, 3.3) + +# This is not. +assert axiom_commutivity((2.2, 3.3), (4.4, 5.5)) diff --git a/tests/concepts/t19730.nim b/tests/concepts/t19730.nim new file mode 100644 index 000000000..575d45dda --- /dev/null +++ b/tests/concepts/t19730.nim @@ -0,0 +1,20 @@ +discard """ + output: '''1.01.01.01.0 +1.01.01.01.0 +''' +""" + +type + Color = concept c + c.r is SomeFloat + c.g is SomeFloat + c.b is SomeFloat + c.a is SomeFloat + +proc useColor(color: Color) = + echo(color.r, color.g, color.b, color.a) + +let color = (r: 1.0, g: 1.0, b: 1.0, a: 1.0) +useColor(color) + +useColor((r: 1.0, g: 1.0, b: 1.0, a: 1.0)) From 927978345bbf57631828287850eda13b82277eb7 Mon Sep 17 00:00:00 2001 From: huantian Date: Mon, 2 May 2022 09:06:57 -0700 Subject: [PATCH 1608/2472] Rework discarding futures documentation in asyncdispatch (#19738) * Rework discarding futures docs in asyncdispatch * Fix typos Co-authored-by: Danil Yarantsev * Use rst note:: Co-authored-by: flywind * Split discarding and handling futures. * Update lib/pure/asyncdispatch.nim * Update lib/pure/asyncdispatch.nim * Update lib/pure/asyncdispatch.nim * Update lib/pure/asyncdispatch.nim Co-authored-by: Danil Yarantsev Co-authored-by: flywind Co-authored-by: Dominik Picheta --- lib/pure/asyncdispatch.nim | 39 ++++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index b1e1c19fd..755f59ff7 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -123,15 +123,42 @@ ## if future.failed: ## # Handle exception ## -## ## Discarding futures ## ================== ## -## Futures should **never** be discarded. This is because they may contain -## errors. If you do not care for the result of a Future then you should -## use the `asyncCheck` procedure instead of the `discard` keyword. Note -## however that this does not wait for completion, and you should use -## `waitFor` for that purpose. +## Futures should **never** be discarded directly because they may contain +## errors. If you do not care for the result of a Future then you should use +## the `asyncCheck` procedure instead of the `discard` keyword. Note that this +## does not wait for completion, and you should use `waitFor` or `await` for that purpose. +## +## .. note:: `await` also checks if the future fails, so you can safely discard +## its result. +## +## Handling futures +## ================ +## +## There are many different operations that apply to a future. +## The three primary high-level operations are `asyncCheck`, +## `waitFor`, and `await`. +## +## * `asyncCheck`: Raises an exception if the future fails. It neither waits +## for the future to finish nor returns the result of the future. +## * `waitFor`: Polls the event loop and blocks the current thread until the +## future finishes. This is often used to call an async procedure from a +## synchronous context and should never be used in an `async` proc. +## * `await`: Pauses execution in the current async procedure until the future +## finishes. While the current procedure is paused, other async procedures will +## continue running. Should be used instead of `waitFor` in an async +## procedure. +## +## Here is a handy quick reference chart showing their high-level differences: +## ============== ===================== ======================= +## Procedure Context Blocking +## ============== ===================== ======================= +## `asyncCheck` non-async and async non-blocking +## `waitFor` non-async blocks current thread +## `await` async suspends current proc +## ============== ===================== ======================= ## ## Examples ## ======== From 832b8c50345eab6e6f3d037d6e521831007cb6ed Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Tue, 3 May 2022 12:08:39 +1000 Subject: [PATCH 1609/2472] Fix link to experimental manual in macro pragmas section [skip ci] (#19753) --- doc/manual.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual.rst b/doc/manual.rst index adc1d9d16..a92d6e18f 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -7814,7 +7814,7 @@ the same way. There are a few more applications of macro pragmas, such as in type, variable and constant declarations, but this behavior is considered to be experimental and is documented in the `experimental manual -` instead. +`_ instead. Foreign function interface From 2ecae20c437179ff84e255f952dae7000f7ff316 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 3 May 2022 18:35:16 +0800 Subject: [PATCH 1610/2472] switch to mainline bigints (#19756) --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 6b0b15fd8..a55327211 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -39,7 +39,7 @@ pkg "arraymancer", "nim c tests/tests_cpu.nim" pkg "ast_pattern_matching", "nim c -r --oldgensym:on tests/test1.nim", allowFailure = true pkg "asyncthreadpool" pkg "awk" -pkg "bigints", url = "https://github.com/Araq/nim-bigints" +pkg "bigints" pkg "binaryheap", "nim c -r binaryheap.nim" pkg "BipBuffer" pkg "blscurve", allowFailure = true # pending https://github.com/status-im/nim-blscurve/issues/39 From 278ecad973c6581aeea0a6ff9372109b0dd6df5e Mon Sep 17 00:00:00 2001 From: Zoom Date: Wed, 4 May 2022 14:24:52 +0000 Subject: [PATCH 1611/2472] Add 'usages' option to the --stylechecks error msg (#19759) --- compiler/commands.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index bab98b376..2e08ad615 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -116,7 +116,7 @@ const errInvalidCmdLineOption = "invalid command line option: '$1'" errOnOrOffExpectedButXFound = "'on' or 'off' expected, but '$1' found" errOnOffOrListExpectedButXFound = "'on', 'off' or 'list' expected, but '$1' found" - errOffHintsError = "'off', 'hint' or 'error' expected, but '$1' found" + errOffHintsError = "'off', 'hint', 'error' or 'usages' expected, but '$1' found" proc invalidCmdLineOption(conf: ConfigRef; pass: TCmdLinePass, switch: string, info: TLineInfo) = if switch == " ": localError(conf, info, errInvalidCmdLineOption % "-") @@ -508,7 +508,7 @@ proc registerArcOrc(pass: TCmdLinePass, conf: ConfigRef, isOrc: bool) = else: conf.selectedGC = gcArc defineSymbol(conf.symbols, "gcarc") - + defineSymbol(conf.symbols, "gcdestructors") incl conf.globalOptions, optSeqDestructors incl conf.globalOptions, optTinyRtti From a4401054cc7fbd199a8dfa3ec0f9760ca7498b8e Mon Sep 17 00:00:00 2001 From: Anthony Dario Date: Fri, 6 May 2022 02:16:00 -0700 Subject: [PATCH 1612/2472] Fix broken link in sets documentation. (#19769) --- lib/pure/collections/sets.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/collections/sets.nim b/lib/pure/collections/sets.nim index 7b1c58ac8..114e4582a 100644 --- a/lib/pure/collections/sets.nim +++ b/lib/pure/collections/sets.nim @@ -61,7 +61,7 @@ type HashSet*[A] {.myShallow.} = object ## \ ## A generic hash set. ## - ## Use `init proc <#init,HashSet[A]>`_ or `initHashSet proc <#initHashSet,int>`_ + ## Use `init proc <#init,HashSet[A]>`_ or `initHashSet proc <#initHashSet>`_ ## before calling other procs on it. data: KeyValuePairSeq[A] counter: int From 0455d24d555e4f83537af535c0b2e44291d1fe97 Mon Sep 17 00:00:00 2001 From: Zoom Date: Fri, 6 May 2022 09:19:27 +0000 Subject: [PATCH 1613/2472] Fix questionable suggestion in `strutils` docs (#19765) * Fix questionable suggestion in `stutils` docs - Removes the recommendation to pass a string slice for getting a relative index for `find` and `rfind` functions, as this currently makes a string copy, while a simple subtraction is enough. - Docstring for `SkipTable` type. * Doc layout fixup --- lib/pure/strutils.nim | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index bf7bd6aa8..b98bd2bec 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -278,7 +278,7 @@ func nimIdentNormalize*(s: string): string = ## except first one. ## ## .. Warning:: Backticks (`) are not handled: they remain *as is* and - ## spaces are preserved. See `nimIdentBackticksNormalize + ## spaces are preserved. See `nimIdentBackticksNormalize ## `_ for ## an alternative approach. runnableExamples: @@ -1808,11 +1808,15 @@ func join*[T: not string](a: openArray[T], sep: string = ""): string = add(result, $x) type - SkipTable* = array[char, int] + SkipTable* = array[char, int] ## Character table for efficient substring search. func initSkipTable*(a: var SkipTable, sub: string) {.rtl, extern: "nsuInitSkipTable".} = - ## Preprocess table `a` for `sub`. + ## Initializes table `a` for efficient search of substring `sub`. + ## + ## See also: + ## * `find func<#find,SkipTable,string,string,Natural,int>`_ + # TODO: this should be the `default()` initializer for the type. let m = len(sub) fill(a, m) @@ -1826,6 +1830,9 @@ func find*(a: SkipTable, s, sub: string, start: Natural = 0, last = 0): int {. ## element). ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. + ## + ## See also: + ## * `initSkipTable func<#initSkipTable,SkipTable,string>`_ let last = if last == 0: s.high else: last subLast = sub.len - 1 @@ -1865,7 +1872,7 @@ func find*(s: string, sub: char, start: Natural = 0, last = 0): int {.rtl, ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. ## Otherwise the index returned is relative to `s[0]`, not `start`. - ## Use `s[start..last].rfind` for a `start`-origin index. + ## Subtract `start` from the result for a `start`-origin index. ## ## See also: ## * `rfind func<#rfind,string,char,Natural,int>`_ @@ -1893,7 +1900,7 @@ func find*(s: string, chars: set[char], start: Natural = 0, last = 0): int {. ## ## If `s` contains none of the characters in `chars`, -1 is returned. ## Otherwise the index returned is relative to `s[0]`, not `start`. - ## Use `s[start..last].find` for a `start`-origin index. + ## Subtract `start` from the result for a `start`-origin index. ## ## See also: ## * `rfind func<#rfind,string,set[char],Natural,int>`_ @@ -1910,7 +1917,7 @@ func find*(s, sub: string, start: Natural = 0, last = 0): int {.rtl, ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. ## Otherwise the index returned is relative to `s[0]`, not `start`. - ## Use `s[start..last].find` for a `start`-origin index. + ## Subtract `start` from the result for a `start`-origin index. ## ## See also: ## * `rfind func<#rfind,string,string,Natural,int>`_ @@ -1950,7 +1957,7 @@ func rfind*(s: string, sub: char, start: Natural = 0, last = -1): int {.rtl, ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. ## Otherwise the index returned is relative to `s[0]`, not `start`. - ## Use `s[start..last].find` for a `start`-origin index. + ## Subtract `start` from the result for a `start`-origin index. ## ## See also: ## * `find func<#find,string,char,Natural,int>`_ @@ -1968,7 +1975,7 @@ func rfind*(s: string, chars: set[char], start: Natural = 0, last = -1): int {. ## ## If `s` contains none of the characters in `chars`, -1 is returned. ## Otherwise the index returned is relative to `s[0]`, not `start`. - ## Use `s[start..last].rfind` for a `start`-origin index. + ## Subtract `start` from the result for a `start`-origin index. ## ## See also: ## * `find func<#find,string,set[char],Natural,int>`_ @@ -1986,7 +1993,7 @@ func rfind*(s, sub: string, start: Natural = 0, last = -1): int {.rtl, ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. ## Otherwise the index returned is relative to `s[0]`, not `start`. - ## Use `s[start..last].rfind` for a `start`-origin index. + ## Subtract `start` from the result for a `start`-origin index. ## ## See also: ## * `find func<#find,string,string,Natural,int>`_ From 2c73e84436a11cae1676c7da0228158ba1a885cc Mon Sep 17 00:00:00 2001 From: kraptor Date: Sat, 7 May 2022 20:46:45 +0200 Subject: [PATCH 1614/2472] "ip" protocol as id 0 is long gone (#19760) IANA is using id 0 for "HOPOPT" instead of "ip" for some time now and those systems that still support the old mapping will stop doing so at some point in the future. Some BSDs and openSUSE are already following this change as per IANA list here: https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml As new unixes (and maybe Windows, who knows) start to adopt the updated IANA list, this will keep failing from time to time, so it's better to remove the "ip" check altogether. --- tests/stdlib/tgetprotobyname.nim | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/stdlib/tgetprotobyname.nim b/tests/stdlib/tgetprotobyname.nim index 0b60d059f..014c18845 100644 --- a/tests/stdlib/tgetprotobyname.nim +++ b/tests/stdlib/tgetprotobyname.nim @@ -1,9 +1,5 @@ import nativesockets -when not defined(netbsd): - # Ref: https://github.com/nim-lang/Nim/issues/15452 - NetBSD doesn't define an `ip` protocol - doAssert getProtoByName("ip") == 0 - doAssert getProtoByName("ipv6") == 41 doAssert getProtoByName("tcp") == 6 doAssert getProtoByName("udp") == 17 From 8c100a37b93607806acec51733fe5e2fda392d44 Mon Sep 17 00:00:00 2001 From: kraptor Date: Sun, 8 May 2022 11:54:04 +0200 Subject: [PATCH 1615/2472] Add new self-signed 4096 bit certificate to testdata (#19758) The test tasyncssl may fail on modern linux versions as they require at least 2048 bit certificates. A new certificate and private key with default values has been added to meet this new requirement. --- tests/testdata/mycert.pem | 105 ++++++++++++++++++++++++++++---------- 1 file changed, 77 insertions(+), 28 deletions(-) diff --git a/tests/testdata/mycert.pem b/tests/testdata/mycert.pem index 69039fe04..61dcb685c 100644 --- a/tests/testdata/mycert.pem +++ b/tests/testdata/mycert.pem @@ -1,32 +1,81 @@ -----BEGIN PRIVATE KEY----- -MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBAL9QXX/nuiFbizdI -Uhg1D9gG0GIANENvKwdWTlOlZAoOqvjXPFLGh+87yhvkq4f5FcICkSDao2SfeZcP -JsgD7T01owt8x48898+d91i7nIpr6IXGPyBxHOuaxAITY1D+MbbkhIGUVrEqKEOm -qfS9cqPZaDNkx8xVef0HPCmqEme9AgMBAAECgYBxqrQCvJFQJG3QiL2N+GjTdyj0 -MR7cOf6cu2CKPifz+ccHVgpXO/Gj6Cgq7nAjt5B/1rqXhI+zxzSc1bm6+OpIfakS -E0DLCFacECmL0v3c+XLxTtMhFZF5u7Yq0UMsuWmDSfRb4sbRjC+s+c51i5N0485k -b3un/MDI/i/jD/YZGQJBAPLtcuMIwEblUR1uw7NFezXdauXCRFkekoSlJNvpdM/Z -XDRcuWioek5yD8FvMpTz7H2e26Ev645JT5lIuN4Eti8CQQDJm+Qt9NYUohRsU279 -GYI3vXsXKKqmA22at4I3KRXPSeYV1vtQLYWWqGAXzgGkUEVBY0chmHyDcNwkUsNw -svHTAkEAwOTpD/vX6bOXOD7GqKgoULozcqNScE2FXExhuzliJtTakT17f+4fyABs -IFWynXIevBUTIqeRbJcr3HRRTwIAwwJBAJQ8XkL4IaxcG/4mPpY0ek13sZiumwKj -xKQcx869E78tS9LFFlW2kuHafYUjQIvLRZC1aWinUO3oPsUqYW9s82cCQFjoods5 -YsWEJB2RKCT5nhyAXEZLehxF+FXr+JjLMHkuEINKTnHHKjHJ7LbMcTCKUJAcKDTA -qZFEq5N1aT6DrAU= +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDgpWhnNdNSUbwJ +39mKf+PA879K1wyT23LC3Ipo5g5vaRjB4vrSh5vzrjK5Q/z42yRDLQ2ojEdUne19 +1xMaM+WqTXnfkf9sou4VJ0HsdQ6jOU1LDRPrPvCjHLZjyMu18sGvQf4FATBwWN3n +QhmvcbleTeyv2pgMNeQDfjuIhRJ/aCIi/WYQ466+Rcj0y3/udYX1yPYf2mszXnSE +i2iWgdsjx0qkDU/nJhnKXhKfucAm9Ds0YfQdZYN3rEzfGkJzZUhDG1n30ghu+vUg +qqLi6HU/HqBxKxBlJf2HdMNK4VLkJuFC+wCI0JfF6VYhcNmtZzlQy3M9WjYvbXUY +nvMkQzUECaSaAtmbhFFfgey1Q3IEQS/j1lsVxtuGUE3YaoyaK22/CkVInDz0aka9 +CKq77x7S/V5PP/wwPV4s8XsbT6/34KemuY3oU84+bf5SBuWdzBY0hnDO9c/FaOnZ +4yjLMQi285HLpK417Z32DmzIkb7GjKtP9WLvsWGgbgquEjcFaIT+erkN09cDjsG/ +URWbKcsQkz8v9E4zj0rxdanXoJEtg1gm+dPsiXokZOLTeaMVtDoTkWeCM/VAp25J +MTc656QRUTSzUODK5tXQ3JgE2XIPGV/PsMi7T/q9xFBQmhUk5kfLNlQ1cy/aiTWK +8pjHcmxbyD121AimuTtv9CFYnYtzyQIDAQABAoICAAMYZFrfs/yzYZrlObMd1f6H +nUAjvGmhIXCr50BQwywnz46EWR5jffOal9pTpH2tT0+ZpFGJNUZmMqqENyAqTOTO +0noRIerWR9+EvfTLHBuFo5oAISEhqeEleSHg12W3ZZHLn/tjq84we0Y/c+kl8P7q +pfM6WNP6Ph0KNTnJU5rrzWScBzb+XB8FCSLOVwHrHqBnV3TS5p07lPFqllNUkLdq +fI3MHSi7LqnKKAmJXqtqvBIZs2pgRrJ0bk64pue+IoPCMbgnbbRRwuTjVQE5YLww +6NcGV+B86IRgSHyUpDa+jmYE3VoFPcIdV/F4A5fPD0wcsYbL4mk+4dkn/4OlZWqV +NZILp2IGejKaKtM1fr7fV2IRUbGUBN/+rX4I0SRnuq6Q4Ipx8VASbpgXQPBo9XTg +SHHsFbEu2DL8BHVgXdOy4PW6jQ0Ux2LhuJk6AQ5nIlFBYA+c8rSlZQXJbEbk1VZu +1i7iSOn/kx3ULMjUfhI/Ddm6rQqtiLbXSubXCzu1HMPT1FG9LUfAbq6EpiVkpAk1 +TqlciBHsyz//mk2RmIEx0Bt+0bX8FFGTIUiGyrp5s4hAHbgQZbXBAUYMRzWxhZ2Q +G0KBXx46bv3hJUb0GOgbNVxcaPnyrXaS/Hafcbx2LXlEtKiwGnC/yKJ7Hmcrt+AQ +RTaqNU1o/bkSYC7vHMZdAoIBAQD79uYPZPv5GLCKPZ64gc3+tbFXNqkmL7qv14yD +Gr3VubRbJe3Fx+T1cS+t9cjgOofhjFnwsDaFRoyWOYqRV/znwFsvvsDhHOLBr1u6 +qWQiF2CT1uMdXR3P6KD8h2DUVNNccxKqqIJNCR5oD/ngnnByWkQobzlsnoIdXgZm +ozBZjGr2XUMO5dJqUxaXZwY3j4I2hk/Ka9uroApyptl+DTVbPHvjk3MzU9QKUNor +vXEtQ8EmM8Oy6v/33HBmNs6cF5fMpgWz6u+B357OTxAfu8B42jZ18OeLrvkHFxzu +phOB1uXvqtQ0tdksSHHWj3IIZRK7GDGudnDEZ23vbCaxH4zbAoIBAQDkPn+5N3px +7UAECWrvT10TD3xKeqMkFhqRA9gmDE4N9AdoN6T6PzD7Tr3gOgGLq1tXjCjBqAlx +ZIDTnih3IK3xoRk2zmhq5+LfM8LQRAxAC8IsoQMXAsmW1KlS6MR70m50pFR8NK6r +UmOdrwVUKp3K6Mecid3LmMVLXGMUKwIJc1k7LJHtwrfi0i5xfBtiqQeaR0lJg4Zr ++zEL/4rHfcq06/P3k0+4uLKZ1LGOvwLPiTA3DADPWZbzUXo7McKOFWF/ycGQBrJq +AJikx15dVLnB16bnHXdxrlrd0LON2R+XfT4+dfRymqZLzrBI3L39t/elTmVYnD3P +punkmZuVwNErAoIBAQD5xOiOPibh6S2n/CmI8XQImIgx0kefSRUhFuV9WVbxtOMq +r9CijONUw1zmb40vahYk6gKGa8fAGg1nJadNKRHVkoNSMx/0h7PpGDIwOZa/jLj6 +FLyS8SmKXiqn6nN8SJI1RQUuE1kHkJCJy7yCg80oLn7+LjOYjxCgmAJ0YDSfsGif +zBebfws0xyTP9RrenO4RqtcR7BWYbk+tE+Tp5aIMzUpqcFJ0gRbjGv8K+QJmQpIH +kqzegcI4LFdnm9D4PxMFlVZ14eCGt+wuy4VKT84efwIZrDN77nmCI9FUaWFRBnxt +NsShc9rS4QWoEg6Sb88/lF47ecGTkIwUGPvJ/WKdAoIBAQClF7/zDPn4Zg+j29wJ +dXJxUwYoKUTP2V0l/43dF5Ft7lFdRMKEcCjR3kbhZZOwnyXW0X65dP4/kt7MMt46 +LN0kpc5DIlHM4iXsJNiJJG9n9BljhqNhhZajDvfbDJrypWdX33Vs0f511YZQjERi +eODh4DZiOCbCGaK7u/u+ns0+YLzuXHLBc9Lmsfj+BTMZzgG9ykpsbkJQ4MS9VP3h +BlAVRYaWUWucxZwKQRqdkfRKgYTqjDgZw0e4f/rVzkxX0YdQk3L65p0up3fB2KOd +BqfGWmJTUbEP/XmkcE0wERkUznazX0aNjucydjJ0wZZ7axIp8+bCjWD4TldoDuPH +Ek05AoIBAECgPfBHLQTAsI+wHbFsu/are28BOiJCSEXjRv88CbUbj/qgAppFuXbx +900WwJ1rVWd5x3LFa3VfyuAqYMi5jzmX9kWgEsC0WhgfyIRFiynw45LlcT4u3fWg +vJEx01lGgFVjnYfFUDS9d1MuiXGxIHrNhzHOP2x2CsS5vrFHav7iwG9YULEk8tJr +My0wzjF3UJ2/5DjGK56WuzauLYKrQ6Faw8dWUy4e/bNYId8wglhQQW548JwJEGmq +nq+EzTfEupXH57Bw7MGEOfdlhv98zNT9VcvBAN09vHeF3Hh6AM4aiGSUIt2HIkto +zvw+fqZ2Sk9O5qva+KE1QMVtY1EICI8= -----END PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIICgDCCAemgAwIBAgIJANpVfZSDAyNgMA0GCSqGSIb3DQEBBQUAMFkxCzAJBgNV -BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX -aWRnaXRzIFB0eSBMdGQxEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0xMjA2MDMxMjI4 -MDhaFw0xMzA2MDMxMjI4MDhaMFkxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21l -LVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxEjAQBgNV -BAMMCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAv1Bdf+e6 -IVuLN0hSGDUP2AbQYgA0Q28rB1ZOU6VkCg6q+Nc8UsaH7zvKG+Srh/kVwgKRINqj -ZJ95lw8myAPtPTWjC3zHjzz3z533WLucimvohcY/IHEc65rEAhNjUP4xtuSEgZRW -sSooQ6ap9L1yo9loM2THzFV5/Qc8KaoSZ70CAwEAAaNQME4wHQYDVR0OBBYEFF2n -Of61swO+XSNrYb4T02tGx8afMB8GA1UdIwQYMBaAFF2nOf61swO+XSNrYb4T02tG -x8afMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAEPHofdf4acaph5/e -+BzZGsMfRqdPgwp5sxjFKeTQI1A49VL7ykkb0iLKGfKZtvE8MjMrYjzt20E2bIZj -8eCivT6TbNrVRoACCly/lH9fZfWOG6dBu/85IrTAhSKi8yjbRzmjWUkdrcEJ+ZtV -1cahfFar4l4QwYgqp2pDd6ie+zE= +MIIFCTCCAvGgAwIBAgIUKqDcJ71wiMObIQ5sga2sZItNseowDQYJKoZIhvcNAQEL +BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTIyMDUwNDExMzAwN1oXDTIyMDYw +MzExMzAwN1owFDESMBAGA1UEAwwJbG9jYWxob3N0MIICIjANBgkqhkiG9w0BAQEF +AAOCAg8AMIICCgKCAgEA4KVoZzXTUlG8Cd/Zin/jwPO/StcMk9tywtyKaOYOb2kY +weL60oeb864yuUP8+NskQy0NqIxHVJ3tfdcTGjPlqk1535H/bKLuFSdB7HUOozlN +Sw0T6z7woxy2Y8jLtfLBr0H+BQEwcFjd50IZr3G5Xk3sr9qYDDXkA347iIUSf2gi +Iv1mEOOuvkXI9Mt/7nWF9cj2H9prM150hItoloHbI8dKpA1P5yYZyl4Sn7nAJvQ7 +NGH0HWWDd6xM3xpCc2VIQxtZ99IIbvr1IKqi4uh1Px6gcSsQZSX9h3TDSuFS5Cbh +QvsAiNCXxelWIXDZrWc5UMtzPVo2L211GJ7zJEM1BAmkmgLZm4RRX4HstUNyBEEv +49ZbFcbbhlBN2GqMmittvwpFSJw89GpGvQiqu+8e0v1eTz/8MD1eLPF7G0+v9+Cn +prmN6FPOPm3+UgblncwWNIZwzvXPxWjp2eMoyzEItvORy6SuNe2d9g5syJG+xoyr +T/Vi77FhoG4KrhI3BWiE/nq5DdPXA47Bv1EVmynLEJM/L/ROM49K8XWp16CRLYNY +JvnT7Il6JGTi03mjFbQ6E5FngjP1QKduSTE3OuekEVE0s1DgyubV0NyYBNlyDxlf +z7DIu0/6vcRQUJoVJOZHyzZUNXMv2ok1ivKYx3JsW8g9dtQIprk7b/QhWJ2Lc8kC +AwEAAaNTMFEwHQYDVR0OBBYEFEZcUeqH6MfIzC56BlD3NSs2mCgGMB8GA1UdIwQY +MBaAFEZcUeqH6MfIzC56BlD3NSs2mCgGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI +hvcNAQELBQADggIBADGc7WONP6I6Trb7ici9fQ9qT3wh/RGDcmUDmDtARn9SFtOF +hsbszOMZg1Flj10fuD6OYDonKz4rv+Ieo5VkAYXxxd3J+bx2x1pqd1YSIsvugTwv +pnx39uBR9cjOmt4W7RyzhFnXoVfuSBE6LpkBUjcrqi5xwrQ31mOSCPwe8uDZYEWS +pX49MiHXGTyjQ481QLiOtTBZJa5igfnHUJbJbyZMa86zBQ/clS7+OeDwkvaEpjov +2VQf3QouVLghfLZYWSxWdEKD9+IWHn8rV6qksEb/Ogu4ZtzDRGqJow4j0DeSSEu7 +ns1YeT2mVTFwHjGXCWS+0iE885NDVX/b5YptlwH5PW7aqeXyCS9Hrd1C1GnXoXGp +NHltYRTyNWm974xWg7eu2gbbB8Ng02chXysdkBq7l+7OyA0a2EfX3Cbz3/49+Mqn +viqwNO5toSHVCdfV9Jd0p0CcqryYgyt2YNpJB+2nUQpiW4jviAs49PZg2PpCVw/2 +0cqtaPeUh26Si8UzDOuT697PIuGkZ9Q9QVwccVXtCyA0UpJ13P0fMrA+yEMhtwSs +k1tRm0pUQa6t3v26/cAy+kMhviHBJFwi5dx+y3OMvqQqpQJrgfZawm/o2ZQHy1KP +8m4ngrJzb13evKf216qCwllmQo6Ts4yeI1Ddx8UpdX7RUWpD8Uw4zSi7Th4r -----END CERTIFICATE----- From 61c1142667fabec11a7440b0404ce5688cad54d2 Mon Sep 17 00:00:00 2001 From: huantian Date: Sun, 8 May 2022 02:54:25 -0700 Subject: [PATCH 1616/2472] Update nimscript `withDir` doc example (#19776) --- lib/system/nimscript.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/system/nimscript.nim b/lib/system/nimscript.nim index 5d1916c7f..0b49ea2e7 100644 --- a/lib/system/nimscript.nim +++ b/lib/system/nimscript.nim @@ -342,17 +342,17 @@ template withDir*(dir: string; body: untyped): untyped = ## Usage example: ## ## .. code-block:: nim + ## # inside /some/path/ ## withDir "foo": - ## # inside foo - ## #back to last dir - var curDir = getCurrentDir() + ## # move to /some/path/foo/ + ## # back in /some/path/ + let curDir = getCurrentDir() try: cd(dir) body finally: cd(curDir) - proc writeTask(name, desc: string) = if desc.len > 0: var spaces = " " From b2b878f4d62ff2dcaf69979bb0f96e6015b47605 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Sun, 8 May 2022 06:56:46 -0300 Subject: [PATCH 1617/2472] Remove deprecated posix proc (#19774) * Remove deprecated posix proc that takes wrong argument types * Remove deprecated posix proc that takes wrong argument types --- changelog.md | 2 +- lib/posix/posix.nim | 10 ---------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/changelog.md b/changelog.md index 8b9d0bc29..3b2685547 100644 --- a/changelog.md +++ b/changelog.md @@ -47,7 +47,7 @@ becomes an alias for `addr`. - Removed deprecated `math.c_frexp`. - Removed deprecated ``` httpcore.`==` ```. - Removed deprecated `std/dom_extensions`. - +- Removed deprecated `std/posix.CMSG_SPACE` and `std/posix.CMSG_LEN` that takes wrong argument types. - Remove deprecated `osproc.poDemon`, symbol with typo. - Deprecated `selfExe` for Nimscript. diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 146ba886f..4ebae4361 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -894,19 +894,9 @@ proc CMSG_NXTHDR*(mhdr: ptr Tmsghdr, cmsg: ptr Tcmsghdr): ptr Tcmsghdr {. proc CMSG_FIRSTHDR*(mhdr: ptr Tmsghdr): ptr Tcmsghdr {. importc, header: "".} -{.push warning[deprecated]: off.} -proc CMSG_SPACE*(len: csize): csize {. - importc, header: "", deprecated: "argument `len` should be of type `csize_t`".} -{.pop.} - proc CMSG_SPACE*(len: csize_t): csize_t {. importc, header: "".} -{.push warning[deprecated]: off.} -proc CMSG_LEN*(len: csize): csize {. - importc, header: "", deprecated: "argument `len` should be of type `csize_t`".} -{.pop.} - proc CMSG_LEN*(len: csize_t): csize_t {. importc, header: "".} From 85bc8326acc9bf18e748055e770a40890e7ac069 Mon Sep 17 00:00:00 2001 From: Alfred Morgan Date: Tue, 10 May 2022 05:21:35 -0700 Subject: [PATCH 1618/2472] varargs example erroneously transformed "abc" to "def" (#19781) --- doc/manual.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual.rst b/doc/manual.rst index a92d6e18f..126b0f0d6 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -1556,7 +1556,7 @@ type conversions in this context: myWriteln(stdout, 123, "abc", 4.0) # is transformed to: - myWriteln(stdout, [$123, $"def", $4.0]) + myWriteln(stdout, [$123, $"abc", $4.0]) In this example `$` is applied to any argument that is passed to the parameter `a`. (Note that `$` applied to strings is a nop.) From b9f243eb2aeb4930ba552f8bf487719d628f4a2a Mon Sep 17 00:00:00 2001 From: Alfred Morgan Date: Tue, 10 May 2022 08:45:57 -0700 Subject: [PATCH 1619/2472] string is missing formatting when calling fmt (#19780) it appears the documentation intends to compare & with .fmt but there is no formatting in the string. even though the assert is true it doesn't quite prove that .fmt is an equivalent formatter. --- lib/pure/strformat.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim index ce3439600..247d0a296 100644 --- a/lib/pure/strformat.nim +++ b/lib/pure/strformat.nim @@ -705,6 +705,6 @@ macro `&`*(pattern: string{lit}): string = runnableExamples: let x = 7 assert &"{x}\n" == "7\n" # regular string literal - assert &"{x}\n" == "7\n".fmt # `fmt` can be used instead - assert &"{x}\n" != fmt"7\n" # see `fmt` docs, this would use a raw string literal + assert &"{x}\n" == "{x}\n".fmt # `fmt` can be used instead + assert &"{x}\n" != fmt"{x}\n" # see `fmt` docs, this would use a raw string literal strformatImpl(pattern.strVal, '{', '}') From c61eb5df32c66431d0d08417c39e268fdf236132 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 13 May 2022 01:03:30 +0800 Subject: [PATCH 1620/2472] disable polypbren (#19787) * disable polypbren * Update important_packages.nim --- testament/important_packages.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index a55327211..c8fb5ad3c 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -103,7 +103,7 @@ pkg "nimfp", "nim c -o:nfp -r src/fp.nim" pkg "nimgame2", "nim c nimgame2/nimgame.nim" # XXX Doesn't work with deprecated 'randomize', will create a PR. pkg "nimgen", "nim c -o:nimgenn -r src/nimgen/runcfg.nim" -pkg "nimlsp", allowFailure = true +pkg "nimlsp" pkg "nimly", "nim c -r tests/test_readme_example.nim" pkg "nimongo", "nimble test_ci", allowFailure = true pkg "nimph", "nimble test", "https://github.com/disruptek/nimph", allowFailure = true @@ -125,7 +125,7 @@ pkg "patty" pkg "pixie" pkg "plotly", "nim c examples/all.nim" pkg "pnm" -pkg "polypbren" +pkg "polypbren", allowFailure = true pkg "prologue", "nimble tcompile" pkg "protobuf", "nim c -o:protobuff -r src/protobuf.nim" pkg "pylib" From 19001c070bbe7645ff45fcbd66ab221235715302 Mon Sep 17 00:00:00 2001 From: Anthony Dario Date: Thu, 12 May 2022 20:19:42 -0700 Subject: [PATCH 1621/2472] Fix typo in sequtils documentation (#19789) Found another small typo. --- lib/pure/collections/sequtils.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index 5e9b492c2..39908e9c1 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -914,7 +914,7 @@ template foldl*(sequence, operation, first): untyped = ## ## The `operation` parameter should be an expression which uses the variables ## `a` and `b` for each step of the fold. The `first` parameter is the - ## start value (the first `a`) and therefor defines the type of the result. + ## start value (the first `a`) and therefore defines the type of the result. ## ## **See also:** ## * `foldr template<#foldr.t,untyped,untyped>`_ From 33888a73840a7f9fa46f79e613d488de2d193916 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 17 May 2022 13:37:26 +0800 Subject: [PATCH 1622/2472] [manual] TLock => Lock (#19785) * [manual] TLock => Lock * minor --- doc/manual.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/doc/manual.rst b/doc/manual.rst index 126b0f0d6..bfcb41b7a 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -8112,7 +8112,9 @@ Object fields and global variables can be annotated via a `guard` pragma: .. code-block:: nim - var glock: TLock + import std/locks + + var glock: Lock var gdata {.guard: glock.}: int The compiler then ensures that every access of `gdata` is within a `locks` @@ -8139,7 +8141,7 @@ that also implement some form of locking at runtime: .. code-block:: nim - template lock(a: TLock; body: untyped) = + template lock(a: Lock; body: untyped) = pthread_mutex_lock(a) {.locks: [a].}: try: @@ -8181,10 +8183,12 @@ the expressivity of the language: .. code-block:: nim + import std/locks + type ProtectedCounter = object v {.guard: L.}: int - L: TLock + L: Lock proc incCounters(counters: var openArray[ProtectedCounter]) = for i in 0..counters.high: From 06f02bb7716066241b04b2a92a3a61f539eadff2 Mon Sep 17 00:00:00 2001 From: Yardanico Date: Tue, 17 May 2022 10:56:39 +0300 Subject: [PATCH 1623/2472] Always use httpclient in nimgrab (#19767) --- tools/nimgrab.nim | 37 +--- tools/urldownloader.nim | 431 ---------------------------------------- 2 files changed, 9 insertions(+), 459 deletions(-) delete mode 100644 tools/urldownloader.nim diff --git a/tools/nimgrab.nim b/tools/nimgrab.nim index ee5eced1e..7e4161faf 100644 --- a/tools/nimgrab.nim +++ b/tools/nimgrab.nim @@ -1,33 +1,14 @@ +import std/[os, httpclient] +proc syncDownload(url, file: string) = + var client = newHttpClient() + proc onProgressChanged(total, progress, speed: BiggestInt) = + echo "Downloading " & url & " " & $(speed div 1000) & "kb/s" + echo clamp(int(progress*100 div total), 0, 100), "%" -when defined(windows): - import os, urldownloader - - proc syncDownload(url, file: string) = - proc progress(status: DownloadStatus, progress: uint, total: uint, - message: string) {.gcsafe.} = - echo "Downloading " & url - let t = total.BiggestInt - if t != 0: - echo clamp(int(progress.BiggestInt*100 div t), 0, 100), "%" - else: - echo "0%" - - downloadToFile(url, file, {optUseCache}, progress) - echo "100%" - -else: - import os, asyncdispatch, httpclient - - proc syncDownload(url, file: string) = - var client = newHttpClient() - proc onProgressChanged(total, progress, speed: BiggestInt) = - echo "Downloading " & url & " " & $(speed div 1000) & "kb/s" - echo clamp(int(progress*100 div total), 0, 100), "%" - - client.onProgressChanged = onProgressChanged - client.downloadFile(url, file) - echo "100%" + client.onProgressChanged = onProgressChanged + client.downloadFile(url, file) + echo "100%" if os.paramCount() != 2: quit "Usage: nimgrab " diff --git a/tools/urldownloader.nim b/tools/urldownloader.nim deleted file mode 100644 index 73e4034c9..000000000 --- a/tools/urldownloader.nim +++ /dev/null @@ -1,431 +0,0 @@ - -# -# -# Windows native FTP/HTTP/HTTPS file downloader -# (c) Copyright 2017 Eugene Kabanov -# -# See the file "LICENSE", included in this -# distribution, for details about the copyright. -# - -## This module implements native Windows FTP/HTTP/HTTPS downloading feature, -## using ``urlmon.UrlDownloadToFile()``. -## -## - -when not (defined(windows) or defined(nimdoc)): - {.error: "Platform is not supported.".} - -import os - -type - DownloadOptions* = enum - ## Available download options - optUseCache, ## Use Windows cache. - optUseProgressCallback, ## Report progress via callback. - optIgnoreSecurity ## Ignore HTTPS security problems. - - DownloadStatus* = enum - ## Available download status sent to ``progress`` callback. - statusProxyDetecting, ## Automatic Proxy detection. - statusCookieSent ## Cookie will be sent with request. - statusResolving, ## Resolving URL with DNS. - statusConnecting, ## Establish connection to server. - statusRedirecting ## HTTP redirection pending. - statusRequesting, ## Sending request to server. - statusMimetypeAvailable, ## Mimetype received from server. - statusBeginDownloading, ## Download process starting. - statusDownloading, ## Download process pending. - statusEndDownloading, ## Download process finished. - statusCacheAvailable ## File found in Windows cache. - statusUnsupported ## Unsupported status. - statusError ## Error happens. - - DownloadProgressCallback* = proc(status: DownloadStatus, progress: uint, - progressMax: uint, - message: string) - ## Progress callback. - ## - ## status - ## Indicate current stage of downloading process. - ## - ## progress - ## Number of bytes currently downloaded. Available only, if ``status`` is - ## ``statusBeginDownloading``, ``statusDownloading`` or - ## ``statusEndDownloading``. - ## - ## progressMax - ## Number of bytes expected to download. Available only, if ``status`` is - ## ``statusBeginDownloading``, ``statusDownloading`` or - ## ``statusEndDownloading``. - ## - ## message - ## Status message, which depends on ``status`` code. - ## - ## Available messages' values: - ## - ## statusResolving - ## URL hostname to be resolved. - ## statusConnecting - ## IP address - ## statusMimetypeAvailable - ## Downloading resource MIME type. - ## statusCacheAvailable - ## Path to filename stored in Windows cache. - -type - UUID = array[4, uint32] - - LONG = clong - ULONG = culong - HRESULT = clong - DWORD = uint32 - OLECHAR = uint16 - OLESTR = ptr OLECHAR - LPWSTR = OLESTR - UINT = cuint - REFIID = ptr UUID - -const - E_NOINTERFACE = 0x80004002'i32 - E_NOTIMPL = 0x80004001'i32 - S_OK = 0x00000000'i32 - - CP_UTF8 = 65001'u32 - - IID_IUnknown = UUID([0'u32, 0'u32, 192'u32, 1174405120'u32]) - IID_IBindStatusCallback = UUID([2045430209'u32, 298760953'u32, - 2852160140'u32, 195644160'u32]) - - BINDF_GETNEWESTVERSION = 0x00000010'u32 - BINDF_IGNORESECURITYPROBLEM = 0x00000100'u32 - BINDF_RESYNCHRONIZE = 0x00000200'u32 - BINDF_NO_UI = 0x00000800'u32 - BINDF_SILENTOPERATION = 0x00001000'u32 - BINDF_PRAGMA_NO_CACHE = 0x00002000'u32 - - ERROR_FILE_NOT_FOUND = 2 - ERROR_ACCESS_DENIED = 5 - - BINDSTATUS_FINDINGRESOURCE = 1 - BINDSTATUS_CONNECTING = 2 - BINDSTATUS_REDIRECTING = 3 - BINDSTATUS_BEGINDOWNLOADDATA = 4 - BINDSTATUS_DOWNLOADINGDATA = 5 - BINDSTATUS_ENDDOWNLOADDATA = 6 - BINDSTATUS_SENDINGREQUEST = 11 - BINDSTATUS_MIMETYPEAVAILABLE = 13 - BINDSTATUS_CACHEFILENAMEAVAILABLE = 14 - BINDSTATUS_PROXYDETECTING = 32 - BINDSTATUS_COOKIE_SENT = 34 - -type - STGMEDIUM = object - tymed: DWORD - pstg: pointer - pUnkForRelease: pointer - - SECURITY_ATTRIBUTES = object - nLength*: uint32 - lpSecurityDescriptor*: pointer - bInheritHandle*: int32 - - BINDINFO = object - cbSize: ULONG - stgmedData: STGMEDIUM - szExtraInfo: LPWSTR - grfBindInfoF: DWORD - dwBindVerb: DWORD - szCustomVerb: LPWSTR - cbstgmedData: DWORD - dwOptions: DWORD - dwOptionsFlags: DWORD - dwCodePage: DWORD - securityAttributes: SECURITY_ATTRIBUTES - iid: UUID - pUnk: pointer - dwReserved: DWORD - - IBindStatusCallback = object - vtable: ptr IBindStatusCallbackVTable - options: set[DownloadOptions] - objectRefCount: ULONG - binfoFlags: DWORD - progressCallback: DownloadProgressCallback - - PIBindStatusCallback = ptr IBindStatusCallback - LPBINDSTATUSCALLBACK = PIBindStatusCallback - - IBindStatusCallbackVTable = object - QueryInterface: proc (self: PIBindStatusCallback, - riid: ptr UUID, - pvObject: ptr pointer): HRESULT {.gcsafe,stdcall.} - AddRef: proc(self: PIBindStatusCallback): ULONG {.gcsafe, stdcall.} - Release: proc(self: PIBindStatusCallback): ULONG {.gcsafe, stdcall.} - OnStartBinding: proc(self: PIBindStatusCallback, - dwReserved: DWORD, pib: pointer): HRESULT - {.gcsafe, stdcall.} - GetPriority: proc(self: PIBindStatusCallback, pnPriority: ptr LONG): HRESULT - {.gcsafe, stdcall.} - OnLowResource: proc(self: PIBindStatusCallback, dwReserved: DWORD): HRESULT - {.gcsafe, stdcall.} - OnProgress: proc(self: PIBindStatusCallback, ulProgress: ULONG, - ulProgressMax: ULONG, ulStatusCode: ULONG, - szStatusText: LPWSTR): HRESULT - {.gcsafe, stdcall.} - OnStopBinding: proc(self: PIBindStatusCallback, hresult: HRESULT, - szError: LPWSTR): HRESULT - {.gcsafe, stdcall.} - GetBindInfo: proc(self: PIBindStatusCallback, grfBINDF: ptr DWORD, - pbindinfo: ptr BINDINFO): HRESULT - {.gcsafe, stdcall.} - OnDataAvailable: proc(self: PIBindStatusCallback, grfBSCF: DWORD, - dwSize: DWORD, pformatetc: pointer, - pstgmed: pointer): HRESULT - {.gcsafe, stdcall.} - OnObjectAvailable: proc(self: PIBindStatusCallback, riid: REFIID, - punk: pointer): HRESULT - {.gcsafe, stdcall.} - -template FAILED(hr: HRESULT): bool = - (hr < 0) - -proc URLDownloadToFile(pCaller: pointer, szUrl: LPWSTR, szFileName: LPWSTR, - dwReserved: DWORD, - lpfnCb: LPBINDSTATUSCALLBACK): HRESULT - {.stdcall, dynlib: "urlmon.dll", importc: "URLDownloadToFileW".} - -proc WideCharToMultiByte(CodePage: UINT, dwFlags: DWORD, - lpWideCharStr: ptr OLECHAR, cchWideChar: cint, - lpMultiByteStr: ptr char, cbMultiByte: cint, - lpDefaultChar: ptr char, - lpUsedDefaultChar: ptr uint32): cint - {.stdcall, dynlib: "kernel32.dll", importc: "WideCharToMultiByte".} - -proc MultiByteToWideChar(CodePage: UINT, dwFlags: DWORD, - lpMultiByteStr: ptr char, cbMultiByte: cint, - lpWideCharStr: ptr OLECHAR, cchWideChar: cint): cint - {.stdcall, dynlib: "kernel32.dll", importc: "MultiByteToWideChar".} -proc DeleteUrlCacheEntry(lpszUrlName: LPWSTR): int32 - {.stdcall, dynlib: "wininet.dll", importc: "DeleteUrlCacheEntryW".} - -proc `==`(a, b: UUID): bool = - result = false - if a[0] == b[0] and a[1] == b[1] and - a[2] == b[2] and a[3] == b[3]: - result = true - -proc `$`(bstr: LPWSTR): string = - var buffer: char - var count = WideCharToMultiByte(CP_UTF8, 0, bstr, -1, addr(buffer), 0, - nil, nil) - if count == 0: - raiseOsError(osLastError()) - else: - result = newString(count + 8) - let res = WideCharToMultiByte(CP_UTF8, 0, bstr, -1, addr(result[0]), count, - nil, nil) - if res == 0: - raiseOsError(osLastError()) - result.setLen(res - 1) - -proc toBstring(str: string): LPWSTR = - var buffer: OLECHAR - var count = MultiByteToWideChar(CP_UTF8, 0, unsafeAddr(str[0]), -1, - addr(buffer), 0) - if count == 0: - raiseOsError(osLastError()) - else: - result = cast[LPWSTR](alloc0((count + 1) * sizeof(OLECHAR))) - let res = MultiByteToWideChar(CP_UTF8, 0, unsafeAddr(str[0]), -1, - result, count) - if res == 0: - raiseOsError(osLastError()) - -proc freeBstring(bstr: LPWSTR) = - dealloc(bstr) - -proc getStatus(scode: ULONG): DownloadStatus = - case scode - of 0: result = statusError - of BINDSTATUS_PROXYDETECTING: result = statusProxyDetecting - of BINDSTATUS_REDIRECTING: result = statusRedirecting - of BINDSTATUS_COOKIE_SENT: result = statusCookieSent - of BINDSTATUS_FINDINGRESOURCE: result = statusResolving - of BINDSTATUS_CONNECTING: result = statusConnecting - of BINDSTATUS_SENDINGREQUEST: result = statusRequesting - of BINDSTATUS_MIMETYPEAVAILABLE: result = statusMimetypeAvailable - of BINDSTATUS_BEGINDOWNLOADDATA: result = statusBeginDownloading - of BINDSTATUS_DOWNLOADINGDATA: result = statusDownloading - of BINDSTATUS_ENDDOWNLOADDATA: result = statusEndDownloading - of BINDSTATUS_CACHEFILENAMEAVAILABLE: result = statusCacheAvailable - else: result = statusUnsupported - -proc addRef(self: PIBindStatusCallback): ULONG {.gcsafe, stdcall.} = - inc(self.objectRefCount) - result = self.objectRefCount - -proc release(self: PIBindStatusCallback): ULONG {.gcsafe, stdcall.} = - dec(self.objectRefCount) - result = self.objectRefCount - -proc queryInterface(self: PIBindStatusCallback, riid: ptr UUID, - pvObject: ptr pointer): HRESULT {.gcsafe,stdcall.} = - pvObject[] = nil - - if riid[] == IID_IUnknown: - pvObject[] = cast[pointer](self) - elif riid[] == IID_IBindStatusCallback: - pvObject[] = cast[pointer](self) - - if not isNil(pvObject[]): - discard addRef(self) - result = S_OK - else: - result = E_NOINTERFACE - -proc onStartBinding(self: PIBindStatusCallback, dwReserved: DWORD, - pib: pointer): HRESULT {.gcsafe, stdcall.} = - result = S_OK - -proc getPriority(self: PIBindStatusCallback, - pnPriority: ptr LONG): HRESULT {.gcsafe, stdcall.} = - result = E_NOTIMPL - -proc onLowResource(self: PIBindStatusCallback, - dwReserved: DWORD): HRESULT {.gcsafe, stdcall.} = - result = S_OK - -proc onStopBinding(self: PIBindStatusCallback, - hresult: HRESULT, szError: LPWSTR): HRESULT - {.gcsafe, stdcall.} = - result = S_OK - -proc getBindInfo(self: PIBindStatusCallback, - grfBINDF: ptr DWORD, pbindinfo: ptr BINDINFO): HRESULT - {.gcsafe, stdcall.} = - var cbSize = pbindinfo.cbSize - zeroMem(cast[pointer](pbindinfo), cbSize) - pbindinfo.cbSize = cbSize - grfBINDF[] = self.binfoFlags - result = S_OK - -proc onDataAvailable(self: PIBindStatusCallback, - grfBSCF: DWORD, dwSize: DWORD, pformatetc: pointer, - pstgmed: pointer): HRESULT {.gcsafe, stdcall.} = - result = S_OK - -proc onObjectAvailable(self: PIBindStatusCallback, - riid: REFIID, punk: pointer): HRESULT - {.gcsafe, stdcall.} = - result = S_OK - -proc onProgress(self: PIBindStatusCallback, - ulProgress: ULONG, ulProgressMax: ULONG, ulStatusCode: ULONG, - szStatusText: LPWSTR): HRESULT {.gcsafe, stdcall.} = - var message: string - if optUseProgressCallback in self.options: - if not isNil(szStatusText): - message = $szStatusText - else: - message = "" - self.progressCallback(getStatus(ulStatusCode), uint(ulProgress), - uint(ulProgressMax), message) - result = S_OK - -proc newBindStatusCallback(): IBindStatusCallback = - result = IBindStatusCallback() - result.vtable = cast[ptr IBindStatusCallbackVTable]( - alloc0(sizeof(IBindStatusCallbackVTable)) - ) - result.vtable.QueryInterface = queryInterface - result.vtable.AddRef = addRef - result.vtable.Release = release - result.vtable.OnStartBinding = onStartBinding - result.vtable.GetPriority = getPriority - result.vtable.OnLowResource = onLowResource - result.vtable.OnStopBinding = onStopBinding - result.vtable.GetBindInfo = getBindInfo - result.vtable.OnDataAvailable = onDataAvailable - result.vtable.OnObjectAvailable = onObjectAvailable - result.vtable.OnProgress = onProgress - result.objectRefCount = 1 - -proc freeBindStatusCallback(v: var IBindStatusCallback) = - dealloc(v.vtable) - -proc downloadToFile*(szUrl: string, szFileName: string, - options: set[DownloadOptions] = {}, - progresscb: DownloadProgressCallback = nil) = - ## Downloads from URL specified in ``szUrl`` to local filesystem path - ## specified in ``szFileName``. - ## - ## szUrl - ## URL to download, international names are supported. - ## szFileName - ## Destination path for downloading resource. - ## options - ## Downloading options. Currently only 2 options supported. - ## progresscb - ## Callback procedure, which will be called throughout the download - ## process, indicating status and progress. - ## - ## Available downloading options: - ## - ## optUseCache - ## Try to use Windows cache when downloading. - ## optIgnoreSecurity - ## Ignore HTTPS security problems, e.g. self-signed HTTPS certificate. - ## - var bszUrl = szUrl.toBstring() - var bszFile = szFileName.toBstring() - var bstatus = newBindStatusCallback() - - bstatus.options = {} - - if optUseCache notin options: - bstatus.options.incl(optUseCache) - let res = DeleteUrlCacheEntry(bszUrl) - if res == 0: - let err = osLastError() - if err.int notin {ERROR_ACCESS_DENIED, ERROR_FILE_NOT_FOUND}: - freeBindStatusCallback(bstatus) - freeBstring(bszUrl) - freeBstring(bszFile) - raiseOsError(err) - - bstatus.binfoFlags = BINDF_GETNEWESTVERSION or BINDF_RESYNCHRONIZE or - BINDF_PRAGMA_NO_CACHE or BINDF_NO_UI or - BINDF_SILENTOPERATION - - if optIgnoreSecurity in options: - bstatus.binfoFlags = bstatus.binfoFlags or BINDF_IGNORESECURITYPROBLEM - - if not isNil(progresscb): - bstatus.options.incl(optUseProgressCallback) - bstatus.progressCallback = progresscb - - let res = URLDownloadToFile(nil, bszUrl, bszFile, 0, addr bstatus) - if FAILED(res): - freeBindStatusCallback(bstatus) - freeBstring(bszUrl) - freeBstring(bszFile) - raiseOsError(OSErrorCode(res)) - - freeBindStatusCallback(bstatus) - freeBstring(bszUrl) - freeBstring(bszFile) - -when isMainModule: - proc progress(status: DownloadStatus, progress: uint, progressMax: uint, - message: string) {.gcsafe.} = - const downset: set[DownloadStatus] = {statusBeginDownloading, - statusDownloading, statusEndDownloading} - if status in downset: - var message = "Downloaded " & $progress & " of " & $progressMax & "\c" - stdout.write(message) - else: - echo "Status [" & $status & "] message = [" & $message & "]" - - downloadToFile("https://nim-lang.org/download/mingw64.7z", - "test.zip", {optUseCache}, progress) From 63cca93ea9bcab3c1fea39ff3789fd3a656d2db2 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Thu, 19 May 2022 17:11:53 +0200 Subject: [PATCH 1624/2472] testament: include extra options in test name (#19801) there's currently no (simple) way to disambiguate which option failed --- testament/categories.nim | 12 +++--- testament/testament.nim | 79 +++++++++++++++++++--------------------- 2 files changed, 44 insertions(+), 47 deletions(-) diff --git a/testament/categories.nim b/testament/categories.nim index 46cc3b210..a092fec84 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -437,7 +437,7 @@ proc testNimblePackages(r: var TResults; cat: Category; packageFilter: string) = if pkg.allowFailure: inc r.passed inc r.failedButAllowed - addResult(r, test, targetC, "", cmd & "\n" & outp, reFailed, allowFailure = pkg.allowFailure) + addResult(r, test, targetC, "", "", cmd & "\n" & outp, reFailed, allowFailure = pkg.allowFailure) continue outp @@ -450,21 +450,21 @@ proc testNimblePackages(r: var TResults; cat: Category; packageFilter: string) = discard tryCommand("nimble install --depsOnly -y", maxRetries = 3) discard tryCommand(pkg.cmd, reFailed = reBuildFailed) inc r.passed - r.addResult(test, targetC, "", "", reSuccess, allowFailure = pkg.allowFailure) + r.addResult(test, targetC, "", "", "", reSuccess, allowFailure = pkg.allowFailure) errors = r.total - r.passed if errors == 0: - r.addResult(packageFileTest, targetC, "", "", reSuccess) + r.addResult(packageFileTest, targetC, "", "", "", reSuccess) else: - r.addResult(packageFileTest, targetC, "", "", reBuildFailed) + r.addResult(packageFileTest, targetC, "", "", "", reBuildFailed) except JsonParsingError: errors = 1 - r.addResult(packageFileTest, targetC, "", "Invalid package file", reBuildFailed) + r.addResult(packageFileTest, targetC, "", "", "Invalid package file", reBuildFailed) raise except ValueError: errors = 1 - r.addResult(packageFileTest, targetC, "", "Unknown package", reBuildFailed) + r.addResult(packageFileTest, targetC, "", "", "Unknown package", reBuildFailed) raise # bug #18805 finally: if errors == 0: removeDir(packagesDir) diff --git a/testament/testament.nim b/testament/testament.nim index 4b5a8a147..a4329b982 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -259,7 +259,8 @@ Tests skipped: $4 / $1
              """ % [$x.total, $x.passed, $x.failedButAllowed, $x.skipped] proc addResult(r: var TResults, test: TTest, target: TTarget, - expected, given: string, successOrig: TResultEnum, allowFailure = false, givenSpec: ptr TSpec = nil) = + extraOptions, expected, given: string, successOrig: TResultEnum, + allowFailure = false, givenSpec: ptr TSpec = nil) = # instead of `ptr TSpec` we could also use `Option[TSpec]`; passing `givenSpec` makes it easier to get what we need # instead of having to pass individual fields, or abusing existing ones like expected vs given. # test.name is easier to find than test.name.extractFilename @@ -269,6 +270,7 @@ proc addResult(r: var TResults, test: TTest, target: TTarget, if allowFailure: name.add " (allowed to fail) " if test.options.len > 0: name.add ' ' & test.options + if extraOptions.len > 0: name.add ' ' & extraOptions let duration = epochTime() - test.startTime let success = if test.spec.timeout > 0.0 and duration > test.spec.timeout: reTimeout @@ -333,7 +335,8 @@ proc addResult(r: var TResults, test: TTest, target: TTarget, discard waitForExit(p) close(p) -proc checkForInlineErrors(r: var TResults, expected, given: TSpec, test: TTest, target: TTarget) = +proc checkForInlineErrors(r: var TResults, expected, given: TSpec, test: TTest, + target: TTarget, extraOptions: string) = let pegLine = peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' {[^:]*} ':' \s* {.*}" var covered = initIntSet() for line in splitLines(given.nimout): @@ -367,10 +370,10 @@ proc checkForInlineErrors(r: var TResults, expected, given: TSpec, test: TTest, e.add ": " e.add expected.inlineErrors[j].msg - r.addResult(test, target, e, given.nimout, reMsgsDiffer) + r.addResult(test, target, extraOptions, e, given.nimout, reMsgsDiffer) break coverCheck - r.addResult(test, target, "", given.msg, reSuccess) + r.addResult(test, target, extraOptions, "", given.msg, reSuccess) inc(r.passed) proc nimoutCheck(expected, given: TSpec): bool = @@ -381,22 +384,23 @@ proc nimoutCheck(expected, given: TSpec): bool = elif expected.nimout.len > 0 and not greedyOrderedSubsetLines(expected.nimout, given.nimout): result = false -proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest, target: TTarget) = +proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest, + target: TTarget, extraOptions: string) = if expected.inlineErrors.len > 0: - checkForInlineErrors(r, expected, given, test, target) + checkForInlineErrors(r, expected, given, test, target, extraOptions) elif strip(expected.msg) notin strip(given.msg): - r.addResult(test, target, expected.msg, given.msg, reMsgsDiffer) + r.addResult(test, target, extraOptions, expected.msg, given.msg, reMsgsDiffer) elif not nimoutCheck(expected, given): - r.addResult(test, target, expected.nimout, given.nimout, reMsgsDiffer) + r.addResult(test, target, extraOptions, expected.nimout, given.nimout, reMsgsDiffer) elif extractFilename(expected.file) != extractFilename(given.file) and "internal error:" notin expected.msg: - r.addResult(test, target, expected.file, given.file, reFilesDiffer) + r.addResult(test, target, extraOptions, expected.file, given.file, reFilesDiffer) elif expected.line != given.line and expected.line != 0 or expected.column != given.column and expected.column != 0: - r.addResult(test, target, $expected.line & ':' & $expected.column, + r.addResult(test, target, extraOptions, $expected.line & ':' & $expected.column, $given.line & ':' & $given.column, reLinesDiffer) else: - r.addResult(test, target, expected.msg, given.msg, reSuccess) + r.addResult(test, target, extraOptions, expected.msg, given.msg, reSuccess) inc(r.passed) proc generatedFile(test: TTest, target: TTarget): string = @@ -434,8 +438,8 @@ proc codegenCheck(test: TTest, target: TTarget, spec: TSpec, expectedMsg: var st given.err = reCodeNotFound echo getCurrentExceptionMsg() -proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec, - expected: TSpec; r: var TResults) = +proc compilerOutputTests(test: TTest, target: TTarget, extraOptions: string, + given: var TSpec, expected: TSpec; r: var TResults) = var expectedmsg: string = "" var givenmsg: string = "" if given.err == reSuccess: @@ -449,7 +453,7 @@ proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec, else: givenmsg = "$ " & given.cmd & '\n' & given.nimout if given.err == reSuccess: inc(r.passed) - r.addResult(test, target, expectedmsg, givenmsg, given.err) + r.addResult(test, target, extraOptions, expectedmsg, givenmsg, given.err) proc getTestSpecTarget(): TTarget = if getEnv("NIM_COMPILE_TO_CPP", "false") == "true": @@ -457,16 +461,6 @@ proc getTestSpecTarget(): TTarget = else: result = targetC -proc checkDisabled(r: var TResults, test: TTest): bool = - if test.spec.err in {reDisabled, reJoined}: - # targetC is a lie, but parameter is required - r.addResult(test, targetC, "", "", test.spec.err) - inc(r.skipped) - inc(r.total) - result = false - else: - result = true - var count = 0 proc equalModuloLastNewline(a, b: string): bool = @@ -474,8 +468,13 @@ proc equalModuloLastNewline(a, b: string): bool = result = a == b or b.endsWith("\n") and a == b[0 ..< ^1] proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, - target: TTarget, nimcache: string, extraOptions = "") = + target: TTarget, extraOptions: string, nimcache: string) = test.startTime = epochTime() + if test.spec.err in {reDisabled, reJoined}: + r.addResult(test, target, extraOptions, "", "", test.spec.err) + inc(r.skipped) + return + template callNimCompilerImpl(): untyped = # xxx this used to also pass: `--stdout --hint:Path:off`, but was done inconsistently # with other branches @@ -483,21 +482,21 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, case expected.action of actionCompile: var given = callNimCompilerImpl() - compilerOutputTests(test, target, given, expected, r) + compilerOutputTests(test, target, extraOptions, given, expected, r) of actionRun: var given = callNimCompilerImpl() if given.err != reSuccess: - r.addResult(test, target, "", "$ " & given.cmd & '\n' & given.nimout, given.err, givenSpec = given.addr) + r.addResult(test, target, extraOptions, "", "$ " & given.cmd & '\n' & given.nimout, given.err, givenSpec = given.addr) else: let isJsTarget = target == targetJS var exeFile = changeFileExt(test.name, if isJsTarget: "js" else: ExeExt) if not fileExists(exeFile): - r.addResult(test, target, expected.output, + r.addResult(test, target, extraOptions, expected.output, "executable not found: " & exeFile, reExeNotFound) else: let nodejs = if isJsTarget: findNodeJs() else: "" if isJsTarget and nodejs == "": - r.addResult(test, target, expected.output, "nodejs binary not in PATH", + r.addResult(test, target, extraOptions, expected.output, "nodejs binary not in PATH", reExeNotFound) else: var exeCmd: string @@ -528,24 +527,24 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, else: buf if exitCode != expected.exitCode: - r.addResult(test, target, "exitcode: " & $expected.exitCode, + r.addResult(test, target, extraOptions, "exitcode: " & $expected.exitCode, "exitcode: " & $exitCode & "\n\nOutput:\n" & bufB, reExitcodesDiffer) elif (expected.outputCheck == ocEqual and not expected.output.equalModuloLastNewline(bufB)) or (expected.outputCheck == ocSubstr and expected.output notin bufB): given.err = reOutputsDiffer - r.addResult(test, target, expected.output, bufB, reOutputsDiffer) + r.addResult(test, target, extraOptions, expected.output, bufB, reOutputsDiffer) else: - compilerOutputTests(test, target, given, expected, r) + compilerOutputTests(test, target, extraOptions, given, expected, r) of actionReject: let given = callNimCompilerImpl() - cmpMsgs(r, expected, given, test, target) + cmpMsgs(r, expected, given, test, target, extraOptions) -proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions = "") = +proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions: string) = for target in expected.targets: inc(r.total) if target notin gTargets: - r.addResult(test, target, "", "", reDisabled) + r.addResult(test, target, extraOptions, "", "", reDisabled) inc(r.skipped) elif simulate: inc count @@ -553,16 +552,15 @@ proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions = else: let nimcache = nimcacheDir(test.name, test.options, target) var testClone = test - testSpecHelper(r, testClone, expected, target, nimcache, extraOptions) + testSpecHelper(r, testClone, expected, target, extraOptions, nimcache) proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) = var expected = test.spec if expected.parseErrors.len > 0: # targetC is a lie, but a parameter is required - r.addResult(test, targetC, "", expected.parseErrors, reInvalidSpec) + r.addResult(test, targetC, "", "", expected.parseErrors, reInvalidSpec) inc(r.total) return - if not checkDisabled(r, test): return expected.targets.incl targets # still no target specified at all @@ -572,14 +570,13 @@ proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) = for m in test.spec.matrix: targetHelper(r, test, expected, m) else: - targetHelper(r, test, expected) + targetHelper(r, test, expected, "") proc testSpecWithNimcache(r: var TResults, test: TTest; nimcache: string) {.used.} = - if not checkDisabled(r, test): return for target in test.spec.targets: inc(r.total) var testClone = test - testSpecHelper(r, testClone, test.spec, target, nimcache) + testSpecHelper(r, testClone, test.spec, target, "", nimcache) proc makeTest(test, options: string, cat: Category): TTest = result.cat = cat From a8426fc7890f1bea1c507ddb2fe42662f7a574f8 Mon Sep 17 00:00:00 2001 From: quantimnot <54247259+quantimnot@users.noreply.github.com> Date: Mon, 23 May 2022 00:12:34 -0400 Subject: [PATCH 1625/2472] Fix default testament target in docs and cli help (#19796) Co-authored-by: quantimnot --- doc/testament.rst | 4 ++-- testament/testament.nim | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/testament.rst b/doc/testament.rst index 5590cc6d7..427a7ff71 100644 --- a/doc/testament.rst +++ b/doc/testament.rst @@ -36,7 +36,7 @@ Options (for debugging) --failing Only show failing/ignored tests --targets:"c cpp js objc" - Run tests for specified targets (default: all) + Run tests for specified targets (default: c) --nim:path Use a particular nim executable (default: $PATH/nim) --directory:dir Change to directory dir before reading the tests or doing anything else. @@ -164,7 +164,7 @@ Example "template" **to edit** and write a Testament unittest: # Timeout seconds to run the test. Fractional values are supported. timeout: 1.5 - # Targets to run the test into (c, cpp, objc, js). + # Targets to run the test into (c, cpp, objc, js). Defaults to c. targets: "c js" # flags with which to run the test, delimited by `;` diff --git a/testament/testament.nim b/testament/testament.nim index a4329b982..d2b0e7fb4 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -54,7 +54,7 @@ Options: --verbose print commands (compiling and running tests) --simulate see what tests would be run but don't run them (for debugging) --failing only show failing/ignored tests - --targets:"c cpp js objc" run tests for specified targets (default: all) + --targets:"c cpp js objc" run tests for specified targets (default: c) --nim:path use a particular nim executable (default: $$PATH/nim) --directory:dir Change to directory dir before reading the tests or doing anything else. --colors:on|off Turn messages coloring on|off. From 05c0419658e3f97c29a12a16e99c0f8b842622c8 Mon Sep 17 00:00:00 2001 From: quantimnot <54247259+quantimnot@users.noreply.github.com> Date: Mon, 23 May 2022 00:17:32 -0400 Subject: [PATCH 1626/2472] Fix global destructor injection for JS backend (#19797) * Fix global destructor injection for JS backend * Moved global destructors injection before the final call to transform and generate JS code. It had previously been after and thus not no JS was generated for them. * Added some internal documentation of `jsgen`. * Enable a current destructor test to cover the JS backend as well. * Fixes the JS aspect of #17237. * Fixed global destructor injection order for JS backend Co-authored-by: quantimnot --- compiler/jsgen.nim | 30 +++++++++++++++++++++++++----- tests/destructor/t5342.nim | 3 ++- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index b9a73daf9..6d34fcae2 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -2392,6 +2392,7 @@ proc optionalLine(p: Rope): Rope = return p & "\L" proc genProc(oldProc: PProc, prc: PSym): Rope = + ## Generate a JS procedure ('function'). var resultSym: PSym a: TCompRes @@ -2673,6 +2674,7 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) = else: internalError(p.config, n.info, "gen: unknown node type: " & $n.kind) proc newModule(g: ModuleGraph; module: PSym): BModule = + ## Create a new JS backend module node. new(result) result.module = module result.sigConflicts = initCountTable[SigHash]() @@ -2684,6 +2686,7 @@ proc newModule(g: ModuleGraph; module: PSym): BModule = PGlobals(g.backend).inSystem = true proc genHeader(): Rope = + ## Generate the JS header. result = rope("""/* Generated by the Nim Compiler v$1 */ var framePtr = null; var excHandler = 0; @@ -2714,6 +2717,8 @@ proc addHcrInitGuards(p: PProc, n: PNode, genStmt(p, n) proc genModule(p: PProc, n: PNode) = + ## Generate the JS module code. + ## Called for each top level node in a Nim module. if optStackTrace in p.options: p.body.add(frameCreate(p, makeJSString("module " & p.module.module.name.s), @@ -2742,6 +2747,7 @@ proc genModule(p: PProc, n: PNode) = p.body.add(frameDestroy(p)) proc myProcess(b: PPassContext, n: PNode): PNode = + ## Generate JS code for a node. result = n let m = BModule(b) if passes.skipCodegen(m.config, n): return n @@ -2754,6 +2760,7 @@ proc myProcess(b: PPassContext, n: PNode): PNode = p.g.code.add(p.body) proc wholeCode(graph: ModuleGraph; m: BModule): Rope = + ## Combine source code from all nodes. let globals = PGlobals(graph.backend) for prc in globals.forwarded: if not globals.generatedSyms.containsOrIncl(prc.id): @@ -2779,28 +2786,41 @@ proc getClassName(t: PType): Rope = else: result = rope(s.name.s) proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode = - result = myProcess(b, n) + ## Finalize JS code generation of a Nim module. + ## Param `n` may contain nodes returned from the last module close call. var m = BModule(b) if sfMainModule in m.module.flags: - for destructorCall in graph.globalDestructors: - n.add destructorCall + # Add global destructors to the module. + # This must come before the last call to `myProcess`. + for i in countdown(high(graph.globalDestructors), 0): + n.add graph.globalDestructors[i] + # Process any nodes left over from the last call to `myClose`. + result = myProcess(b, n) + # Some codegen is different (such as no stacktraces; see `initProcOptions`) + # when `std/system` is being processed. if sfSystemModule in m.module.flags: PGlobals(graph.backend).inSystem = false + # Check if codegen should continue before any files are generated. + # It may bail early is if too many errors have been raised. if passes.skipCodegen(m.config, n): return n + # Nim modules are compiled into a single JS file. + # If this is the main module, then this is the final call to `myClose`. if sfMainModule in m.module.flags: var code = genHeader() & wholeCode(graph, m) let outFile = m.config.prepareToWriteOutput() - + # Generate an optional source map. if optSourcemap in m.config.globalOptions: var map: SourceMap (code, map) = genSourceMap($(code), outFile.string) writeFile(outFile.string & ".map", $(%map)) + # Check if the generated JS code matches the output file, or else + # write it to the file. if not equalsFile(code, outFile): if not writeRope(code, outFile): rawMessage(m.config, errCannotOpenFile, outFile.string) - proc myOpen(graph: ModuleGraph; s: PSym; idgen: IdGenerator): PPassContext = + ## Create the JS backend pass context `BModule` for a Nim module. result = newModule(graph, s) result.idgen = idgen diff --git a/tests/destructor/t5342.nim b/tests/destructor/t5342.nim index 19354ea64..0acd5ef9d 100644 --- a/tests/destructor/t5342.nim +++ b/tests/destructor/t5342.nim @@ -1,5 +1,6 @@ discard """ - matrix: "--gc:refc; --gc:arc" + matrix: "--mm:refc; --mm:arc" + targets: "c js" output: ''' 1 2 From b1b5e3ab10749851e942d624ee85188d8b06822e Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Tue, 24 May 2022 12:37:19 -0300 Subject: [PATCH 1627/2472] Add Array.shift (#19811) * Add Array.shift for JavaScript targets * Add Array.shift for JavaScript targets --- changelog.md | 2 +- lib/js/jscore.nim | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 3b2685547..e120dcfb8 100644 --- a/changelog.md +++ b/changelog.md @@ -37,7 +37,7 @@ becomes an alias for `addr`. - Added `initDateTime` in `times` to create a datetime from a weekday, and ISO 8601 week number and week-based year. - Added `getIsoWeekAndYear` in `times` to get an ISO week number along with the corresponding ISO week-based year from a datetime. - Added `getIsoWeeksInYear` in `times` to return the number of weeks in an ISO week-based year. - +- Added [`Array.shift`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/shift) for JavaScript targets. - Added `std/oserrors` for OS error reporting. Added `std/envvars` for environment variables handling. - Removed deprecated `oids.oidToString`. - Remove define `nimExperimentalAsyncjsThen` for `std/asyncjs.then` and `std/jsfetch`. diff --git a/lib/js/jscore.nim b/lib/js/jscore.nim index 61c188431..4518f32ce 100644 --- a/lib/js/jscore.nim +++ b/lib/js/jscore.nim @@ -123,3 +123,12 @@ since (1, 5): assert [1, 2, 3, 4, 5].copyWithin(0, 3) == @[4, 5, 3, 4, 5] assert [1, 2, 3, 4, 5].copyWithin(0, 3, 4) == @[4, 2, 3, 4, 5] assert [1, 2, 3, 4, 5].copyWithin(-2, -3, -1) == @[1, 2, 3, 3, 4] + + +since (1, 7): + func shift*[T](self: seq[T]): T {.importjs: "#.$1()".} = + ## https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/shift + runnableExamples: + var arrai = @[1, 2, 3] + assert arrai.shift() == 1 + assert arrai == @[2, 3] From ec0cec3170a3f175b9e02ca98e82c963fe64df4f Mon Sep 17 00:00:00 2001 From: tandy1000 Date: Tue, 24 May 2022 16:37:39 +0100 Subject: [PATCH 1628/2472] Add `document.hidden` and `document.visibilityState` properties (#19817) --- lib/js/dom.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/js/dom.nim b/lib/js/dom.nim index c1a677258..1a62780a7 100644 --- a/lib/js/dom.nim +++ b/lib/js/dom.nim @@ -216,11 +216,13 @@ type defaultCharset*: cstring fgColor*: cstring head*: Element + hidden*: bool lastModified*: cstring linkColor*: cstring referrer*: cstring title*: cstring URL*: cstring + visibilityState*: cstring vlinkColor*: cstring anchors*: seq[AnchorElement] forms*: seq[FormElement] From d81edcacc6740a99b0360cf31ddc2f2a1626006c Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Tue, 24 May 2022 23:53:01 +0800 Subject: [PATCH 1629/2472] Add link to std/tempfiles in the docs; fix #19155 (#19807) --- doc/lib.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/lib.rst b/doc/lib.rst index f72012777..2f3a315a8 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -251,6 +251,10 @@ Generic Operating System Services This module contains a few procedures to control the *terminal* (also called *console*). The implementation simply uses ANSI escape sequences and does not depend on any other module. + +* `tempfiles `_ + This module provides some utils to generate temporary path names and + create temporary files and directories. Math libraries From 14960fa754f42c820727bed9308e075b2cb923f6 Mon Sep 17 00:00:00 2001 From: Michael New Date: Sun, 29 May 2022 19:40:15 -0700 Subject: [PATCH 1630/2472] Fix typo and incorrect pragma name (#19847) --- doc/manual_experimental_strictnotnil.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual_experimental_strictnotnil.rst b/doc/manual_experimental_strictnotnil.rst index b14e5f9f3..ebfca7e51 100644 --- a/doc/manual_experimental_strictnotnil.rst +++ b/doc/manual_experimental_strictnotnil.rst @@ -172,7 +172,7 @@ We might need to check for `strictFuncs` pure funcs and not do that then. For field expressions `a.field`, we calculate an integer value based on a hash of the tree and just accept equivalent trees as equivalent expressions. For item expression `a[index]`, we also calculate an integer value based on a hash of the tree and accept equivalent trees as equivalent expressions: for static values only. -For now we support only constant indices: we dont track expression with no-const indices. For those we just report a warning even if they are safe for now: one can use a local variable to workaround. For loops this might be annoying: so one should be able to turn off locally the warning using the `{.warning[StrictCheckNotNil]:off}.`. +For now we support only constant indices: we dont track expression with no-const indices. For those we just report a warning even if they are safe for now: one can use a local variable to workaround. For loops this might be annoying: so one should be able to turn off locally the warning using the `{.warning[StrictNotNil]:off.}`. For bracket expressions, in the future we might count `a[]` as the same general expression. This means we should should the index but otherwise handle it the same for assign (maybe "aliasing" all the non-static elements) and differentiate only for static: e.g. `a[0]` and `a[1]`. From 004fc23a3fa79fc32e0fb5d4f438ddc22964b0da Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Mon, 30 May 2022 16:41:24 +0800 Subject: [PATCH 1631/2472] [vm] remove unused opcSubstr opcode (#19834) remove unused opcSubstr --- compiler/vm.nim | 8 -------- compiler/vmdef.nim | 4 ++-- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/compiler/vm.nim b/compiler/vm.nim index fa1c71c85..bbcff77ba 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -1185,14 +1185,6 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = of opcContainsSet: decodeBC(rkInt) regs[ra].intVal = ord(inSet(regs[rb].node, regs[rc].regToNode)) - of opcSubStr: - decodeBC(rkNode) - inc pc - assert c.code[pc].opcode == opcSubStr - let rd = c.code[pc].regA - createStr regs[ra] - regs[ra].node.strVal = substr(regs[rb].node.strVal, - regs[rc].intVal.int, regs[rd].intVal.int) of opcParseFloat: decodeBC(rkInt) inc pc diff --git a/compiler/vmdef.nim b/compiler/vmdef.nim index ecdbeff89..c653501ca 100644 --- a/compiler/vmdef.nim +++ b/compiler/vmdef.nim @@ -102,7 +102,7 @@ type opcMulSet, opcPlusSet, opcMinusSet, opcConcatStr, opcContainsSet, opcRepr, opcSetLenStr, opcSetLenSeq, opcIsNil, opcOf, opcIs, - opcSubStr, opcParseFloat, opcConv, opcCast, + opcParseFloat, opcConv, opcCast, opcQuit, opcInvalidField, opcNarrowS, opcNarrowU, opcSignExtend, @@ -306,7 +306,7 @@ proc registerCallback*(c: PCtx; name: string; callback: VmCallback): int {.disca const firstABxInstr* = opcTJmp largeInstrs* = { # instructions which use 2 int32s instead of 1: - opcSubStr, opcConv, opcCast, opcNewSeq, opcOf + opcConv, opcCast, opcNewSeq, opcOf } slotSomeTemp* = slotTempUnknown relativeJumps* = {opcTJmp, opcFJmp, opcJmp, opcJmpBack} From 497af2c0d9f311e050ba57842a6326ca70489d55 Mon Sep 17 00:00:00 2001 From: quantimnot <54247259+quantimnot@users.noreply.github.com> Date: Mon, 30 May 2022 05:07:47 -0400 Subject: [PATCH 1632/2472] Doc preferred import style in compiler (#19832) Doc prefered import style in compiler Co-authored-by: quantimnot --- doc/intern.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/intern.rst b/doc/intern.rst index a8846ae02..4ca7eff20 100644 --- a/doc/intern.rst +++ b/doc/intern.rst @@ -186,6 +186,7 @@ Coding Guidelines * Use a space after a colon, but not before it. * [deprecated] Start types with a capital `T`, unless they are pointers/references which start with `P`. +* Prefer `import package`:nim: over `from package import symbol`:nim:. See also the `API naming design `_ document. From 15f0b4867679120580b2f14bbb7a8b302505b34d Mon Sep 17 00:00:00 2001 From: Alfred Morgan Date: Mon, 30 May 2022 03:09:18 -0700 Subject: [PATCH 1633/2472] Zectbumo fixes 19824 (#19825) * borrowed `$` to make Time string friendly * added sep character parameter * Revert "added sep character parameter" This reverts commit 45f4b019a4883b6ba577ade1f94677266beb5960. * added sep character parameter * Revert "borrowed `$` to make Time string friendly" This reverts commit 10e2e44c9a04970f38cf66556635bdbb50b69136. * added uri tests and made changelong entry * Update lib/pure/uri.nim Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Update lib/pure/uri.nim Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Update tests/stdlib/turi.nim Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Update tests/stdlib/turi.nim Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- changelog.md | 1 + lib/pure/uri.nim | 14 ++++++++------ tests/stdlib/turi.nim | 4 ++++ 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/changelog.md b/changelog.md index e120dcfb8..9548f3034 100644 --- a/changelog.md +++ b/changelog.md @@ -39,6 +39,7 @@ becomes an alias for `addr`. - Added `getIsoWeeksInYear` in `times` to return the number of weeks in an ISO week-based year. - Added [`Array.shift`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/shift) for JavaScript targets. - Added `std/oserrors` for OS error reporting. Added `std/envvars` for environment variables handling. +- Added `sep` parameter in `std/uri` to specify the query separator. - Removed deprecated `oids.oidToString`. - Remove define `nimExperimentalAsyncjsThen` for `std/asyncjs.then` and `std/jsfetch`. diff --git a/lib/pure/uri.nim b/lib/pure/uri.nim index 7583dbd1e..1dbb018c2 100644 --- a/lib/pure/uri.nim +++ b/lib/pure/uri.nim @@ -123,13 +123,13 @@ func decodeUrl*(s: string, decodePlus = true): string = setLen(result, j) func encodeQuery*(query: openArray[(string, string)], usePlus = true, - omitEq = true): string = + omitEq = true, sep = '&'): string = ## Encodes a set of (key, value) parameters into a URL query string. ## ## Every (key, value) pair is URL-encoded and written as `key=value`. If the ## value is an empty string then the `=` is omitted, unless `omitEq` is ## false. - ## The pairs are joined together by a `&` character. + ## The pairs are joined together by the `sep` character. ## ## The `usePlus` parameter is passed down to the `encodeUrl` function that ## is used for the URL encoding of the string values. @@ -140,9 +140,10 @@ func encodeQuery*(query: openArray[(string, string)], usePlus = true, assert encodeQuery({: }) == "" assert encodeQuery({"a": "1", "b": "2"}) == "a=1&b=2" assert encodeQuery({"a": "1", "b": ""}) == "a=1&b" + assert encodeQuery({"a": "1", "b": ""}, omitEq = false, sep = ';') == "a=1;b=" for elem in query: - # Encode the `key = value` pairs and separate them with a '&' - if result.len > 0: result.add('&') + # Encode the `key = value` pairs and separate them with 'sep' + if result.len > 0: result.add(sep) let (key, val) = elem result.add(encodeUrl(key, usePlus)) # Omit the '=' if the value string is empty @@ -150,7 +151,7 @@ func encodeQuery*(query: openArray[(string, string)], usePlus = true, result.add('=') result.add(encodeUrl(val, usePlus)) -iterator decodeQuery*(data: string): tuple[key, value: string] = +iterator decodeQuery*(data: string, sep = '&'): tuple[key, value: string] = ## Reads and decodes the query string `data` and yields the `(key, value)` pairs ## the data consists of. If compiled with `-d:nimLegacyParseQueryStrict`, ## a `UriParseError` is raised when there is an unencoded `=` character in a decoded @@ -158,6 +159,7 @@ iterator decodeQuery*(data: string): tuple[key, value: string] = runnableExamples: import std/sequtils assert toSeq(decodeQuery("foo=1&bar=2=3")) == @[("foo", "1"), ("bar", "2=3")] + assert toSeq(decodeQuery("foo=1;bar=2=3", ';')) == @[("foo", "1"), ("bar", "2=3")] assert toSeq(decodeQuery("&a&=b&=&&")) == @[("", ""), ("a", ""), ("", "b"), ("", ""), ("", "")] proc parseData(data: string, i: int, field: var string, sep: char): int = @@ -186,7 +188,7 @@ iterator decodeQuery*(data: string): tuple[key, value: string] = when defined(nimLegacyParseQueryStrict): i = parseData(data, i, value, '=') else: - i = parseData(data, i, value, '&') + i = parseData(data, i, value, sep) yield (name, value) if i < data.len: when defined(nimLegacyParseQueryStrict): diff --git a/tests/stdlib/turi.nim b/tests/stdlib/turi.nim index a3b6afe2c..79ddd773b 100644 --- a/tests/stdlib/turi.nim +++ b/tests/stdlib/turi.nim @@ -274,7 +274,9 @@ template main() = doAssert encodeQuery({"foo": ""}) == "foo" doAssert encodeQuery({"foo": ""}, omitEq = false) == "foo=" doAssert encodeQuery({"a": "1", "b": "", "c": "3"}) == "a=1&b&c=3" + doAssert encodeQuery({"a": "1", "b": "", "c": "3"}, sep = ';') == "a=1;b;c=3" doAssert encodeQuery({"a": "1", "b": "", "c": "3"}, omitEq = false) == "a=1&b=&c=3" + doAssert encodeQuery({"a": "1", "b": "", "c": "3"}, omitEq = false, sep = ';') == "a=1;b=;c=3" block: # `?` block: @@ -300,7 +302,9 @@ template main() = block: # decodeQuery doAssert toSeq(decodeQuery("a=1&b=0")) == @[("a", "1"), ("b", "0")] + doAssert toSeq(decodeQuery("a=1;b=0", sep = ';')) == @[("a", "1"), ("b", "0")] doAssert toSeq(decodeQuery("a=1&b=2c=6")) == @[("a", "1"), ("b", "2c=6")] + doAssert toSeq(decodeQuery("a=1;b=2c=6", sep = ';')) == @[("a", "1"), ("b", "2c=6")] block: # bug #17481 let u1 = parseUri("./") From d30c6419a051a815e3fdb354ac79522f17e55bda Mon Sep 17 00:00:00 2001 From: quantimnot <54247259+quantimnot@users.noreply.github.com> Date: Mon, 30 May 2022 12:52:19 -0400 Subject: [PATCH 1634/2472] Refactor and doc package handling, module name mangling (#19821) * Refactor and doc package handling, module name mangling * Consolidate, de-duplicate and extend package handling * Alter how duplicate module names of a package are handled * Alter how module names are mangled * Fix crash when another package is named 'stdlib' (test case added) * Doc what defines a package in the manual Modules with duplicate names within a package used to be given 'fake' packages to resolve conflicts. That prevented the ability to discern if a module belonged to the current project package or a foreign package. They now have the proper package owner and the names are mangled in a consistent manner to prevent codegen clashes. All module names are now mangled the same. Stdlib was treated special before, but now it is same as any other package. This fixes a crash when a foreign package is named 'stdlib'. Module mangling is altered for both file paths and symbols used by the backends. Removed an unused module name to package mapping that may have been intended for IC. The mapping was removed because it wasn't being used and was complicating the issue of package modules with duplicate names not having the proper package owner assigned. * Fix some tests * Refactor `packagehandling` * Remove `packagehandling.withPackageName` and its uses * Move module path mangling from `packagehandling` to `modulepaths` * Move `options.toRodFile` to `ic` to break import cycle * Changed import style to match preferred style Co-authored-by: quantimnot --- compiler/ast.nim | 21 ---- compiler/cgen.nim | 27 ++--- compiler/docgen.nim | 13 +-- compiler/docgen2.nim | 4 +- compiler/extccomp.nim | 11 ++- compiler/ic/cbackend.nim | 5 +- compiler/ic/ic.nim | 19 ++-- compiler/modulegraphs.nim | 18 +++- compiler/modulepaths.nim | 109 +++------------------ compiler/modules.nim | 51 +--------- compiler/options.nim | 7 +- compiler/packagehandling.nim | 23 +---- compiler/packages.nim | 49 +++++++++ compiler/passes.nim | 12 +-- compiler/pragmas.nim | 3 +- doc/manual.rst | 13 +++ tests/ccgbugs/tforward_decl_only.nim | 2 +- tests/modules/a/module_name_clashes.nim | 8 ++ tests/modules/b/module_name_clashes.nim | 3 + tests/modules/tmodule_name_clashes.nim | 16 +++ tests/package/stdlib/stdlib.nimble | 0 tests/package/stdlib/system.nim | 2 + tests/package/tstdlib_name_not_special.nim | 3 + 23 files changed, 175 insertions(+), 244 deletions(-) create mode 100644 compiler/packages.nim create mode 100644 tests/modules/a/module_name_clashes.nim create mode 100644 tests/modules/b/module_name_clashes.nim create mode 100644 tests/modules/tmodule_name_clashes.nim create mode 100644 tests/package/stdlib/stdlib.nimble create mode 100644 tests/package/stdlib/system.nim create mode 100644 tests/package/tstdlib_name_not_special.nim diff --git a/compiler/ast.nim b/compiler/ast.nim index d1e5ae2bf..f8343c1a3 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -1108,21 +1108,6 @@ proc getPIdent*(a: PNode): PIdent {.inline.} = of nkIdent: a.ident else: nil -proc getnimblePkg*(a: PSym): PSym = - result = a - while result != nil: - case result.kind - of skModule: - result = result.owner - assert result.kind == skPackage - of skPackage: - if result.owner == nil: - break - else: - result = result.owner - else: - assert false, $result.kind - const moduleShift = when defined(cpu32): 20 else: 24 @@ -1167,13 +1152,7 @@ when false: assert dest.ItemId.item <= src.ItemId.item dest = src -proc getnimblePkgId*(a: PSym): int = - let b = a.getnimblePkg - result = if b == nil: -1 else: b.id - var ggDebug* {.deprecated.}: bool ## convenience switch for trying out things -#var -# gMainPackageId*: int proc isCallExpr*(n: PNode): bool = result = n.kind in nkCallKinds diff --git a/compiler/cgen.nim b/compiler/cgen.nim index ad59b759f..8d24f30c5 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -15,8 +15,7 @@ import ccgutils, os, ropes, math, passes, wordrecg, treetab, cgmeth, rodutils, renderer, cgendata, aliases, lowerings, tables, sets, ndi, lineinfos, pathutils, transf, - injectdestructors, astmsgs - + injectdestructors, astmsgs, modulepaths when defined(nimPreviewSlimSystem): import std/assertions @@ -1306,17 +1305,19 @@ proc getFileHeader(conf: ConfigRef; cfile: Cfile): Rope = if conf.hcrOn: result.add("#define NIM_HOT_CODE_RELOADING\L") addNimDefines(result, conf) -proc getSomeNameForModule(m: PSym): Rope = - assert m.kind == skModule - assert m.owner.kind == skPackage - if {sfSystemModule, sfMainModule} * m.flags == {}: - result = m.owner.name.s.mangle.rope - result.add "_" - result.add m.name.s.mangle +proc getSomeNameForModule(conf: ConfigRef, filename: AbsoluteFile): Rope = + ## Returns a mangled module name. + result.add mangleModuleName(conf, filename).mangle + +proc getSomeNameForModule(m: BModule): Rope = + ## Returns a mangled module name. + assert m.module.kind == skModule + assert m.module.owner.kind == skPackage + result.add mangleModuleName(m.g.config, m.filename).mangle proc getSomeInitName(m: BModule, suffix: string): Rope = if not m.hcrOn: - result = getSomeNameForModule(m.module) + result = getSomeNameForModule(m) result.add suffix proc getInitName(m: BModule): Rope = @@ -1555,11 +1556,11 @@ proc genMainProc(m: BModule) = proc registerInitProcs*(g: BModuleList; m: PSym; flags: set[ModuleBackendFlag]) = ## Called from the IC backend. if HasDatInitProc in flags: - let datInit = getSomeNameForModule(m) & "DatInit000" + let datInit = getSomeNameForModule(g.config, g.config.toFullPath(m.info.fileIndex).AbsoluteFile) & "DatInit000" g.mainModProcs.addf("N_LIB_PRIVATE N_NIMCALL(void, $1)(void);$N", [datInit]) g.mainDatInit.addf("\t$1();$N", [datInit]) if HasModuleInitProc in flags: - let init = getSomeNameForModule(m) & "Init000" + let init = getSomeNameForModule(g.config, g.config.toFullPath(m.info.fileIndex).AbsoluteFile) & "Init000" g.mainModProcs.addf("N_LIB_PRIVATE N_NIMCALL(void, $1)(void);$N", [init]) let initCall = "\t$1();$N" % [init] if sfMainModule in m.flags: @@ -1940,7 +1941,7 @@ proc getCFile(m: BModule): AbsoluteFile = if m.compileToCpp: ".nim.cpp" elif m.config.backend == backendObjc or sfCompileToObjc in m.module.flags: ".nim.m" else: ".nim.c" - result = changeFileExt(completeCfilePath(m.config, withPackageName(m.config, m.cfilename)), ext) + result = changeFileExt(completeCfilePath(m.config, mangleModuleName(m.config, m.cfilename).AbsoluteFile), ext) when false: proc myOpenCached(graph: ModuleGraph; module: PSym, rd: PRodReader): PPassContext = diff --git a/compiler/docgen.nim b/compiler/docgen.nim index ecf98d0b6..f633a57a0 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -16,7 +16,7 @@ import packages/docutils/[rst, rstgen, dochelpers], json, xmltree, trees, types, typesrenderer, astalgo, lineinfos, intsets, - pathutils, tables, nimpaths, renderverbatim, osproc + pathutils, tables, nimpaths, renderverbatim, osproc, packages import packages/docutils/rstast except FileIndex, TLineInfo from uri import encodeUrl @@ -413,9 +413,6 @@ proc getPlainDocstring(n: PNode): string = result = getPlainDocstring(n[i]) if result.len > 0: return -proc belongsToPackage(conf: ConfigRef; module: PSym): bool = - result = module.kind == skModule and module.getnimblePkgId == conf.mainPackageId - proc externalDep(d: PDoc; module: PSym): string = if optWholeProject in d.conf.globalOptions or d.conf.docRoot.len > 0: let full = AbsoluteFile toFullPath(d.conf, FileIndex module.position) @@ -471,7 +468,7 @@ proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var string; "\\spanIdentifier{$1}", [escLit, procLink]) elif s != nil and s.kind in {skType, skVar, skLet, skConst} and sfExported in s.flags and s.owner != nil and - belongsToPackage(d.conf, s.owner) and d.target == outHtml: + belongsToProjectPackage(d.conf, s.owner) and d.target == outHtml: let external = externalDep(d, s.owner) result.addf "$3", [changeFileExt(external, "html"), literal, @@ -1131,7 +1128,7 @@ proc traceDeps(d: PDoc, it: PNode) = for x in it[2]: a[2] = x traceDeps(d, a) - elif it.kind == nkSym and belongsToPackage(d.conf, it.sym): + elif it.kind == nkSym and belongsToProjectPackage(d.conf, it.sym): let external = externalDep(d, it.sym) if d.section[k].finalMarkup != "": d.section[k].finalMarkup.add(", ") dispA(d.conf, d.section[k].finalMarkup, @@ -1141,7 +1138,7 @@ proc traceDeps(d: PDoc, it: PNode) = proc exportSym(d: PDoc; s: PSym) = const k = exportSection - if s.kind == skModule and belongsToPackage(d.conf, s): + if s.kind == skModule and belongsToProjectPackage(d.conf, s): let external = externalDep(d, s) if d.section[k].finalMarkup != "": d.section[k].finalMarkup.add(", ") dispA(d.conf, d.section[k].finalMarkup, @@ -1150,7 +1147,7 @@ proc exportSym(d: PDoc; s: PSym) = changeFileExt(external, "html")]) elif s.kind != skModule and s.owner != nil: let module = originatingModule(s) - if belongsToPackage(d.conf, module): + if belongsToProjectPackage(d.conf, module): let complexSymbol = complexName(s.kind, s.ast, s.name.s) symbolOrId = d.newUniquePlainSymbol(complexSymbol) diff --git a/compiler/docgen2.nim b/compiler/docgen2.nim index bfdb4568c..9abde9f52 100644 --- a/compiler/docgen2.nim +++ b/compiler/docgen2.nim @@ -11,7 +11,7 @@ # semantic checking. import - options, ast, msgs, passes, docgen, lineinfos, pathutils + options, ast, msgs, passes, docgen, lineinfos, pathutils, packages from modulegraphs import ModuleGraph, PPassContext @@ -23,7 +23,7 @@ type PGen = ref TGen proc shouldProcess(g: PGen): bool = - (optWholeProject in g.doc.conf.globalOptions and g.module.getnimblePkgId == g.doc.conf.mainPackageId) or + (optWholeProject in g.doc.conf.globalOptions and g.doc.conf.belongsToProjectPackage(g.module)) or sfMainModule in g.module.flags or g.config.projectMainIdx == g.module.info.fileIndex template closeImpl(body: untyped) {.dirty.} = diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 09a1dfbb3..515c7ba29 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -12,7 +12,7 @@ # from a lineinfos file, to provide generalized procedures to compile # nim files. -import ropes, platform, condsyms, options, msgs, lineinfos, pathutils +import ropes, platform, condsyms, options, msgs, lineinfos, pathutils, modulepaths import std/[os, strutils, osproc, sha1, streams, sequtils, times, strtabs, json, jsonutils, sugar] @@ -370,6 +370,7 @@ proc initVars*(conf: ConfigRef) = proc completeCfilePath*(conf: ConfigRef; cfile: AbsoluteFile, createSubDir: bool = true): AbsoluteFile = + ## Generate the absolute file path to the generated modules. result = completeGeneratedFilePath(conf, cfile, createSubDir) proc toObjFile*(conf: ConfigRef; filename: AbsoluteFile): AbsoluteFile = @@ -380,7 +381,7 @@ proc addFileToCompile*(conf: ConfigRef; cf: Cfile) = conf.toCompile.add(cf) proc addLocalCompileOption*(conf: ConfigRef; option: string; nimfile: AbsoluteFile) = - let key = completeCfilePath(conf, withPackageName(conf, nimfile)).string + let key = completeCfilePath(conf, mangleModuleName(conf, nimfile).AbsoluteFile).string var value = conf.cfileSpecificOptions.getOrDefault(key) if strutils.find(value, option, 0) < 0: addOpt(value, option) @@ -637,7 +638,7 @@ proc footprint(conf: ConfigRef; cfile: Cfile): SecureHash = proc externalFileChanged(conf: ConfigRef; cfile: Cfile): bool = if conf.backend == backendJs: return false # pre-existing behavior, but not sure it's good - let hashFile = toGeneratedFile(conf, conf.withPackageName(cfile.cname), "sha1") + let hashFile = toGeneratedFile(conf, conf.mangleModuleName(cfile.cname).AbsoluteFile, "sha1") let currentHash = footprint(conf, cfile) var f: File if open(f, hashFile.string, fmRead): @@ -845,9 +846,9 @@ proc hcrLinkTargetName(conf: ConfigRef, objFile: string, isMain = false): Absolu proc displayProgressCC(conf: ConfigRef, path, compileCmd: string): string = if conf.hasHint(hintCC): if optListCmd in conf.globalOptions or conf.verbosity > 1: - result = MsgKindToStr[hintCC] % (demanglePackageName(path.splitFile.name) & ": " & compileCmd) + result = MsgKindToStr[hintCC] % (demangleModuleName(path.splitFile.name) & ": " & compileCmd) else: - result = MsgKindToStr[hintCC] % demanglePackageName(path.splitFile.name) + result = MsgKindToStr[hintCC] % demangleModuleName(path.splitFile.name) proc callCCompiler*(conf: ConfigRef) = var diff --git a/compiler/ic/cbackend.nim b/compiler/ic/cbackend.nim index e7ab000e6..815078a36 100644 --- a/compiler/ic/cbackend.nim +++ b/compiler/ic/cbackend.nim @@ -24,7 +24,7 @@ when defined(nimPreviewSlimSystem): import std/assertions import ".."/[ast, options, lineinfos, modulegraphs, cgendata, cgen, - pathutils, extccomp, msgs] + pathutils, extccomp, msgs, modulepaths] import packed_ast, ic, dce, rodfiles @@ -64,7 +64,8 @@ proc addFileToLink(config: ConfigRef; m: PSym) = if config.backend == backendCpp: ".nim.cpp" elif config.backend == backendObjc: ".nim.m" else: ".nim.c" - let cfile = changeFileExt(completeCfilePath(config, withPackageName(config, filename)), ext) + let cfile = changeFileExt(completeCfilePath(config, + mangleModuleName(config, filename).AbsoluteFile), ext) let objFile = completeCfilePath(config, toObjFile(config, cfile)) if fileExists(objFile): var cf = Cfile(nimname: m.name.s, cname: cfile, diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim index 193e5f517..38b6987f9 100644 --- a/compiler/ic/ic.nim +++ b/compiler/ic/ic.nim @@ -10,7 +10,7 @@ import hashes, tables, intsets, std/sha1 import packed_ast, bitabs, rodfiles import ".." / [ast, idents, lineinfos, msgs, ropes, options, - pathutils, condsyms] + pathutils, condsyms, packages, modulepaths] #import ".." / [renderer, astalgo] from os import removeFile, isAbsolute @@ -551,6 +551,10 @@ proc loadError(err: RodFileError; filename: AbsoluteFile; config: ConfigRef;) = rawMessage(config, warnCannotOpenFile, filename.string & " reason: " & $err) #echo "Error: ", $err, " loading file: ", filename.string +proc toRodFile*(conf: ConfigRef; f: AbsoluteFile; ext = RodExt): AbsoluteFile = + result = changeFileExt(completeGeneratedFilePath(conf, + mangleModuleName(conf, f).AbsoluteFile), ext) + proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef; ignoreConfig = false): RodFileError = var f = rodfiles.open(filename.string) @@ -930,17 +934,6 @@ proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; t result = g[si].types[t.item] assert result.itemId.item > 0 -proc newPackage(config: ConfigRef; cache: IdentCache; fileIdx: FileIndex): PSym = - let filename = AbsoluteFile toFullPath(config, fileIdx) - let name = getIdent(cache, splitFile(filename).name) - let info = newLineInfo(fileIdx, 1, 1) - let - pck = getPackageName(config, filename.string) - pck2 = if pck.len > 0: pck else: "unknown" - pack = getIdent(cache, pck2) - result = newSym(skPackage, getIdent(cache, pck2), - ItemId(module: PackageModuleId, item: int32(fileIdx)), nil, info) - proc setupLookupTables(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; fileIdx: FileIndex; m: var LoadedModule) = m.iface = initTable[PIdent, seq[PackedItemId]]() @@ -968,7 +961,7 @@ proc setupLookupTables(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCa name: getIdent(cache, splitFile(filename).name), info: newLineInfo(fileIdx, 1, 1), position: int(fileIdx)) - m.module.owner = newPackage(conf, cache, fileIdx) + m.module.owner = getPackage(conf, cache, fileIdx) m.module.flags = m.fromDisk.moduleFlags proc loadToReplayNodes(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index c44908dc3..147381910 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -12,7 +12,7 @@ ## or stored in a rod-file. import intsets, tables, hashes, md5_old -import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils +import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils, packages import ic / [packed_ast, ic] when defined(nimPreviewSlimSystem): @@ -67,7 +67,6 @@ type startupPackedConfig*: PackedConfig packageSyms*: TStrTable - modulesPerPackage*: Table[ItemId, TStrTable] deps*: IntSet # the dependency graph or potentially its transitive closure. importDeps*: Table[FileIndex, seq[FileIndex]] # explicit import module dependencies suggestMode*: bool # whether we are in nimsuggest mode or not. @@ -597,3 +596,18 @@ proc onProcessing*(graph: ModuleGraph, fileIdx: FileIndex, moduleStatus: string, let fromModule2 = if fromModule != nil: $fromModule.name.s else: "(toplevel)" let mode = if isNimscript: "(nims) " else: "" rawMessage(conf, hintProcessing, "$#$# $#: $#: $#" % [mode, indent, fromModule2, moduleStatus, path]) + +proc getPackage*(graph: ModuleGraph; fileIdx: FileIndex): PSym = + ## Returns a package symbol for yet to be defined module for fileIdx. + ## The package symbol is added to the graph if it doesn't exist. + let pkgSym = getPackage(graph.config, graph.cache, fileIdx) + # check if the package is already in the graph + result = graph.packageSyms.strTableGet(pkgSym.name) + if result == nil: + # the package isn't in the graph, so create and add it + result = pkgSym + graph.packageSyms.strTableAdd(pkgSym) + +func belongsToStdlib*(graph: ModuleGraph, sym: PSym): bool = + ## Check if symbol belongs to the 'stdlib' package. + sym.getPackageSymbol.getPackageId == graph.systemModule.getPackageId diff --git a/compiler/modulepaths.nim b/compiler/modulepaths.nim index a16b669c4..e80ea3fa6 100644 --- a/compiler/modulepaths.nim +++ b/compiler/modulepaths.nim @@ -10,100 +10,6 @@ import ast, renderer, strutils, msgs, options, idents, os, lineinfos, pathutils -when false: - const - considerParentDirs = not defined(noParentProjects) - considerNimbleDirs = not defined(noNimbleDirs) - - proc findInNimbleDir(pkg, subdir, dir: string): string = - var best = "" - var bestv = "" - for k, p in os.walkDir(dir, relative=true): - if k == pcDir and p.len > pkg.len+1 and - p[pkg.len] == '-' and p.startsWith(pkg): - let (_, a, _) = getPathVersionChecksum(p) - if bestv.len == 0 or bestv < a: - bestv = a - best = dir / p - - if best.len > 0: - var f: File - if open(f, best / changeFileExt(pkg, ".nimble-link")): - # the second line contains what we're interested in, see: - # https://github.com/nim-lang/nimble#nimble-link - var override = "" - discard readLine(f, override) - discard readLine(f, override) - close(f) - if not override.isAbsolute(): - best = best / override - else: - best = override - let f = if subdir.len == 0: pkg else: subdir - let res = addFileExt(best / f, "nim") - if best.len > 0 and fileExists(res): - result = res - -when false: - proc resolveDollar(project, source, pkg, subdir: string; info: TLineInfo): string = - template attempt(a) = - let x = addFileExt(a, "nim") - if fileExists(x): return x - - case pkg - of "stdlib": - if subdir.len == 0: - return options.libpath - else: - for candidate in stdlibDirs: - attempt(options.libpath / candidate / subdir) - of "root": - let root = project.splitFile.dir - if subdir.len == 0: - return root - else: - attempt(root / subdir) - else: - when considerParentDirs: - var p = parentDir(source.splitFile.dir) - # support 'import $karax': - let f = if subdir.len == 0: pkg else: subdir - - while p.len > 0: - let dir = p / pkg - if dirExists(dir): - attempt(dir / f) - # 2nd attempt: try to use 'karax/karax' - attempt(dir / pkg / f) - # 3rd attempt: try to use 'karax/src/karax' - attempt(dir / "src" / f) - attempt(dir / "src" / pkg / f) - p = parentDir(p) - - when considerNimbleDirs: - if not options.gNoNimblePath: - var nimbleDir = getEnv("NIMBLE_DIR") - if nimbleDir.len == 0: nimbleDir = getHomeDir() / ".nimble" - result = findInNimbleDir(pkg, subdir, nimbleDir / "pkgs") - if result.len > 0: return result - when not defined(windows): - result = findInNimbleDir(pkg, subdir, "/opt/nimble/pkgs") - if result.len > 0: return result - - proc scriptableImport(pkg, sub: string; info: TLineInfo): string = - resolveDollar(gProjectFull, info.toFullPath(), pkg, sub, info) - - proc lookupPackage(pkg, subdir: PNode): string = - let sub = if subdir != nil: renderTree(subdir, {renderNoComments}).replace(" ") else: "" - case pkg.kind - of nkStrLit, nkRStrLit, nkTripleStrLit: - result = scriptableImport(pkg.strVal, sub, pkg.info) - of nkIdent: - result = scriptableImport(pkg.ident.s, sub, pkg.info) - else: - localError(pkg.info, "package name must be an identifier or string literal") - result = "" - proc getModuleName*(conf: ConfigRef; n: PNode): string = # This returns a short relative module name without the nim extension # e.g. like "system", "importer" or "somepath/module" @@ -163,3 +69,18 @@ proc checkModuleName*(conf: ConfigRef; n: PNode; doLocalError=true): FileIndex = result = InvalidFileIdx else: result = fileInfoIdx(conf, fullPath) + +proc mangleModuleName*(conf: ConfigRef; path: AbsoluteFile): string = + ## Mangle a relative module path to avoid path and symbol collisions. + ## + ## Used by backends that need to generate intermediary files from Nim modules. + ## This is needed because the compiler uses a flat cache file hierarchy. + ## + ## Example: + ## `foo-#head/../bar` becomes `@foo-@hhead@s..@sbar` + "@m" & relativeTo(path, conf.projectPath).string.multiReplace( + {$os.DirSep: "@s", $os.AltSep: "@s", "#": "@h", "@": "@@", ":": "@c"}) + +proc demangleModuleName*(path: string): string = + ## Demangle a relative module path. + result = path.multiReplace({"@@": "@", "@h": "#", "@s": "/", "@m": "", "@c": ":"}) diff --git a/compiler/modules.nim b/compiler/modules.nim index dd5db63fa..2becef38f 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -12,7 +12,7 @@ import ast, astalgo, magicsys, msgs, options, idents, lexer, passes, syntaxes, llstream, modulegraphs, - lineinfos, pathutils, tables + lineinfos, pathutils, tables, packages when defined(nimPreviewSlimSystem): import std/[syncio, assertions] @@ -25,56 +25,11 @@ proc resetSystemArtifacts*(g: ModuleGraph) = template getModuleIdent(graph: ModuleGraph, filename: AbsoluteFile): PIdent = getIdent(graph.cache, splitFile(filename).name) -template packageId(): untyped {.dirty.} = ItemId(module: PackageModuleId, item: int32(fileIdx)) - -proc getPackage(graph: ModuleGraph; fileIdx: FileIndex): PSym = - ## returns package symbol (skPackage) for yet to be defined module for fileIdx - let filename = AbsoluteFile toFullPath(graph.config, fileIdx) - let name = getModuleIdent(graph, filename) - let info = newLineInfo(fileIdx, 1, 1) - let - pck = getPackageName(graph.config, filename.string) - pck2 = if pck.len > 0: pck else: "unknown" - pack = getIdent(graph.cache, pck2) - result = graph.packageSyms.strTableGet(pack) - if result == nil: - result = newSym(skPackage, getIdent(graph.cache, pck2), packageId(), nil, info) - #initStrTable(packSym.tab) - graph.packageSyms.strTableAdd(result) - else: - let modules = graph.modulesPerPackage.getOrDefault(result.itemId) - let existing = if modules.data.len > 0: strTableGet(modules, name) else: nil - if existing != nil and existing.info.fileIndex != info.fileIndex: - when false: - # we used to produce an error: - localError(graph.config, info, - "module names need to be unique per Nimble package; module clashes with " & - toFullPath(graph.config, existing.info.fileIndex)) - else: - # but starting with version 0.20 we now produce a fake Nimble package instead - # to resolve the conflicts: - let pck3 = fakePackageName(graph.config, filename) - # this makes the new `result`'s owner be the original `result` - result = newSym(skPackage, getIdent(graph.cache, pck3), packageId(), result, info) - #initStrTable(packSym.tab) - graph.packageSyms.strTableAdd(result) - proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; filename: AbsoluteFile) = let packSym = getPackage(graph, fileIdx) result.owner = packSym result.position = int fileIdx - #initStrTable(result.tab(graph)) - when false: - strTableAdd(result.tab, result) # a module knows itself - # This is now implemented via - # c.moduleScope.addSym(module) # a module knows itself - # in sem.nim, around line 527 - - if graph.modulesPerPackage.getOrDefault(packSym.itemId).data.len == 0: - graph.modulesPerPackage[packSym.itemId] = newStrTable() - graph.modulesPerPackage[packSym.itemId].strTableAdd(result) - proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym = let filename = AbsoluteFile toFullPath(graph.config, fileIdx) # We cannot call ``newSym`` here, because we have to circumvent the ID @@ -136,7 +91,7 @@ proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym = # localError(result.info, errAttemptToRedefine, result.name.s) # restore the notes for outer module: graph.config.notes = - if s.getnimblePkgId == graph.config.mainPackageId or isDefined(graph.config, "booting"): graph.config.mainPackageNotes + if graph.config.belongsToProjectPackage(s) or isDefined(graph.config, "booting"): graph.config.mainPackageNotes else: graph.config.foreignPackageNotes proc includeModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PNode = @@ -171,7 +126,7 @@ proc compileProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) = conf.projectMainIdx2 = projectFile let packSym = getPackage(graph, projectFile) - graph.config.mainPackageId = packSym.getnimblePkgId + graph.config.mainPackageId = packSym.getPackageId graph.importStack.add projectFile if projectFile == systemFileIdx: diff --git a/compiler/options.nim b/compiler/options.nim index 69eafb67f..545dfb1d1 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -807,6 +807,8 @@ proc toGeneratedFile*(conf: ConfigRef; path: AbsoluteFile, proc completeGeneratedFilePath*(conf: ConfigRef; f: AbsoluteFile, createSubDir: bool = true): AbsoluteFile = + ## Return an absolute path of a generated intermediary file. + ## Optionally creates the cache directory if `createSubDir` is `true`. let subdir = getNimcacheDir(conf) if createSubDir: try: @@ -814,11 +816,6 @@ proc completeGeneratedFilePath*(conf: ConfigRef; f: AbsoluteFile, except OSError: conf.quitOrRaise "cannot create directory: " & subdir.string result = subdir / RelativeFile f.string.splitPath.tail - #echo "completeGeneratedFilePath(", f, ") = ", result - -proc toRodFile*(conf: ConfigRef; f: AbsoluteFile; ext = RodExt): AbsoluteFile = - result = changeFileExt(completeGeneratedFilePath(conf, - withPackageName(conf, f)), ext) proc rawFindFile(conf: ConfigRef; f: RelativeFile; suppressStdlib: bool): AbsoluteFile = for it in conf.searchPaths: diff --git a/compiler/packagehandling.nim b/compiler/packagehandling.nim index 4af0c28fa..8cf209779 100644 --- a/compiler/packagehandling.nim +++ b/compiler/packagehandling.nim @@ -37,24 +37,7 @@ proc getNimbleFile*(conf: ConfigRef; path: string): string = proc getPackageName*(conf: ConfigRef; path: string): string = ## returns nimble package name, e.g.: `cligen` let path = getNimbleFile(conf, path) - result = path.splitFile.name - -proc fakePackageName*(conf: ConfigRef; path: AbsoluteFile): string = - # Convert `path` so that 2 modules with same name - # in different directory get different name and they can be - # placed in a directory. - # foo-#head/../bar becomes @foo-@hhead@s..@sbar - result = "@m" & relativeTo(path, conf.projectPath).string.multiReplace( - {$os.DirSep: "@s", $os.AltSep: "@s", "#": "@h", "@": "@@", ":": "@c"}) - -proc demanglePackageName*(path: string): string = - result = path.multiReplace({"@@": "@", "@h": "#", "@s": "/", "@m": "", "@c": ":"}) - -proc withPackageName*(conf: ConfigRef; path: AbsoluteFile): AbsoluteFile = - let x = getPackageName(conf, path.string) - let (p, file, ext) = path.splitFile - if x == "stdlib": - # Hot code reloading now relies on 'stdlib_system' names etc. - result = p / RelativeFile((x & '_' & file) & ext) + if path.len > 0: + return path.splitFile.name else: - result = p / RelativeFile(fakePackageName(conf, path)) + return "unknown" diff --git a/compiler/packages.nim b/compiler/packages.nim new file mode 100644 index 000000000..6ceeb1ccc --- /dev/null +++ b/compiler/packages.nim @@ -0,0 +1,49 @@ +# +# +# The Nim Compiler +# (c) Copyright 2022 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Package related procs. +## +## See Also: +## * `packagehandling` for package path handling +## * `modulegraphs.getPackage` +## * `modulegraphs.belongsToStdlib` + +import "." / [options, ast, lineinfos, idents, pathutils, msgs] + +proc getPackage*(conf: ConfigRef; cache: IdentCache; fileIdx: FileIndex): PSym = + ## Return a new package symbol. + ## + ## See Also: + ## * `modulegraphs.getPackage` + let + filename = AbsoluteFile toFullPath(conf, fileIdx) + name = getIdent(cache, splitFile(filename).name) + info = newLineInfo(fileIdx, 1, 1) + pkgName = getPackageName(conf, filename.string) + pkgIdent = getIdent(cache, pkgName) + newSym(skPackage, pkgIdent, ItemId(module: PackageModuleId, item: int32(fileIdx)), nil, info) + +func getPackageSymbol*(sym: PSym): PSym = + ## Return the owning package symbol. + assert sym != nil + result = sym + while result.kind != skPackage: + result = result.owner + assert result != nil, repr(sym.info) + +func getPackageId*(sym: PSym): int = + ## Return the owning package ID. + sym.getPackageSymbol.id + +func belongsToProjectPackage*(conf: ConfigRef, sym: PSym): bool = + ## Return whether the symbol belongs to the project's package. + ## + ## See Also: + ## * `modulegraphs.belongsToStdlib` + conf.mainPackageId == sym.getPackageId diff --git a/compiler/passes.nim b/compiler/passes.nim index 7fb2842f5..3de27575b 100644 --- a/compiler/passes.nim +++ b/compiler/passes.nim @@ -14,7 +14,7 @@ import options, ast, llstream, msgs, idents, syntaxes, modulegraphs, reorder, - lineinfos, pathutils + lineinfos, pathutils, packages when defined(nimPreviewSlimSystem): import std/syncio @@ -104,7 +104,7 @@ const proc prepareConfigNotes(graph: ModuleGraph; module: PSym) = # don't be verbose unless the module belongs to the main package: - if module.getnimblePkgId == graph.config.mainPackageId: + if graph.config.belongsToProjectPackage(module): graph.config.notes = graph.config.mainPackageNotes else: if graph.config.mainPackageNotes == {}: graph.config.mainPackageNotes = graph.config.notes @@ -114,12 +114,6 @@ proc moduleHasChanged*(graph: ModuleGraph; module: PSym): bool {.inline.} = result = true #module.id >= 0 or isDefined(graph.config, "nimBackendAssumesChange") -proc partOfStdlib(x: PSym): bool = - var it = x.owner - while it != nil and it.kind == skPackage and it.owner != nil: - it = it.owner - result = it != nil and it.name.s == "stdlib" - proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator; stream: PLLStream): bool {.discardable.} = if graph.stopCompile(): return true @@ -141,7 +135,7 @@ proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator; while true: openParser(p, fileIdx, s, graph.cache, graph.config) - if not partOfStdlib(module) or module.name.s == "distros": + if not belongsToStdlib(graph, module) or (belongsToStdlib(graph, module) and module.name.s == "distros"): # XXX what about caching? no processing then? what if I change the # modules to include between compilation runs? we'd need to track that # in ROD files. I think we should enable this feature only diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 458c7547a..2262e441b 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -641,12 +641,13 @@ proc pragmaLine(c: PContext, n: PNode) = n.info = getInfoContext(c.config, -1) proc processPragma(c: PContext, n: PNode, i: int) = + ## Create and add a new custom pragma `{.pragma: name.}` node to the module's context. let it = n[i] if it.kind notin nkPragmaCallKinds and it.safeLen == 2: invalidPragma(c, n) elif it.safeLen != 2 or it[0].kind != nkIdent or it[1].kind != nkIdent: invalidPragma(c, n) - var userPragma = newSym(skTemplate, it[1].ident, nextSymId(c.idgen), nil, it.info, c.config.options) + var userPragma = newSym(skTemplate, it[1].ident, nextSymId(c.idgen), c.module, it.info, c.config.options) userPragma.ast = newTreeI(nkPragma, n.info, n.sons[i+1..^1]) strTableAdd(c.userPragmas, userPragma) diff --git a/doc/manual.rst b/doc/manual.rst index bfcb41b7a..5ab257fec 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -6551,6 +6551,19 @@ iterator in which case the overloading resolution takes place: write(stdout, x) # not ambiguous: uses the module C's x +Packages +-------- +A collection of modules in a file tree with an ``identifier.nimble`` file in the +root of the tree is called a Nimble package. A valid package name can only be a +valid Nim identifier and thus its filename is ``identifier.nimble`` where +``identifier`` is the desired package name. A module without a ``.nimble`` file +is assigned the package identifier: `unknown`. + +The distinction between packages allows diagnostic compiler messages to be +scoped to the current project's package vs foreign packages. + + + Compiler Messages ================= diff --git a/tests/ccgbugs/tforward_decl_only.nim b/tests/ccgbugs/tforward_decl_only.nim index 74fbae303..416e50eb5 100644 --- a/tests/ccgbugs/tforward_decl_only.nim +++ b/tests/ccgbugs/tforward_decl_only.nim @@ -1,7 +1,7 @@ discard """ ccodecheck: "\\i !@('struct tyObject_MyRefObject'[0-z]+' {')" ccodecheck: "\\i !@('mymoduleInit')" -ccodecheck: "\\i @('mymoduleDatInit')" +ccodecheck: "\\i @('atmmymoduledotnim_DatInit000')" output: "hello" """ diff --git a/tests/modules/a/module_name_clashes.nim b/tests/modules/a/module_name_clashes.nim new file mode 100644 index 000000000..209526e22 --- /dev/null +++ b/tests/modules/a/module_name_clashes.nim @@ -0,0 +1,8 @@ +# See `tmodule_name_clashes` + +import ../b/module_name_clashes +type A* = object + b*: B + +proc print*(a: A) = + echo repr a diff --git a/tests/modules/b/module_name_clashes.nim b/tests/modules/b/module_name_clashes.nim new file mode 100644 index 000000000..6a10cac33 --- /dev/null +++ b/tests/modules/b/module_name_clashes.nim @@ -0,0 +1,3 @@ +# See `tmodule_name_clashes` + +type B* = object diff --git a/tests/modules/tmodule_name_clashes.nim b/tests/modules/tmodule_name_clashes.nim new file mode 100644 index 000000000..73b166c77 --- /dev/null +++ b/tests/modules/tmodule_name_clashes.nim @@ -0,0 +1,16 @@ +discard """ +targets: "c" +ccodecheck: "\\i @('atmaatsmodule_name_clashesdotnim_DatInit000')" +ccodecheck: "\\i @('atmbatsmodule_name_clashesdotnim_DatInit000')" +joinable: false +""" + +# Test module name clashes within same package. +# This was created to test that module symbol mangling functioned correctly +# for the C backend when there are one or more modules with the same name in +# a package, and more than one of them require module initialization procs. +# I'm not sure of the simplest method to cause the init procs to be generated. + +import a/module_name_clashes + +print A() diff --git a/tests/package/stdlib/stdlib.nimble b/tests/package/stdlib/stdlib.nimble new file mode 100644 index 000000000..e69de29bb diff --git a/tests/package/stdlib/system.nim b/tests/package/stdlib/system.nim new file mode 100644 index 000000000..475f8ec5b --- /dev/null +++ b/tests/package/stdlib/system.nim @@ -0,0 +1,2 @@ +# this module is part of tstdlib_name_not_special +doAssert true \ No newline at end of file diff --git a/tests/package/tstdlib_name_not_special.nim b/tests/package/tstdlib_name_not_special.nim new file mode 100644 index 000000000..e8226a82d --- /dev/null +++ b/tests/package/tstdlib_name_not_special.nim @@ -0,0 +1,3 @@ +# Test whether a another package named 'stdlib' can be imported and used. +# This caused a crash in the past. +import stdlib/system \ No newline at end of file From 4e3eb7414cfedad3d29b2eeab451de4206d91df4 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Wed, 1 Jun 2022 20:44:26 +0800 Subject: [PATCH 1635/2472] [Minor] remove unused and unnecessary local variable (#19853) --- lib/system/reprjs.nim | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/system/reprjs.nim b/lib/system/reprjs.nim index 28935a4ef..f5a0ed3ea 100644 --- a/lib/system/reprjs.nim +++ b/lib/system/reprjs.nim @@ -115,7 +115,6 @@ proc reprArray(a: pointer, typ: PNimType, # We prepend @ to seq, the C backend prepends the pointer to the seq. result = if typ.kind == tySequence: "@[" else: "[" var len: int = 0 - var i: int = 0 {. emit: "`len` = `a`.length;\n" .} var dereffed: pointer = a From 68aeb4c1a64ce73f38a471372277f5ec788f5a6e Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Wed, 1 Jun 2022 21:01:17 +0800 Subject: [PATCH 1636/2472] change the type of mangleJsName; fixes CStringConv warnings (#19852) change the type of mangleJsName since mangleJsName is used in macros, there is no need to use cstring. Using cstring may increase conversions and cause warnings. --- lib/js/jsffi.nim | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/js/jsffi.nim b/lib/js/jsffi.nim index ac963eb89..35cbf2864 100644 --- a/lib/js/jsffi.nim +++ b/lib/js/jsffi.nim @@ -64,7 +64,7 @@ proc validJsName(name: string): bool = if chr notin {'A'..'Z','a'..'z','_','$','0'..'9'}: return false -template mangleJsName(name: cstring): cstring = +template mangleJsName(name: string): string = inc nameCounter "mangledName" & $nameCounter @@ -233,7 +233,7 @@ macro `.`*(obj: JsObject, field: untyped): JsObject = helper(`obj`) else: if not mangledNames.hasKey($field): - mangledNames[$field] = $mangleJsName($field) + mangledNames[$field] = mangleJsName($field) let importString = "#." & mangledNames[$field] result = quote do: proc helper(o: JsObject): JsObject @@ -251,7 +251,7 @@ macro `.=`*(obj: JsObject, field, value: untyped): untyped = helper(`obj`, `value`) else: if not mangledNames.hasKey($field): - mangledNames[$field] = $mangleJsName($field) + mangledNames[$field] = mangleJsName($field) let importString = "#." & mangledNames[$field] & " = #" result = quote do: proc helper(o: JsObject, v: auto) @@ -282,7 +282,7 @@ macro `.()`*(obj: JsObject, importString = "#." & $field & "(@)" else: if not mangledNames.hasKey($field): - mangledNames[$field] = $mangleJsName($field) + mangledNames[$field] = mangleJsName($field) importString = "#." & mangledNames[$field] & "(@)" result = quote: proc helper(o: JsObject): JsObject @@ -302,7 +302,7 @@ macro `.`*[K: cstring, V](obj: JsAssoc[K, V], importString = "#." & $field else: if not mangledNames.hasKey($field): - mangledNames[$field] = $mangleJsName($field) + mangledNames[$field] = mangleJsName($field) importString = "#." & mangledNames[$field] result = quote do: proc helper(o: type(`obj`)): `obj`.V @@ -319,7 +319,7 @@ macro `.=`*[K: cstring, V](obj: JsAssoc[K, V], importString = "#." & $field & " = #" else: if not mangledNames.hasKey($field): - mangledNames[$field] = $mangleJsName($field) + mangledNames[$field] = mangleJsName($field) importString = "#." & mangledNames[$field] & " = #" result = quote do: proc helper(o: type(`obj`), v: `obj`.V) From f7a13f62d634300c3cf68e36dd7926a6a235d52f Mon Sep 17 00:00:00 2001 From: quantimnot <54247259+quantimnot@users.noreply.github.com> Date: Sat, 4 Jun 2022 00:25:21 -0400 Subject: [PATCH 1637/2472] Stop type aliases from inheriting sfUsed (#19861) Fixes #18201 Co-authored-by: quantimnot --- compiler/ast.nim | 2 +- tests/errmsgs/treportunused.nim | 53 +++++++++++++++++++++++++-------- 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index f8343c1a3..6610a1333 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -1506,7 +1506,7 @@ proc assignType*(dest, src: PType) = # this fixes 'type TLock = TSysLock': if src.sym != nil: if dest.sym != nil: - dest.sym.flags.incl src.sym.flags-{sfExported} + dest.sym.flags.incl src.sym.flags-{sfUsed, sfExported} if dest.sym.annex == nil: dest.sym.annex = src.sym.annex mergeLoc(dest.sym.loc, src.sym.loc) else: diff --git a/tests/errmsgs/treportunused.nim b/tests/errmsgs/treportunused.nim index f5ee79afa..46afe163d 100644 --- a/tests/errmsgs/treportunused.nim +++ b/tests/errmsgs/treportunused.nim @@ -2,19 +2,27 @@ discard """ matrix: "--hint:all:off --hint:XDeclaredButNotUsed" nimoutFull: true nimout: ''' -treportunused.nim(23, 10) Hint: 's1' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(24, 10) Hint: 's2' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(25, 10) Hint: 's3' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(26, 6) Hint: 's4' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(27, 6) Hint: 's5' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(28, 7) Hint: 's6' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(29, 7) Hint: 's7' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(30, 5) Hint: 's8' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(31, 5) Hint: 's9' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(32, 6) Hint: 's10' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(33, 6) Hint: 's11' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(37, 3) Hint: 'v0.99' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(38, 3) Hint: 'v0.99.99' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(51, 5) Hint: 'A' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(52, 5) Hint: 'B' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(55, 5) Hint: 'D' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(56, 5) Hint: 'E' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(59, 5) Hint: 'G' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(60, 5) Hint: 'H' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(64, 5) Hint: 'K' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(65, 5) Hint: 'L' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(31, 10) Hint: 's1' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(32, 10) Hint: 's2' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(33, 10) Hint: 's3' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(34, 6) Hint: 's4' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(35, 6) Hint: 's5' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(36, 7) Hint: 's6' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(37, 7) Hint: 's7' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(38, 5) Hint: 's8' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(39, 5) Hint: 's9' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(40, 6) Hint: 's10' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(41, 6) Hint: 's11' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(45, 3) Hint: 'v0.99' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(46, 3) Hint: 'v0.99.99' is declared but not used [XDeclaredButNotUsed] ''' action: compile """ @@ -36,3 +44,22 @@ type s11 = type(1.2) let `v0.99` = "0.99" `v0.99.99` = "0.99.99" + +block: # bug #18201 + # Test that unused type aliases raise hint XDeclaredButNotUsed. + type + A = int + B = distinct int + + C = object + D = C + E = distinct C + + F = string + G = F + H = distinct F + + J = enum + Foo + K = J + L = distinct J From 4341b06f65396c63eec3689d0f8bb00db26c362f Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Sat, 4 Jun 2022 08:03:03 +0300 Subject: [PATCH 1638/2472] RST: improve simple tables (#19859) * RST: improve simple tables * nim 1.0 gotchas * Still allow legacy boundaries like `----` --- compiler/docgen.nim | 1 + compiler/lineinfos.nim | 2 + doc/apis.rst | 6 +- doc/astspec.txt | 12 +- doc/manual.rst | 2 +- doc/nep1.rst | 6 +- doc/nimdoc.cls | 8 +- doc/niminst.rst | 22 +-- doc/tut1.rst | 6 +- lib/packages/docutils/rst.nim | 223 +++++++++++++++++++++--------- lib/packages/docutils/rstast.nim | 13 ++ lib/packages/docutils/rstgen.nim | 55 +++++--- lib/pure/coro.nim | 4 +- lib/pure/os.nim | 12 +- lib/pure/xmltree.nim | 6 +- tests/stdlib/trst.nim | 229 +++++++++++++++++++++++++++++++ 16 files changed, 487 insertions(+), 120 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index f633a57a0..390f44f2e 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -232,6 +232,7 @@ template declareClosures = of meExpected: k = errXExpected of meGridTableNotImplemented: k = errRstGridTableNotImplemented of meMarkdownIllformedTable: k = errRstMarkdownIllformedTable + of meIllformedTable: k = errRstIllformedTable of meNewSectionExpected: k = errRstNewSectionExpected of meGeneralParseError: k = errRstGeneralParseError of meInvalidDirective: k = errRstInvalidDirectiveX diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index 575be6196..105de1636 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -34,6 +34,7 @@ type errXExpected, errRstGridTableNotImplemented, errRstMarkdownIllformedTable, + errRstIllformedTable, errRstNewSectionExpected, errRstGeneralParseError, errRstInvalidDirectiveX, @@ -106,6 +107,7 @@ const errXExpected: "'$1' expected", errRstGridTableNotImplemented: "grid table is not implemented", errRstMarkdownIllformedTable: "illformed delimiter row of a markdown table", + errRstIllformedTable: "Illformed table: $1", errRstNewSectionExpected: "new section expected $1", errRstGeneralParseError: "general parse error", errRstInvalidDirectiveX: "invalid directive: '$1'", diff --git a/doc/apis.rst b/doc/apis.rst index e8313749d..f0b8c93e5 100644 --- a/doc/apis.rst +++ b/doc/apis.rst @@ -18,9 +18,9 @@ been renamed to fit this scheme. The ultimate goal is that the programmer can *guess* a name. -------------------- ------------ -------------------------------------- +=================== ============ ====================================== English word To use Notes -------------------- ------------ -------------------------------------- +=================== ============ ====================================== initialize initT `init` is used to create a value type `T` new newP `new` is used to create a @@ -82,4 +82,4 @@ literal lit string str identifier ident indentation indent -------------------- ------------ -------------------------------------- +=================== ============ ====================================== diff --git a/doc/astspec.txt b/doc/astspec.txt index 6d3fa1f8c..dbbe2799d 100644 --- a/doc/astspec.txt +++ b/doc/astspec.txt @@ -50,9 +50,9 @@ A leaf of the AST often corresponds to a terminal symbol in the concrete syntax. Note that the default ``float`` in Nim maps to ``float64`` such that the default AST for a float is ``nnkFloat64Lit`` as below. ------------------ --------------------------------------------- +================= ============================================= Nim expression Corresponding AST ------------------ --------------------------------------------- +================= ============================================= ``42`` ``nnkIntLit(intVal = 42)`` ``42'i8`` ``nnkInt8Lit(intVal = 42)`` ``42'i16`` ``nnkInt16Lit(intVal = 42)`` @@ -72,7 +72,7 @@ Nim expression Corresponding AST ``nil`` ``nnkNilLit()`` ``myIdentifier`` ``nnkIdent(strVal = "myIdentifier")`` ``myIdentifier`` after lookup pass: ``nnkSym(strVal = "myIdentifier", ...)`` ------------------ --------------------------------------------- +================= ============================================= Identifiers are ``nnkIdent`` nodes. After the name lookup pass these nodes get transferred into ``nnkSym`` nodes. @@ -1211,9 +1211,9 @@ AST: In general, declaring types mirrors this syntax (i.e., ``nnkStaticTy`` for ``static``, etc.). Examples follow (exceptions marked by ``*``): -------------- --------------------------------------------- +============= ============================================= Nim type Corresponding AST -------------- --------------------------------------------- +============= ============================================= ``static`` ``nnkStaticTy`` ``tuple`` ``nnkTupleTy`` ``var`` ``nnkVarTy`` @@ -1226,7 +1226,7 @@ Nim type Corresponding AST ``proc`` ``nnkProcTy`` ``iterator`` ``nnkIteratorTy`` ``object`` ``nnkObjectTy`` -------------- --------------------------------------------- +============= ============================================= Take special care when declaring types as ``proc``. The behavior is similar to ``Procedure declaration``, below, but does not treat ``nnkGenericParams``. diff --git a/doc/manual.rst b/doc/manual.rst index 5ab257fec..82c9f5758 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -363,7 +363,7 @@ contain the following `escape sequences`:idx:\ : ``\\`` `backslash`:idx: ``\"`` `quotation mark`:idx: ``\'`` `apostrophe`:idx: - ``\`` '0'..'9'+ `character with decimal value d`:idx:; + ``\`` '0'..'9'+ `character with decimal value d`:idx:; all decimal digits directly following are used for the character ``\a`` `alert`:idx: diff --git a/doc/nep1.rst b/doc/nep1.rst index 8eb31332f..9627071bf 100644 --- a/doc/nep1.rst +++ b/doc/nep1.rst @@ -153,9 +153,9 @@ The library uses a simple naming scheme that makes use of common abbreviations to keep the names short but meaningful. -------------------- ------------ -------------------------------------- +=================== ============ ====================================== English word To use Notes -------------------- ------------ -------------------------------------- +=================== ============ ====================================== initialize initFoo initializes a value type `Foo` new newFoo initializes a reference type `Foo` via `new` or a value type `Foo` @@ -220,7 +220,7 @@ literal lit string str identifier ident indentation indent -------------------- ------------ -------------------------------------- +=================== ============ ====================================== Coding Conventions diff --git a/doc/nimdoc.cls b/doc/nimdoc.cls index 271dc5dc9..37039f130 100644 --- a/doc/nimdoc.cls +++ b/doc/nimdoc.cls @@ -63,7 +63,7 @@ \usepackage{scrextend} % for the `addmargin` environment -\usepackage{xcolor} +\usepackage[table]{xcolor} \usepackage[urlbordercolor=blue,linkbordercolor=cyan, pdfborderstyle={/S/U/W 1}]{hyperref} \usepackage{enumitem} % for option list, enumList, and rstfootnote @@ -84,6 +84,11 @@ \definecolor{rstframecolor}{rgb}{0.85, 0.8, 0.6} +\usepackage{booktabs} +\belowrulesep=0ex +\aboverulesep=0ex +\renewcommand{\arraystretch}{1.1} + \newtcolorbox{rstprebox}[1][]{blanker, breakable, left=3mm, right=3mm, top=1mm, bottom=1mm, borderline ={0.1em}{0pt}{rstframecolor}, @@ -127,6 +132,7 @@ \newenvironment{rstoptlist}{% \begin{description}[font=\sffamily\bfseries,style=nextline,leftmargin=\rstoptleftmargin,labelwidth=\rstoptlabelwidth]}{\end{description}} +\usepackage{multirow} \usepackage{tabulary} % tables with adjustable cell width and no overflow % make tabulary prevent overflows (https://tex.stackexchange.com/a/195088) \tymin=60pt diff --git a/doc/niminst.rst b/doc/niminst.rst index 8fff1f0e8..2da8664a9 100644 --- a/doc/niminst.rst +++ b/doc/niminst.rst @@ -49,20 +49,20 @@ contain the following key-value pairs: ==================== ======================================================= Key description ==================== ======================================================= -`Name` the project's name; this needs to be a single word -`DisplayName` the project's long name; this can contain spaces. If +`Name` the project's name; this needs to be a single word +`DisplayName` the project's long name; this can contain spaces. If not specified, this is the same as `Name`. -`Version` the project's version -`OS` the OSes to generate C code for; for example: +`Version` the project's version +`OS` the OSes to generate C code for; for example: `"windows;linux;macosx"` -`CPU` the CPUs to generate C code for; for example: +`CPU` the CPUs to generate C code for; for example: `"i386;amd64;powerpc"` -`Authors` the project's authors -`Description` the project's description -`App` the application's type: "Console" or "GUI". If +`Authors` the project's authors +`Description` the project's description +`App` the application's type: "Console" or "GUI". If "Console", niminst generates a special batch file for Windows to open up the command-line shell. -`License` the filename of the application's license +`License` the filename of the application's license ==================== ======================================================= @@ -149,9 +149,9 @@ Possible options are: ==================== ======================================================= Key description ==================== ======================================================= -`InstallScript` boolean flag whether an installation shell script +`InstallScript` boolean flag whether an installation shell script should be generated. Example: `InstallScript: "Yes"` -`UninstallScript` boolean flag whether a de-installation shell script +`UninstallScript` boolean flag whether a de-installation shell script should be generated. Example: `UninstallScript: "Yes"` ==================== ======================================================= diff --git a/doc/tut1.rst b/doc/tut1.rst index 405df57b1..66a4c3274 100644 --- a/doc/tut1.rst +++ b/doc/tut1.rst @@ -1185,9 +1185,9 @@ subranges) are called ordinal types. Ordinal types have quite a few special operations: ------------------ -------------------------------------------------------- +================= ======================================================== Operation Comment ------------------ -------------------------------------------------------- +================= ======================================================== `ord(x)` returns the integer value that is used to represent `x`'s value `inc(x)` increments `x` by one @@ -1198,7 +1198,7 @@ Operation Comment `succ(x, n)` returns the `n`'th successor of `x` `pred(x)` returns the predecessor of `x` `pred(x, n)` returns the `n`'th predecessor of `x` ------------------ -------------------------------------------------------- +================= ======================================================== The `inc `_, `dec `_, `succ diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index d4bfae48f..cd5f26278 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -255,6 +255,7 @@ type meExpected = "'$1' expected", meGridTableNotImplemented = "grid table is not implemented", meMarkdownIllformedTable = "illformed delimiter row of a Markdown table", + meIllformedTable = "Illformed table: $1", meNewSectionExpected = "new section expected $1", meGeneralParseError = "general parse error", meInvalidDirective = "invalid directive: '$1'", @@ -2467,81 +2468,175 @@ proc parseOverline(p: var RstParser): PRstNode = anchorType=headlineAnchor) type - IntSeq = seq[int] - ColumnLimits = tuple + ColSpec = object + start, stop: int + RstCols = seq[ColSpec] + ColumnLimits = tuple # for Markdown first, last: int ColSeq = seq[ColumnLimits] -proc tokEnd(p: RstParser): int = - result = currentTok(p).col + currentTok(p).symbol.len - 1 +proc tokStart(p: RstParser, idx: int): int = + result = p.tok[idx].col -proc getColumns(p: var RstParser, cols: var IntSeq) = +proc tokStart(p: RstParser): int = + result = tokStart(p, p.idx) + +proc tokEnd(p: RstParser, idx: int): int = + result = p.tok[idx].col + p.tok[idx].symbol.len - 1 + +proc tokEnd(p: RstParser): int = + result = tokEnd(p, p.idx) + +proc getColumns(p: RstParser, cols: var RstCols, startIdx: int): int = + # Fills table column specification (or separator) `cols` and returns + # the next parser index after it. var L = 0 + result = startIdx while true: inc L setLen(cols, L) - cols[L - 1] = tokEnd(p) - assert(currentTok(p).kind == tkAdornment) - inc p.idx - if currentTok(p).kind != tkWhite: break - inc p.idx - if currentTok(p).kind != tkAdornment: break - if currentTok(p).kind == tkIndent: inc p.idx - # last column has no limit: - cols[L - 1] = 32000 + cols[L - 1].start = tokStart(p, result) + cols[L - 1].stop = tokEnd(p, result) + assert(p.tok[result].kind == tkAdornment) + inc result + if p.tok[result].kind != tkWhite: break + inc result + if p.tok[result].kind != tkAdornment: break + if p.tok[result].kind == tkIndent: inc result + +proc checkColumns(p: RstParser, cols: RstCols) = + var + i = p.idx + col = 0 + if p.tok[i].symbol[0] != '=': + rstMessage(p, mwRstStyle, + "only tables with `=` columns specification are allowed") + for col in 0 ..< cols.len: + if tokEnd(p, i) != cols[col].stop: + rstMessage(p, meIllformedTable, + "end of table column #$1 should end at position $2" % [ + $(col+1), $(cols[col].stop+ColRstOffset)], + p.tok[i].line, tokEnd(p, i)) + inc i + if col == cols.len - 1: + if p.tok[i].kind == tkWhite: + inc i + if p.tok[i].kind notin {tkIndent, tkEof}: + rstMessage(p, meIllformedTable, "extraneous column specification") + elif p.tok[i].kind == tkWhite: + inc i + else: + rstMessage(p, meIllformedTable, "no enough table columns", + p.tok[i].line, p.tok[i].col) + +proc getSpans(p: RstParser, nextLine: int, + cols: RstCols, unitedCols: RstCols): seq[int] = + ## Calculates how many columns a joined cell occupies. + if unitedCols.len > 0: + result = newSeq[int](unitedCols.len) + var + iCell = 0 + jCell = 0 + uCell = 0 + while jCell < cols.len: + if cols[jCell].stop < unitedCols[uCell].stop: + inc jCell + elif cols[jCell].stop == unitedCols[uCell].stop: + result[uCell] = jCell - iCell + 1 + iCell = jCell + 1 + jCell = jCell + 1 + inc uCell + else: + rstMessage(p, meIllformedTable, + "spanning underline does not match main table columns", + p.tok[nextLine].line, p.tok[nextLine].col) + +proc parseSimpleTableRow(p: var RstParser, cols: RstCols, colChar: char): PRstNode = + ## Parses 1 row in RST simple table. + # Consider that columns may be spanning (united by using underline like ----): + let nextLine = tokenAfterNewline(p) + var unitedCols: RstCols + var afterSpan: int + if p.tok[nextLine].kind == tkAdornment and p.tok[nextLine].symbol[0] == '-': + afterSpan = getColumns(p, unitedCols, nextLine) + if unitedCols == cols and p.tok[nextLine].symbol[0] == colChar: + # legacy rst.nim compat.: allow punctuation like `----` in main boundaries + afterSpan = nextLine + unitedCols.setLen 0 + else: + afterSpan = nextLine + template colEnd(i): int = + if i == cols.len - 1: high(int) # last column has no limit + elif unitedCols.len > 0: unitedCols[i].stop else: cols[i].stop + template colStart(i): int = + if unitedCols.len > 0: unitedCols[i].start else: cols[i].start + var row = newSeq[string](if unitedCols.len > 0: unitedCols.len else: cols.len) + var spans: seq[int] = getSpans(p, nextLine, cols, unitedCols) + + let line = currentTok(p).line + # Iterate over the lines a single cell may span: + while true: + var nCell = 0 + # distribute tokens between cells in the current line: + while currentTok(p).kind notin {tkIndent, tkEof}: + if tokEnd(p) <= colEnd(nCell): + if tokStart(p) < colStart(nCell): + if currentTok(p).kind != tkWhite: + rstMessage(p, meIllformedTable, + "this word crosses table column from the left") + else: + inc p.idx + else: + row[nCell].add(currentTok(p).symbol) + inc p.idx + else: + if tokStart(p) < colEnd(nCell) and currentTok(p).kind != tkWhite: + rstMessage(p, meIllformedTable, + "this word crosses table column from the right") + inc nCell + if currentTok(p).kind == tkIndent: inc p.idx + if tokEnd(p) <= colEnd(0): break + # Continued current cells because the 1st column is empty. + if currentTok(p).kind in {tkEof, tkAdornment}: + break + for nCell in countup(1, high(row)): row[nCell].add('\n') + result = newRstNode(rnTableRow) + var q: RstParser + for uCell in 0 ..< row.len: + initParser(q, p.s) + q.col = colStart(uCell) + q.line = line - 1 + getTokens(row[uCell], q.tok) + let cell = newRstNode(rnTableDataCell) + cell.span = if spans.len == 0: 0 else: spans[uCell] + cell.add(parseDoc(q)) + result.add(cell) + if afterSpan > p.idx: + p.idx = afterSpan proc parseSimpleTable(p: var RstParser): PRstNode = - var - cols: IntSeq - row: seq[string] - i, last, line: int - c: char - q: RstParser - a, b: PRstNode + var cols: RstCols result = newRstNodeA(p, rnTable) - cols = @[] - row = @[] - a = nil - c = currentTok(p).symbol[0] + let startIdx = getColumns(p, cols, p.idx) + let colChar = currentTok(p).symbol[0] + checkColumns(p, cols) + p.idx = startIdx + result.colCount = cols.len while true: if currentTok(p).kind == tkAdornment: - last = tokenAfterNewline(p) - if p.tok[last].kind in {tkEof, tkIndent}: + checkColumns(p, cols) + p.idx = tokenAfterNewline(p) + if currentTok(p).kind in {tkEof, tkIndent}: # skip last adornment line: - p.idx = last break - getColumns(p, cols) - setLen(row, cols.len) - if a != nil: - for j in 0 ..< a.len: # fix rnTableDataCell -> rnTableHeaderCell - a.sons[j] = newRstNode(rnTableHeaderCell, a.sons[j].sons) + if result.sons.len > 0: result.sons[^1].endsHeader = true + # fix rnTableDataCell -> rnTableHeaderCell for previous table rows: + for nRow in 0 ..< result.sons.len: + for nCell in 0 ..< result.sons[nRow].len: + result.sons[nRow].sons[nCell].kind = rnTableHeaderCell if currentTok(p).kind == tkEof: break - for j in countup(0, high(row)): row[j] = "" - # the following while loop iterates over the lines a single cell may span: - line = currentTok(p).line - while true: - i = 0 - while currentTok(p).kind notin {tkIndent, tkEof}: - if tokEnd(p) <= cols[i]: - row[i].add(currentTok(p).symbol) - inc p.idx - else: - if currentTok(p).kind == tkWhite: inc p.idx - inc i - if currentTok(p).kind == tkIndent: inc p.idx - if tokEnd(p) <= cols[0]: break - if currentTok(p).kind in {tkEof, tkAdornment}: break - for j in countup(1, high(row)): row[j].add('\n') - a = newRstNode(rnTableRow) - for j in countup(0, high(row)): - initParser(q, p.s) - q.col = cols[j] - q.line = line - 1 - getTokens(row[j], q.tok) - b = newRstNode(rnTableDataCell) - b.add(parseDoc(q)) - a.add(b) - result.add(a) + let tabRow = parseSimpleTableRow(p, cols, colChar) + result.add tabRow proc readTableRow(p: var RstParser): ColSeq = if currentTok(p).symbol == "|": inc p.idx @@ -2574,17 +2669,16 @@ proc isValidDelimiterRow(p: var RstParser, colNum: int): bool = proc parseMarkdownTable(p: var RstParser): PRstNode = var row: ColSeq - colNum: int a, b: PRstNode q: RstParser result = newRstNodeA(p, rnMarkdownTable) proc parseRow(p: var RstParser, cellKind: RstNodeKind, result: PRstNode) = row = readTableRow(p) - if colNum == 0: colNum = row.len # table header - elif row.len < colNum: row.setLen(colNum) + if result.colCount == 0: result.colCount = row.len # table header + elif row.len < result.colCount: row.setLen(result.colCount) a = newRstNode(rnTableRow) - for j in 0 ..< colNum: + for j in 0 ..< result.colCount: b = newRstNode(cellKind) initParser(q, p.s) q.col = p.col @@ -2595,7 +2689,8 @@ proc parseMarkdownTable(p: var RstParser): PRstNode = result.add(a) parseRow(p, rnTableHeaderCell, result) - if not isValidDelimiterRow(p, colNum): rstMessage(p, meMarkdownIllformedTable) + if not isValidDelimiterRow(p, result.colCount): + rstMessage(p, meMarkdownIllformedTable) while predNL(p) and currentTok(p).symbol == "|": parseRow(p, rnTableDataCell, result) diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim index 1d5da5e1c..bf0bc1be5 100644 --- a/lib/packages/docutils/rstast.nim +++ b/lib/packages/docutils/rstast.nim @@ -112,6 +112,12 @@ type ## nodes that are post-processed after parsing of rnNimdocRef: tooltip*: string + of rnTable, rnGridTable, rnMarkdownTable: + colCount*: int ## Number of (not-united) cells in the table + of rnTableRow: + endsHeader*: bool ## Is last row in the header of table? + of rnTableHeaderCell, rnTableDataCell: + span*: int ## Number of table columns that the cell occupies else: discard anchor*: string ## anchor, internal link target @@ -416,6 +422,13 @@ proc treeRepr*(node: PRstNode, indent=0): string = result.add (if node.order == 0: "" else: " order=" & $node.order) of rnMarkdownBlockQuoteItem: result.add " quotationDepth=" & $node.quotationDepth + of rnTable, rnGridTable, rnMarkdownTable: + result.add " colCount=" & $node.colCount + of rnTableHeaderCell, rnTableDataCell: + if node.span > 0: + result.add " span=" & $node.span + of rnTableRow: + if node.endsHeader: result.add " endsHeader" else: discard result.add (if node.anchor == "": "" else: " anchor='" & node.anchor & "'") diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index d662a667c..8eb4be675 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -1091,10 +1091,6 @@ proc renderContainer(d: PDoc, n: PRstNode, result: var string) = else: dispA(d.target, result, "
              $2
              ", "$2", [arg, tmp]) -proc texColumns(n: PRstNode): string = - let nColumns = if n.sons.len > 0: len(n.sons[0]) else: 1 - result = "L".repeat(nColumns) - proc renderField(d: PDoc, n: PRstNode, result: var string) = var b = false if d.target == outLatex: @@ -1323,24 +1319,49 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = renderAux(d, n, "$1", "\n$2\n\\begin{rsttab}{" & - texColumns(n) & "}\n\\hline\n$1\\end{rsttab}", result) + "L".repeat(n.colCount) & "}\n\\toprule\n$1" & + "\\addlinespace[0.1em]\\bottomrule\n\\end{rsttab}", result) of rnTableRow: if len(n) >= 1: - if d.target == outLatex: - #var tmp = "" - renderRstToOut(d, n.sons[0], result) - for i in countup(1, len(n) - 1): - result.add(" & ") - renderRstToOut(d, n.sons[i], result) - result.add("\\\\\n\\hline\n") - else: + case d.target + of outHtml: result.add("") renderAux(d, n, result) result.add("\n") - of rnTableDataCell: - renderAux(d, n, "$1", "$1", result) - of rnTableHeaderCell: - renderAux(d, n, "$1", "\\textbf{$1}", result) + of outLatex: + if n.sons[0].kind == rnTableHeaderCell: + result.add "\\rowcolor{gray!15} " + var spanLines: seq[(int, int)] + var nCell = 0 + for uCell in 0 .. n.len - 1: + renderRstToOut(d, n.sons[uCell], result) + if n.sons[uCell].span > 0: + spanLines.add (nCell + 1, nCell + n.sons[uCell].span) + nCell += n.sons[uCell].span + else: + nCell += 1 + if uCell != n.len - 1: + result.add(" & ") + result.add("\\\\") + if n.endsHeader: result.add("\\midrule\n") + for (start, stop) in spanLines: + result.add("\\cmidrule(lr){$1-$2}" % [$start, $stop]) + result.add("\n") + of rnTableHeaderCell, rnTableDataCell: + case d.target + of outHtml: + let tag = if n.kind == rnTableHeaderCell: "th" else: "td" + var spanSpec: string + if n.span <= 1: spanSpec = "" + else: + spanSpec = " colspan=\"" & $n.span & "\" style=\"text-align: center\"" + renderAux(d, n, "<$1$2>$$1" % [tag, spanSpec], "", result) + of outLatex: + let text = if n.kind == rnTableHeaderCell: "\\textbf{$1}" else: "$1" + var latexStr: string + if n.span <= 1: latexStr = text + else: latexStr = "\\multicolumn{" & $n.span & "}{c}{" & text & "}" + renderAux(d, n, "", latexStr, result) of rnFootnoteGroup: renderAux(d, n, "
              " & diff --git a/lib/pure/coro.nim b/lib/pure/coro.nim index f4495a536..aaf442a83 100644 --- a/lib/pure/coro.nim +++ b/lib/pure/coro.nim @@ -8,11 +8,11 @@ # ## Nim coroutines implementation, supports several context switching methods: -## -------- ------------ +## ======== ============ ## ucontext available on unix and alike (default) ## setjmp available on unix and alike (x86/64 only) ## fibers available and required on windows. -## -------- ------------ +## ======== ============ ## ## -d:nimCoroutines Required to build this module. ## -d:nimCoroutinesUcontext Use ucontext backend. diff --git a/lib/pure/os.nim b/lib/pure/os.nim index fa379a228..247a0d089 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -2383,21 +2383,21 @@ iterator walkDirRec*(dir: string, ## ## Walking is recursive. `followFilter` controls the behaviour of the iterator: ## - ## --------------------- --------------------------------------------- + ## ===================== ============================================= ## yieldFilter meaning - ## --------------------- --------------------------------------------- + ## ===================== ============================================= ## ``pcFile`` yield real files (default) ## ``pcLinkToFile`` yield symbolic links to files ## ``pcDir`` yield real directories ## ``pcLinkToDir`` yield symbolic links to directories - ## --------------------- --------------------------------------------- + ## ===================== ============================================= ## - ## --------------------- --------------------------------------------- + ## ===================== ============================================= ## followFilter meaning - ## --------------------- --------------------------------------------- + ## ===================== ============================================= ## ``pcDir`` follow real directories (default) ## ``pcLinkToDir`` follow symbolic links to directories - ## --------------------- --------------------------------------------- + ## ===================== ============================================= ## ## ## See also: diff --git a/lib/pure/xmltree.nim b/lib/pure/xmltree.nim index fdb2c3fc5..9a9ecde57 100644 --- a/lib/pure/xmltree.nim +++ b/lib/pure/xmltree.nim @@ -552,15 +552,15 @@ proc escape*(s: string): string = ## ## Escapes these characters: ## - ## ------------ ------------------- + ## ============ =================== ## char is converted to - ## ------------ ------------------- + ## ============ =================== ## ``<`` ``<`` ## ``>`` ``>`` ## ``&`` ``&`` ## ``"`` ``"`` ## ``'`` ``'`` - ## ------------ ------------------- + ## ============ =================== ## ## You can also use `addEscaped proc <#addEscaped,string,string>`_. result = newStringOfCap(s.len) diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index 9787c13eb..24bc03027 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -3,6 +3,8 @@ discard """ [Suite] RST parsing +[Suite] RST tables + [Suite] RST indentation [Suite] Warnings @@ -618,6 +620,233 @@ suite "RST parsing": rnLeaf 'code' """) +suite "RST tables": + + test "formatting in tables works": + check( + dedent""" + ========= === + `build` `a` + ========= === + """.toAst == + dedent""" + rnTable colCount=2 + rnTableRow + rnTableDataCell + rnInlineCode + rnDirArg + rnLeaf 'nim' + [nil] + rnLiteralBlock + rnLeaf 'build' + rnTableDataCell + rnInlineCode + rnDirArg + rnLeaf 'nim' + [nil] + rnLiteralBlock + rnLeaf 'a' + """) + + test "tables with slightly overflowed cells cause an error (1)": + var error = new string + check( + dedent""" + ====== ====== + Inputs Output + ====== ====== + """.toAst(error=error) == "") + check(error[] == "input(2, 2) Error: Illformed table: " & + "this word crosses table column from the right") + + test "tables with slightly overflowed cells cause an error (2)": + var error = new string + check("" == dedent""" + ===== ===== ====== + Input Output + ===== ===== ====== + False False False + ===== ===== ====== + """.toAst(error=error)) + check(error[] == "input(2, 8) Error: Illformed table: " & + "this word crosses table column from the right") + + test "tables with slightly underflowed cells cause an error": + var error = new string + check("" == dedent""" + ===== ===== ====== + Input Output + ===== ===== ====== + False False False + ===== ===== ====== + """.toAst(error=error)) + check(error[] == "input(2, 7) Error: Illformed table: " & + "this word crosses table column from the left") + + test "tables with unequal underlines should be reported (1)": + var error = new string + error[] = "none" + check("" == dedent""" + ===== ====== + Input Output + ===== ====== + False False + ===== ======= + """.toAst(error=error)) + check(error[] == "input(5, 14) Error: Illformed table: " & + "end of table column #2 should end at position 13") + + test "tables with unequal underlines should be reported (2)": + var error = new string + check("" == dedent""" + ===== ====== + Input Output + ===== ======= + False False + ===== ====== + """.toAst(error=error)) + check(error[] == "input(3, 14) Error: Illformed table: " & + "end of table column #2 should end at position 13") + + test "tables with empty first cells": + check( + dedent""" + = = = + x y z + t + = = = + """.toAst == + dedent""" + rnTable colCount=3 + rnTableRow + rnTableDataCell + rnLeaf 'x' + rnTableDataCell + rnInner + rnLeaf 'y' + rnLeaf ' ' + rnTableDataCell + rnInner + rnLeaf 'z' + rnLeaf ' ' + rnLeaf 't' + """) + + test "tables with spanning cells & separators": + check( + dedent""" + ===== ===== ====== + Inputs Output + ------------ ------ + A B A or B + ===== ===== ====== + False False False + True False True + ----- ----- ------ + False True True + True True True + ===== ===== ====== + """.toAst == + dedent""" + rnTable colCount=3 + rnTableRow + rnTableHeaderCell span=2 + rnLeaf 'Inputs' + rnTableHeaderCell span=1 + rnLeaf 'Output' + rnTableRow endsHeader + rnTableHeaderCell + rnLeaf 'A' + rnTableHeaderCell + rnLeaf 'B' + rnTableHeaderCell + rnInner + rnLeaf 'A' + rnLeaf ' ' + rnLeaf 'or' + rnLeaf ' ' + rnLeaf 'B' + rnTableRow + rnTableDataCell + rnLeaf 'False' + rnTableDataCell + rnLeaf 'False' + rnTableDataCell + rnLeaf 'False' + rnTableRow + rnTableDataCell span=1 + rnLeaf 'True' + rnTableDataCell span=1 + rnLeaf 'False' + rnTableDataCell span=1 + rnLeaf 'True' + rnTableRow + rnTableDataCell + rnLeaf 'False' + rnTableDataCell + rnLeaf 'True' + rnTableDataCell + rnLeaf 'True' + rnTableRow + rnTableDataCell + rnLeaf 'True' + rnTableDataCell + rnLeaf 'True' + rnTableDataCell + rnLeaf 'True' + """) + + test "tables with spanning cells with uneqal underlines cause an error": + var error = new string + check( + dedent""" + ===== ===== ====== + Inputs Output + ------------- ------ + A B A or B + ===== ===== ====== + """.toAst(error=error) == "") + check(error[] == "input(3, 1) Error: Illformed table: " & + "spanning underline does not match main table columns") + + let expTable = dedent""" + rnTable colCount=2 + rnTableRow + rnTableDataCell + rnLeaf 'Inputs' + rnTableDataCell + rnLeaf 'Output' + """ + + test "only tables with `=` columns specs are allowed (1)": + var warnings = new seq[string] + check( + dedent""" + ------ ------ + Inputs Output + ------ ------ + """.toAst(warnings=warnings) == + expTable) + check(warnings[] == + @["input(1, 1) Warning: RST style: " & + "only tables with `=` columns specification are allowed", + "input(3, 1) Warning: RST style: " & + "only tables with `=` columns specification are allowed"]) + + test "only tables with `=` columns specs are allowed (2)": + var warnings = new seq[string] + check( + dedent""" + ====== ====== + Inputs Output + ~~~~~~ ~~~~~~ + """.toAst(warnings=warnings) == + expTable) + check(warnings[] == + @["input(3, 1) Warning: RST style: "& + "only tables with `=` columns specification are allowed"]) + + suite "RST indentation": test "nested bullet lists": let input = dedent """ From d0232f0e5b97b697a640c2f622c7596ce6d10eb5 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Sat, 4 Jun 2022 02:03:25 -0300 Subject: [PATCH 1639/2472] Add Microtasks (#19860) * Add Microtasks * Add Microtasks --- changelog.md | 2 +- lib/js/jscore.nim | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 9548f3034..2ae64d543 100644 --- a/changelog.md +++ b/changelog.md @@ -50,7 +50,7 @@ becomes an alias for `addr`. - Removed deprecated `std/dom_extensions`. - Removed deprecated `std/posix.CMSG_SPACE` and `std/posix.CMSG_LEN` that takes wrong argument types. - Remove deprecated `osproc.poDemon`, symbol with typo. - +- Added [`queueMicrotask`](https://developer.mozilla.org/en-US/docs/Web/API/queueMicrotask) for JavaScript targets. - Deprecated `selfExe` for Nimscript. diff --git a/lib/js/jscore.nim b/lib/js/jscore.nim index 4518f32ce..e6060535b 100644 --- a/lib/js/jscore.nim +++ b/lib/js/jscore.nim @@ -132,3 +132,8 @@ since (1, 7): var arrai = @[1, 2, 3] assert arrai.shift() == 1 assert arrai == @[2, 3] + + func queueMicrotask*(function: proc) {.importjs: "$1(#)".} = + ## * https://developer.mozilla.org/en-US/docs/Web/API/queueMicrotask + ## * https://developer.mozilla.org/en-US/docs/Web/API/HTML_DOM_API/Microtask_guide + runnableExamples: queueMicrotask(proc() = echo "Microtask") From b024a45163ce1a2224ec5891c81cf2cc7d399f11 Mon Sep 17 00:00:00 2001 From: Zoom Date: Tue, 7 Jun 2022 08:35:06 +0000 Subject: [PATCH 1640/2472] Fix `find` routines' api to default to `last=-1` (#19761) This changes the default for the `last` parameter of various `find` routines from `0` to `-1`. Previous default prevents limiting the search to the first character. This is a logic error, as full text search was performed for 2 *valid* values of `last`: `0` and `last.high()`. Adds an overload for `initSkipTable` which returns a newly initialized table. This encapsulates every single usage of a `var`-acting original func in this module. Co-authored-by: flywind --- lib/pure/strutils.nim | 91 +++++++++++++++++++++----------------- tests/stdlib/tstrutils.nim | 42 ++++++++++-------- 2 files changed, 73 insertions(+), 60 deletions(-) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index b98bd2bec..ffbefd0fb 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -1815,6 +1815,7 @@ func initSkipTable*(a: var SkipTable, sub: string) {.rtl, ## Initializes table `a` for efficient search of substring `sub`. ## ## See also: + ## * `initSkipTable func<#initSkipTable,string>`_ ## * `find func<#find,SkipTable,string,string,Natural,int>`_ # TODO: this should be the `default()` initializer for the type. let m = len(sub) @@ -1823,7 +1824,16 @@ func initSkipTable*(a: var SkipTable, sub: string) {.rtl, for i in 0 ..< m - 1: a[sub[i]] = m - 1 - i -func find*(a: SkipTable, s, sub: string, start: Natural = 0, last = 0): int {. +func initSkipTable*(sub: string): SkipTable {.noinit, rtl, + extern: "nsuInitNewSkipTable".} = + ## Returns a new table initialized for `sub`. + ## + ## See also: + ## * `initSkipTable func<#initSkipTable,SkipTable,string>`_ + ## * `find func<#find,SkipTable,string,string,Natural,int>`_ + initSkipTable(result, sub) + +func find*(a: SkipTable, s, sub: string, start: Natural = 0, last = -1): int {. rtl, extern: "nsuFindStrA".} = ## Searches for `sub` in `s` inside range `start..last` using preprocessed ## table `a`. If `last` is unspecified, it defaults to `s.high` (the last @@ -1832,9 +1842,10 @@ func find*(a: SkipTable, s, sub: string, start: Natural = 0, last = 0): int {. ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. ## ## See also: + ## * `initSkipTable func<#initSkipTable,string>`_ ## * `initSkipTable func<#initSkipTable,SkipTable,string>`_ let - last = if last == 0: s.high else: last + last = if last < 0: s.high else: last subLast = sub.len - 1 if subLast == -1: @@ -1844,6 +1855,7 @@ func find*(a: SkipTable, s, sub: string, start: Natural = 0, last = 0): int {. # This is an implementation of the Boyer-Moore Horspool algorithms # https://en.wikipedia.org/wiki/Boyer%E2%80%93Moore%E2%80%93Horspool_algorithm + result = -1 var skip = start while last - skip >= subLast: @@ -1853,7 +1865,6 @@ func find*(a: SkipTable, s, sub: string, start: Natural = 0, last = 0): int {. return skip dec i inc skip, a[s[skip + subLast]] - return -1 when not (defined(js) or defined(nimdoc) or defined(nimscript)): func c_memchr(cstr: pointer, c: char, n: csize_t): pointer {. @@ -1865,10 +1876,10 @@ when not (defined(js) or defined(nimdoc) or defined(nimscript)): else: const hasCStringBuiltin = false -func find*(s: string, sub: char, start: Natural = 0, last = 0): int {.rtl, +func find*(s: string, sub: char, start: Natural = 0, last = -1): int {.rtl, extern: "nsuFindChar".} = ## Searches for `sub` in `s` inside range `start..last` (both ends included). - ## If `last` is unspecified, it defaults to `s.high` (the last element). + ## If `last` is unspecified or negative, it defaults to `s.high` (the last element). ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. ## Otherwise the index returned is relative to `s[0]`, not `start`. @@ -1877,26 +1888,30 @@ func find*(s: string, sub: char, start: Natural = 0, last = 0): int {.rtl, ## See also: ## * `rfind func<#rfind,string,char,Natural,int>`_ ## * `replace func<#replace,string,char,char>`_ - let last = if last == 0: s.high else: last - when nimvm: + result = -1 + let last = if last < 0: s.high else: last + + template findImpl = for i in int(start)..last: - if sub == s[i]: return i + if s[i] == sub: + return i + + when nimvm: + findImpl() else: when hasCStringBuiltin: - let L = last-start+1 - if L > 0: - let found = c_memchr(s[start].unsafeAddr, sub, cast[csize_t](L)) + let length = last-start+1 + if length > 0: + let found = c_memchr(s[start].unsafeAddr, sub, cast[csize_t](length)) if not found.isNil: return cast[ByteAddress](found) -% cast[ByteAddress](s.cstring) else: - for i in int(start)..last: - if sub == s[i]: return i - return -1 + findImpl() -func find*(s: string, chars: set[char], start: Natural = 0, last = 0): int {. +func find*(s: string, chars: set[char], start: Natural = 0, last = -1): int {. rtl, extern: "nsuFindCharSet".} = ## Searches for `chars` in `s` inside range `start..last` (both ends included). - ## If `last` is unspecified, it defaults to `s.high` (the last element). + ## If `last` is unspecified or negative, it defaults to `s.high` (the last element). ## ## If `s` contains none of the characters in `chars`, -1 is returned. ## Otherwise the index returned is relative to `s[0]`, not `start`. @@ -1905,15 +1920,16 @@ func find*(s: string, chars: set[char], start: Natural = 0, last = 0): int {. ## See also: ## * `rfind func<#rfind,string,set[char],Natural,int>`_ ## * `multiReplace func<#multiReplace,string,varargs[]>`_ - let last = if last == 0: s.high else: last + result = -1 + let last = if last < 0: s.high else: last for i in int(start)..last: - if s[i] in chars: return i - return -1 + if s[i] in chars: + return i -func find*(s, sub: string, start: Natural = 0, last = 0): int {.rtl, +func find*(s, sub: string, start: Natural = 0, last = -1): int {.rtl, extern: "nsuFindStr".} = ## Searches for `sub` in `s` inside range `start..last` (both ends included). - ## If `last` is unspecified, it defaults to `s.high` (the last element). + ## If `last` is unspecified or negative, it defaults to `s.high` (the last element). ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. ## Otherwise the index returned is relative to `s[0]`, not `start`. @@ -1925,28 +1941,23 @@ func find*(s, sub: string, start: Natural = 0, last = 0): int {.rtl, if sub.len > s.len - start: return -1 if sub.len == 1: return find(s, sub[0], start, last) - template useSkipTable {.dirty.} = - var a {.noinit.}: SkipTable - initSkipTable(a, sub) - result = find(a, s, sub, start, last) + template useSkipTable = + result = find(initSkipTable(sub), s, sub, start, last) - when not hasCStringBuiltin: + when nimvm: useSkipTable() else: - when nimvm: - useSkipTable() - else: - when hasCStringBuiltin: - if last == 0 and s.len > start: - let found = c_strstr(s[start].unsafeAddr, sub) - if not found.isNil: - result = cast[ByteAddress](found) -% cast[ByteAddress](s.cstring) + when hasCStringBuiltin: + if last < 0 and start < s.len: + let found = c_strstr(s[start].unsafeAddr, sub) + result = if not found.isNil: + cast[ByteAddress](found) -% cast[ByteAddress](s.cstring) else: - result = -1 - else: - useSkipTable() + -1 else: useSkipTable() + else: + useSkipTable() func rfind*(s: string, sub: char, start: Natural = 0, last = -1): int {.rtl, extern: "nsuRFindChar".} = @@ -2121,8 +2132,7 @@ func replace*(s, sub: string, by = ""): string {.rtl, # copy the rest: add result, substr(s, i) else: - var a {.noinit.}: SkipTable - initSkipTable(a, sub) + var a = initSkipTable(sub) let last = s.high var i = 0 while true: @@ -2162,9 +2172,8 @@ func replaceWord*(s, sub: string, by = ""): string {.rtl, ## replaced. if sub.len == 0: return s const wordChars = {'a'..'z', 'A'..'Z', '0'..'9', '_', '\128'..'\255'} - var a {.noinit.}: SkipTable result = "" - initSkipTable(a, sub) + var a = initSkipTable(sub) var i = 0 let last = s.high let sublen = sub.len diff --git a/tests/stdlib/tstrutils.nim b/tests/stdlib/tstrutils.nim index 26fc0c7b0..e17277ef2 100644 --- a/tests/stdlib/tstrutils.nim +++ b/tests/stdlib/tstrutils.nim @@ -232,18 +232,22 @@ template main() = {.pop.} block: # find - doAssert "0123456789ABCDEFGH".find('A') == 10 - doAssert "0123456789ABCDEFGH".find('A', 5) == 10 - doAssert "0123456789ABCDEFGH".find('A', 5, 10) == 10 - doAssert "0123456789ABCDEFGH".find('A', 5, 9) == -1 - doAssert "0123456789ABCDEFGH".find("A") == 10 - doAssert "0123456789ABCDEFGH".find("A", 5) == 10 - doAssert "0123456789ABCDEFGH".find("A", 5, 10) == 10 - doAssert "0123456789ABCDEFGH".find("A", 5, 9) == -1 - doAssert "0123456789ABCDEFGH".find({'A'..'C'}) == 10 - doAssert "0123456789ABCDEFGH".find({'A'..'C'}, 5) == 10 - doAssert "0123456789ABCDEFGH".find({'A'..'C'}, 5, 10) == 10 - doAssert "0123456789ABCDEFGH".find({'A'..'C'}, 5, 9) == -1 + const haystack: string = "0123456789ABCDEFGH" + doAssert haystack.find('A') == 10 + doAssert haystack.find('A', 5) == 10 + doAssert haystack.find('A', 5, 10) == 10 + doAssert haystack.find('A', 5, 9) == -1 + doAssert haystack.find("A") == 10 + doAssert haystack.find("A", 5) == 10 + doAssert haystack.find("A", 5, 10) == 10 + doAssert haystack.find("A", 5, 9) == -1 + doAssert haystack.find({'A'..'C'}) == 10 + doAssert haystack.find({'A'..'C'}, 5) == 10 + doAssert haystack.find({'A'..'C'}, 5, 10) == 10 + doAssert haystack.find({'A'..'C'}, 5, 9) == -1 + doAssert haystack.find('A', 0, 0) == -1 # search limited to the first char + doAssert haystack.find('A', 5, 0) == -1 # last < start + doAssert haystack.find('A', 5, 4) == -1 # last < start block: const haystack: string = "ABCABABABABCAB" @@ -290,17 +294,17 @@ template main() = # when last <= start, searching for non-empty string block: - let last: int = -1 - doAssert "abcd".find("ab", start=0, last=last) == -1 - doAssert "abcd".find("ab", start=1, last=last) == -1 - doAssert "abcd".find("bc", start=1, last=last) == -1 - doAssert "abcd".find("bc", start=2, last=last) == -1 - block: - let last: int = 0 + let last: int = -1 # searching through whole line doAssert "abcd".find("ab", start=0, last=last) == 0 doAssert "abcd".find("ab", start=1, last=last) == -1 doAssert "abcd".find("bc", start=1, last=last) == 1 doAssert "abcd".find("bc", start=2, last=last) == -1 + block: + let last: int = 0 + doAssert "abcd".find("ab", start=0, last=last) == -1 + doAssert "abcd".find("ab", start=1, last=last) == -1 + doAssert "abcd".find("bc", start=1, last=last) == -1 + doAssert "abcd".find("bc", start=2, last=last) == -1 block: let last: int = 1 doAssert "abcd".find("ab", start=0, last=last) == 0 From f2b16c490df858328cbd884ff9ca9d1a675a4a31 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Wed, 8 Jun 2022 11:10:52 -0300 Subject: [PATCH 1641/2472] Add Microtasks docs (#19870) * Add Microtasks * Add Microtasks * Workaround to build js docs in older NodeJS versions --- lib/js/jscore.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/js/jscore.nim b/lib/js/jscore.nim index e6060535b..781e8fd57 100644 --- a/lib/js/jscore.nim +++ b/lib/js/jscore.nim @@ -136,4 +136,4 @@ since (1, 7): func queueMicrotask*(function: proc) {.importjs: "$1(#)".} = ## * https://developer.mozilla.org/en-US/docs/Web/API/queueMicrotask ## * https://developer.mozilla.org/en-US/docs/Web/API/HTML_DOM_API/Microtask_guide - runnableExamples: queueMicrotask(proc() = echo "Microtask") + runnableExamples"-r:off": queueMicrotask(proc() = echo "Microtask") From e1702ae1e6da5bf560d7b692bcd5fa513ece8a69 Mon Sep 17 00:00:00 2001 From: Zoom Date: Wed, 8 Jun 2022 14:11:22 +0000 Subject: [PATCH 1642/2472] `changelog.md`: Add `strutils.find` changes (#19868) Added `strutils.find` changes Standard library section restructured to group by `changed/added/deprecated/removed` with paragraphs marked by markdown comments. --- changelog.md | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/changelog.md b/changelog.md index 2ae64d543..356ee9263 100644 --- a/changelog.md +++ b/changelog.md @@ -24,35 +24,39 @@ becomes an alias for `addr`. ## Standard library additions and changes -- `macros.parseExpr` and `macros.parseStmt` now accept an optional +[//]: # "Changes:" +- `macros.parseExpr` and `macros.parseStmt` now accept an optional. filename argument for more informative errors. - Module `colors` expanded with missing colors from the CSS color standard. - Fixed `lists.SinglyLinkedList` being broken after removing the last node ([#19353](https://github.com/nim-lang/Nim/pull/19353)). - `md5` now works at compile time and in JavaScript. - - `std/smtp` sends `ehlo` first. If the mail server does not understand, it sends `helo` as a fallback. +- Changed mimedb to use an `OrderedTable` instead of `OrderedTableRef`, to use it in a const. +- `strutils.find` now use and default to `last=-1` for whole string searches, making limiting it to just the first char (`last=0`) valid. +[//]: # "Additions:" - Added `IsoWeekRange`, a range type to represent the number of weeks in an ISO week-based year. - Added `IsoYear`, a distinct int type to prevent bugs from confusing the week-based year and the regular year. - Added `initDateTime` in `times` to create a datetime from a weekday, and ISO 8601 week number and week-based year. - Added `getIsoWeekAndYear` in `times` to get an ISO week number along with the corresponding ISO week-based year from a datetime. - Added `getIsoWeeksInYear` in `times` to return the number of weeks in an ISO week-based year. -- Added [`Array.shift`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/shift) for JavaScript targets. - Added `std/oserrors` for OS error reporting. Added `std/envvars` for environment variables handling. - Added `sep` parameter in `std/uri` to specify the query separator. -- Removed deprecated `oids.oidToString`. -- Remove define `nimExperimentalAsyncjsThen` for `std/asyncjs.then` and `std/jsfetch`. - -- Changed mimedb to use an `OrderedTable` instead of `OrderedTableRef`, to use it in a const. -- Removed deprecated `jsre.test` and `jsre.toString`. -- Removed deprecated `math.c_frexp`. -- Removed deprecated ``` httpcore.`==` ```. -- Removed deprecated `std/dom_extensions`. -- Removed deprecated `std/posix.CMSG_SPACE` and `std/posix.CMSG_LEN` that takes wrong argument types. -- Remove deprecated `osproc.poDemon`, symbol with typo. +- Added [`Array.shift`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/shift) for JavaScript targets. - Added [`queueMicrotask`](https://developer.mozilla.org/en-US/docs/Web/API/queueMicrotask) for JavaScript targets. + +[//]: # "Deprecations:" - Deprecated `selfExe` for Nimscript. +[//]: # "Removals:" +- Removed deprecated `oids.oidToString`. +- Removed define `nimExperimentalAsyncjsThen` for `std/asyncjs.then` and `std/jsfetch`. +- Removed deprecated `jsre.test` and `jsre.toString`. +- Removed deprecated `math.c_frexp`. +- Removed deprecated `` httpcore.`==` ``. +- Removed deprecated `std/dom_extensions`. +- Removed deprecated `std/posix.CMSG_SPACE` and `std/posix.CMSG_LEN` that takes wrong argument types. +- Removed deprecated `osproc.poDemon`, symbol with typo. ## Language changes From 415689323c625afb7d39d923c4165a541beb9927 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Thu, 9 Jun 2022 00:17:25 +0800 Subject: [PATCH 1643/2472] clarify how to restart CI (#19845) --- doc/contributing.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/contributing.rst b/doc/contributing.rst index d3b2770f3..0ca0d0cbc 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -539,6 +539,7 @@ Debugging CI failures, flaky tests, etc 2. If CI failure seems unrelated to your PR, it could be caused by a flaky test. File a bug for it if it isn't already reported. A PR push (or opening/closing PR) will re-trigger all CI jobs (even successful ones, which can be wasteful). Instead, + request collaboration from the Nim team. The Nim team should follow these instructions to only restart the jobs that failed: * Azure: if on your own fork, it's possible from inside azure console From 836b95c896087a3f24bedad9bc1bcb2c5db1c5a8 Mon Sep 17 00:00:00 2001 From: Antonis Geralis <43617260+planetis-m@users.noreply.github.com> Date: Thu, 9 Jun 2022 09:27:31 +0300 Subject: [PATCH 1644/2472] Deprecate std/sums (#18439) * Deprecate sums * Update changelog.md * Update lib/std/sums.nim * log * format * remove * Update changelog.md Co-authored-by: sandytypical <43030857+xflywind@users.noreply.github.com> --- changelog.md | 2 ++ lib/std/sums.nim | 2 ++ 2 files changed, 4 insertions(+) diff --git a/changelog.md b/changelog.md index 356ee9263..6bd3c99e3 100644 --- a/changelog.md +++ b/changelog.md @@ -11,6 +11,8 @@ or define your own `Math.trunc` polyfill using the [`emit` pragma](https://nim-lang.org/docs/manual.html#implementation-specific-pragmas-emit-pragma). Nim uses `Math.trunc` for the division and modulo operators for integers. +- Deprecated `std/sums`. + - Optional parameters in combination with `: body` syntax (RFC #405) are now opt-in via `experimental:flexibleOptionalParams`. diff --git a/lib/std/sums.nim b/lib/std/sums.nim index b68858ef7..a6ce1b85d 100644 --- a/lib/std/sums.nim +++ b/lib/std/sums.nim @@ -8,6 +8,8 @@ ## Accurate summation functions. +{.deprecated: "use the nimble package `sums` instead.".} + runnableExamples: import std/math From 25d89269eb7fe7ebed142f2d3136959db11c9f02 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 9 Jun 2022 14:21:18 +0200 Subject: [PATCH 1645/2472] give a better error message for some template expansions (#19871) --- compiler/semstmts.nim | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 3513fd767..0516d08ce 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -140,7 +140,9 @@ proc discardCheck(c: PContext, result: PNode, flags: TExprFlags) = n[0] = result elif result.typ.kind != tyError and c.config.cmd != cmdInteractive: var n = result - while n.kind in skipForDiscardable: n = n.lastSon + while n.kind in skipForDiscardable: + if n.kind == nkTryStmt: n = n[0] + else: n = n.lastSon var s = "expression '" & $n & "' is of type '" & result.typ.typeToString & "' and has to be used (or discarded)" if result.info.line != n.info.line or @@ -483,7 +485,7 @@ proc semVarMacroPragma(c: PContext, a: PNode, n: PNode): PNode = for i in 0 ..< pragmas.len: let it = pragmas[i] let key = if it.kind in nkPragmaCallKinds and it.len >= 1: it[0] else: it - + when false: let lhs = b[0] let clash = strTableGet(c.currentScope.symbols, lhs.ident) From 1972005439306a12088400ef338c40c3d3c2ce28 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Thu, 9 Jun 2022 22:51:17 +0800 Subject: [PATCH 1646/2472] fix #19862; make widestrs consistent between refc and orc (#19874) [backport] fix #19862; make widestrs consistent in refc and orc --- lib/system/widestrs.nim | 3 +++ tests/arc/t19862.nim | 13 +++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 tests/arc/t19862.nim diff --git a/lib/system/widestrs.nim b/lib/system/widestrs.nim index 8b08959b5..bb348fd67 100644 --- a/lib/system/widestrs.nim +++ b/lib/system/widestrs.nim @@ -224,3 +224,6 @@ when defined(nimv2): proc `$`*(s: WideCStringObj): string = $(s.data) + + proc len*(w: WideCStringObj): int {.inline.} = + len(w.data) diff --git a/tests/arc/t19862.nim b/tests/arc/t19862.nim new file mode 100644 index 000000000..f7146ec26 --- /dev/null +++ b/tests/arc/t19862.nim @@ -0,0 +1,13 @@ +discard """ + matrix: "--gc:refc; --gc:arc" +""" + +# bug #19862 +type NewString = object + +proc len(s: NewString): int = 10 + +converter toNewString(x: WideCStringObj): NewString = discard + +let w = newWideCString("test") +doAssert len(w) == 4 From eefca1b81f334a5ccdb7d180de5b6a9969f12793 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Fri, 10 Jun 2022 21:24:28 +0800 Subject: [PATCH 1647/2472] not generate initStackBottomWith in arc/orc [backport] (#19875) not generate initStackBottomWith in arc/orc --- compiler/cgen.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 8d24f30c5..862f31107 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -2094,7 +2094,7 @@ proc finalCodegenActions*(graph: ModuleGraph; m: BModule; n: PNode) = discard cgsym(m, "rawWrite") # raise dependencies on behalf of genMainProc - if m.config.target.targetOS != osStandalone and m.config.selectedGC != gcNone: + if m.config.target.targetOS != osStandalone and m.config.selectedGC notin {gcNone, gcArc, gcOrc}: discard cgsym(m, "initStackBottomWith") if emulatedThreadVars(m.config) and m.config.target.targetOS != osStandalone: discard cgsym(m, "initThreadVarsEmulation") From 8cde6ba6bcf236478f1d5d5aa722977ce9723a57 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Sat, 11 Jun 2022 02:32:27 +0800 Subject: [PATCH 1648/2472] remove noop option `gc:v2` (#19810) * remove noop option gc:v2 * changelog --- changelog.md | 2 ++ compiler/ccgstmts.nim | 2 +- compiler/ccgtypes.nim | 4 ++-- compiler/commands.nim | 2 -- compiler/options.nim | 1 - lib/system/sysstr.nim | 14 ++------------ 6 files changed, 7 insertions(+), 18 deletions(-) diff --git a/changelog.md b/changelog.md index 6bd3c99e3..98655eaa3 100644 --- a/changelog.md +++ b/changelog.md @@ -24,6 +24,8 @@ becomes an alias for `addr`. - io is about to move out of system; use `-d:nimPreviewSlimSystem` and import `std/syncio`. +- The `gc:v2` option is removed. + ## Standard library additions and changes [//]: # "Changes:" diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index 240fa55c8..e791318ee 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -16,7 +16,7 @@ const # above X strings a hash-switch for strings is generated proc getTraverseProc(p: BProc, v: PSym): Rope = - if p.config.selectedGC in {gcMarkAndSweep, gcHooks, gcV2, gcRefc} and + if p.config.selectedGC in {gcMarkAndSweep, gcHooks, gcRefc} and optOwnedRefs notin p.config.globalOptions and containsGarbageCollectedRef(v.loc.t): # we register a specialized marked proc here; this has the advantage diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index c5b4d3857..7c26db40d 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -1483,12 +1483,12 @@ proc genTypeInfoV1(m: BModule, t: PType; info: TLineInfo): Rope = genTupleInfo(m, x, x, result, info) of tySequence: genTypeInfoAux(m, t, t, result, info) - if m.config.selectedGC in {gcMarkAndSweep, gcRefc, gcV2, gcGo}: + if m.config.selectedGC in {gcMarkAndSweep, gcRefc, gcGo}: let markerProc = genTraverseProc(m, origType, sig) m.s[cfsTypeInit3].addf("$1.marker = $2;$n", [tiNameForHcr(m, result), markerProc]) of tyRef: genTypeInfoAux(m, t, t, result, info) - if m.config.selectedGC in {gcMarkAndSweep, gcRefc, gcV2, gcGo}: + if m.config.selectedGC in {gcMarkAndSweep, gcRefc, gcGo}: let markerProc = genTraverseProc(m, origType, sig) m.s[cfsTypeInit3].addf("$1.marker = $2;$n", [tiNameForHcr(m, result), markerProc]) of tyPtr, tyRange, tyUncheckedArray: genTypeInfoAux(m, t, t, result, info) diff --git a/compiler/commands.nim b/compiler/commands.nim index 2e08ad615..31e637aba 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -262,7 +262,6 @@ proc testCompileOptionArg*(conf: ConfigRef; switch, arg: string, info: TLineInfo of "go": result = conf.selectedGC == gcGo of "none": result = conf.selectedGC == gcNone of "stack", "regions": result = conf.selectedGC == gcRegions - of "v2", "generational": warningOptionNoop(arg) else: localError(conf, info, errNoneBoehmRefcExpectedButXFound % arg) of "opt": case arg.normalize @@ -568,7 +567,6 @@ proc processMemoryManagementOption(switch, arg: string, pass: TCmdLinePass, unregisterArcOrc(conf) conf.selectedGC = gcRegions defineSymbol(conf.symbols, "gcregions") - of "v2": warningOptionNoop(arg) else: localError(conf, info, errNoneBoehmRefcExpectedButXFound % arg) proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; diff --git a/compiler/options.nim b/compiler/options.nim index 545dfb1d1..89fb66d5f 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -183,7 +183,6 @@ type gcMarkAndSweep = "markAndSweep" gcHooks = "hooks" gcRefc = "refc" - gcV2 = "v2" gcGo = "go" # gcRefc and the GCs that follow it use a write barrier, # as far as usesWriteBarrier() is concerned diff --git a/lib/system/sysstr.nim b/lib/system/sysstr.nim index 49fff41e9..c4f0db718 100644 --- a/lib/system/sysstr.nim +++ b/lib/system/sysstr.nim @@ -304,20 +304,10 @@ proc setLengthSeq(seq: PGenericSeq, elemSize, elemAlign, newLen: int): PGenericS when not defined(boehmGC) and not defined(nogc) and not defined(gcMarkAndSweep) and not defined(gogc) and not defined(gcRegions): - when false: # deadcode: was used by `compileOption("gc", "v2")` + if ntfNoRefs notin extGetCellType(result).base.flags: for i in newLen..result.len-1: - let len0 = gch.tempStack.len forAllChildrenAux(dataPointer(result, elemAlign, elemSize, i), - extGetCellType(result).base, waPush) - let len1 = gch.tempStack.len - for i in len0 ..< len1: - doDecRef(gch.tempStack.d[i], LocalHeap, MaybeCyclic) - gch.tempStack.len = len0 - else: - if ntfNoRefs notin extGetCellType(result).base.flags: - for i in newLen..result.len-1: - forAllChildrenAux(dataPointer(result, elemAlign, elemSize, i), - extGetCellType(result).base, waZctDecRef) + extGetCellType(result).base, waZctDecRef) # XXX: zeroing out the memory can still result in crashes if a wiped-out # cell is aliased by another pointer (ie proc parameter or a let variable). From 1e5dd9022b0d86d63d616431a0f9959ca4c10f40 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Sat, 11 Jun 2022 02:33:44 +0800 Subject: [PATCH 1649/2472] [js] add testcase for array indexDefect and remove todo (#19838) * remove unused opcSubstr * [js] add testcase for array indexDefect * Revert "remove unused opcSubstr" This reverts commit cb461f2545234d62c1e0b83318f3e5495c97de52. --- lib/system/jssys.nim | 4 ---- tests/js/tindexdefect.nim | 9 +++++++++ 2 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 tests/js/tindexdefect.nim diff --git a/lib/system/jssys.nim b/lib/system/jssys.nim index 00a4a8ab6..a31de0d86 100644 --- a/lib/system/jssys.nim +++ b/lib/system/jssys.nim @@ -731,10 +731,6 @@ proc parseFloatNative(a: string): float = `result` = Number(`a2`); """ -#[ -xxx how come code like this doesn't give IndexDefect ? -let z = s[10000] == 'a' -]# proc nimParseBiggestFloat(s: string, number: var BiggestFloat, start: int): int {.compilerproc.} = var sign: bool var i = start diff --git a/tests/js/tindexdefect.nim b/tests/js/tindexdefect.nim new file mode 100644 index 000000000..37994ec2e --- /dev/null +++ b/tests/js/tindexdefect.nim @@ -0,0 +1,9 @@ +discard """ + outputsub: "unhandled exception: index 10000 not in 0 .. 0 [IndexDefect]" + exitcode: 1 + joinable: false +""" + +var s = ['a'] +let z = s[10000] == 'a' +echo z \ No newline at end of file From 6f4bacff67e2e219ef914e24d9f9aaf34a6d1eb1 Mon Sep 17 00:00:00 2001 From: quantimnot <54247259+quantimnot@users.noreply.github.com> Date: Fri, 10 Jun 2022 14:40:08 -0400 Subject: [PATCH 1650/2472] Extend and document compiler debugging utilities (#19841) * Add two debugutils procs that native debuggers can break on use to execute commands when code of interest is being compiled. * Add GDB and LLDB programs to disable and enable breakpoints and watchpoints when code of interest is being compiled. * Extend the `intern.rst` docs regarding debugging the compiler. Co-authored-by: quantimnot --- compiler/debugutils.nim | 16 +++ doc/intern.rst | 222 +++++++++++++++++++++++++++++++++------- doc/koch.rst | 3 + tools/compiler.gdb | 39 +++++++ tools/compiler.lldb | 40 ++++++++ 5 files changed, 281 insertions(+), 39 deletions(-) create mode 100644 tools/compiler.gdb create mode 100644 tools/compiler.lldb diff --git a/compiler/debugutils.nim b/compiler/debugutils.nim index d109d2121..adbb0517f 100644 --- a/compiler/debugutils.nim +++ b/compiler/debugutils.nim @@ -54,3 +54,19 @@ proc isCompilerDebug*(): bool = {.undef(nimCompilerDebug).} echo 'x' conf0.isDefined("nimCompilerDebug") + +proc enteringDebugSection*() {.exportc, dynlib.} = + ## Provides a way for native debuggers to enable breakpoints, watchpoints, etc + ## when code of interest is being compiled. + ## + ## Set your debugger to break on entering `nimCompilerIsEnteringDebugSection` + ## and then execute a desired command. + discard + +proc exitingDebugSection*() {.exportc, dynlib.} = + ## Provides a way for native debuggers to disable breakpoints, watchpoints, etc + ## when code of interest is no longer being compiled. + ## + ## Set your debugger to break on entering `exitingDebugSection` + ## and then execute a desired command. + discard diff --git a/doc/intern.rst b/doc/intern.rst index 4ca7eff20..9103c694c 100644 --- a/doc/intern.rst +++ b/doc/intern.rst @@ -79,55 +79,216 @@ Set the compilation timestamp with the `SOURCE_DATE_EPOCH` environment variable. koch boot # or `./build_all.sh` -Developing the compiler -======================= +Debugging the compiler +====================== -To create a new compiler for each run, use `koch temp`:cmd:\: + +Bisecting for regressions +------------------------- + +There are often times when there is a bug that is caused by a regression in the +compiler or stdlib. Bisecting the Nim repo commits is a usefull tool to identify +what commit introduced the regression. + +Even if it's not known whether a bug is caused by a regression, bisection can reduce +debugging time by ruling it out. If the bug is found to be a regression, then you +focus on the changes introduced by that one specific commit. + +`koch temp`:cmd: returns 125 as the exit code in case the compiler +compilation fails. This exit code tells `git bisect`:cmd: to skip the +current commit: .. code:: cmd - koch temp c test.nim + git bisect start bad-commit good-commit + git bisect run ./koch temp -r c test-source.nim -`koch temp`:cmd: creates a debug build of the compiler, which is useful -to create stacktraces for compiler debugging. +You can also bisect using custom options to build the compiler, for example if +you don't need a debug version of the compiler (which runs slower), you can replace +`./koch temp`:cmd: by explicit compilation command, see `Bootstrapping the compiler`_. -You can of course use GDB or Visual Studio to debug the -compiler (via `--debuginfo --lineDir:on`:option:). However, there -are also lots of procs that aid in debugging: +Building an instrumented compiler +--------------------------------- + +Considering that a useful method of debugging the compiler is inserting debug +logging, or changing code and then observing the outcome of a testcase, it is +fastest to build a compiler that is instrumented for debugging from an +existing release build. `koch temp`:cmd: provides a convenient method of doing +just that. + +By default running `koch temp`:cmd: will build a lean version of the compiler +with `-d:debug`:option: enabled. The compiler is written to `bin/nim_temp` by +default. A lean version of the compiler lacks JS and documentation generation. + +`bin/nim_temp` can be directly used to run testcases, or used with testament +with `testament --nim:bin/nim_temp r tests/category/tsometest`:cmd:. + +`koch temp`:cmd: will build the temporary compiler with the `-d:debug`:option: +enabled. Here are compiler options that are of interest for debugging: + +* `-d:debug`:option:\: enables `assert` statements and stacktraces and all + runtime checks +* `--opt:speed`:option:\: build with optimizations enabled +* `--debugger:native`:option:\: enables `--debuginfo --lineDir:on`:option: for using + a native debugger like GDB, LLDB or CDB +* `-d:nimDebug`:option: cause calls to `quit` to raise an assertion exception +* `-d:nimDebugUtils`:option:\: enables various debugging utilities; + see `compiler/debugutils` +* `-d:stacktraceMsgs -d:nimCompilerStacktraceHints`:option:\: adds some additional + stacktrace hints; see https://github.com/nim-lang/Nim/pull/13351 +* `-u:leanCompiler`:option:\: enable JS and doc generation + +Another method to build and run the compiler is directly through `koch`:cmd:\: + +.. code:: cmd + + koch temp [options] c test.nim + + # (will build with js support) + koch temp [options] js test.nim + + # (will build with doc support) + koch temp [options] doc test.nim + +Debug logging +------------- + +"Printf debugging" is still the most appropriate way to debug many problems +arising in compiler development. The typical usage of breakpoints to debug +the code is often less practical, because almost all of the code paths in the +compiler will be executed hundreds of times before a particular section of the +tested program is reached where the newly developed code must be activated. + +To work-around this problem, you'll typically introduce an if statement in the +compiler code detecting more precisely the conditions where the tested feature +is being used. One very common way to achieve this is to use the `mdbg` condition, +which will be true only in contexts, processing expressions and statements from +the currently compiled main module: .. code-block:: nim - # dealing with PNode: - echo renderTree(someNode) - debug(someNode) # some JSON representation + # inside some compiler module + if mdbg: + debug someAstNode - # dealing with PType: +Using the `isCompilerDebug`:nim: condition along with inserting some statements +into the testcase provides more granular logging: + +.. code-block:: nim + + # compilermodule.nim + if isCompilerDebug(): + debug someAstNode + + # testcase.nim + proc main = + {.define(nimCompilerDebug).} + let a = 2.5 * 3 + {.undef(nimCompilerDebug).} + +Logging can also be scoped to a specific filename as well. This will of course +match against every module with that name. + +.. code-block:: nim + + if `??`(conf, n.info, "module.nim"): + debug(n) + +The above examples also makes use of the `debug`:nim: proc, which is able to +print a human-readable form of an arbitrary AST tree. Other common ways to print +information about the internal compiler types include: + +.. code-block:: nim + + # pretty print PNode + + # pretty prints the Nim ast + echo renderTree(someNode) + + # pretty prints the Nim ast, but annotates symbol IDs + echo renderTree(someNode, {renderIds}) + + # pretty print ast as JSON + debug(someNode) + + # print as YAML + echo treeToYaml(config, someNode) + + + # pretty print PType + + # print type name echo typeToString(someType) + + # pretty print as JSON debug(someType) - # dealing with PSym: + # print as YAML + echo typeToYaml(config, someType) + + + # pretty print PSym + + # print the symbol's name echo symbol.name.s + + # pretty print as JSON debug(symbol) - # pretty prints the Nim ast, but annotates symbol IDs: - echo renderTree(someNode, {renderIds}) - if `??`(conf, n.info, "temp.nim"): - # only output when it comes from "temp.nim" - echo renderTree(n) - if `??`(conf, n.info, "temp.nim"): - # why does it process temp.nim here? - writeStackTrace() + # print as YAML + echo symToYaml(config, symbol) + + + # pretty print TLineInfo + lineInfoToStr(lineInfo) + + + # print the structure of any type + repr(someVar) + +Here are some other helpful utilities: + +.. code-block:: nim + + # how did execution reach this location? + writeStackTrace() These procs may not already be imported by the module you're editing. You can import them directly for debugging: .. code-block:: nim + from astalgo import debug from types import typeToString from renderer import renderTree from msgs import `??` +Native debugging +---------------- + +Stepping through the compiler with a native debugger is a very powerful tool to +both learn and debug it. However, there is still the need to constrain when +breakpoints are triggered. The same methods as in `Debug logging`_ can be applied +here when combined with calls to the debug helpers `enteringDebugSection()`:nim: +and `exitingDebugSection()`:nim:. + +#. Compile the temp compiler with `--debugger:native -d:nimDebugUtils`:option: +#. Set your desired breakpoints or watchpoints. +#. Configure your debugger: + * GDB: execute `source tools/compiler.gdb` at startup + * LLDB execute `command source tools/compiler.lldb` at startup +#. Use one of the scoping helpers like so: + +.. code-block:: nim + + if isCompilerDebug(): + enteringDebugSection() + else: + exitingDebugSection() + +A caveat of this method is that all breakpoints and watchpoints are enabled or +disabled. Also, due to a bug, only breakpoints can be constrained for LLDB. The compiler's architecture =========================== @@ -152,23 +313,6 @@ for the type definitions. The `macros `_ module contains many examples how the AST represents each syntactic structure. -Bisecting for regressions -========================= - -`koch temp`:cmd: returns 125 as the exit code in case the compiler -compilation fails. This exit code tells `git bisect`:cmd: to skip the -current commit: - -.. code:: cmd - - git bisect start bad-commit good-commit - git bisect run ./koch temp -r c test-source.nim - -You can also bisect using custom options to build the compiler, for example if -you don't need a debug version of the compiler (which runs slower), you can replace -`./koch temp`:cmd: by explicit compilation command, see `Bootstrapping the compiler`_. - - Runtimes ======== diff --git a/doc/koch.rst b/doc/koch.rst index b8d85dff4..91dd5d570 100644 --- a/doc/koch.rst +++ b/doc/koch.rst @@ -38,6 +38,9 @@ options: unless you are debugging the compiler. -d:nimUseLinenoise Use the linenoise library for interactive mode (not needed on Windows). +-d:leanCompiler Produce a compiler without JS codegen or + documentation generator in order to use less RAM + for bootstrapping. After compilation is finished you will hopefully end up with the nim compiler in the `bin` directory. You can add Nim's `bin` directory to diff --git a/tools/compiler.gdb b/tools/compiler.gdb new file mode 100644 index 000000000..c81f47152 --- /dev/null +++ b/tools/compiler.gdb @@ -0,0 +1,39 @@ +# create a breakpoint on `debugutils.enteringDebugSection` +define enable_enteringDebugSection + break -function enteringDebugSection + # run these commands once breakpoint enteringDebugSection is hit + command + # enable all breakpoints and watchpoints + enable + # continue execution + cont + end +end + +# create a breakpoint on `debugutils.exitingDebugSection` named exitingDebugSection +define enable_exitingDebugSection + break -function exitingDebugSection + # run these commands once breakpoint exitingDebugSection is hit + command + # disable all breakpoints and watchpoints + disable + # but enable the enteringDebugSection breakpoint + enable_enteringDebugSection + # continue execution + cont + end +end + +# some commands can't be set until the process is running, so set an entry breakpoint +break -function NimMain +# run these commands once breakpoint NimMain is hit +command + # disable all breakpoints and watchpoints + disable + # but enable the enteringDebugSection breakpoint + enable_enteringDebugSection + # no longer need this breakpoint + delete -function NimMain + # continue execution + cont +end diff --git a/tools/compiler.lldb b/tools/compiler.lldb new file mode 100644 index 000000000..e0b375055 --- /dev/null +++ b/tools/compiler.lldb @@ -0,0 +1,40 @@ +# create a breakpoint on `debugutils.enteringDebugSection` named enteringDebugSection +breakpoint set -n 'enteringDebugSection' -N enteringDebugSection +# run these commands once breakpoint enteringDebugSection is hit +breakpoint command add enteringDebugSection + # enable all breakpoints + breakpoint enable + # enable all watchpoints + # watchpoint enable # FIXME: not currently working for unknown reason + # continue execution + continue +DONE + +# create a breakpoint on `debugutils.exitingDebugSection` named exitingDebugSection +breakpoint set -n 'exitingDebugSection' -N exitingDebugSection +# run these commands once breakpoint exitingDebugSection is hit +breakpoint command add exitingDebugSection + # disable all breakpoints + breakpoint disable + # disable all watchpoints + # watchpoint disable # FIXME: not currently working for unknown reason + breakpoint enable enteringDebugSection + # continue execution + continue +DONE + +# some commands can't be set until the process is running, so set an entry breakpoint +breakpoint set -n NimMain -N NimMain +# run these commands once breakpoint NimMain is hit +breakpoint command add NimMain + # disable all breakpoints + breakpoint disable + # disable all watchpoints + # watchpoint disable # FIXME: not currently working for unknown reason + # enable the enteringDebugSection breakpoint though + breakpoint enable enteringDebugSection + # no longer need this breakpoint + breakpoint delete NimMain + # continue execution + continue +DONE From ac89eead8a40a1a8ee170a4650a1d47e203da14c Mon Sep 17 00:00:00 2001 From: Bung Date: Sat, 11 Jun 2022 02:40:34 +0800 Subject: [PATCH 1651/2472] test for #19046 (#19839) * test for #19046 * add threads flag --- tests/misc/t19046.nim | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 tests/misc/t19046.nim diff --git a/tests/misc/t19046.nim b/tests/misc/t19046.nim new file mode 100644 index 000000000..b3bfec7ae --- /dev/null +++ b/tests/misc/t19046.nim @@ -0,0 +1,19 @@ +discard """ + targets: "c cpp" + matrix: "--threads:on" + disabled: "win" + disabled: "osx" + action: compile +""" + +# bug #19046 + +import std/os + +var t: Thread[void] + +proc test = discard +proc main = + createThread(t, test) + pinToCpu(t, 1) +main() \ No newline at end of file From e2e663a14389500b07e2f6fcd5c231536bdf76a0 Mon Sep 17 00:00:00 2001 From: Carlo Capocasa Date: Sat, 11 Jun 2022 18:23:31 +0200 Subject: [PATCH 1652/2472] Friendlier error message with solution (#19880) * Add helpful suggestion, should always apply * mention var param limitation in async docs * Update compiler/lambdalifting.nim whoops thanks Co-authored-by: flywind <43030857+xflywind@users.noreply.github.com> Co-authored-by: flywind <43030857+xflywind@users.noreply.github.com> --- compiler/lambdalifting.nim | 3 ++- lib/pure/asyncdispatch.nim | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 4b7ca89cd..96edba8c8 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -288,7 +288,8 @@ proc markAsClosure(g: ModuleGraph; owner: PSym; n: PNode) = if illegalCapture(s): localError(g.config, n.info, ("'$1' is of type <$2> which cannot be captured as it would violate memory" & - " safety, declared here: $3; using '-d:nimNoLentIterators' helps in some cases") % + " safety, declared here: $3; using '-d:nimNoLentIterators' helps in some cases." & + " Consider using a which can be captured.") % [s.name.s, typeToString(s.typ), g.config$s.info]) elif not (owner.typ.callConv == ccClosure or owner.typ.callConv == ccNimCall and tfExplicitCallConv notin owner.typ.flags): localError(g.config, n.info, "illegal capture '$1' because '$2' has the calling convention: <$3>" % diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 755f59ff7..0d406b271 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -98,6 +98,10 @@ ## `await`. The following section shows different ways that you can handle ## exceptions in async procs. ## +## .. caution:: +## Procedures marked {.async.} do not support mutable parameters such +## as `var int`. References such as `ref int` should be used instead. +## ## Handling Exceptions ## ------------------- ## @@ -192,6 +196,7 @@ ## ================ ## ## * The effect system (`raises: []`) does not work with async procedures. +## * Mutable parameters are not supported by async procedures. ## ## ## Multiple async backend support From 8fa2c0b532be9fdfd7682e5a8cc846b60bd4ea6c Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Mon, 13 Jun 2022 16:03:40 +1000 Subject: [PATCH 1653/2472] Pass headers and body correctly to FetchOptions (#19884) [backport] * Pass headers to FetchOptions Don't pass body if method is HttpGet or HttpHead * Syntax fixes * Restart CI --- lib/std/jsfetch.nim | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/std/jsfetch.nim b/lib/std/jsfetch.nim index 034bb6332..7fe154325 100644 --- a/lib/std/jsfetch.nim +++ b/lib/std/jsfetch.nim @@ -90,8 +90,9 @@ func newfetchOptions*(metod: HttpMethod; body: cstring; headers: Headers = newHeaders()): FetchOptions = ## Constructor for `FetchOptions`. result = FetchOptions( - body: body, mode: cstring($mode), credentials: cstring($credentials), cache: cstring($cache), referrerPolicy: cstring($referrerPolicy), - keepalive: keepalive, redirect: cstring($redirect), referrer: referrer, integrity: integrity, + body: if metod notin {HttpHead, HttpGet}: body else: nil, + mode: cstring($mode), credentials: cstring($credentials), cache: cstring($cache), referrerPolicy: cstring($referrerPolicy), + keepalive: keepalive, redirect: cstring($redirect), referrer: referrer, integrity: integrity, headers: headers, metod: (case metod of HttpHead: "HEAD".cstring of HttpGet: "GET".cstring From 2f4900615ab0c42c9608910dd30a14be52455f91 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Mon, 13 Jun 2022 14:10:40 +0800 Subject: [PATCH 1654/2472] [cleanup] remove unnecessary procs in vm (#19888) remove unused procs --- compiler/vm.nim | 1 - compiler/vmgen.nim | 7 ++----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/compiler/vm.nim b/compiler/vm.nim index bbcff77ba..1b11cb4f3 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -2252,7 +2252,6 @@ proc setupMacroParam(x: PNode, typ: PType): TFullReg = else: var n = x if n.kind in {nkHiddenSubConv, nkHiddenStdConv}: n = n[1] - n = n.canonValue n.flags.incl nfIsRef n.typ = x.typ result = TFullReg(kind: rkNode, node: n) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index dadcc3865..2e7f6d8b0 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -439,14 +439,11 @@ proc genAndOr(c: PCtx; n: PNode; opc: TOpcode; dest: var TDest) = c.gABC(n, opcAsgnInt, dest, tmp) freeTemp(c, tmp) -proc canonValue*(n: PNode): PNode = - result = n - proc rawGenLiteral(c: PCtx; n: PNode): int = result = c.constants.len #assert(n.kind != nkCall) n.flags.incl nfAllConst - c.constants.add n.canonValue + c.constants.add n internalAssert c.config, result < regBxMax proc sameConstant*(a, b: PNode): bool = @@ -1857,7 +1854,7 @@ proc genVarSection(c: PCtx; n: PNode) = else: let sa = getNullValue(s.typ, a.info, c.config) #if s.ast.isNil: getNullValue(s.typ, a.info) - #else: canonValue(s.ast) + #else: s.ast assert sa.kind != nkCall c.globals.add(sa) s.position = c.globals.len From ab477075860364db2690a0411c0b2c330532042f Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Mon, 13 Jun 2022 15:01:40 +0800 Subject: [PATCH 1655/2472] [semfold] fix #19199; properly fold uint to float conversion (#19890) [backport] fix #19199; properly fold float conversion --- compiler/semfold.nim | 2 +- tests/vm/t19199.nim | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 tests/vm/t19199.nim diff --git a/compiler/semfold.nim b/compiler/semfold.nim index d2bbc63be..72c8abaa9 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -409,7 +409,7 @@ proc foldConv(n, a: PNode; idgen: IdGenerator; g: ModuleGraph; check = false): P rangeCheck(n, getInt(result), g) of tyFloat..tyFloat64: case srcTyp.kind - of tyInt..tyInt64, tyEnum, tyBool, tyChar: + of tyInt..tyInt64, tyUInt..tyUInt64, tyEnum, tyBool, tyChar: result = newFloatNodeT(toFloat64(getOrdValue(a)), n, g) else: result = a diff --git a/tests/vm/t19199.nim b/tests/vm/t19199.nim new file mode 100644 index 000000000..6ae48cb54 --- /dev/null +++ b/tests/vm/t19199.nim @@ -0,0 +1,6 @@ +# bug #19199 +proc mikasa(x: float) = doAssert x == 42 + +static: + mikasa 42.uint.float +mikasa 42.uint.float From a4fdaa88ccfa03bf30113ffc194284fcb45d3054 Mon Sep 17 00:00:00 2001 From: Ardek Romak Date: Mon, 13 Jun 2022 10:22:20 +0300 Subject: [PATCH 1656/2472] Correctly import libcrypto functions using dynlib (#19881) --- lib/wrappers/openssl.nim | 72 +++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 35 deletions(-) diff --git a/lib/wrappers/openssl.nim b/lib/wrappers/openssl.nim index 00ec160cb..1db3fc239 100644 --- a/lib/wrappers/openssl.nim +++ b/lib/wrappers/openssl.nim @@ -93,6 +93,9 @@ else: import dynlib +{.pragma: lcrypto, cdecl, dynlib: DLLUtilName, importc.} +{.pragma: lssl, cdecl, dynlib: DLLSSLName, importc.} + type SslStruct {.final, pure.} = object SslPtr* = ptr SslStruct @@ -102,7 +105,6 @@ type PSTACK* = SslPtr PX509* = SslPtr PX509_NAME* = SslPtr - PEVP_MD* = SslPtr PBIO_METHOD* = SslPtr BIO* = SslPtr EVP_PKEY* = SslPtr @@ -684,48 +686,48 @@ proc RSA_free*(rsa: PRSA) {.cdecl, dynlib: DLLUtilName, importc.} proc RSA_size*(rsa: PRSA): cint {.cdecl, dynlib: DLLUtilName, importc.} # sha types -proc EVP_md_null*(): EVP_MD {.cdecl, importc.} -proc EVP_md2*(): EVP_MD {.cdecl, importc.} -proc EVP_md4*(): EVP_MD {.cdecl, importc.} -proc EVP_md5*(): EVP_MD {.cdecl, importc.} -proc EVP_sha*(): EVP_MD {.cdecl, importc.} -proc EVP_sha1*(): EVP_MD {.cdecl, importc.} -proc EVP_dss*(): EVP_MD {.cdecl, importc.} -proc EVP_dss1*(): EVP_MD {.cdecl, importc.} -proc EVP_ecdsa*(): EVP_MD {.cdecl, importc.} -proc EVP_sha224*(): EVP_MD {.cdecl, importc.} -proc EVP_sha256*(): EVP_MD {.cdecl, importc.} -proc EVP_sha384*(): EVP_MD {.cdecl, importc.} -proc EVP_sha512*(): EVP_MD {.cdecl, importc.} -proc EVP_mdc2*(): EVP_MD {.cdecl, importc.} -proc EVP_ripemd160*(): EVP_MD {.cdecl, importc.} -proc EVP_whirlpool*(): EVP_MD {.cdecl, importc.} -proc EVP_MD_size*(md: EVP_MD): cint {.cdecl, importc.} +proc EVP_md_null*(): EVP_MD {.lcrypto.} +proc EVP_md2*(): EVP_MD {.lcrypto.} +proc EVP_md4*(): EVP_MD {.lcrypto.} +proc EVP_md5*(): EVP_MD {.lcrypto.} +proc EVP_sha*(): EVP_MD {.lcrypto.} +proc EVP_sha1*(): EVP_MD {.lcrypto.} +proc EVP_dss*(): EVP_MD {.lcrypto.} +proc EVP_dss1*(): EVP_MD {.lcrypto.} +proc EVP_ecdsa*(): EVP_MD {.lcrypto.} +proc EVP_sha224*(): EVP_MD {.lcrypto.} +proc EVP_sha256*(): EVP_MD {.lcrypto.} +proc EVP_sha384*(): EVP_MD {.lcrypto.} +proc EVP_sha512*(): EVP_MD {.lcrypto.} +proc EVP_mdc2*(): EVP_MD {.lcrypto.} +proc EVP_ripemd160*(): EVP_MD {.lcrypto.} +proc EVP_whirlpool*(): EVP_MD {.lcrypto.} +proc EVP_MD_size*(md: EVP_MD): cint {.lcrypto.} # hmac functions -proc HMAC*(evp_md: EVP_MD; key: pointer; key_len: cint; d: cstring; n: csize_t; md: cstring; md_len: ptr cuint): cstring {.cdecl, importc.} +proc HMAC*(evp_md: EVP_MD; key: pointer; key_len: cint; d: cstring; n: csize_t; md: cstring; md_len: ptr cuint): cstring {.lcrypto.} # RSA key functions -proc PEM_read_bio_PrivateKey*(bp: BIO, x: ptr EVP_PKEY, cb: pointer, u: pointer): EVP_PKEY {.cdecl, importc.} -proc EVP_PKEY_free*(p: EVP_PKEY) {.cdecl, importc.} -proc EVP_DigestSignInit*(ctx: EVP_MD_CTX, pctx: ptr EVP_PKEY_CTX, typ: EVP_MD, e: ENGINE, pkey: EVP_PKEY): cint {.cdecl, importc.} -proc EVP_DigestInit_ex*(ctx: EVP_MD_CTX, typ: PEVP_MD, engine: SslPtr = nil): cint {.cdecl, importc.} -proc EVP_DigestUpdate*(ctx: EVP_MD_CTX, data: pointer, len: cuint): cint {.cdecl, importc.} -proc EVP_DigestFinal_ex*(ctx: EVP_MD_CTX, buffer: pointer, size: ptr cuint): cint {.cdecl, importc.} -proc EVP_DigestSignFinal*(ctx: EVP_MD_CTX, data: pointer, len: ptr csize_t): cint {.cdecl, importc.} -proc EVP_PKEY_CTX_new*(pkey: EVP_PKEY, e: ENGINE): EVP_PKEY_CTX {.cdecl, importc.} -proc EVP_PKEY_CTX_free*(pkeyCtx: EVP_PKEY_CTX) {.cdecl, importc.} -proc EVP_PKEY_sign_init*(c: EVP_PKEY_CTX): cint {.cdecl, importc.} +proc PEM_read_bio_PrivateKey*(bp: BIO, x: ptr EVP_PKEY, cb: pointer, u: pointer): EVP_PKEY {.lcrypto.} +proc EVP_PKEY_free*(p: EVP_PKEY) {.lcrypto.} +proc EVP_DigestSignInit*(ctx: EVP_MD_CTX, pctx: ptr EVP_PKEY_CTX, typ: EVP_MD, e: ENGINE, pkey: EVP_PKEY): cint {.lcrypto.} +proc EVP_DigestInit_ex*(ctx: EVP_MD_CTX, typ: EVP_MD, engine: SslPtr = nil): cint {.lcrypto.} +proc EVP_DigestUpdate*(ctx: EVP_MD_CTX, data: pointer, len: cuint): cint {.lcrypto.} +proc EVP_DigestFinal_ex*(ctx: EVP_MD_CTX, buffer: pointer, size: ptr cuint): cint {.lcrypto.} +proc EVP_DigestSignFinal*(ctx: EVP_MD_CTX, data: pointer, len: ptr csize_t): cint {.lcrypto.} +proc EVP_PKEY_CTX_new*(pkey: EVP_PKEY, e: ENGINE): EVP_PKEY_CTX {.lcrypto.} +proc EVP_PKEY_CTX_free*(pkeyCtx: EVP_PKEY_CTX) {.lcrypto.} +proc EVP_PKEY_sign_init*(c: EVP_PKEY_CTX): cint {.lcrypto.} when defined(macosx) or defined(windows): - proc EVP_MD_CTX_create*(): EVP_MD_CTX {.cdecl, importc.} - proc EVP_MD_CTX_destroy*(ctx: EVP_MD_CTX) {.cdecl, importc.} - proc EVP_MD_CTX_cleanup*(ctx: EVP_MD_CTX): cint {.cdecl, importc.} + proc EVP_MD_CTX_create*(): EVP_MD_CTX {.lcrypto.} + proc EVP_MD_CTX_destroy*(ctx: EVP_MD_CTX) {.lcrypto.} + proc EVP_MD_CTX_cleanup*(ctx: EVP_MD_CTX): cint {.lcrypto.} else: # some times you will need this instead: - proc EVP_MD_CTX_create*(): EVP_MD_CTX {.cdecl, importc: "EVP_MD_CTX_new".} - proc EVP_MD_CTX_destroy*(ctx: EVP_MD_CTX) {.cdecl, importc: "EVP_MD_CTX_free".} - proc EVP_MD_CTX_cleanup*(ctx: EVP_MD_CTX): cint {.cdecl, importc: "EVP_MD_CTX_cleanup".} + proc EVP_MD_CTX_create*(): EVP_MD_CTX {.cdecl, importc: "EVP_MD_CTX_new", dynlib: DLLUtilName.} + proc EVP_MD_CTX_destroy*(ctx: EVP_MD_CTX) {.cdecl, importc: "EVP_MD_CTX_free", dynlib: DLLUtilName.} + proc EVP_MD_CTX_cleanup*(ctx: EVP_MD_CTX): cint {.cdecl, importc: "EVP_MD_CTX_cleanup", dynlib: DLLUtilName.} # type From b41226001ce596fe520fd326a302c6f4e91c2504 Mon Sep 17 00:00:00 2001 From: Ivan Yonchovski Date: Mon, 13 Jun 2022 12:33:44 +0300 Subject: [PATCH 1657/2472] Initial implementation of nimsuggest v3 (#19826) * Initial implementation of nimsuggest v3 Rework `nimsuggest` to use caching to make usage of ide commands more efficient. Previously, all commands no matter what the state of the process is were causing clean build. In the context of Language Server Protocol(LSP) and lsp clients this was causing perf issues and overall instability. Overall, the goal of v3 is to fit to LSP Server needs - added two new commands: - `recompile` to do clean compilation - `changed` which can be used by the IDEs to notify that a particular file has been changed. The later can be utilized when using LSP file watches. - `globalSymbols` - searching global references - added `segfaults` dependency to allow fallback to clean build when incremental fails. I wish the error to be propagated to the client so we can work on fixing the incremental build failures (typically hitting pointer) - more efficient rebuild flow. ATM incremental rebuild is triggered when the command needs that(i. e. it is global) while the commands that work on the current source rebuild only it Things missing in this PR: - Documentation - Extensive unit testing. Although functional I still see this more as a POC that this approach can work. Next steps: - Implement `sug` request. - Rework/extend the protocol to allow better client/server communication. Ideally we will need push events, diagnostics should be restructored to allow per file notifications, etc. - implement v3 test suite. - better logging * Add tests for v3 and implement ideSug * Remove typeInstCache/procInstCache cleanup * Add ideChkFile command * Avoid contains call when adding symbol info * Remove log * Remove segfaults --- compiler/modulegraphs.nim | 82 ++++++++++--- compiler/options.nim | 13 +- compiler/passes.nim | 7 +- compiler/suggest.nim | 12 +- nimsuggest/nimsuggest.nim | 242 +++++++++++++++++++++++++++++++++++--- nimsuggest/tester.nim | 5 +- nimsuggest/tests/tv3.nim | 25 ++++ 7 files changed, 349 insertions(+), 37 deletions(-) create mode 100644 nimsuggest/tests/tv3.nim diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index 147381910..8294d863e 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -11,7 +11,7 @@ ## represents a complete Nim project. Single modules can either be kept in RAM ## or stored in a rod-file. -import intsets, tables, hashes, md5_old +import intsets, tables, hashes, md5_old, sequtils import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils, packages import ic / [packed_ast, ic] @@ -83,6 +83,8 @@ type doStopCompile*: proc(): bool {.closure.} usageSym*: PSym # for nimsuggest owners*: seq[PSym] + suggestSymbols*: Table[FileIndex, seq[tuple[sym: PSym, info: TLineInfo]]] + suggestErrors*: Table[FileIndex, seq[Suggest]] methods*: seq[tuple[methods: seq[PSym], dispatcher: PSym]] # needs serialization! systemModule*: PSym sysTypes*: array[TTypeKind, PType] @@ -385,9 +387,19 @@ when defined(nimfind): c.graph.onDefinitionResolveForward(c.graph, s, info) else: - template onUse*(info: TLineInfo; s: PSym) = discard - template onDef*(info: TLineInfo; s: PSym) = discard - template onDefResolveForward*(info: TLineInfo; s: PSym) = discard + when defined(nimsuggest): + template onUse*(info: TLineInfo; s: PSym) = discard + + template onDef*(info: TLineInfo; s: PSym) = + let c = getPContext() + if c.graph.config.suggestVersion == 3: + suggestSym(c.graph, info, s, c.graph.usageSym) + + template onDefResolveForward*(info: TLineInfo; s: PSym) = discard + else: + template onUse*(info: TLineInfo; s: PSym) = discard + template onDef*(info: TLineInfo; s: PSym) = discard + template onDefResolveForward*(info: TLineInfo; s: PSym) = discard proc stopCompile*(g: ModuleGraph): bool {.inline.} = result = g.doStopCompile != nil and g.doStopCompile() @@ -434,8 +446,7 @@ proc initOperators*(g: ModuleGraph): Operators = result.opNot = createMagic(g, "not", mNot) result.opContains = createMagic(g, "contains", mInSet) -proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph = - result = ModuleGraph() +proc initModuleGraphFields(result: ModuleGraph) = # A module ID of -1 means that the symbol is not attached to a module at all, # but to the module graph: result.idgen = IdGenerator(module: -1'i32, symId: 0'i32, typeId: 0'i32) @@ -445,9 +456,9 @@ proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph = result.ifaces = @[] result.importStack = @[] result.inclToMod = initTable[FileIndex, FileIndex]() - result.config = config - result.cache = cache result.owners = @[] + result.suggestSymbols = initTable[FileIndex, seq[tuple[sym: PSym, info: TLineInfo]]]() + result.suggestErrors = initTable[FileIndex, seq[Suggest]]() result.methods = @[] initStrTable(result.compilerprocs) initStrTable(result.exposed) @@ -461,6 +472,12 @@ proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph = result.operators = initOperators(result) result.emittedTypeInfo = initTable[string, FileIndex]() +proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph = + result = ModuleGraph() + result.config = config + result.cache = cache + initModuleGraphFields(result) + proc resetAllModules*(g: ModuleGraph) = initStrTable(g.packageSyms) g.deps = initIntSet() @@ -472,6 +489,7 @@ proc resetAllModules*(g: ModuleGraph) = g.methods = @[] initStrTable(g.compilerprocs) initStrTable(g.exposed) + initModuleGraphFields(g) proc getModule*(g: ModuleGraph; fileIdx: FileIndex): PSym = if fileIdx.int32 >= 0: @@ -550,7 +568,19 @@ proc transitiveClosure(g: var IntSet; n: int) = proc markDirty*(g: ModuleGraph; fileIdx: FileIndex) = let m = g.getModule fileIdx - if m != nil: incl m.flags, sfDirty + if m != nil: + g.suggestSymbols.del(fileIdx) + g.suggestErrors.del(fileIdx) + incl m.flags, sfDirty + +proc unmarkAllDirty*(g: ModuleGraph) = + for i in 0i32.. 100: + break + else: + myLog fmt "Discarding {cmd}" + +# v3 end when isMainModule: handleCmdLine(newIdentCache(), newConfigRef()) else: @@ -726,8 +939,9 @@ else: if self.loadConfigsAndProcessCmdLine(cache, conf, graph): mockCommand(graph) if gLogging: + log("Search paths:") for it in conf.searchPaths: - log(it.string) + log(" " & it.string) retval.doStopCompile = proc (): bool = false return NimSuggest(graph: retval, idle: 0, cachedMsgs: @[]) diff --git a/nimsuggest/tester.nim b/nimsuggest/tester.nim index 1db33706a..fea0a8d45 100644 --- a/nimsuggest/tester.nim +++ b/nimsuggest/tester.nim @@ -252,7 +252,10 @@ proc runEpcTest(filename: string): int = for cmd in s.startup: if not runCmd(cmd, s.dest): quit "invalid command: " & cmd - let epccmd = s.cmd.replace("--tester", "--epc --v2 --log") + let epccmd = if s.cmd.contains("--v3"): + s.cmd.replace("--tester", "--epc --log") + else: + s.cmd.replace("--tester", "--epc --v2 --log") let cl = parseCmdLine(epccmd) var p = startProcess(command=cl[0], args=cl[1 .. ^1], options={poStdErrToStdOut, poUsePath, diff --git a/nimsuggest/tests/tv3.nim b/nimsuggest/tests/tv3.nim new file mode 100644 index 000000000..99caa987b --- /dev/null +++ b/nimsuggest/tests/tv3.nim @@ -0,0 +1,25 @@ +# tests v3 + +type + Foo* = ref object of RootObj + bar*: string + +proc test(f: Foo) = + echo f.ba#[!]#r + +discard """ +$nimsuggest --v3 --tester $file +>use $1 +def skField tv3.Foo.bar string $file 5 4 "" 100 +use skField tv3.Foo.bar string $file 8 9 "" 100 +>def $1 +def skField tv3.Foo.bar string $file 5 4 "" 100 +>outline $1 +outline skType tv3.Foo Foo $file 4 2 "" 100 +outline skField tv3.Foo.bar string $file 5 4 "" 100 +outline skProc tv3.test proc (f: Foo){.gcsafe, locks: 0.} $file 7 5 "" 100 +>sug $1 +sug skField bar string $file 5 4 "" 100 Prefix +>globalSymbols test +def skProc tv3.test proc (f: Foo){.gcsafe, locks: 0.} $file 7 5 "" 100 +""" From 251bdc1d5aed98fccb31677dda3c3ead9be2f6bf Mon Sep 17 00:00:00 2001 From: Tanguy Date: Tue, 14 Jun 2022 12:37:31 +0200 Subject: [PATCH 1658/2472] Windows: enable nimRawSetjmp by default [backport] (#19891) * Windows: enable nimRawSetjmp by default See #19197. The default setjmp can randomly segfault on windows * Attempt to disable the flag for bootstraping * Disable styleCheck for c_setjmp --- config/config.nims | 5 +++++ lib/system/ansi_c.nim | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/config/config.nims b/config/config.nims index 08d9d9f55..b50181eaf 100644 --- a/config/config.nims +++ b/config/config.nims @@ -20,5 +20,10 @@ when defined(nimStrictMode): switch("hintAsError", "ConvFromXtoItselfNotNeeded") # future work: XDeclaredButNotUsed +when defined(windows) and not defined(booting): + # Avoid some rare stack corruption while using exceptions with a SEH-enabled + # toolchain: https://github.com/nim-lang/Nim/pull/19197 + switch("define", "nimRawSetjmp") + switch("define", "nimVersion:" & NimVersion) switch("define", "nimPreviewDotLikeOps") diff --git a/lib/system/ansi_c.nim b/lib/system/ansi_c.nim index 23fb9fdef..5d0ecc01c 100644 --- a/lib/system/ansi_c.nim +++ b/lib/system/ansi_c.nim @@ -116,6 +116,7 @@ elif defined(nimBuiltinSetjmp): proc c_builtin_setjmp(jmpb: ptr pointer): cint {. importc: "__builtin_setjmp", nodecl.} c_builtin_setjmp(unsafeAddr jmpb[0]) + elif defined(nimRawSetjmp) and not defined(nimStdSetjmp): when defined(windows): # No `_longjmp()` on Windows. @@ -127,10 +128,16 @@ elif defined(nimRawSetjmp) and not defined(nimStdSetjmp): # prone to stack corruption during unwinding, so we disable that by setting # it to NULL. # More details: https://github.com/status-im/nimbus-eth2/issues/3121 + when defined(nimHasStyleChecks): + {.push styleChecks: off.} + proc c_setjmp*(jmpb: C_JmpBuf): cint = proc c_setjmp_win(jmpb: C_JmpBuf, ctx: pointer): cint {. header: "", importc: "_setjmp".} c_setjmp_win(jmpb, nil) + + when defined(nimHasStyleChecks): + {.pop.} else: proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {. header: "", importc: "_longjmp".} From 789b1bcbb60278b335a81df416be1d5a663d7e30 Mon Sep 17 00:00:00 2001 From: Don-Duong Quach Date: Tue, 14 Jun 2022 23:20:34 -0700 Subject: [PATCH 1659/2472] Fixes #19883 so genscript works with vcc (#19893) --- compiler/extccomp.nim | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 515c7ba29..5c074d284 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -434,7 +434,13 @@ proc noAbsolutePaths(conf: ConfigRef): bool {.inline.} = # really: Cross compilation from Linux to Linux for example is entirely # reasonable. # `optGenMapping` is included here for niminst. - result = conf.globalOptions * {optGenScript, optGenMapping} != {} + # We use absolute paths for vcc / cl, see issue #19883. + let options = + if conf.cCompiler == ccVcc: + {optGenMapping} + else: + {optGenScript, optGenMapping} + result = conf.globalOptions * options != {} proc cFileSpecificOptions(conf: ConfigRef; nimname, fullNimFile: string): string = result = conf.compileOptions @@ -921,7 +927,7 @@ proc callCCompiler*(conf: ConfigRef) = objfiles.add(' ') objfiles.add(quoteShell(objFile)) let mainOutput = if optGenScript notin conf.globalOptions: conf.prepareToWriteOutput - else: AbsoluteFile(conf.projectName) + else: AbsoluteFile(conf.outFile) linkCmd = getLinkCmd(conf, mainOutput, objfiles, removeStaticFile = true) extraCmds = getExtraCmds(conf, mainOutput) From e7e8f437c4f95f4f5d038cdddf5036462733288a Mon Sep 17 00:00:00 2001 From: JJ <35242550+j-james@users.noreply.github.com> Date: Wed, 15 Jun 2022 06:40:56 -0700 Subject: [PATCH 1660/2472] Keep the doc sidebar on the screen while scrolling (#19851) * [docgen] Group sidebar sections into
              (open by default) * [docgen] Consistent indentation in generated HTML (this is a boon for working on docgen's html/css output) * [docgen] Move Source/Edit buttons inside main div This makes styling the documentation significantly easier. * [docgen] Somewhat consistent CSS formatting * [docgen] Keep the sidebar onscreen while scrolling * [docgen] Tweak CSS for the sticky sidebar * [docgen] search type=text ==> type=search * [docgen] Update expected doc output * [docgen] Fix Group by Type sidebar placement bug * [docgen] Curse you, whitespace (fix tests) * [docgen] Fix rst2html tests Co-authored-by: sandytypical <43030857+xflywind@users.noreply.github.com> --- config/nimdoc.cfg | 216 ++- doc/nimdoc.css | 164 +- nimdoc/rst2html/expected/rst_examples.html | 85 +- .../expected/index.html | 122 +- .../expected/theindex.html | 24 +- nimdoc/testproject/expected/nimdoc.out.css | 164 +- .../expected/subdir/subdir_b/utils.html | 758 ++++----- nimdoc/testproject/expected/testproject.html | 1454 ++++++++--------- nimdoc/testproject/expected/theindex.html | 24 +- 9 files changed, 1401 insertions(+), 1610 deletions(-) diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index 4efa1f637..725f9e0a5 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -9,25 +9,28 @@ split.item.toc = "20" doc.section = """
              -

              $sectionTitle

              -
              -$content -
              +

              $sectionTitle

              +
              + $content +
              +
              """ doc.section.toc = """
            • - $sectionTitle -
                - $content -
              +
              + $sectionTitle +
                + $content +
              +
              • """ doc.section.toc2 = """ -
                $plainName - $content -
              +
                $plainName + $content +
              """ # Chunk of HTML emitted for each entry in the HTML table of contents. @@ -47,12 +50,12 @@ doc.section.toc2 = """ doc.item = """
              -
              $header
              -
              -$deprecationMsg -$desc -$seeSrc -
              + 
              $header
              +
              + $deprecationMsg + $desc + $seeSrc +
              """ @@ -61,9 +64,8 @@ $seeSrc # * $overloadGroupName - the anchor for this whole group # * $content - string containing `doc.item`s themselves doc.item2 = """ -
              -$content + $content
              """ @@ -73,18 +75,14 @@ $content # This is used for TOC items which are not overloadable (e.g. types). # `$header_plain` would be too verbose here, so we use $name. doc.item.toc = """ -
            • $name
              • +
            • $name
              • """ # This is used for TOC items which are grouped by the same name (e.g. procs). doc.item.tocTable = """ -
            • $header_plain
              • +
            • $header_plain
              • """ - - # HTML rendered for doc.item's seeSrc variable. Note that this will render to # the empty string if you don't pass anything through --git.url. Available # substitutaion variables here are: @@ -94,32 +92,31 @@ doc.item.tocTable = """ # * $line: line of the item in the original source file. # * $url: whatever you did pass through the --git.url switch (which also # gets variables path/line replaced!) -doc.item.seesrc = """  Source -  Edit +doc.item.seesrc = """ +Source   +Edit   """ doc.deprecationmsg = """ -
              - $label $message -
              +
              + $label $message +
              """ doc.toc = """
                -$content + $content
              """ doc.body_toc_groupsection = """ -
              - Group by: - -
              +
              + Group by: + +
              """ @if boot: @@ -130,36 +127,36 @@ doc.body_toc_groupsection = """ doc.body_toc_group = """
              -
              - -     Dark Mode +
              + +     Dark Mode +
              + +
              + Search: +
              + $body_toc_groupsection +
              + $tableofcontents
              - -
              - Search: -
              - $body_toc_groupsection - $tableofcontents -
              - $seeSrc
              -
              - $deprecationMsg -

              $moduledesc

              - $content + $seeSrc + $deprecationMsg +

              $moduledesc

              + $content
              """ @@ -169,39 +166,36 @@ doc.body_toc_group = """ doc.body_toc_group = """
              -
              - -     Dark Mode +
              + +     Dark Mode +
              + +
              + Search: +
              +
              + Group by: + +
              +
              + $tableofcontents
              - -
              - Search: -
              -
              - Group by: - -
              - $tableofcontents -
              - $seeSrc
              -
              - $deprecationMsg -

              $moduledesc

              - $content + $seeSrc + $deprecationMsg +

              $moduledesc

              + $content
              """ @@ -220,14 +214,13 @@ doc.listing_end = "" # * $analytics: Google analytics location, includes - -
              -
              -

              $title

              $subtitle - $content -
              +
              +
              +

              $title

              $subtitle + $content
              -
              +
              Made with Nim. Generated: $date $time UTC
              -
              -$analytics + $analytics """ diff --git a/doc/nimdoc.css b/doc/nimdoc.css index 0014cf196..e72c4a213 100644 --- a/doc/nimdoc.css +++ b/doc/nimdoc.css @@ -159,24 +159,28 @@ body { padding: 0; box-sizing: border-box; } -.column, -.columns { +.column, .columns { width: 100%; float: left; box-sizing: border-box; - margin-left: 1%; -} + margin-left: 1%; } -.column:first-child, -.columns:first-child { +.column:first-child, .columns:first-child { margin-left: 0; } +.container .row { + display: flex; } + .three.columns { - width: 22%; -} + width: 25.0%; + height: 100vh; + position: sticky; + top: 0px; + overflow-y: auto; } .nine.columns { - width: 77.0%; } + width: 75.0%; + padding-left: 1.5em; } .twelve.columns { width: 100%; @@ -269,25 +273,26 @@ a.nimdoc { a.toc-backref { text-decoration: none; - color: var(--text); } + color: var(--text); +} a.link-seesrc { color: #607c9f; font-size: 0.9em; - font-style: italic; } + font-style: italic; +} -a:hover, -a:focus { +a:hover, a:focus { color: var(--anchor-focus); - text-decoration: underline; } + text-decoration: underline; +} a:hover span.Identifier { color: var(--anchor); } -sub, -sup { +sub, sup { position: relative; font-size: 75%; line-height: 0; @@ -314,8 +319,7 @@ img { background: transparent !important; box-shadow: none !important; } - a, - a:visited { + a, a:visited { text-decoration: underline; } a[href]:after { @@ -329,16 +333,14 @@ img { a[href^="#"]:after { content: ""; } - pre, - blockquote { + pre, blockquote { border: 1px solid #999; page-break-inside: avoid; } thead { display: table-header-group; } - tr, - img { + tr, img { page-break-inside: avoid; } img { @@ -353,22 +355,18 @@ img { h1.title { page-break-before: avoid; } - p, - h2, - h3 { + p, h2, h3 { orphans: 3; widows: 3; } - h2, - h3 { + h2, h3 { page-break-after: avoid; } } p { margin-top: 0.5em; - margin-bottom: 0.5em; -} + margin-bottom: 0.5em; } small { font-size: 85%; } @@ -376,8 +374,7 @@ small { strong { font-weight: 600; font-size: 0.95em; - color: var(--strong); -} + color: var(--strong); } em { font-style: italic; } @@ -398,8 +395,7 @@ h1.title { text-align: center; font-weight: 900; margin-top: 0.75em; - margin-bottom: 0em; -} + margin-bottom: 0em; } h2 { font-size: 1.3em; @@ -426,36 +422,29 @@ h6 { font-size: 1.1em; } -ul, -ol { +ul, ol { padding: 0; margin-top: 0.5em; margin-left: 0.75em; } -ul ul, -ul ol, -ol ol, -ol ul { +ul ul, ul ol, ol ol, ol ul { margin-bottom: 0; margin-left: 1.25em; } ul.simple > li { - list-style-type: circle; -} + list-style-type: circle; } ul.simple-boot li { - list-style-type: none; - margin-left: 0em; - margin-bottom: 0.5em; -} + list-style-type: none; + margin-left: 0em; + margin-bottom: 0.5em; } ol.simple > li, ul.simple > li { margin-bottom: 0.2em; margin-left: 0.4em } ul.simple.simple-toc > li { - margin-top: 1em; -} + margin-top: 1em; } ul.simple-toc { list-style: none; @@ -464,8 +453,7 @@ ul.simple-toc { margin-top: 1em; } ul.simple-toc > li { - list-style-type: none; -} + list-style-type: none; } ul.simple-toc-section { list-style-type: circle; @@ -475,12 +463,10 @@ ul.simple-toc-section { ul.nested-toc-section { list-style-type: circle; margin-left: -0.75em; - color: var(--text); -} + color: var(--text); } ul.nested-toc-section > li { - margin-left: 1.25em; -} + margin-left: 1.25em; } ol.arabic { @@ -527,7 +513,8 @@ hr.footnote { margin-top: 0.15em; } div.footnote-group { - margin-left: 1em; } + margin-left: 1em; +} div.footnote-label { display: inline-block; min-width: 1.7em; @@ -611,7 +598,7 @@ pre { border: 1px solid var(--border); -webkit-border-radius: 6px; -moz-border-radius: 6px; - border-radius: 6px; + border-radius: 6px; } .copyToClipBoardBtn { @@ -629,7 +616,7 @@ pre { .copyToClipBoard:hover .copyToClipBoardBtn { visibility: visible; -} +} .pre-scrollable { max-height: 340px; @@ -694,8 +681,8 @@ table th { font-weight: bold; } table th.docinfo-name { - background-color: transparent; - text-align: right; + background-color: transparent; + text-align: right; } table tr:hover { @@ -712,31 +699,31 @@ table.borderless td, table.borderless th { padding: 0 0.5em 0 0 !important; } .admonition { - padding: 0.3em; - background-color: var(--secondary-background); - border-left: 0.4em solid #7f7f84; - margin-bottom: 0.5em; - -webkit-box-shadow: 0 5px 8px -6px rgba(0,0,0,.2); - -moz-box-shadow: 0 5px 8px -6px rgba(0,0,0,.2); - box-shadow: 0 5px 8px -6px rgba(0,0,0,.2); + padding: 0.3em; + background-color: var(--secondary-background); + border-left: 0.4em solid #7f7f84; + margin-bottom: 0.5em; + -webkit-box-shadow: 0 5px 8px -6px rgba(0,0,0,.2); + -moz-box-shadow: 0 5px 8px -6px rgba(0,0,0,.2); + box-shadow: 0 5px 8px -6px rgba(0,0,0,.2); } .admonition-info { - border-color: var(--info-background); + border-color: var(--info-background); } .admonition-info-text { - color: var(--info-background); + color: var(--info-background); } .admonition-warning { - border-color: var(--warning-background); + border-color: var(--warning-background); } .admonition-warning-text { - color: var(--warning-background); + color: var(--warning-background); } .admonition-error { - border-color: var(--error-background); + border-color: var(--error-background); } .admonition-error-text { - color: var(--error-background); + color: var(--error-background); } .first { @@ -770,8 +757,7 @@ div.footer, div.header { font-size: smaller; } div.footer { - padding-top: 5em; -} + padding-top: 5em; } div.line-block { display: block; @@ -790,17 +776,14 @@ div.search_results { background-color: var(--third-background); margin: 3em; padding: 1em; - border: 1px solid #4d4d4d; -} + border: 1px solid #4d4d4d; } div#global-links ul { margin-left: 0; - list-style-type: none; -} + list-style-type: none; } div#global-links > simple-boot { - margin-left: 3em; -} + margin-left: 3em; } hr.docutils { width: 75%; } @@ -980,8 +963,7 @@ span.Directive { span.option { font-weight: bold; font-family: "Source Code Pro", Monaco, Menlo, Consolas, "Courier New", monospace; - color: var(--option); -} + color: var(--option); } span.Prompt { font-weight: bold; @@ -997,11 +979,10 @@ span.program { text-decoration: underline; text-decoration-color: var(--hint); text-decoration-thickness: 0.05em; - text-underline-offset: 0.15em; -} + text-underline-offset: 0.15em; } -span.Command, span.Rule, span.Hyperlink, span.Label, span.Reference, -span.Other { +span.Command, span.Rule, span.Hyperlink, +span.Label, span.Reference, span.Other { color: var(--other); } /* Pop type, const, proc, and iterator defs in nim def blocks */ @@ -1039,17 +1020,14 @@ span.pragmadots { border-radius: 4px; margin: 0 2px; cursor: pointer; - font-size: 0.8em; -} + font-size: 0.8em; } span.pragmadots:hover { - background-color: var(--hint); -} + background-color: var(--hint); } + span.pragmawrap { - display: none; -} + display: none; } span.attachedType { display: none; - visibility: hidden; -} + visibility: hidden; } diff --git a/nimdoc/rst2html/expected/rst_examples.html b/nimdoc/rst2html/expected/rst_examples.html index 2b5218d9f..532917055 100644 --- a/nimdoc/rst2html/expected/rst_examples.html +++ b/nimdoc/rst2html/expected/rst_examples.html @@ -1,12 +1,11 @@ - + - + - +Not a Nim Manual @@ -17,45 +16,42 @@ -Not a Nim Manual + - -
              -
              -

              Not a Nim Manual

              -
              +
              +
              +

              Not a Nim Manual

              +
              -
              - -     Dark Mode -
              - -
              - Search: -
              -
              - Group by: - -
              -
              -
              -
              - -

              + + +

              Authors:Andreas Rumpf, Zahary Karadjov
              Authors:Andreas Rumpf, Zahary Karadjov
              Version:|nimversion|

              "Complexity" seems to be a lot like "energy": you can transfer it from the end-user to one/some of the other players, but the total amount seems to remain pretty much constant for a given task. -- Ran

              @@ -262,19 +257,17 @@ stmt = IND{>} stmt ^+ IND{=} DED # list of statements F2 without pipe

              not in table

              - +
              -
              -
              +
              Made with Nim. Generated: 1970-01-02 03:46:40 UTC
              -
              - + diff --git a/nimdoc/test_out_index_dot_html/expected/index.html b/nimdoc/test_out_index_dot_html/expected/index.html index c6a116bd6..0c0dc2268 100644 --- a/nimdoc/test_out_index_dot_html/expected/index.html +++ b/nimdoc/test_out_index_dot_html/expected/index.html @@ -1,12 +1,11 @@ - + - + - +nimdoc/test_out_index_dot_html/foo @@ -17,94 +16,89 @@ -nimdoc/test_out_index_dot_html/foo + - -
              -
              -

              nimdoc/test_out_index_dot_html/foo

              -
              +
              +
              +

              nimdoc/test_out_index_dot_html/foo

              +
              -
              - -     Dark Mode -
              - -
              - Search: -
              -
              - Group by: - -
              -
                -
              • - Procs -
                  +
                  + +     Dark Mode +
                  + +
                  + Search: +
                  +
                  + Group by: + +
                  +
                  + +
                +
              -
              -
              - -

              -
              -

              Procs

              -
              - -
              -
              -
              proc foo() {....raises: [], tags: [].}
              -
              - -I do foo - -
              + + +

              +
              +

              Procs

              +
              +
              +
              + 
              proc foo() {....raises: [], tags: [].}
              +
              + + I do foo + +
              -
              +
              +
              -
              -
              +
              Made with Nim. Generated: 1970-01-02 03:46:40 UTC
              -
              - + diff --git a/nimdoc/test_out_index_dot_html/expected/theindex.html b/nimdoc/test_out_index_dot_html/expected/theindex.html index 8ee62a330..00c81189e 100644 --- a/nimdoc/test_out_index_dot_html/expected/theindex.html +++ b/nimdoc/test_out_index_dot_html/expected/theindex.html @@ -1,12 +1,11 @@ - + - + - +Index @@ -17,31 +16,28 @@ -Index + - -
              -
              -

              Index

              - Modules: index.

              API symbols

              +
              +
              +

              Index

              + Modules: index.

              API symbols

              foo:
              -
              -
              +
              Made with Nim. Generated: 1970-01-02 03:46:40 UTC
              -
              - + diff --git a/nimdoc/testproject/expected/nimdoc.out.css b/nimdoc/testproject/expected/nimdoc.out.css index 0014cf196..e72c4a213 100644 --- a/nimdoc/testproject/expected/nimdoc.out.css +++ b/nimdoc/testproject/expected/nimdoc.out.css @@ -159,24 +159,28 @@ body { padding: 0; box-sizing: border-box; } -.column, -.columns { +.column, .columns { width: 100%; float: left; box-sizing: border-box; - margin-left: 1%; -} + margin-left: 1%; } -.column:first-child, -.columns:first-child { +.column:first-child, .columns:first-child { margin-left: 0; } +.container .row { + display: flex; } + .three.columns { - width: 22%; -} + width: 25.0%; + height: 100vh; + position: sticky; + top: 0px; + overflow-y: auto; } .nine.columns { - width: 77.0%; } + width: 75.0%; + padding-left: 1.5em; } .twelve.columns { width: 100%; @@ -269,25 +273,26 @@ a.nimdoc { a.toc-backref { text-decoration: none; - color: var(--text); } + color: var(--text); +} a.link-seesrc { color: #607c9f; font-size: 0.9em; - font-style: italic; } + font-style: italic; +} -a:hover, -a:focus { +a:hover, a:focus { color: var(--anchor-focus); - text-decoration: underline; } + text-decoration: underline; +} a:hover span.Identifier { color: var(--anchor); } -sub, -sup { +sub, sup { position: relative; font-size: 75%; line-height: 0; @@ -314,8 +319,7 @@ img { background: transparent !important; box-shadow: none !important; } - a, - a:visited { + a, a:visited { text-decoration: underline; } a[href]:after { @@ -329,16 +333,14 @@ img { a[href^="#"]:after { content: ""; } - pre, - blockquote { + pre, blockquote { border: 1px solid #999; page-break-inside: avoid; } thead { display: table-header-group; } - tr, - img { + tr, img { page-break-inside: avoid; } img { @@ -353,22 +355,18 @@ img { h1.title { page-break-before: avoid; } - p, - h2, - h3 { + p, h2, h3 { orphans: 3; widows: 3; } - h2, - h3 { + h2, h3 { page-break-after: avoid; } } p { margin-top: 0.5em; - margin-bottom: 0.5em; -} + margin-bottom: 0.5em; } small { font-size: 85%; } @@ -376,8 +374,7 @@ small { strong { font-weight: 600; font-size: 0.95em; - color: var(--strong); -} + color: var(--strong); } em { font-style: italic; } @@ -398,8 +395,7 @@ h1.title { text-align: center; font-weight: 900; margin-top: 0.75em; - margin-bottom: 0em; -} + margin-bottom: 0em; } h2 { font-size: 1.3em; @@ -426,36 +422,29 @@ h6 { font-size: 1.1em; } -ul, -ol { +ul, ol { padding: 0; margin-top: 0.5em; margin-left: 0.75em; } -ul ul, -ul ol, -ol ol, -ol ul { +ul ul, ul ol, ol ol, ol ul { margin-bottom: 0; margin-left: 1.25em; } ul.simple > li { - list-style-type: circle; -} + list-style-type: circle; } ul.simple-boot li { - list-style-type: none; - margin-left: 0em; - margin-bottom: 0.5em; -} + list-style-type: none; + margin-left: 0em; + margin-bottom: 0.5em; } ol.simple > li, ul.simple > li { margin-bottom: 0.2em; margin-left: 0.4em } ul.simple.simple-toc > li { - margin-top: 1em; -} + margin-top: 1em; } ul.simple-toc { list-style: none; @@ -464,8 +453,7 @@ ul.simple-toc { margin-top: 1em; } ul.simple-toc > li { - list-style-type: none; -} + list-style-type: none; } ul.simple-toc-section { list-style-type: circle; @@ -475,12 +463,10 @@ ul.simple-toc-section { ul.nested-toc-section { list-style-type: circle; margin-left: -0.75em; - color: var(--text); -} + color: var(--text); } ul.nested-toc-section > li { - margin-left: 1.25em; -} + margin-left: 1.25em; } ol.arabic { @@ -527,7 +513,8 @@ hr.footnote { margin-top: 0.15em; } div.footnote-group { - margin-left: 1em; } + margin-left: 1em; +} div.footnote-label { display: inline-block; min-width: 1.7em; @@ -611,7 +598,7 @@ pre { border: 1px solid var(--border); -webkit-border-radius: 6px; -moz-border-radius: 6px; - border-radius: 6px; + border-radius: 6px; } .copyToClipBoardBtn { @@ -629,7 +616,7 @@ pre { .copyToClipBoard:hover .copyToClipBoardBtn { visibility: visible; -} +} .pre-scrollable { max-height: 340px; @@ -694,8 +681,8 @@ table th { font-weight: bold; } table th.docinfo-name { - background-color: transparent; - text-align: right; + background-color: transparent; + text-align: right; } table tr:hover { @@ -712,31 +699,31 @@ table.borderless td, table.borderless th { padding: 0 0.5em 0 0 !important; } .admonition { - padding: 0.3em; - background-color: var(--secondary-background); - border-left: 0.4em solid #7f7f84; - margin-bottom: 0.5em; - -webkit-box-shadow: 0 5px 8px -6px rgba(0,0,0,.2); - -moz-box-shadow: 0 5px 8px -6px rgba(0,0,0,.2); - box-shadow: 0 5px 8px -6px rgba(0,0,0,.2); + padding: 0.3em; + background-color: var(--secondary-background); + border-left: 0.4em solid #7f7f84; + margin-bottom: 0.5em; + -webkit-box-shadow: 0 5px 8px -6px rgba(0,0,0,.2); + -moz-box-shadow: 0 5px 8px -6px rgba(0,0,0,.2); + box-shadow: 0 5px 8px -6px rgba(0,0,0,.2); } .admonition-info { - border-color: var(--info-background); + border-color: var(--info-background); } .admonition-info-text { - color: var(--info-background); + color: var(--info-background); } .admonition-warning { - border-color: var(--warning-background); + border-color: var(--warning-background); } .admonition-warning-text { - color: var(--warning-background); + color: var(--warning-background); } .admonition-error { - border-color: var(--error-background); + border-color: var(--error-background); } .admonition-error-text { - color: var(--error-background); + color: var(--error-background); } .first { @@ -770,8 +757,7 @@ div.footer, div.header { font-size: smaller; } div.footer { - padding-top: 5em; -} + padding-top: 5em; } div.line-block { display: block; @@ -790,17 +776,14 @@ div.search_results { background-color: var(--third-background); margin: 3em; padding: 1em; - border: 1px solid #4d4d4d; -} + border: 1px solid #4d4d4d; } div#global-links ul { margin-left: 0; - list-style-type: none; -} + list-style-type: none; } div#global-links > simple-boot { - margin-left: 3em; -} + margin-left: 3em; } hr.docutils { width: 75%; } @@ -980,8 +963,7 @@ span.Directive { span.option { font-weight: bold; font-family: "Source Code Pro", Monaco, Menlo, Consolas, "Courier New", monospace; - color: var(--option); -} + color: var(--option); } span.Prompt { font-weight: bold; @@ -997,11 +979,10 @@ span.program { text-decoration: underline; text-decoration-color: var(--hint); text-decoration-thickness: 0.05em; - text-underline-offset: 0.15em; -} + text-underline-offset: 0.15em; } -span.Command, span.Rule, span.Hyperlink, span.Label, span.Reference, -span.Other { +span.Command, span.Rule, span.Hyperlink, +span.Label, span.Reference, span.Other { color: var(--other); } /* Pop type, const, proc, and iterator defs in nim def blocks */ @@ -1039,17 +1020,14 @@ span.pragmadots { border-radius: 4px; margin: 0 2px; cursor: pointer; - font-size: 0.8em; -} + font-size: 0.8em; } span.pragmadots:hover { - background-color: var(--hint); -} + background-color: var(--hint); } + span.pragmawrap { - display: none; -} + display: none; } span.attachedType { display: none; - visibility: hidden; -} + visibility: hidden; } diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html index f94da7f40..d5a3b84c7 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html @@ -1,12 +1,11 @@ - + - + - +subdir/subdir_b/utils @@ -17,215 +16,189 @@ -subdir/subdir_b/utils + - -
              -
              -

              subdir/subdir_b/utils

              -
              +
              +
              +

              subdir/subdir_b/utils

              +
              -
              - -     Dark Mode -
              - -
              - Search: -
              -
              - Group by: - -
              - +
            • - Templates - +
            +
          -
          -
          - -

          This is a description of the utils module.

          + + +

          This is a description of the utils module.

          Links work:

          • other module: iterators (not in this dir, just an example)
          • internal: fn2(x)
            • @@ -256,377 +229,356 @@

            Ref. type like G and type G and G[T] and type G*[T].

            Group ref. with capital letters works: fN11 or fn11

            Ref. [] is the same as proc `[]`(G[T]) because there are no overloads. The full form: proc `[]`*[T](x: G[T]): TRef. []= aka `[]=`(G[T], int, T).Ref. $ aka proc $ or proc `$`.Ref. $(a: ref SomeType).Ref. foo_bar aka iterator foo_bar_.Ref. fn[T; U,V: SomeFloat]().Ref. 'big or func `'big` or `'big`(string).

            -
            -

            Types

            -
            -
            -
            G[T] = object
+    
            
+  
            Types
            
+  
            
+    
            
+  
            G[T] = object
   val: T
 
            
-
            
-
-
-
-
            
+  
            
+    
+    
+    
+  
            
 
            
 
            
-
            SomeType = enum
+  
            SomeType = enum
   enumValueA, enumValueB, enumValueC
            
-
            
-
-
-
-
            
+  
            
+    
+    
+    
+  
            
 
            
 
-
            
+
            +
            -

            Procs

            -
            - -
            -
            -
            proc `$`[T](a: G[T]): string
            -
            - - - -
            +

            Procs

            +
            +
            +
            + 
            proc `$`[T](a: G[T]): string
            +
            + + + +
            -
            proc `$`[T](a: ref SomeType): string
            -
            - - - -
            + 
            proc `$`[T](a: ref SomeType): string
            +
            + + + +
            -
            -
            -
            func `'big`(a: string): SomeType {....raises: [], tags: [].}
            -
            - - - -
            +
            + 
            func `'big`(a: string): SomeType {....raises: [], tags: [].}
            +
            + + + +
            -
            -
            -
            proc `[]`[T](x: G[T]): T
            -
            - - - -
            +
            + 
            proc `[]`[T](x: G[T]): T
            +
            + + + +
            -
            -
            -
            proc `[]=`[T](a: var G[T]; index: int; value: T)
            -
            - - - -
            +
            + 
            proc `[]=`[T](a: var G[T]; index: int; value: T)
            +
            + + + +
            -
            -
            -
            proc binarySearch[T, K](a: openArray[T]; key: K;
+  
            
+  
            proc binarySearch[T, K](a: openArray[T]; key: K;
                         cmp: proc (x: T; y: K): int {.closure.}): int
            
-
            
-
-
-
-
            
+  
            
+    
+    
+    
+
            -
            -
            -
            proc f(x: G[int]) {....raises: [], tags: [].}
            -
            - -There is also variant f(G[string]) - -
            +
            + 
            proc f(x: G[int]) {....raises: [], tags: [].}
            +
            + + There is also variant f(G[string]) + +
            -
            proc f(x: G[string]) {....raises: [], tags: [].}
            -
            - -See also f(G[int]). - -
            + 
            proc f(x: G[string]) {....raises: [], tags: [].}
            +
            + + See also f(G[int]). + +
            -
            -
            -
            proc fn[T; U, V: SomeFloat]()
            -
            - - - -
            +
            + 
            proc fn[T; U, V: SomeFloat]()
            +
            + + + +
            -
            -
            -
            proc fn2() {....raises: [], tags: [].}
            -
            - -comment - -
            +
            + 
            proc fn2() {....raises: [], tags: [].}
            +
            + + comment + +
            -
            proc fn2(x: int) {....raises: [], tags: [].}
            -
            - -fn2 comment - -
            + 
            proc fn2(x: int) {....raises: [], tags: [].}
            +
            + + fn2 comment + +
            -
            proc fn2(x: int; y: float) {....raises: [], tags: [].}
            -
            - - - -
            + 
            proc fn2(x: int; y: float) {....raises: [], tags: [].}
            +
            + + + +
            -
            -
            -
            proc fn3(): auto {....raises: [], tags: [].}
            -
            - -comment - -
            +
            + 
            proc fn3(): auto {....raises: [], tags: [].}
            +
            + + comment + +
            -
            -
            -
            proc fn4(): auto {....raises: [], tags: [].}
            -
            - -comment - -
            +
            + 
            proc fn4(): auto {....raises: [], tags: [].}
            +
            + + comment + +
            -
            -
            -
            proc fn5() {....raises: [], tags: [].}
            -
            - -comment - -
            +
            + 
            proc fn5() {....raises: [], tags: [].}
            +
            + + comment + +
            -
            -
            -
            proc fn6() {....raises: [], tags: [].}
            -
            - -comment - -
            +
            + 
            proc fn6() {....raises: [], tags: [].}
            +
            + + comment + +
            -
            -
            -
            proc fn7() {....raises: [], tags: [].}
            -
            - -comment - -
            +
            + 
            proc fn7() {....raises: [], tags: [].}
            +
            + + comment + +
            -
            -
            -
            proc fn8(): auto {....raises: [], tags: [].}
            -
            - -comment - -
            +
            + 
            proc fn8(): auto {....raises: [], tags: [].}
            +
            + + comment + +
            -
            -
            -
            func fn9(a: int): int {....raises: [], tags: [].}
            -
            - -comment - -
            +
            + 
            func fn9(a: int): int {....raises: [], tags: [].}
            +
            + + comment + +
            -
            -
            -
            func fn10(a: int): int {....raises: [], tags: [].}
            -
            - -comment - -
            +
            + 
            func fn10(a: int): int {....raises: [], tags: [].}
            +
            + + comment + +
            -
            -
            -
            func fN11() {....raises: [], tags: [].}
            -
            - - - -
            +
            + 
            func fN11() {....raises: [], tags: [].}
            +
            + + + +
            -
            func fN11(x: int) {....raises: [], tags: [].}
            -
            - - - -
            + 
            func fN11(x: int) {....raises: [], tags: [].}
            +
            + + + +
            -
            -
            -
            proc funWithGenerics[T, U: SomeFloat](a: T; b: U)
            -
            - - - -
            +
            + 
            proc funWithGenerics[T, U: SomeFloat](a: T; b: U)
            +
            + + + +
            -
            -
            -
            proc someType(): SomeType {....raises: [], tags: [].}
            -
            - -constructor. - -
            +
            + 
            proc someType(): SomeType {....raises: [], tags: [].}
            +
            + + constructor. + +
            -
            +
            +
            -

            Iterators

            -
            - -
            -
            -
            iterator fooBar(a: seq[SomeType]): int {....raises: [], tags: [].}
            -
            - - - -
            +

            Iterators

            +
            +
            +
            + 
            iterator fooBar(a: seq[SomeType]): int {....raises: [], tags: [].}
            +
            + + + +
            -
            +
            +
            -

            Templates

            -
            - -
            -
            -
            template aEnum(): untyped
            -
            - - - -
            +

            Templates

            +
            +
            +
            + 
            template aEnum(): untyped
            +
            + + + +
            -
            -
            -
            template bEnum(): untyped
            -
            - - - -
            +
            + 
            template bEnum(): untyped
            +
            + + + +
            -
            -
            -
            template fromUtilsGen(): untyped
            -
            - -should be shown in utils.html only +
            + 
            template fromUtilsGen(): untyped
            +
            + + should be shown in utils.html only

            Example:

            discard "should be in utils.html only, not in module that calls fromUtilsGen"
            ditto - -
            + +
            -
            +
            +
          -
          -
          +
          Made with Nim. Generated: 1970-01-02 03:46:40 UTC
          - - + diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html index cba9391af..49f24d204 100644 --- a/nimdoc/testproject/expected/testproject.html +++ b/nimdoc/testproject/expected/testproject.html @@ -1,12 +1,11 @@ - + - + - +testproject @@ -17,372 +16,327 @@ -testproject + - -
          -
          -

          testproject

          -
          +
          +
          +

          testproject

          +
          -
          - -     Dark Mode -
          - -
          - Search: -
          -
          - Group by: - -
          - +
        • - Methods - +
        +
      • - Iterators - +
      +
    • - Macros - +
    +
  • - Templates - + +
    • -
    -
    - -

    This is the top level module. + + +

    This is the top level module.

    Example:

    import testproject
 import subdir / subdir_b / utils
@@ -396,133 +350,136 @@
 
    Example:
    
 
    import testproject
 discard "in top3"
    top3 after
    
-  
+    
 
    
-
    Types
    
-
    
-
    
-
    A {.inject.} = enum
+  
    Types
    
+  
    
+    
    
+  
    A {.inject.} = enum
   aA
    
-
    
-
-The enum A.
-
-
    
+  
    
+    
+    The enum A.
+    
+  
    
 
    
 
    
-
    B {.inject.} = enum
+  
    B {.inject.} = enum
   bB
    
-
    
-
-The enum B.
-
-
    
+  
    
+    
+    The enum B.
+    
+  
    
 
    
 
    
-
    Foo = enum
+  
    Foo = enum
   enumValueA2
    
-
    
-
-
-
-
    
+  
    
+    
+    
+    
+  
    
 
    
 
    
-
    FooBuzz {....deprecated: "FooBuzz msg".} = int
    
-
    
-  
    
-    Deprecated: FooBuzz msg
-  
    
+  
    FooBuzz {....deprecated: "FooBuzz msg".} = int
    
+  
    
+    
    
+  Deprecated: FooBuzz msg
+
    
 
-
-
-
    
+    
+    
+  
 
    
 
    
-
    Shapes = enum
+  
    Shapes = enum
   Circle,                   ## A circle
   Triangle,                 ## A three-sided shape
   Rectangle                  ## A four-sided shape
    
-
    
-
-Some shapes.
-
-
    
+  
    
+    
+    Some shapes.
+    
+  
    
 
    
 
-
    
+  
    
+
    
 
    
-
    Vars
    
-
    
-
    
-
    aVariable: array[1, int]
    
-
    
-
-
-
-
    
+  
    Vars
    
+  
    
+    
    
+  
    aVariable: array[1, int]
    
+  
    
+    
+    
+    
+  
    
 
    
 
    
-
    someVariable: bool
    
-
    
-
-This should be visible.
-
-
    
+  
    someVariable: bool
    
+  
    
+    
+    This should be visible.
+    
+  
    
 
    
 
-
    
+  
    
+
    
 
    
-
    Consts
    
-
    
-
    
-
    C_A = 0x7FF0000000000000'f64
    
-
    
-
-
-
-
    
+  
    Consts
    
+  
    
+    
    
+  
    C_A = 0x7FF0000000000000'f64
    
+  
    
+    
+    
+    
+  
    
 
    
 
    
-
    C_B = 0o377'i8
    
-
    
-
-
-
-
    
+  
    C_B = 0o377'i8
    
+  
    
+    
+    
+    
+  
    
 
    
 
    
-
    C_C = 0o277'i8
    
-
    
-
-
-
-
    
+  
    C_C = 0o277'i8
    
+  
    
+    
+    
+    
+  
    
 
    
 
    
-
    C_D = 0o177777'i16
    
-
    
-
-
-
-
    
+  
    C_D = 0o177777'i16
    
+  
    
+    
+    
+    
+  
    
 
    
 
-
    
+  
    
+
    
 
    
-
    Procs
    
-
    
-
-
    
-
    
-
    proc addfBug14485() {....raises: [], tags: [].}
    
-
    
-
-Some proc
+  
    Procs
    
+  
    
+    
    
+  
    
+  
    proc addfBug14485() {....raises: [], tags: [].}
    
+  
    
+    
+    Some proc
 
    Example:
    
 
    discard "foo() = " & $[1]
 #[
@@ -535,208 +492,194 @@ Some proc
 6: </script
 7: end of broken html
 ]#
    
-
-
    
+    
+  
    
 
    
 
 
    
-
 
    
-
    
-
    proc anything() {....raises: [], tags: [].}
    
-
    
-
-There is no block quote after blank lines at the beginning.
-
-
    
+  
    
+  
    proc anything() {....raises: [], tags: [].}
    
+  
    
+    
+    There is no block quote after blank lines at the beginning.
+    
+  
    
 
    
 
 
    
-
 
    
-
    
-
    proc asyncFun1(): Future[int] {....raises: [Exception, ValueError],
+  
    
+  
    proc asyncFun1(): Future[int] {....raises: [Exception, ValueError],
                                 tags: [RootEffect].}
    
-
    
-
-ok1
-
-
    
+  
    
+    
+    ok1
+    
+  
    
 
    
 
 
    
-
 
    
-
    
-
    proc asyncFun2(): owned(Future[void]) {....raises: [Exception], tags: [RootEffect].}
    
-
    
-
-
-
-
    
+  
    
+  
    proc asyncFun2(): owned(Future[void]) {....raises: [Exception], tags: [RootEffect].}
    
+  
    
+    
+    
+    
+  
    
 
    
 
 
    
-
 
    
-
    
-
    proc asyncFun3(): owned(Future[void]) {....raises: [Exception], tags: [RootEffect].}
    
-
    
-
-
+  
    
+  
    proc asyncFun3(): owned(Future[void]) {....raises: [Exception], tags: [RootEffect].}
    
+  
    
+    
+    
 
    Example:
    
 
    discard
    ok1
-
-
    
+    
+  
 
    
 
 
    
-
 
    
-
    
-
    proc bar[T](a, b: T): T
    
-
    
-
-
-
-
    
+  
    
+  
    proc bar[T](a, b: T): T
    
+  
    
+    
+    
+    
+  
    
 
    
 
 
    
-
 
    
-
    
-
    proc baz() {....raises: [], tags: [].}
    
-
    
-
-
-
-
    
+  
    
+  
    proc baz() {....raises: [], tags: [].}
    
+  
    
+    
+    
+    
+  
    
 
    
 
    
-
    proc baz[T](a, b: T): T {....deprecated.}
    
-
    
-  
    
-    Deprecated 
-  
    
+  
    proc baz[T](a, b: T): T {....deprecated.}
    
+  
    
+    
    
+  Deprecated 
+
    
 
-This is deprecated without message.
-
-
    
+    This is deprecated without message.
+    
+  
 
    
 
 
    
-
 
    
-
    
-
    proc buzz[T](a, b: T): T {....deprecated: "since v0.20".}
    
-
    
-  
    
-    Deprecated: since v0.20
-  
    
+  
    
+  
    proc buzz[T](a, b: T): T {....deprecated: "since v0.20".}
    
+  
    
+    
    
+  Deprecated: since v0.20
+
    
 
-This is deprecated with a message.
-
-
    
+    This is deprecated with a message.
+    
+  
 
    
 
 
    
-
 
    
-
    
-
    proc c_nonexistent(frmt: cstring): cint {.importc: "nonexistent",
+  
    
+  
    proc c_nonexistent(frmt: cstring): cint {.importc: "nonexistent",
     header: "<stdio.h>", varargs, discardable, ...raises: [], tags: [].}
    
-
    
-
-
-
-
    
+  
    
+    
+    
+    
+  
    
 
    
 
 
    
-
 
    
-
    
-
    proc c_printf(frmt: cstring): cint {.importc: "printf", header: "<stdio.h>",
+  
    
+  
    proc c_printf(frmt: cstring): cint {.importc: "printf", header: "<stdio.h>",
                                      varargs, discardable, ...raises: [], tags: [].}
    
-
    
-
-the c printf. etc.
-
-
    
+  
    
+    
+    the c printf. etc.
+    
+  
    
 
    
 
 
    
-
 
    
-
    
-
    proc fromUtils3() {....raises: [], tags: [].}
    
-
    
-
-came form utils but should be shown where fromUtilsGen is called
+  
    
+  
    proc fromUtils3() {....raises: [], tags: [].}
    
+  
    
+    
+    came form utils but should be shown where fromUtilsGen is called
 
    Example:
    
 
    discard """should be shown as examples for fromUtils3
        in module calling fromUtilsGen"""
    
-
-
    
+    
+  
 
    
 
 
    
-
 
    
-
    
-
    proc isValid[T](x: T): bool
    
-
    
-
-
-
-
    
+  
    
+  
    proc isValid[T](x: T): bool
    
+  
    
+    
+    
+    
+  
    
 
    
 
 
    
-
 
    
-
    
-
    proc low[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect,
+  
    
+  
    proc low[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect,
     ...raises: [], tags: [].}
    
-
    
-
-
    Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.
    
+  
    
+    
+    
    Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.
    
 
    See also:
    
 
 
    low(2) # => -9223372036854775808
    
-
-
    
+    
+  
 
    
 
 
    
-
 
    
-
    
-
    proc low2[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect,
+  
    
+  
    proc low2[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect,
     ...raises: [], tags: [].}
    
-
    
-
-
    Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.
    
+  
    
+    
+    
    Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.
    
 
    See also:
    
 
 
    low2(2) # => -9223372036854775808
    
 
    Example:
    
 
    discard "in low2"
    
-
-
    
+    
+  
 
    
 
 
    
-
 
    
-
    
-
    proc p1() {....raises: [], tags: [].}
    
-
    
-
-cp1
+  
    
+  
    proc p1() {....raises: [], tags: [].}
    
+  
    
+    
+    cp1
 
    Example:
    
 
    doAssert 1 == 1 # regular comments work here
    c4
 
    Example:
    
@@ -753,30 +696,28 @@ this is a nested doc comment
 ]##
 discard "c9"
 # also work after
    - - + +
    -
    -
    -
    func someFunc() {....raises: [], tags: [].}
    -
    - -My someFunc. Stuff in quotes here. Some link - -
    +
    + 
    func someFunc() {....raises: [], tags: [].}
    +
    + + My someFunc. Stuff in quotes here. Some link + +
    -
    -
    -
    proc tripleStrLitTest() {....raises: [], tags: [].}
    -
    - - +
    + 
    proc tripleStrLitTest() {....raises: [], tags: [].}
    +
    + +

    Example: cmd: --hint:XDeclaredButNotUsed:off

    ## mullitline string litterals are tricky as their indentation can span
 ## below that of the runnableExamples
@@ -813,365 +754,343 @@ at indent 0
   """ ]
 discard
 # should be in
    - -
    + +
    -
    -
    -
    proc z1(): Foo {....raises: [], tags: [].}
    -
    - -cz1 - -
    +
    + 
    proc z1(): Foo {....raises: [], tags: [].}
    +
    + + cz1 + +
    -
    -
    -
    proc z2() {....raises: [], tags: [].}
    -
    - -cz2 +
    + 
    proc z2() {....raises: [], tags: [].}
    +
    + + cz2

    Example:

    discard "in cz2"
    - -
    + +
    -
    -
    -
    proc z3() {....raises: [], tags: [].}
    -
    - -cz3 - -
    +
    + 
    proc z3() {....raises: [], tags: [].}
    +
    + + cz3 + +
    -
    -
    -
    proc z4() {....raises: [], tags: [].}
    -
    - -cz4 - -
    +
    + 
    proc z4() {....raises: [], tags: [].}
    +
    + + cz4 + +
    -
    -
    -
    proc z5(): int {....raises: [], tags: [].}
    -
    - -cz5 - -
    +
    + 
    proc z5(): int {....raises: [], tags: [].}
    +
    + + cz5 + +
    -
    -
    -
    proc z6(): int {....raises: [], tags: [].}
    -
    - -cz6 - -
    +
    + 
    proc z6(): int {....raises: [], tags: [].}
    +
    + + cz6 + +
    -
    -
    -
    proc z7(): int {....raises: [], tags: [].}
    -
    - -cz7 - -
    +
    + 
    proc z7(): int {....raises: [], tags: [].}
    +
    + + cz7 + +
    -
    -
    -
    proc z8(): int {....raises: [], tags: [].}
    -
    - -cz8 - -
    +
    + 
    proc z8(): int {....raises: [], tags: [].}
    +
    + + cz8 + +
    -
    -
    -
    proc z9() {....raises: [], tags: [].}
    -
    - - +
    + 
    proc z9() {....raises: [], tags: [].}
    +
    + +

    Example:

    doAssert 1 + 1 == 2
    - -
    + +
    -
    -
    -
    proc z10() {....raises: [], tags: [].}
    -
    - - +
    + 
    proc z10() {....raises: [], tags: [].}
    +
    + +

    Example: cmd: -d:foobar

    discard 1
    cz10 - -
    + +
    -
    -
    -
    proc z11() {....raises: [], tags: [].}
    -
    - - +
    + 
    proc z11() {....raises: [], tags: [].}
    +
    + +

    Example:

    discard 1
    - -
    + +
    -
    -
    -
    proc z12(): int {....raises: [], tags: [].}
    -
    - - +
    + 
    proc z12(): int {....raises: [], tags: [].}
    +
    + +

    Example:

    discard 1
    - -
    + +
    -
    -
    -
    proc z13() {....raises: [], tags: [].}
    -
    - -cz13 +
    + 
    proc z13() {....raises: [], tags: [].}
    +
    + + cz13

    Example:

    discard
    - -
    + +
    -
    -
    -
    proc z17() {....raises: [], tags: [].}
    -
    - -cz17 rest +
    + 
    proc z17() {....raises: [], tags: [].}
    +
    + + cz17 rest

    Example:

    discard 1
    rest - -
    + +
    -
    + +
    -

    Methods

    -
    - -
    -
    -
    method method1(self: Moo) {.base, ...raises: [], tags: [].}
    -
    - -foo1 - -
    +

    Methods

    +
    +
    +
    + 
    method method1(self: Moo) {.base, ...raises: [], tags: [].}
    +
    + + foo1 + +
    -
    -
    -
    method method2(self: Moo): int {.base, ...raises: [], tags: [].}
    -
    - -foo2 - -
    +
    + 
    method method2(self: Moo): int {.base, ...raises: [], tags: [].}
    +
    + + foo2 + +
    -
    -
    -
    method method3(self: Moo): int {.base, ...raises: [], tags: [].}
    -
    - -foo3 - -
    +
    + 
    method method3(self: Moo): int {.base, ...raises: [], tags: [].}
    +
    + + foo3 + +
    -
    +
    +
    -

    Iterators

    -
    - -
    -
    -
    iterator fromUtils1(): int {....raises: [], tags: [].}
    -
    - - +

    Iterators

    +
    +
    +
    + 
    iterator fromUtils1(): int {....raises: [], tags: [].}
    +
    + +

    Example:

    # ok1
 assert 1 == 1
 # ok2
    - -
    + +
    -
    -
    -
    iterator iter1(n: int): int {....raises: [], tags: [].}
    -
    - -foo1 - -
    +
    + 
    iterator iter1(n: int): int {....raises: [], tags: [].}
    +
    + + foo1 + +
    -
    -
    -
    iterator iter2(n: int): int {....raises: [], tags: [].}
    -
    - -foo2 +
    + 
    iterator iter2(n: int): int {....raises: [], tags: [].}
    +
    + + foo2

    Example:

    discard # bar
    - -
    + +
    -
    + +
    -

    Macros

    -
    - -
    -
    -
    macro bar(): untyped
    -
    - - - -
    +

    Macros

    +
    +
    +
    + 
    macro bar(): untyped
    +
    + + + +
    -
    -
    -
    macro z16()
    -
    - - +
    + 
    macro z16()
    +
    + +

    Example:

    discard 1
    cz16 after

    Example:

    doAssert 2 == 1 + 1
    - -
    + +
    -
    -
    -
    macro z18(): int
    -
    - -cz18 - -
    +
    + 
    macro z18(): int
    +
    + + cz18 + +
    -
    +
    +
    -

    Templates

    -
    - -
    -
    -
    template foo(a, b: SomeType)
    -
    - -This does nothing - -
    +

    Templates

    +
    +
    +
    + 
    template foo(a, b: SomeType)
    +
    + + This does nothing + +
    -
    -
    -
    template fromUtils2()
    -
    - -ok3 +
    + 
    template fromUtils2()
    +
    + + ok3

    Example:

    discard """should be shown as examples for fromUtils2
        in module calling fromUtilsGen"""
    - -
    + +
    -
    -
    -
    template myfn()
    -
    - - +
    + 
    template myfn()
    +
    + +

    Example:

    import std/strutils
 ## issue #8871 preserve formatting
@@ -1187,58 +1106,54 @@ bar
 block:
   discard 0xff # elu par cette crapule
 # should be in
    should be still in - -
    + +
    -
    -
    -
    template testNimDocTrailingExample()
    -
    - - +
    + 
    template testNimDocTrailingExample()
    +
    + +

    Example:

    discard 2
    - -
    + +
    -
    -
    -
    template z6t(): int
    -
    - -cz6t - -
    +
    + 
    template z6t(): int
    +
    + + cz6t + +
    -
    -
    -
    template z14()
    -
    - -cz14 +
    + 
    template z14()
    +
    + + cz14

    Example:

    discard
    - -
    + +
    -
    -
    -
    template z15()
    -
    - -cz15 +
    + 
    template z15()
    +
    + + cz15

    Example:

    discard

    Example:

    @@ -1249,26 +1164,25 @@ cz15 
    assert true

    Example:

    discard 1
    in or out? - -
    + +
    -
    +
    +
    -
    -
    +
    Made with Nim. Generated: 1970-01-02 03:46:40 UTC
    - - + diff --git a/nimdoc/testproject/expected/theindex.html b/nimdoc/testproject/expected/theindex.html index 47fae2491..c62b4c7db 100644 --- a/nimdoc/testproject/expected/theindex.html +++ b/nimdoc/testproject/expected/theindex.html @@ -1,12 +1,11 @@ - + - + - +Index @@ -17,17 +16,16 @@ -Index + - -
    -
    -

    Index

    - Modules: subdir/subdir_b/utils, testproject.

    API symbols

    +
    +
    +

    Index

    + Modules: subdir/subdir_b/utils, testproject.

    API symbols

    `$`:
    -
    -
    +
    Made with Nim. Generated: 1970-01-02 03:46:40 UTC
    -
    - + From d33e1127666be23ad2dc879c2f89a41c179e2093 Mon Sep 17 00:00:00 2001 From: Tanguy Date: Wed, 15 Jun 2022 16:38:12 +0200 Subject: [PATCH 1661/2472] Better range error messages (#19867) * Better range error messages * Revert to old behavior for arrays * Small corrections --- compiler/ccgexprs.nim | 6 +++--- lib/system/chcks.nim | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 8e7a21c67..4c15101a9 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -988,12 +988,12 @@ proc genBoundsCheck(p: BProc; arr, a, b: TLoc) = if reifiedOpenArray(arr.lode): linefmt(p, cpsStmts, "if ($2-$1 != -1 && " & - "($1 < 0 || $1 >= $3.Field1 || $2 < 0 || $2 >= $3.Field1)){ #raiseIndexError(); $4}$n", + "($1 < 0 || $1 >= $3.Field1 || $2 < 0 || $2 >= $3.Field1)){ #raiseIndexError4($1, $2, $3.Field1); $4}$n", [rdLoc(a), rdLoc(b), rdLoc(arr), raiseInstr(p)]) else: linefmt(p, cpsStmts, "if ($2-$1 != -1 && ($1 < 0 || $1 >= $3Len_0 || $2 < 0 || $2 >= $3Len_0))" & - "{ #raiseIndexError(); $4}$n", + "{ #raiseIndexError4($1, $2, $3Len_0); $4}$n", [rdLoc(a), rdLoc(b), rdLoc(arr), raiseInstr(p)]) of tyArray: let first = intLiteral(firstOrd(p.config, ty)) @@ -1004,7 +1004,7 @@ proc genBoundsCheck(p: BProc; arr, a, b: TLoc) = of tySequence, tyString: linefmt(p, cpsStmts, "if ($2-$1 != -1 && " & - "($1 < 0 || $1 >= $3 || $2 < 0 || $2 >= $3)){ #raiseIndexError(); $4}$n", + "($1 < 0 || $1 >= $3 || $2 < 0 || $2 >= $3)){ #raiseIndexError4($1, $2, $3); $4}$n", [rdLoc(a), rdLoc(b), lenExpr(p, arr), raiseInstr(p)]) else: discard diff --git a/lib/system/chcks.nim b/lib/system/chcks.nim index a404e9d40..1e1ce9c87 100644 --- a/lib/system/chcks.nim +++ b/lib/system/chcks.nim @@ -16,6 +16,9 @@ proc raiseRangeError(val: BiggestInt) {.compilerproc, noinline.} = else: sysFatal(RangeDefect, "value out of range: ", $val) +proc raiseIndexError4(l1, h1, h2: int) {.compilerproc, noinline.} = + sysFatal(IndexDefect, "index out of bounds: " & $l1 & ".." & $h1 & " notin 0.." & $(h2 - 1)) + proc raiseIndexError3(i, a, b: int) {.compilerproc, noinline.} = sysFatal(IndexDefect, formatErrorIndexBound(i, a, b)) From ef2dd2e473d7082d60088ece05beef81062850bb Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Thu, 16 Jun 2022 14:46:59 +0800 Subject: [PATCH 1662/2472] document nimTestsNimdocFixup for rsttester (#19894) --- nimdoc/rsttester.nim | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/nimdoc/rsttester.nim b/nimdoc/rsttester.nim index daca3dfc7..a0bdfca1e 100644 --- a/nimdoc/rsttester.nim +++ b/nimdoc/rsttester.nim @@ -1,9 +1,14 @@ +# To run this, cd to the git repo root, and run "nim r nimdoc/rsttester.nim". +# to change expected results (after carefully verifying everything), use -d:nimTestsNimdocFixup + import os, strutils from std/private/gitutils import diffFiles const baseDir = "nimdoc/rst2html" +const fixup = defined(nimTestsNimdocFixup) + var failures = 0 proc exec(cmd: string) = @@ -29,7 +34,7 @@ proc testRst2Html(fixup = false) = if failures == 0: removeDir(baseDir / "source/htmldocs") -testRst2Html(defined(fixup)) +testRst2Html(fixup) # Check for failures if failures > 0: quit($failures & " failures occurred.") From 332aa0af75baf328416068459931d5a6e4292d34 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Sun, 19 Jun 2022 13:35:25 +0800 Subject: [PATCH 1663/2472] [vm] refactor `stdlib.marshal.load` (#19905) refactor stdlib.marshal.load --- compiler/vmops.nim | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/compiler/vmops.nim b/compiler/vmops.nim index 640826cc8..04b753ef6 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -364,21 +364,5 @@ proc registerAdditionalOps*(c: PCtx) = let typ = a.getNode(0).typ let p = a.getReg(1) var res: string - - var node: PNode - case p.kind - of rkNone: - node = newNode(nkEmpty) - of rkInt: - node = newIntNode(nkIntLit, p.intVal) - of rkFloat: - node = newFloatNode(nkFloatLit, p.floatVal) - of rkNode: - node = p.node - of rkRegisterAddr: - node = p.regAddr.node - of rkNodeAddr: - node = p.nodeAddr[] - - storeAny(res, typ, node, c.config) + storeAny(res, typ, regToNode(p[]), c.config) setResult(a, res) From dd4cc266cd39d7cecfd7215e683b0c6499fc4a36 Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Sun, 19 Jun 2022 15:36:33 +1000 Subject: [PATCH 1664/2472] Fixes #19900 (#19906) Fixed tocRoot placement --- config/nimdoc.cfg | 4 ++-- nimdoc/rst2html/expected/rst_examples.html | 2 +- nimdoc/test_out_index_dot_html/expected/index.html | 2 +- nimdoc/testproject/expected/subdir/subdir_b/utils.html | 2 +- nimdoc/testproject/expected/testproject.html | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index 725f9e0a5..3b0afe8d6 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -149,11 +149,11 @@ doc.body_toc_group = """ onkeyup="search()" />
    $body_toc_groupsection -
    $tableofcontents
    $seeSrc +
    $deprecationMsg

    $moduledesc

    $content @@ -188,11 +188,11 @@ doc.body_toc_group = """
    -
    $tableofcontents
    $seeSrc +
    $deprecationMsg

    $moduledesc

    $content diff --git a/nimdoc/rst2html/expected/rst_examples.html b/nimdoc/rst2html/expected/rst_examples.html index 532917055..efc8ac414 100644 --- a/nimdoc/rst2html/expected/rst_examples.html +++ b/nimdoc/rst2html/expected/rst_examples.html @@ -49,7 +49,6 @@
    -
    • About this document
      • Encoding
        • @@ -75,6 +74,7 @@
        +

        diff --git a/nimdoc/test_out_index_dot_html/expected/index.html b/nimdoc/test_out_index_dot_html/expected/index.html index 0c0dc2268..43285db6b 100644 --- a/nimdoc/test_out_index_dot_html/expected/index.html +++ b/nimdoc/test_out_index_dot_html/expected/index.html @@ -49,7 +49,6 @@ -
        • @@ -69,6 +68,7 @@
          +

          diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html index d5a3b84c7..4d753ad8f 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html @@ -49,7 +49,6 @@
          -
          +

          This is a description of the utils module.

          Links work:

          diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html index 49f24d204..24dfdc13f 100644 --- a/nimdoc/testproject/expected/testproject.html +++ b/nimdoc/testproject/expected/testproject.html @@ -49,7 +49,6 @@
          -
          • @@ -335,6 +334,7 @@
            +

            This is the top level module.

            Example:

            From 5e32fc785527d9b71ebab643f142fbab13eb29ce Mon Sep 17 00:00:00 2001 From: tersec Date: Mon, 20 Jun 2022 06:01:41 +0000 Subject: [PATCH 1665/2472] Recommend gcc 5.x or newer (#19910) --- readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index 7ff056a20..8999091e6 100644 --- a/readme.md +++ b/readme.md @@ -51,8 +51,8 @@ Nim programming language. Those C sources are available within the Next, to build from source you will need: - * A C compiler such as ``gcc`` 3.x/later or an alternative such as ``clang``, - ``Visual C++`` or ``Intel C++``. It is recommended to use ``gcc`` 3.x or + * A C compiler such as ``gcc`` 5.x/later or an alternative such as ``clang``, + ``Visual C++`` or ``Intel C++``. It is recommended to use ``gcc`` 5.x or later. * Either ``git`` or ``wget`` to download the needed source repositories. * The ``build-essential`` package when using ``gcc`` on Ubuntu (and likely From 40464fa7626a616a3881c0b16a74b9ec7d788720 Mon Sep 17 00:00:00 2001 From: Tanguy Date: Mon, 20 Jun 2022 08:21:20 +0200 Subject: [PATCH 1666/2472] Fix nimRawSetjmp for VCC [backport: 1.2] (#19899) --- lib/system/ansi_c.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/system/ansi_c.nim b/lib/system/ansi_c.nim index 5d0ecc01c..74f79167a 100644 --- a/lib/system/ansi_c.nim +++ b/lib/system/ansi_c.nim @@ -118,7 +118,7 @@ elif defined(nimBuiltinSetjmp): c_builtin_setjmp(unsafeAddr jmpb[0]) elif defined(nimRawSetjmp) and not defined(nimStdSetjmp): - when defined(windows): + when defined(windows) and not defined(vcc): # No `_longjmp()` on Windows. proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {. header: "", importc: "longjmp".} From 0ae44e562f693d270e3cad77675ce5f68474921d Mon Sep 17 00:00:00 2001 From: flintforge Date: Tue, 21 Jun 2022 03:53:46 +0200 Subject: [PATCH 1667/2472] fix typo in nre.nim (#19915) Update nre.nim typo in proc replace description --- lib/impure/nre.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/impure/nre.nim b/lib/impure/nre.nim index 03cbf5220..1012c7c36 100644 --- a/lib/impure/nre.nim +++ b/lib/impure/nre.nim @@ -695,8 +695,7 @@ proc replace*(str: string, pattern: Regex, ## each match and the return value is the replacement value. ## ## If `subproc` is a `proc (string): string`, then it is executed with the - ## full text of the match and and the return value is the replacement - ## value. + ## full text of the match and the return value is the replacement value. ## ## If `subproc` is a string, the syntax is as follows: ## From a65db5e2e924c51af7c654f8e564332299b95cb3 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Tue, 21 Jun 2022 15:37:23 +0800 Subject: [PATCH 1668/2472] [Tiny] correct comment opcDeref => opcLdDeref (#19908) correct comment opcDeref => opcLdDeref --- compiler/vm.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/vm.nim b/compiler/vm.nim index 1b11cb4f3..656508793 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -642,7 +642,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = of opcNodeToReg: let ra = instr.regA let rb = instr.regB - # opcDeref might already have loaded it into a register. XXX Let's hope + # opcLdDeref might already have loaded it into a register. XXX Let's hope # this is still correct this way: if regs[rb].kind != rkNode: regs[ra] = regs[rb] From 3cb2d7af0591b12044d4be20dfb6d7f7ff79a4ff Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Wed, 22 Jun 2022 14:43:58 +0800 Subject: [PATCH 1669/2472] [vm]fixes #15974 #12551 #19464 #16020 #16780 #16613 #14553 #19909 #18641 (#19902) [backport] * revert #12217 since the root problem seems to have been fixed; fix #15974;fix #12551; fix #19464 * fix #16020; fix #16780 * fix tests and #16613 * fix #14553 * fix #19909; skip skipRegisterAddr * fix #18641 --- compiler/vm.nim | 28 +++--- compiler/vmgen.nim | 3 - tests/vm/tissues.nim | 50 ++++++++++- tests/vm/tmisc_vm.nim | 196 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 263 insertions(+), 14 deletions(-) diff --git a/compiler/vm.nim b/compiler/vm.nim index 656508793..e681bbc96 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -523,8 +523,7 @@ when not defined(nimHasSinkInference): template takeAddress(reg, source) = reg.nodeAddr = addr source - when defined(gcDestructors): - GC_ref source + GC_ref source proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = var pc = start @@ -1011,6 +1010,12 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = decodeBC(rkInt) template getTyp(n): untyped = n.typ.skipTypes(abstractInst) + template skipRegisterAddr(n: TFullReg): TFullReg = + var tmp = n + while tmp.kind == rkRegisterAddr: + tmp = tmp.regAddr[] + tmp + proc ptrEquality(n1: ptr PNode, n2: PNode): bool = ## true if n2.intVal represents a ptr equal to n1 let p1 = cast[int](n1) @@ -1024,16 +1029,19 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = return t2.kind in PtrLikeKinds and n2.intVal == p1 else: return false - if regs[rb].kind == rkNodeAddr: - if regs[rc].kind == rkNodeAddr: - ret = regs[rb].nodeAddr == regs[rc].nodeAddr + let rbReg = skipRegisterAddr(regs[rb]) + let rcReg = skipRegisterAddr(regs[rc]) + + if rbReg.kind == rkNodeAddr: + if rcReg.kind == rkNodeAddr: + ret = rbReg.nodeAddr == rcReg.nodeAddr else: - ret = ptrEquality(regs[rb].nodeAddr, regs[rc].node) - elif regs[rc].kind == rkNodeAddr: - ret = ptrEquality(regs[rc].nodeAddr, regs[rb].node) + ret = ptrEquality(rbReg.nodeAddr, rcReg.node) + elif rcReg.kind == rkNodeAddr: + ret = ptrEquality(rcReg.nodeAddr, rbReg.node) else: - let nb = regs[rb].node - let nc = regs[rc].node + let nb = rbReg.node + let nc = rcReg.node if nb.kind != nc.kind: discard elif (nb == nc) or (nb.kind == nkNilLit): ret = true # intentional elif nb.kind in {nkSym, nkTupleConstr, nkClosure} and nb.typ != nil and nb.typ.kind == tyProc and sameConstant(nb, nc): diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 2e7f6d8b0..aa2bdbe4c 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1382,9 +1382,6 @@ proc unneededIndirection(n: PNode): bool = n.typ.skipTypes(abstractInstOwned-{tyTypeDesc}).kind == tyRef proc canElimAddr(n: PNode): PNode = - if n[0].typ.skipTypes(abstractInst).kind in {tyObject, tyTuple, tyArray}: - # objects are reference types in the VM - return n[0] case n[0].kind of nkObjUpConv, nkObjDownConv, nkChckRange, nkChckRangeF, nkChckRange64: var m = n[0][0] diff --git a/tests/vm/tissues.nim b/tests/vm/tissues.nim index 1cf3afc00..f0ae6c296 100644 --- a/tests/vm/tissues.nim +++ b/tests/vm/tissues.nim @@ -1,6 +1,6 @@ import macros -block t9043: # issue #9043 +block t9043: # bug #9043 proc foo[N: static[int]](dims: array[N, int]): string = const N1 = N const N2 = dims.len @@ -26,3 +26,51 @@ block t4952: let tree = newTree(nnkExprColonExpr) let t = (n: tree) doAssert: t.n.kind == tree.kind + + +# bug #19909 +type + SinglyLinkedList[T] = ref object + SinglyLinkedListObj[T] = ref object + + +proc addMoved[T](a, b: var SinglyLinkedList[T]) = + if a.addr != b.addr: discard + +proc addMoved[T](a, b: var SinglyLinkedListObj[T]) = + if a.addr != b.addr: discard + +proc main = + var a: SinglyLinkedList[int]; new a + var b: SinglyLinkedList[int]; new b + a.addMoved b + + var a0: SinglyLinkedListObj[int] + var b0: SinglyLinkedListObj[int] + a0.addMoved b0 + +static: main() + + +# bug #18641 + +type A = object + ha1: int +static: + var a = A() + var a2 = a.addr + a2.ha1 = 11 + doAssert a2.ha1 == 11 + a.ha1 = 12 + doAssert a.ha1 == 12 + doAssert a2.ha1 == 12 # ok +static: + proc fn() = + var a = A() + var a2 = a.addr + a2.ha1 = 11 + doAssert a2.ha1 == 11 + a.ha1 = 12 + doAssert a.ha1 == 12 + doAssert a2.ha1 == 12 # fails + fn() diff --git a/tests/vm/tmisc_vm.nim b/tests/vm/tmisc_vm.nim index bbf618622..af09e8871 100644 --- a/tests/vm/tmisc_vm.nim +++ b/tests/vm/tmisc_vm.nim @@ -1,7 +1,12 @@ discard """ + targets: "c js" output: ''' [127, 127, 0, 255][127, 127, 0, 255] (data: 1) +(2, 1) +(2, 1) +(2, 1) +(f0: 5) ''' nimout: '''caught Exception @@ -17,6 +22,19 @@ foo4 (a: 0, b: 0) (a: 0, b: 0) (a: 0, b: 0) +z1 m: (lo: 12) +z2 a: (lo: 3) +x1 a: (lo: 3) +x2 a: (lo: 6) +x3 a: (lo: 0) +z3 a: (lo: 3) +x1 a: (lo: 3) +x2 a: (lo: 6) +x3 a: (lo: 0) +(2, 1) +(2, 1) +(2, 1) +(f0: 5) ''' """ import std/sets @@ -249,3 +267,181 @@ static: echo x2[] let x3 = new(ref MyObject) # cannot generate VM code for ref MyObject echo x3[] + +# bug #19464 +type + Wrapper = object + inner: int + +proc assign(r: var Wrapper, a: Wrapper) = + r = a + +proc myEcho(a: Wrapper) = + var tmp = a + assign(tmp, Wrapper(inner: 0)) # this shouldn't modify `a` + doAssert a.inner == 1 + +static: + var result: Wrapper + assign(result, Wrapper(inner: 1)) + myEcho(result) + +when true: + # bug #15974 + type Foo = object + f0: int + + proc fn(a: var Foo) = + var s: Foo + a = Foo(f0: 2) + s = a + doAssert s.f0 == 2 + a = Foo(f0: 3) + doAssert s.f0 == 2 + + proc test2()= + var a = Foo(f0: 1) + fn(a) + + static: test2() + test2() + +# bug #12551 +type + StUint = object + lo: uint64 + +func `+=`(x: var Stuint, y: Stuint) = + x.lo += y.lo + +func `-`(x, y: Stuint): Stuint = + result.lo = x.lo - y.lo + +func `+`(x, y: Stuint): Stuint = + result.lo = x.lo + y.lo + +func `-=`(x: var Stuint, y: Stuint) = + x = x - y + +func `<`(x, y: Stuint): bool= + x.lo < y.lo + +func `==`(x, y: Stuint): bool = + x.lo == y.lo + +func `<=`(x, y: Stuint): bool = + x.lo <= y.lo + +proc div3n2n(r: var Stuint, b: Stuint) = + var d: Stuint + r = d + r += b + +func div2n1n(r: var Stuint, b: Stuint) = + div3n2n(r, b) + +func divmodBZ(x, y: Stuint, r: var Stuint)= + div2n1n(r, y) + r.lo = 3 + +func `mod`(x, y: Stuint): Stuint = + divmodBZ(x, y, result) + +func doublemod_internal(a, m: Stuint): Stuint = + result = a + if a >= m - a: + result -= m + result += a + +func mulmod_internal(a, b, m: Stuint): Stuint = + var (a, b) = (a, b) + swap(a, b) + debugEcho "x1 a: ", a + a = doublemod_internal(a, m) + debugEcho "x2 a: ", a + a = doublemod_internal(a, m) + debugEcho "x3 a: ", a + +func powmod_internal(a, m: Stuint): Stuint = + var a = a + debugEcho "z1 m: ", m + debugEcho "z2 a: ", a + result = mulmod_internal(result, a, m) + debugEcho "z3 a: ", a + a = mulmod_internal(a, a, m) + +func powmod*(a, m: Stuint) = + discard powmod_internal(a mod m, m) + +static: + var x = Stuint(lo: high(uint64)) + var y = Stuint(lo: 12) + + powmod(x, y) + +# bug #16780 +when true: + template swap*[T](a, b: var T) = + var a2 = addr(a) + var b2 = addr(b) + var aOld = a2[] + a2[] = b2[] + b2[] = aOld + + proc rather = + block: + var a = 1 + var b = 2 + swap(a, b) + echo (a,b) + + block: + type Foo = ref object + x: int + var a = Foo(x:1) + var b = Foo(x:2) + swap(a, b) + echo (a.x, b.x) + + block: + type Foo = object + x: int + var a = Foo(x:1) + var b = Foo(x:2) + swap(a, b) + echo (a.x,b.x) + + static: rather() + rather() + +# bug #16020 +when true: + block: + type Foo = object + f0: int + proc main= + var f = Foo(f0: 3) + var f2 = f.addr + f2[].f0 += 1 + f2.f0 += 1 + echo f + static: main() + main() + +import tables, strutils + +# bug #14553 +const PpcPatterns = @[("aaaa", "bbbb"), ("aaaaa", "bbbbb"), ("aaaaaa", "bbbbbb"), ("aaaaaaa", "bbbbbbb"), ("aaaaaaaa", "bbbbb")] + +static: + var + needSecondIdentifier = initTable[uint32, seq[(string, string)]]() + + for (name, pattern) in PpcPatterns: + let + firstPart = 0'u32 + lastPart = "test" + + needSecondIdentifier.mgetOrPut(firstPart, @[]).add((name, pattern)) + + doAssert needSecondIdentifier[0] == @[("aaaa", "bbbb"), ("aaaaa", "bbbbb"), ("aaaaaa", "bbbbbb"), ("aaaaaaa", "bbbbbbb"), ("aaaaaaaa", "bbbbb")] From caf6aff06b6dd36ef0bbe8c1f8b527952790e208 Mon Sep 17 00:00:00 2001 From: Khaled Hammouda Date: Wed, 22 Jun 2022 06:36:30 -0400 Subject: [PATCH 1670/2472] Fix distinct requiresInit test and manual (#19901) fix distinct test and manual --- doc/manual.rst | 11 +++++++---- tests/distinct/tdistinct.nim | 6 +++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/doc/manual.rst b/doc/manual.rst index 82c9f5758..b63a2f68f 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -2851,7 +2851,10 @@ Given the following distinct type definitions: .. code-block:: nim type - DistinctObject {.requiresInit, borrow: `.`.} = distinct MyObject + Foo = object + x: string + + DistinctFoo {.requiresInit, borrow: `.`.} = distinct Foo DistinctString {.requiresInit.} = distinct string The following code blocks will fail to compile: @@ -2864,7 +2867,7 @@ The following code blocks will fail to compile: .. code-block:: nim var s: DistinctString s = "test" - doAssert s == "test" + doAssert string(s) == "test" But these ones will compile successfully: @@ -2873,8 +2876,8 @@ But these ones will compile successfully: doAssert foo.x == "test" .. code-block:: nim - let s = "test" - doAssert s == "test" + let s = DistinctString("test") + doAssert string(s) == "test" Let statement ------------- diff --git a/tests/distinct/tdistinct.nim b/tests/distinct/tdistinct.nim index dd8237854..8ec083020 100644 --- a/tests/distinct/tdistinct.nim +++ b/tests/distinct/tdistinct.nim @@ -135,11 +135,11 @@ block tRequiresInit: reject: var s: DistinctString s = "test" - doAssert s == "test" + doAssert string(s) == "test" accept: - let s = "test" - doAssert s == "test" + let s = DistinctString("test") + doAssert string(s) == "test" block: #17322 type From 0189122d4fadc73a3de18a7a781997c506f837aa Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Mon, 27 Jun 2022 23:57:07 +0800 Subject: [PATCH 1671/2472] ref #19830; multiple definition of in Nim generated static libraries (#19934) * ref #19830; multiple definition of in Nim generated static libraries * fix compile errors --- compiler/cgen.nim | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 862f31107..c0b94ebbb 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1323,7 +1323,7 @@ proc getSomeInitName(m: BModule, suffix: string): Rope = proc getInitName(m: BModule): Rope = if sfMainModule in m.module.flags: # generate constant name for main module, for "easy" debugging. - result = rope"NimMainModule" + result = rope(m.config.nimMainPrefix) & rope"NimMainModule" else: result = getSomeInitName(m, "Init000") @@ -1356,35 +1356,35 @@ proc genMainProc(m: BModule) = preMainCode.add("\tinitStackBottomWith_actual((void *)&inner);\L") preMainCode.add("\t(*inner)();\L") else: - preMainCode.add("\tPreMain();\L") + preMainCode.add("\t$1PreMain();\L" % [rope m.config.nimMainPrefix]) + + var posixCmdLine: Rope + if optNoMain notin m.config.globalOptions: + posixCmdLine.add "\tN_LIB_PRIVATE int cmdCount;\L" + posixCmdLine.add "\tN_LIB_PRIVATE char** cmdLine;\L" + posixCmdLine.add "\tN_LIB_PRIVATE char** gEnv;\L" const - # not a big deal if we always compile these 3 global vars... makes the HCR code easier - PosixCmdLine = - "N_LIB_PRIVATE int cmdCount;$N" & - "N_LIB_PRIVATE char** cmdLine;$N" & - "N_LIB_PRIVATE char** gEnv;$N" - # The use of a volatile function pointer to call Pre/NimMainInner # prevents inlining of the NimMainInner function and dependent # functions, which might otherwise merge their stack frames. PreMainVolatileBody = "\tvoid (*volatile inner)(void);$N" & - "\tinner = PreMainInner;$N" & + "\tinner = $3PreMainInner;$N" & "$1" & "\t(*inner)();$N" PreMainNonVolatileBody = "$1" & - "\tPreMainInner();$N" + "\t$3PreMainInner();$N" PreMainBodyStart = "$N" & - "N_LIB_PRIVATE void PreMainInner(void) {$N" & + "N_LIB_PRIVATE void $3PreMainInner(void) {$N" & "$2" & "}$N$N" & - PosixCmdLine & - "N_LIB_PRIVATE void PreMain(void) {$N" + "$4" & + "N_LIB_PRIVATE void $3PreMain(void) {$N" PreMainBodyEnd = "}$N$N" @@ -1395,21 +1395,21 @@ proc genMainProc(m: BModule) = MainProcsWithResult = MainProcs & ("\treturn $1nim_program_result;$N") - NimMainInner = "N_LIB_PRIVATE N_CDECL(void, NimMainInner)(void) {$N" & + NimMainInner = "N_LIB_PRIVATE N_CDECL(void, $5NimMainInner)(void) {$N" & "$1" & "}$N$N" NimMainVolatileBody = "\tvoid (*volatile inner)(void);$N" & "$4" & - "\tinner = NimMainInner;$N" & + "\tinner = $5NimMainInner;$N" & "$2" & "\t(*inner)();$N" NimMainNonVolatileBody = "$4" & "$2" & - "\tNimMainInner();$N" + "\t$5NimMainInner();$N" NimMainProcStart = "N_CDECL(void, $5NimMain)(void) {$N" @@ -1489,9 +1489,9 @@ proc genMainProc(m: BModule) = else: ropecg(m, "\t#initStackBottomWith((void *)&inner);$N", []) inc(m.labels) if m.config.selectedGC notin {gcNone, gcArc, gcOrc}: - appcg(m, m.s[cfsProcs], PreMainBodyStart & PreMainVolatileBody & PreMainBodyEnd, [m.g.mainDatInit, m.g.otherModsInit]) + appcg(m, m.s[cfsProcs], PreMainBodyStart & PreMainVolatileBody & PreMainBodyEnd, [m.g.mainDatInit, m.g.otherModsInit, m.config.nimMainPrefix, posixCmdLine]) else: - appcg(m, m.s[cfsProcs], PreMainBodyStart & PreMainNonVolatileBody & PreMainBodyEnd, [m.g.mainDatInit, m.g.otherModsInit]) + appcg(m, m.s[cfsProcs], PreMainBodyStart & PreMainNonVolatileBody & PreMainBodyEnd, [m.g.mainDatInit, m.g.otherModsInit, m.config.nimMainPrefix, posixCmdLine]) if m.config.target.targetOS == osWindows and m.config.globalOptions * {optGenGuiApp, optGenDynLib} != {}: From 7c31b6a47b5cfb6f2d1f296390eb457bd75e55c2 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Tue, 28 Jun 2022 03:13:17 -0300 Subject: [PATCH 1672/2472] Fix jsre (#19917) * Fixes for jsre to make it more safe at runtime on some edge cases * https://github.com/nim-lang/Nim/pull/19917#issuecomment-1162692893 --- lib/js/jsre.nim | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/js/jsre.nim b/lib/js/jsre.nim index cd8fb2be7..0cc2f8a87 100644 --- a/lib/js/jsre.nim +++ b/lib/js/jsre.nim @@ -33,13 +33,13 @@ func compile*(self: RegExp; pattern: cstring; flags: cstring) {.importjs: "#.com func replace*(pattern: cstring; self: RegExp; replacement: cstring): cstring {.importjs: "#.replace(#, #)".} ## Returns a new string with some or all matches of a pattern replaced by given replacement -func split*(pattern: cstring; self: RegExp): seq[cstring] {.importjs: "#.split(#)".} +func split*(pattern: cstring; self: RegExp): seq[cstring] {.importjs: "(#.split(#) || [])".} ## Divides a string into an ordered list of substrings and returns the array -func match*(pattern: cstring; self: RegExp): seq[cstring] {.importjs: "#.match(#)".} +func match*(pattern: cstring; self: RegExp): seq[cstring] {.importjs: "(#.match(#) || [])".} ## Returns an array of matches of a RegExp against given string -func exec*(self: RegExp; pattern: cstring): seq[cstring] {.importjs: "#.exec(#)".} +func exec*(self: RegExp; pattern: cstring): seq[cstring] {.importjs: "(#.exec(#) || [])".} ## Executes a search for a match in its string parameter. func toCstring*(self: RegExp): cstring {.importjs: "#.toString()".} @@ -87,3 +87,5 @@ runnableExamples: assert "do1ne".split(jsregex) == @["do".cstring, "ne".cstring] jsregex.compile(r"[lw]", r"i") assert "hello world".replace(jsregex,"X") == "heXlo world" + let digitsRegex: RegExp = newRegExp(r"\d") + assert "foo".match(digitsRegex) == @[] From 8a344cb25b9b9d1b185ba6e5a8e1cf14421fedb4 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Tue, 28 Jun 2022 16:49:31 +0800 Subject: [PATCH 1673/2472] closes #11042; add testcase (#19935) close #11042; add testcase --- tests/closure/t11042.nim | 55 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 tests/closure/t11042.nim diff --git a/tests/closure/t11042.nim b/tests/closure/t11042.nim new file mode 100644 index 000000000..6a3928316 --- /dev/null +++ b/tests/closure/t11042.nim @@ -0,0 +1,55 @@ +discard """ + output:''' +foo: 1 +foo: 2 +bar: 1 +bar: 2 +foo: 1 +foo: 2 +bar: 1 +bar: 2 +bar: 3 +bar: 4 +bar: 5 +bar: 6 +bar: 7 +bar: 8 +bar: 9 +''' +""" + +# bug #11042 +block: + iterator foo: int = + for x in 1..2: + echo "foo: ", x + for y in 1..2: + discard + + for x in foo(): discard + + let bar = iterator: int = + for x in 1..2: + echo "bar: ", x + for y in 1..2: + discard + + for x in bar(): discard + + +block: + iterator foo: int = + for x in 1..2: + echo "foo: ", x + for y in 1..2: + discard + + for x in foo(): discard + + let bar = iterator: int = + for x in 1..9: + echo "bar: ", x + for y in 1..2: + discard + + for x in bar(): discard \ No newline at end of file From bcff13debcbdd8108237c8033e4dc9c38bb331e0 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Wed, 29 Jun 2022 22:37:24 +0800 Subject: [PATCH 1674/2472] dec inLoop after exiting the while scope in computeLiveRanges [backport] (#19918) * dec inLoop after exiting the while scope in computeLiveRanges * add testcase --- compiler/varpartitions.nim | 2 +- tests/arc/tcursorloop.nim | 45 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 tests/arc/tcursorloop.nim diff --git a/compiler/varpartitions.nim b/compiler/varpartitions.nim index d04a7b40c..a93104b06 100644 --- a/compiler/varpartitions.nim +++ b/compiler/varpartitions.nim @@ -851,7 +851,7 @@ proc computeLiveRanges(c: var Partitions; n: PNode) = # connect(graph, cursorVar) inc c.inLoop for child in n: computeLiveRanges(c, child) - inc c.inLoop + dec c.inLoop of nkElifBranch, nkElifExpr, nkElse, nkOfBranch: inc c.inConditional for child in n: computeLiveRanges(c, child) diff --git a/tests/arc/tcursorloop.nim b/tests/arc/tcursorloop.nim new file mode 100644 index 000000000..a37a6a036 --- /dev/null +++ b/tests/arc/tcursorloop.nim @@ -0,0 +1,45 @@ +discard """ + cmd: '''nim c --gc:arc --expandArc:traverse --hint:Performance:off $file''' + nimout: ''' +--expandArc: traverse + +var + it + jt_cursor +try: + `=copy`(it, root) + block :tmp: + while ( + not (it == nil)): + if true: + echo [it.s] + `=copy`(it, it.ri) + jt_cursor = root + if ( + not (jt_cursor == nil)): + echo [jt_cursor.s] + jt_cursor = jt_cursor.ri +finally: + `=destroy`(it) +-- end of expandArc ------------------------ +''' +""" + +type + Node = ref object + le, ri: Node + s: string + +proc traverse(root: Node) = + var it = root + while it != nil: + if true: + echo it.s + it = it.ri + + var jt = root + if jt != nil: + echo jt.s + jt = jt.ri + +traverse(nil) From ad430c0daad16019a6c440ab735a61856523329a Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Thu, 30 Jun 2022 10:20:19 +0200 Subject: [PATCH 1675/2472] once C++, always C++ [backport] (#19938) * once C++, always C++ When using `{.compile: "file.cc".}` in a nim module, even when compiling with `nim c` the C++ compiler should be used - once any C++ file has been compiled, the C++ linker also needs to be used. * more strict C++ check * simplify code --- compiler/extccomp.nim | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 5c074d284..f1dc01f44 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -493,7 +493,10 @@ proc needsExeExt(conf: ConfigRef): bool {.inline.} = (conf.target.hostOS == osWindows) proc useCpp(conf: ConfigRef; cfile: AbsoluteFile): bool = - conf.backend == backendCpp and not cfile.string.endsWith(".c") + # List of possible file extensions taken from gcc + for ext in [".C", ".cc", ".cpp", ".CPP", ".c++", ".cp", ".cxx"]: + if cfile.string.endsWith(ext): return true + false proc envFlags(conf: ConfigRef): string = result = if conf.backend == backendCpp: @@ -501,14 +504,14 @@ proc envFlags(conf: ConfigRef): string = else: getEnv("CFLAGS") -proc getCompilerExe(conf: ConfigRef; compiler: TSystemCC; cfile: AbsoluteFile): string = +proc getCompilerExe(conf: ConfigRef; compiler: TSystemCC; isCpp: bool): string = if compiler == ccEnv: - result = if useCpp(conf, cfile): + result = if isCpp: getEnv("CXX") else: getEnv("CC") else: - result = if useCpp(conf, cfile): + result = if isCpp: CC[compiler].cppCompiler else: CC[compiler].compilerExe @@ -547,22 +550,25 @@ proc ccHasSaneOverflow*(conf: ConfigRef): bool = proc getLinkerExe(conf: ConfigRef; compiler: TSystemCC): string = result = if CC[compiler].linkerExe.len > 0: CC[compiler].linkerExe - elif optMixedMode in conf.globalOptions and conf.backend != backendCpp: CC[compiler].cppCompiler - else: getCompilerExe(conf, compiler, AbsoluteFile"") + else: getCompilerExe(conf, compiler, optMixedMode in conf.globalOptions or conf.backend == backendCpp) proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile, isMainFile = false; produceOutput = false): string = - let c = conf.cCompiler + let + c = conf.cCompiler + isCpp = useCpp(conf, cfile.cname) # We produce files like module.nim.cpp, so the absolute Nim filename is not # cfile.name but `cfile.cname.changeFileExt("")`: var options = cFileSpecificOptions(conf, cfile.nimname, cfile.cname.changeFileExt("").string) - if useCpp(conf, cfile.cname): + if isCpp: # needs to be prepended so that --passc:-std=c++17 can override default. # we could avoid allocation by making cFileSpecificOptions inplace options = CC[c].cppXsupport & ' ' & options + # If any C++ file was compiled, we need to use C++ driver for linking as well + incl conf.globalOptions, optMixedMode var exe = getConfigVar(conf, c, ".exe") - if exe.len == 0: exe = getCompilerExe(conf, c, cfile.cname) + if exe.len == 0: exe = getCompilerExe(conf, c, isCpp) if needsExeExt(conf): exe = addFileExt(exe, "exe") if (optGenDynLib in conf.globalOptions or (conf.hcrOn and not isMainFile)) and @@ -582,7 +588,7 @@ proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile, compilePattern = joinPath(conf.cCompilerPath, exe) else: - compilePattern = getCompilerExe(conf, c, cfile.cname) + compilePattern = getCompilerExe(conf, c, isCpp) includeCmd.add(join([CC[c].includeCmd, quoteShell(conf.projectPath.string)])) From 306810a748ab3e0182f798bea7d2792c64ab856a Mon Sep 17 00:00:00 2001 From: Sam Zaydel Date: Thu, 30 Jun 2022 14:16:50 -0700 Subject: [PATCH 1676/2472] Enable nim-lang to build correctly on illumos-based systems (#19952) --- tools/niminst/buildsh.nimf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/niminst/buildsh.nimf b/tools/niminst/buildsh.nimf index 7a05ef342..6b99c49ee 100644 --- a/tools/niminst/buildsh.nimf +++ b/tools/niminst/buildsh.nimf @@ -164,7 +164,7 @@ esac case $ucpu in *i386* | *i486* | *i586* | *i686* | *bepc* | *i86pc* ) - if [ "$isOpenIndiana" = "yes" ] ; then + if [ "$isOpenIndiana" = "yes" ] || [ `uname -o` == "illumos" ] ; then mycpu="amd64" else mycpu="i386" From 2c0aaac3045a63b5e4a24468341bf9ccfc8820cc Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Thu, 30 Jun 2022 18:18:11 -0300 Subject: [PATCH 1677/2472] jsffi add missing braces (#19948) js codegen add missing whitespaces and braces --- lib/js/jsffi.nim | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/js/jsffi.nim b/lib/js/jsffi.nim index 35cbf2864..aca4fc292 100644 --- a/lib/js/jsffi.nim +++ b/lib/js/jsffi.nim @@ -348,8 +348,8 @@ iterator pairs*(obj: JsObject): (cstring, JsObject) = var k: cstring var v: JsObject {.emit: "for (var `k` in `obj`) {".} - {.emit: " if (!`obj`.hasOwnProperty(`k`)) continue;".} - {.emit: " `v`=`obj`[`k`];".} + {.emit: " if (!`obj`.hasOwnProperty(`k`)) { continue; }".} + {.emit: " `v` = `obj`[`k`];".} yield (k, v) {.emit: "}".} @@ -357,8 +357,8 @@ iterator items*(obj: JsObject): JsObject = ## Yields the `values` of each field in a JsObject, wrapped into a JsObject. var v: JsObject {.emit: "for (var k in `obj`) {".} - {.emit: " if (!`obj`.hasOwnProperty(k)) continue;".} - {.emit: " `v`=`obj`[k];".} + {.emit: " if (!`obj`.hasOwnProperty(k)) { continue; }".} + {.emit: " `v` = `obj`[k];".} yield v {.emit: "}".} @@ -366,7 +366,7 @@ iterator keys*(obj: JsObject): cstring = ## Yields the `names` of each field in a JsObject. var k: cstring {.emit: "for (var `k` in `obj`) {".} - {.emit: " if (!`obj`.hasOwnProperty(`k`)) continue;".} + {.emit: " if (!`obj`.hasOwnProperty(`k`)) { continue; }".} yield k {.emit: "}".} @@ -376,8 +376,8 @@ iterator pairs*[K: JsKey, V](assoc: JsAssoc[K, V]): (K,V) = var k: cstring var v: V {.emit: "for (var `k` in `assoc`) {".} - {.emit: " if (!`assoc`.hasOwnProperty(`k`)) continue;".} - {.emit: " `v`=`assoc`[`k`];".} + {.emit: " if (!`assoc`.hasOwnProperty(`k`)) { continue; }".} + {.emit: " `v` = `assoc`[`k`];".} yield (k.toJsKey(K), v) {.emit: "}".} @@ -385,8 +385,8 @@ iterator items*[K, V](assoc: JsAssoc[K, V]): V = ## Yields the `values` in a JsAssoc. var v: V {.emit: "for (var k in `assoc`) {".} - {.emit: " if (!`assoc`.hasOwnProperty(k)) continue;".} - {.emit: " `v`=`assoc`[k];".} + {.emit: " if (!`assoc`.hasOwnProperty(k)) { continue; }".} + {.emit: " `v` = `assoc`[k];".} yield v {.emit: "}".} @@ -394,7 +394,7 @@ iterator keys*[K: JsKey, V](assoc: JsAssoc[K, V]): K = ## Yields the `keys` in a JsAssoc. var k: cstring {.emit: "for (var `k` in `assoc`) {".} - {.emit: " if (!`assoc`.hasOwnProperty(`k`)) continue;".} + {.emit: " if (!`assoc`.hasOwnProperty(`k`)) { continue; }".} yield k.toJsKey(K) {.emit: "}".} From ce4078acd40bb27de5d05832f486dbc95918a3c9 Mon Sep 17 00:00:00 2001 From: Tanguy Date: Thu, 30 Jun 2022 23:19:04 +0200 Subject: [PATCH 1678/2472] Allow recursive closure iterators (#19939) --- compiler/semexprs.nim | 3 ++- doc/manual.rst | 5 +---- tests/iter/titer_issues.nim | 13 +++++++++++++ 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index ed28d8145..e07a98417 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -886,7 +886,8 @@ proc semOverloadedCallAnalyseEffects(c: PContext, n: PNode, nOrig: PNode, case callee.kind of skMacro, skTemplate: discard else: - if callee.kind == skIterator and callee.id == c.p.owner.id: + if callee.kind == skIterator and callee.id == c.p.owner.id and + not isClosureIterator(c.p.owner.typ): localError(c.config, n.info, errRecursiveDependencyIteratorX % callee.name.s) # error correction, prevents endless for loop elimination in transf. # See bug #2051: diff --git a/doc/manual.rst b/doc/manual.rst index b63a2f68f..571379a87 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -4326,13 +4326,10 @@ Closure iterators and inline iterators have some restrictions: 1. For now, a closure iterator cannot be executed at compile time. 2. `return` is allowed in a closure iterator but not in an inline iterator (but rarely useful) and ends the iteration. -3. Neither inline nor closure iterators can be (directly)* recursive. +3. Inline iterators cannot be recursive. 4. Neither inline nor closure iterators have the special `result` variable. 5. Closure iterators are not supported by the JS backend. -(*) Closure iterators can be co-recursive with a factory proc which results -in similar syntax to a recursive iterator. More details follow. - Iterators that are neither marked `{.closure.}` nor `{.inline.}` explicitly default to being inline, but this may change in future versions of the implementation. diff --git a/tests/iter/titer_issues.nim b/tests/iter/titer_issues.nim index 15fe867c8..1f7e41e69 100644 --- a/tests/iter/titer_issues.nim +++ b/tests/iter/titer_issues.nim @@ -29,6 +29,7 @@ end 9018 @[1, 2] @[1, 2, 3] +1 ''' """ @@ -274,3 +275,15 @@ iterator cc() {.closure.} = break var a2 = cc + +# bug #16876 +block: + iterator a(num: int): int {.closure.} = + if num == 1: + yield num + else: + for i in a(num - 1): + yield i + + for i in a(5): + echo i From 4897c47c8004e85474b760d770d9d09ae1d58771 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Sat, 2 Jul 2022 16:51:21 +0800 Subject: [PATCH 1679/2472] closes #15897; add testcase (#19961) --- tests/views/tviews1.nim | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/views/tviews1.nim b/tests/views/tviews1.nim index 249058eb6..b81b17f30 100644 --- a/tests/views/tviews1.nim +++ b/tests/views/tviews1.nim @@ -66,3 +66,14 @@ proc mainB = assert foo.x.y == @[1, 2, 3] mainB() + + +# bug #15897 +type Outer = ref object + value: int +type Inner = object + owner: var Outer + +var o = Outer(value: 1234) +var v = Inner(owner: o).owner.value +doAssert v == 1234 From d2d8f1342b3125aac8e0592da25df61e6feb95b2 Mon Sep 17 00:00:00 2001 From: rockcavera Date: Mon, 4 Jul 2022 08:52:44 -0300 Subject: [PATCH 1680/2472] Fixing `nimRawSetJmp` for vcc and clangcl on Windows (#19959) * fix vcc rawsetjmp * changing `_longjmp()` to `longjmp()` and `_setjmp()` to `setjmp()` * fix * fix setjmp to clangcl on Windows * fix genTrySetjmp() to clangcl on Windows --- compiler/ccgstmts.nim | 19 ++++++++++++------- lib/system/ansi_c.nim | 34 +++++++++++++++++++--------------- 2 files changed, 31 insertions(+), 22 deletions(-) diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index e791318ee..3ecf68402 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -1367,13 +1367,18 @@ proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) = linefmt(p, cpsStmts, "$1.status = __builtin_setjmp($1.context);$n", [safePoint]) elif isDefined(p.config, "nimRawSetjmp"): if isDefined(p.config, "mswindows"): - # The Windows `_setjmp()` takes two arguments, with the second being an - # undocumented buffer used by the SEH mechanism for stack unwinding. - # Mingw-w64 has been trying to get it right for years, but it's still - # prone to stack corruption during unwinding, so we disable that by setting - # it to NULL. - # More details: https://github.com/status-im/nimbus-eth2/issues/3121 - linefmt(p, cpsStmts, "$1.status = _setjmp($1.context, 0);$n", [safePoint]) + if isDefined(p.config, "vcc") or isDefined(p.config, "clangcl"): + # For the vcc compiler, use `setjmp()` with one argument. + # See https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/setjmp?view=msvc-170 + linefmt(p, cpsStmts, "$1.status = setjmp($1.context);$n", [safePoint]) + else: + # The Windows `_setjmp()` takes two arguments, with the second being an + # undocumented buffer used by the SEH mechanism for stack unwinding. + # Mingw-w64 has been trying to get it right for years, but it's still + # prone to stack corruption during unwinding, so we disable that by setting + # it to NULL. + # More details: https://github.com/status-im/nimbus-eth2/issues/3121 + linefmt(p, cpsStmts, "$1.status = _setjmp($1.context, 0);$n", [safePoint]) else: linefmt(p, cpsStmts, "$1.status = _setjmp($1.context);$n", [safePoint]) else: diff --git a/lib/system/ansi_c.nim b/lib/system/ansi_c.nim index 74f79167a..0dbded126 100644 --- a/lib/system/ansi_c.nim +++ b/lib/system/ansi_c.nim @@ -118,26 +118,30 @@ elif defined(nimBuiltinSetjmp): c_builtin_setjmp(unsafeAddr jmpb[0]) elif defined(nimRawSetjmp) and not defined(nimStdSetjmp): - when defined(windows) and not defined(vcc): + when defined(windows): # No `_longjmp()` on Windows. proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {. header: "", importc: "longjmp".} - # The Windows `_setjmp()` takes two arguments, with the second being an - # undocumented buffer used by the SEH mechanism for stack unwinding. - # Mingw-w64 has been trying to get it right for years, but it's still - # prone to stack corruption during unwinding, so we disable that by setting - # it to NULL. - # More details: https://github.com/status-im/nimbus-eth2/issues/3121 - when defined(nimHasStyleChecks): - {.push styleChecks: off.} + when defined(vcc) or defined(clangcl): + proc c_setjmp*(jmpb: C_JmpBuf): cint {. + header: "", importc: "setjmp".} + else: + # The Windows `_setjmp()` takes two arguments, with the second being an + # undocumented buffer used by the SEH mechanism for stack unwinding. + # Mingw-w64 has been trying to get it right for years, but it's still + # prone to stack corruption during unwinding, so we disable that by setting + # it to NULL. + # More details: https://github.com/status-im/nimbus-eth2/issues/3121 + when defined(nimHasStyleChecks): + {.push styleChecks: off.} - proc c_setjmp*(jmpb: C_JmpBuf): cint = - proc c_setjmp_win(jmpb: C_JmpBuf, ctx: pointer): cint {. - header: "", importc: "_setjmp".} - c_setjmp_win(jmpb, nil) + proc c_setjmp*(jmpb: C_JmpBuf): cint = + proc c_setjmp_win(jmpb: C_JmpBuf, ctx: pointer): cint {. + header: "", importc: "_setjmp".} + c_setjmp_win(jmpb, nil) - when defined(nimHasStyleChecks): - {.pop.} + when defined(nimHasStyleChecks): + {.pop.} else: proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {. header: "", importc: "_longjmp".} From 7d0285853f4327a0a4f683e13971dc3429aad1f3 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Tue, 5 Jul 2022 16:11:34 +0800 Subject: [PATCH 1681/2472] rename gc to mm (#19971) --- compiler/msgs.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 9d111b2e2..371a886f4 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -677,7 +677,7 @@ proc genSuccessX*(conf: ConfigRef) = const debugModeHints = "none (DEBUG BUILD, `-d:release` generates faster code)" if conf.cmd in cmdBackends: if conf.backend != backendJs: - build.add "gc: $#; " % $conf.selectedGC + build.add "mm: $#; " % $conf.selectedGC if optThreads in conf.globalOptions: build.add "threads: on; " build.add "opt: " if optOptimizeSpeed in conf.options: build.add "speed" From 01b40dc1d7c7c5331361341bdf305084c799c05b Mon Sep 17 00:00:00 2001 From: Daniel Clarke Date: Wed, 6 Jul 2022 06:29:05 +1000 Subject: [PATCH 1682/2472] Fixes return values of execCmd on macos (#19963) * Fixes return values of execCmd on macos * update tests to use existing structure Co-authored-by: daniel --- lib/pure/osproc.nim | 2 +- tests/stdlib/tosproc.nim | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 512db92a4..666ccdda8 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -1523,7 +1523,7 @@ elif not defined(useNimRtl): header: "".} proc execCmd(command: string): int = - when defined(linux): + when defined(posix): let tmp = csystem(command) result = if tmp == -1: tmp else: exitStatusLikeShell(tmp) else: diff --git a/tests/stdlib/tosproc.nim b/tests/stdlib/tosproc.nim index c54e0d112..f55dd3e21 100644 --- a/tests/stdlib/tosproc.nim +++ b/tests/stdlib/tosproc.nim @@ -29,6 +29,12 @@ when defined(case_testfile): # compiled test file for child process case arg of "exit_0": if true: quit(0) + of "exit_1": + if true: quit(1) + of "exit_2": + if true: quit(2) + of "exit_42": + if true: quit(42) of "exitnow_139": if true: exitnow(139) of "c_exit2_139": @@ -115,6 +121,13 @@ else: # main driver runTest("c_exit2_139", 139) runTest("quit_139", 139) + block execCmdTest: + let output = compileNimProg("-d:release -d:case_testfile", "D20220705T221100") + doAssert execCmd(output & " exit_0") == 0 + doAssert execCmd(output & " exit_1") == 1 + doAssert execCmd(output & " exit_2") == 2 + doAssert execCmd(output & " exit_42") == 42 + import std/streams block execProcessTest: From 430a1793075866179b34790a461936023fca1c0a Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Wed, 6 Jul 2022 19:06:41 +0800 Subject: [PATCH 1683/2472] default threads on (#19368) * default threads on * make rst gcsafe * ignore threads option for nimscript * threads off * use createShared for threads * test without threads * avr threds off * avr threads off * async threads off * threads off * fix ci * restore option * make CI pleased * fix ic tests * Update config.nims * add changelog * Update changelog.md Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- changelog.md | 2 ++ config/nim.cfg | 2 ++ testament/categories.nim | 10 +++++----- tests/avr/thello.nim | 2 +- tests/destructor/trecursive.nim | 2 +- tests/ic/config.nims | 2 ++ tests/manyloc/standalone2/tavr.nim.cfg | 1 + tests/niminaction/Chapter7/Tweeter/src/tweeter.nim | 1 + tests/stdlib/ttasks.nim | 2 +- tests/system/tgcnone.nim | 2 +- 10 files changed, 17 insertions(+), 9 deletions(-) create mode 100644 tests/ic/config.nims diff --git a/changelog.md b/changelog.md index 98655eaa3..c35a403e6 100644 --- a/changelog.md +++ b/changelog.md @@ -26,6 +26,8 @@ becomes an alias for `addr`. - The `gc:v2` option is removed. +- The `threads:on` option becomes the default. + ## Standard library additions and changes [//]: # "Changes:" diff --git a/config/nim.cfg b/config/nim.cfg index 7fe279f56..37d12a02d 100644 --- a/config/nim.cfg +++ b/config/nim.cfg @@ -17,6 +17,8 @@ cc = gcc hint[LineTooLong]=off #hint[XDeclaredButNotUsed]=off +threads:on + # Examples of how to setup a cross-compiler: # Nim can target architectures and OSes different than the local host # Syntax: ..gcc.exe = "" diff --git a/testament/categories.nim b/testament/categories.nim index a092fec84..43ea9f0ee 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -59,10 +59,10 @@ proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) = var test2 = makeTest("tests/dll/server.nim", options & " --threads:on" & rpath, cat) test2.spec.action = actionCompile testSpec c, test2 - var test3 = makeTest("lib/nimhcr.nim", options & " --outdir:tests/dll" & rpath, cat) + var test3 = makeTest("lib/nimhcr.nim", options & " --threads:off --outdir:tests/dll" & rpath, cat) test3.spec.action = actionCompile testSpec c, test3 - var test4 = makeTest("tests/dll/visibility.nim", options & " --app:lib" & rpath, cat) + var test4 = makeTest("tests/dll/visibility.nim", options & " --threads:off --app:lib" & rpath, cat) test4.spec.action = actionCompile testSpec c, test4 @@ -77,13 +77,13 @@ proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) = defer: putEnv(libpathenv, libpath) testSpec r, makeTest("tests/dll/client.nim", options & " --threads:on" & rpath, cat) - testSpec r, makeTest("tests/dll/nimhcr_unit.nim", options & rpath, cat) - testSpec r, makeTest("tests/dll/visibility.nim", options & rpath, cat) + testSpec r, makeTest("tests/dll/nimhcr_unit.nim", options & " --threads:off" & rpath, cat) + testSpec r, makeTest("tests/dll/visibility.nim", options & " --threads:off" & rpath, cat) if "boehm" notin options: # force build required - see the comments in the .nim file for more details var hcri = makeTest("tests/dll/nimhcr_integration.nim", - options & " --forceBuild --hotCodeReloading:on" & rpath, cat) + options & " --threads:off --forceBuild --hotCodeReloading:on" & rpath, cat) let nimcache = nimcacheDir(hcri.name, hcri.options, getTestSpecTarget()) let cmd = prepareTestCmd(hcri.spec.getCmd, hcri.name, hcri.options, nimcache, getTestSpecTarget()) diff --git a/tests/avr/thello.nim b/tests/avr/thello.nim index a0191815c..7ebaeae5f 100644 --- a/tests/avr/thello.nim +++ b/tests/avr/thello.nim @@ -1,5 +1,5 @@ discard """ - cmd: "nim c --compileOnly --os:standalone --exceptions:quirky -d:noSignalHandler -d:danger $file" + cmd: "nim c --compileOnly --os:standalone --exceptions:quirky -d:noSignalHandler -d:danger --threads:off $file" action: "compile" """ diff --git a/tests/destructor/trecursive.nim b/tests/destructor/trecursive.nim index 17a40e5a9..e7afa6ba9 100644 --- a/tests/destructor/trecursive.nim +++ b/tests/destructor/trecursive.nim @@ -47,7 +47,7 @@ proc `=destroy`(x: var MyObject) = proc `=`(x: var MyObject, y: MyObject) {.error.} proc newMyObject(i: int): MyObject = - result.p = create(int) + result.p = createShared(int) result.p[] = i proc test: seq[MyObject] = diff --git a/tests/ic/config.nims b/tests/ic/config.nims new file mode 100644 index 000000000..76b29a6aa --- /dev/null +++ b/tests/ic/config.nims @@ -0,0 +1,2 @@ +when defined(windows): + --threads:off diff --git a/tests/manyloc/standalone2/tavr.nim.cfg b/tests/manyloc/standalone2/tavr.nim.cfg index e5291969d..2a31618d0 100644 --- a/tests/manyloc/standalone2/tavr.nim.cfg +++ b/tests/manyloc/standalone2/tavr.nim.cfg @@ -2,3 +2,4 @@ --cpu:avr --os:standalone --compileOnly +--threads:off diff --git a/tests/niminaction/Chapter7/Tweeter/src/tweeter.nim b/tests/niminaction/Chapter7/Tweeter/src/tweeter.nim index fe39278fb..2fac949b9 100644 --- a/tests/niminaction/Chapter7/Tweeter/src/tweeter.nim +++ b/tests/niminaction/Chapter7/Tweeter/src/tweeter.nim @@ -1,5 +1,6 @@ discard """ action: compile +matrix: "--threads:off" """ import asyncdispatch, times diff --git a/tests/stdlib/ttasks.nim b/tests/stdlib/ttasks.nim index 75fed9f9b..e90823aba 100644 --- a/tests/stdlib/ttasks.nim +++ b/tests/stdlib/ttasks.nim @@ -1,6 +1,6 @@ discard """ targets: "c cpp" - matrix: "--gc:orc" + matrix: "--gc:orc --threads:off" """ import std/[tasks, strformat] diff --git a/tests/system/tgcnone.nim b/tests/system/tgcnone.nim index 700176d5f..47c6c6014 100644 --- a/tests/system/tgcnone.nim +++ b/tests/system/tgcnone.nim @@ -1,5 +1,5 @@ discard """ - matrix: "--gc:none -d:useMalloc" + matrix: "--gc:none -d:useMalloc --threads:off" """ # bug #15617 let x = 4 From dcb28fd061743ba141b3914501512b5eb352aa1b Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Thu, 7 Jul 2022 20:14:05 +0800 Subject: [PATCH 1684/2472] cache rope when threads are enabled (#19981) * cache rope * add threadvar --- compiler/ropes.nim | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/compiler/ropes.nim b/compiler/ropes.nim index a44d84ddc..610159c75 100644 --- a/compiler/ropes.nim +++ b/compiler/ropes.nim @@ -86,13 +86,12 @@ proc newRope(data: string = ""): Rope = result.L = -data.len result.data = data -when not compileOption("threads"): - var - cache: array[0..2048*2 - 1, Rope] +var + cache {.threadvar.} : array[0..2048*2 - 1, Rope] - proc resetRopeCache* = - for i in low(cache)..high(cache): - cache[i] = nil +proc resetRopeCache* = + for i in low(cache)..high(cache): + cache[i] = nil proc ropeInvariant(r: Rope): bool = if r == nil: From ad0aee535435ac9b22c2ee9ef02085ffdc901d48 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Thu, 7 Jul 2022 15:26:58 +0200 Subject: [PATCH 1685/2472] sysrand: fix syscall signature [backport] (#19982) sysrand: fix syscall signature `syscall` is a `C` varags function --- lib/std/sysrand.nim | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/std/sysrand.nim b/lib/std/sysrand.nim index 1b7b2c024..4ee25d01e 100644 --- a/lib/std/sysrand.nim +++ b/lib/std/sysrand.nim @@ -170,9 +170,8 @@ elif defined(linux) and not defined(nimNoGetRandom) and not defined(emscripten): const syscallHeader = """#include #include """ - proc syscall( - n: clong, buf: pointer, bufLen: cint, flags: cuint - ): clong {.importc: "syscall", header: syscallHeader.} + proc syscall(n: clong): clong {. + importc: "syscall", varargs, header: syscallHeader.} # When reading from the urandom source (GRND_RANDOM is not set), # getrandom() will block until the entropy pool has been # initialized (unless the GRND_NONBLOCK flag was specified). If a @@ -211,7 +210,7 @@ elif defined(zephyr): proc sys_csrand_get(dst: pointer, length: csize_t): cint {.importc: "sys_csrand_get", header: "".} # Fill the destination buffer with cryptographically secure # random data values - # + # proc getRandomImpl(p: pointer, size: int): int {.inline.} = # 0 if success, -EIO if entropy reseed error From 7a29a782f84ebceec0fb5c3d39bb484621831fca Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 9 Jul 2022 06:46:05 +0200 Subject: [PATCH 1686/2472] removed caching logic; saves 400MB for an ORC booting compiler (#19989) --- compiler/injectdestructors.nim | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/compiler/injectdestructors.nim b/compiler/injectdestructors.nim index 6500c5bc7..867d30d96 100644 --- a/compiler/injectdestructors.nim +++ b/compiler/injectdestructors.nim @@ -80,12 +80,6 @@ import sets, hashes proc hash(n: PNode): Hash = hash(cast[pointer](n)) -proc aliasesCached(cache: var Table[(PNode, PNode), AliasKind], obj, field: PNode): AliasKind = - let key = (obj, field) - if not cache.hasKey(key): - cache[key] = aliases(obj, field) - cache[key] - type State = ref object lastReads: IntSet @@ -116,9 +110,8 @@ proc mergeStates(a: var State, b: sink State) = a.alreadySeen.incl b.alreadySeen proc computeLastReadsAndFirstWrites(cfg: ControlFlowGraph) = - var cache = initTable[(PNode, PNode), AliasKind]() template aliasesCached(obj, field: PNode): AliasKind = - aliasesCached(cache, obj, field) + aliases(obj, field) var cfg = cfg preprocessCfg(cfg) From e8ee2f9c2ad06cad2f62fe7505acbb43530f28d7 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Sat, 9 Jul 2022 17:28:14 +0800 Subject: [PATCH 1687/2472] update section regarding `std/assertions` in changelog (#19992) Update changelog.md --- changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index c35a403e6..770393e13 100644 --- a/changelog.md +++ b/changelog.md @@ -22,7 +22,7 @@ - `addr` is now available for all addressable locations, `unsafeAddr` is deprecated and becomes an alias for `addr`. -- io is about to move out of system; use `-d:nimPreviewSlimSystem` and import `std/syncio`. +- `io` and `assertions` are about to move out of system; use `-d:nimPreviewSlimSystem`, import `std/syncio` and import `std/assertions`. - The `gc:v2` option is removed. From ed2bf02a511ae230243ce384ad16c318ca3fe661 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Sun, 10 Jul 2022 15:37:15 +0800 Subject: [PATCH 1688/2472] remove `when declared(cache)`; cache is always there (#19991) --- compiler/ropes.nim | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/compiler/ropes.nim b/compiler/ropes.nim index 610159c75..677a3ce09 100644 --- a/compiler/ropes.nim +++ b/compiler/ropes.nim @@ -111,16 +111,13 @@ var gCacheMisses* = 0 var gCacheIntTries* = 0 proc insertInCache(s: string): Rope = - when declared(cache): - inc gCacheTries - var h = hash(s) and high(cache) - result = cache[h] - if isNil(result) or result.data != s: - inc gCacheMisses - result = newRope(s) - cache[h] = result - else: + inc gCacheTries + var h = hash(s) and high(cache) + result = cache[h] + if isNil(result) or result.data != s: + inc gCacheMisses result = newRope(s) + cache[h] = result proc rope*(s: string): Rope = ## Converts a string to a rope. From a90763ebd762a4862e2af1dab7d4429f4e984a8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20M=20G=C3=B3mez?= Date: Sun, 10 Jul 2022 15:40:26 +0200 Subject: [PATCH 1689/2472] Fixes Compilation error with --app:lib (#19965) Fixes Compilation error with --app:lib when a module tries to pull os.paramStr on posix by throwing a runtime exception instead. More details here: #19964 --- lib/pure/os.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 247a0d089..a60e913f1 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -2939,7 +2939,7 @@ elif defined(genode): proc paramCount*(): int = raise newException(OSError, "paramCount is not implemented on Genode") -elif weirdTarget: +elif weirdTarget or (defined(posix) and appType == "lib"): proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = raise newException(OSError, "paramStr is not implemented on current platform") From fb5fbf1e087563f0288b8ed684c8dcc1891730b0 Mon Sep 17 00:00:00 2001 From: Tanguy Date: Mon, 11 Jul 2022 11:28:52 +0200 Subject: [PATCH 1690/2472] Fix nested finally handling in closureiters [backport] (#19933) * Fix nested finally handling in closureiters * Fix CI * review comment * third time the charm * Update compiler/closureiters.nim Co-authored-by: Dominik Picheta Co-authored-by: Dominik Picheta --- compiler/closureiters.nim | 31 ++++++++++++--- tests/iter/titer_issues.nim | 75 +++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 6 deletions(-) diff --git a/compiler/closureiters.nim b/compiler/closureiters.nim index 6370d0dcb..2848942fa 100644 --- a/compiler/closureiters.nim +++ b/compiler/closureiters.nim @@ -121,7 +121,10 @@ # yield 2 # if :unrollFinally: # This node is created by `newEndFinallyNode` # if :curExc.isNil: -# return :tmpResult +# if nearestFinally == 0: +# return :tmpResult +# else: +# :state = nearestFinally # bubble up # else: # closureIterSetupExc(nil) # raise @@ -807,7 +810,10 @@ proc newEndFinallyNode(ctx: var Ctx, info: TLineInfo): PNode = # Generate the following code: # if :unrollFinally: # if :curExc.isNil: - # return :tmpResult + # if nearestFinally == 0: + # return :tmpResult + # else: + # :state = nearestFinally # bubble up # else: # raise let curExc = ctx.newCurExcAccess() @@ -816,11 +822,17 @@ proc newEndFinallyNode(ctx: var Ctx, info: TLineInfo): PNode = let cmp = newTree(nkCall, newSymNode(ctx.g.getSysMagic(info, "==", mEqRef), info), curExc, nilnode) cmp.typ = ctx.g.getSysType(info, tyBool) - let asgn = newTree(nkFastAsgn, - newSymNode(getClosureIterResult(ctx.g, ctx.fn, ctx.idgen), info), - ctx.newTmpResultAccess()) + let retStmt = + if ctx.nearestFinally == 0: + # last finally, we can return + let asgn = newTree(nkFastAsgn, + newSymNode(getClosureIterResult(ctx.g, ctx.fn, ctx.idgen), info), + ctx.newTmpResultAccess()) + newTree(nkReturnStmt, asgn) + else: + # bubble up to next finally + newTree(nkGotoState, ctx.g.newIntLit(info, ctx.nearestFinally)) - let retStmt = newTree(nkReturnStmt, asgn) let branch = newTree(nkElifBranch, cmp, retStmt) let nullifyExc = newTree(nkCall, newSymNode(ctx.g.getCompilerProc("closureIterSetupExc")), nilnode) @@ -864,6 +876,13 @@ proc transformReturnsInTry(ctx: var Ctx, n: PNode): PNode = of nkSkip: discard + of nkTryStmt: + if n.hasYields: + # the inner try will handle these transformations + discard + else: + for i in 0.. try + iterator p1: int {.closure.} = + try: + yield 0 + try: + return + finally: + echo "nested finally" + echo "shouldn't run" + finally: + echo "outer finally" + echo "shouldn't run" + + for _ in p1(): + discard + + # try -> try yield + iterator p2: int {.closure.} = + try: + try: + yield 0 + return + finally: + echo "nested finally" + echo "shouldn't run" + finally: + echo "outer finally" + echo "shouldn't run" + + for _ in p2(): + discard + + # try yield -> try yield + iterator p3: int {.closure.} = + try: + yield 0 + try: + yield 0 + return + finally: + echo "nested finally" + echo "shouldn't run" + finally: + echo "outer finally" + echo "shouldn't run" + + for _ in p3(): + discard + + # try -> try + iterator p4: int {.closure.} = + try: + try: + return + finally: + echo "nested finally" + echo "shouldn't run" + finally: + echo "outer finally" + echo "shouldn't run" + + for _ in p4(): + discard From cf1c14936670ef0c175149f6fba4dffc1cf4ba43 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Mon, 11 Jul 2022 23:27:01 +0800 Subject: [PATCH 1691/2472] tracking the memory usage of orc-booting compiler for each commit (#19941) * yaml * pub * redo * let's comment * now action * newly * code name * build * ready * remove submodule * build * modify name * fix * rephrase * trigger when PR is merged --- .github/workflows/ci_publish.yml | 90 ++++++++++++++++++++++++++++++++ ci/action.nim | 28 ++++++++++ 2 files changed, 118 insertions(+) create mode 100644 .github/workflows/ci_publish.yml create mode 100644 ci/action.nim diff --git a/.github/workflows/ci_publish.yml b/.github/workflows/ci_publish.yml new file mode 100644 index 000000000..94ea5ff8f --- /dev/null +++ b/.github/workflows/ci_publish.yml @@ -0,0 +1,90 @@ +name: Tracking orc-booting compiler memory usage + +on: + push: + branches: + - devel + + +jobs: + build: + strategy: + fail-fast: false + matrix: + os: [ubuntu-20.04] + cpu: [amd64] + name: '${{ matrix.os }}' + runs-on: ${{ matrix.os }} + steps: + - name: 'Checkout' + uses: actions/checkout@v2 + with: + fetch-depth: 2 + + - name: 'Install node.js 16.x' + uses: actions/setup-node@v2 + with: + node-version: '16.x' + + - name: 'Install dependencies (Linux amd64)' + if: runner.os == 'Linux' && matrix.cpu == 'amd64' + run: | + sudo apt-fast update -qq + DEBIAN_FRONTEND='noninteractive' \ + sudo apt-fast install --no-install-recommends -yq \ + libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev \ + valgrind libc6-dbg libblas-dev xorg-dev + - name: 'Install dependencies (macOS)' + if: runner.os == 'macOS' + run: brew install boehmgc make sfml gtk+3 + - name: 'Install dependencies (Windows)' + if: runner.os == 'Windows' + shell: bash + run: | + set -e + . ci/funs.sh + nimInternalInstallDepsWindows + echo_run echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}" + + - name: 'Add build binaries to PATH' + shell: bash + run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}" + + - name: 'System information' + shell: bash + run: . ci/funs.sh && nimCiSystemInfo + + - name: 'Build csourcesAny' + shell: bash + run: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc ucpu='${{ matrix.cpu }}' + + - name: 'Build koch' + shell: bash + run: nim c koch + + - name: 'Build Nim' + shell: bash + run: ./koch boot -d:release -d:nimStrictMode --lib:lib + + - name: 'Action' + shell: bash + run: nim c -r -d:release ci/action.nim + + - name: 'Comment' + uses: actions/github-script@v6 + with: + script: | + const fs = require('fs'); + + try { + const data = fs.readFileSync('ci/nimcache/results.txt', 'utf8'); + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: data + }) + } catch (err) { + console.error(err); + } + diff --git a/ci/action.nim b/ci/action.nim new file mode 100644 index 000000000..8c3260096 --- /dev/null +++ b/ci/action.nim @@ -0,0 +1,28 @@ +import std/[strutils, os, osproc, parseutils, strformat] + + +proc main() = + var msg = "" + const cmd = "./koch boot --gc:orc -d:release" + + let (output, exitCode) = execCmdEx(cmd) + + doAssert exitCode == 0, output + + var start = rfind(output, "Hint: gc") + if start < 0: + start = rfind(output, "Hint: mm") + doAssert parseUntil(output, msg, "; proj", start) > 0, output + + let (commitHash, _) = execCmdEx("""git log --format="%H" -n 1""") + + let welcomeMessage = fmt"""Thanks for your hard work on this PR! +The lines below are statistics of the Nim compiler built from {commitHash} + +{msg} +""" + createDir "ci/nimcache" + writeFile "ci/nimcache/results.txt", welcomeMessage + +when isMainModule: + main() \ No newline at end of file From d0bae989d63f91345cd0991e00502c05d18dbf6b Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Tue, 12 Jul 2022 09:05:50 +0800 Subject: [PATCH 1692/2472] fix github script failure (#20006) * yaml * pub * redo * let's comment * now action * newly * code name * build * ready * remove submodule * build * modify name * fix * rephrase * trigger when PR is merged * fix CI failure --- .github/workflows/ci_publish.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_publish.yml b/.github/workflows/ci_publish.yml index 94ea5ff8f..90a3438a7 100644 --- a/.github/workflows/ci_publish.yml +++ b/.github/workflows/ci_publish.yml @@ -1,13 +1,13 @@ name: Tracking orc-booting compiler memory usage on: - push: - branches: - - devel + pull_request: + types: [closed] jobs: build: + if: github.event.pull_request.merged == true strategy: fail-fast: false matrix: From a97b00ad8d18d836d693c224909b607d2af0c4b1 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Tue, 12 Jul 2022 19:07:18 +0800 Subject: [PATCH 1693/2472] try to optimize hot spots for orc-booting compiler (#20001) * optimize hot spots fro orc-booting compiler * remove GC_ref * minor * remove comments * Revert "minor" This reverts commit 4965a190a2d6457044faa1442795e55bdad57602. * emulate cursor --- compiler/astalgo.nim | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/compiler/astalgo.nim b/compiler/astalgo.nim index bc5b7d6e1..22f4257fc 100644 --- a/compiler/astalgo.nim +++ b/compiler/astalgo.nim @@ -18,6 +18,9 @@ import when defined(nimPreviewSlimSystem): import std/assertions +when not defined(nimHasCursor): + {.pragma: cursor.} + proc hashNode*(p: RootRef): Hash proc treeToYaml*(conf: ConfigRef; n: PNode, indent: int = 0, maxRecDepth: int = - 1): Rope # Convert a tree into its YAML representation; this is used by the @@ -815,16 +818,21 @@ type name*: PIdent proc nextIdentIter*(ti: var TIdentIter, tab: TStrTable): PSym = + # hot spots var h = ti.h and high(tab.data) var start = h - result = tab.data[h] - while result != nil: - if result.name.id == ti.name.id: break + var p {.cursor.} = tab.data[h] + while p != nil: + if p.name.id == ti.name.id: break h = nextTry(h, high(tab.data)) if h == start: - result = nil + p = nil break - result = tab.data[h] + p = tab.data[h] + if p != nil: + result = p # increase the count + else: + result = nil ti.h = nextTry(h, high(tab.data)) proc initIdentIter*(ti: var TIdentIter, tab: TStrTable, s: PIdent): PSym = From 5c510a9ab96f265ffb0323a18deadb5bb175cfdc Mon Sep 17 00:00:00 2001 From: metagn <10591326+metagn@users.noreply.github.com> Date: Tue, 12 Jul 2022 19:03:58 +0300 Subject: [PATCH 1694/2472] allow dots in defined() (#20010) * allow dots in defined() refs https://github.com/nim-lang/RFCs/issues/181 * mention accents in older versions --- changelog.md | 3 +++ compiler/semexprs.nim | 14 +++++++++++++- tests/misc/tdefine.nim | 19 +++++++++++++++++-- 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/changelog.md b/changelog.md index 770393e13..7498a9567 100644 --- a/changelog.md +++ b/changelog.md @@ -101,6 +101,9 @@ becomes an alias for `addr`. - Full command syntax and block arguments i.e. `foo a, b: c` are now allowed for the right-hand side of type definitions in type sections. Previously they would error with "invalid indentation". +- `defined` now accepts identifiers separated by dots, i.e. `defined(a.b.c)`. + In the command line, this is defined as `-d:a.b.c`. Older versions can + use accents as in ``defined(`a.b.c`)`` to access such defines. ## Compiler changes diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index e07a98417..fed9cae21 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1938,11 +1938,23 @@ proc semYield(c: PContext, n: PNode): PNode = elif c.p.owner.typ[0] != nil: localError(c.config, n.info, errGenerated, "yield statement must yield a value") +proc considerQuotedIdentOrDot(c: PContext, n: PNode, origin: PNode = nil): PIdent = + if n.kind == nkDotExpr: + let a = considerQuotedIdentOrDot(c, n[0], origin).s + let b = considerQuotedIdentOrDot(c, n[1], origin).s + var s = newStringOfCap(a.len + b.len + 1) + s.add(a) + s.add('.') + s.add(b) + result = getIdent(c.cache, s) + else: + result = considerQuotedIdent(c, n, origin) + proc semDefined(c: PContext, n: PNode): PNode = checkSonsLen(n, 2, c.config) # we replace this node by a 'true' or 'false' node: result = newIntNode(nkIntLit, 0) - result.intVal = ord isDefined(c.config, considerQuotedIdent(c, n[1], n).s) + result.intVal = ord isDefined(c.config, considerQuotedIdentOrDot(c, n[1], n).s) result.info = n.info result.typ = getSysType(c.graph, n.info, tyBool) diff --git a/tests/misc/tdefine.nim b/tests/misc/tdefine.nim index f1c6e7a96..c4d11c941 100644 --- a/tests/misc/tdefine.nim +++ b/tests/misc/tdefine.nim @@ -1,6 +1,6 @@ discard """ joinable: false -cmd: "nim c -d:booldef -d:booldef2=false -d:intdef=2 -d:strdef=foobar -r $file" +cmd: "nim c -d:booldef -d:booldef2=false -d:intdef=2 -d:strdef=foobar -d:namespaced.define=false -d:double.namespaced.define -r $file" """ const booldef {.booldefine.} = false @@ -27,4 +27,19 @@ type T = object when intdef2 == 1: field2: int when strdef2 == "abc": - field3: int \ No newline at end of file + field3: int + +doAssert not defined(booldef3) +doAssert not defined(intdef2) +doAssert not defined(strdef2) +discard T(field1: 1, field2: 2, field3: 3) + +doAssert defined(namespaced.define) +const `namespaced.define` {.booldefine.} = true +doAssert not `namespaced.define` + +doAssert defined(double.namespaced.define) +const `double.namespaced.define` {.booldefine.} = false +doAssert `double.namespaced.define` + +doAssert not defined(namespaced.butnotdefined) From 0180c6179aa0ff2dacaa6c063c0fa24cf459a84d Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Wed, 13 Jul 2022 01:35:08 +0800 Subject: [PATCH 1695/2472] fix #18735; genDepend broken for duplicate module names in separate folders (#19988) --- compiler/depends.nim | 58 ++++++++++++++++++++++++++++++++++++++------ compiler/options.nim | 2 +- 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/compiler/depends.nim b/compiler/depends.nim index 30fc961c5..1dcdec989 100644 --- a/compiler/depends.nim +++ b/compiler/depends.nim @@ -9,10 +9,12 @@ # This module implements a dependency file generator. -import - options, ast, ropes, idents, passes, modulepaths, pathutils +import options, ast, ropes, passes, pathutils, msgs, lineinfos -from modulegraphs import ModuleGraph, PPassContext +import modulegraphs + +import std/[os, strutils, parseutils] +import std/private/globs type TGen = object of PPassContext @@ -28,6 +30,50 @@ proc addDependencyAux(b: Backend; importing, imported: string) = b.dotGraph.addf("\"$1\" -> \"$2\";$n", [rope(importing), rope(imported)]) # s1 -> s2_4[label="[0-9]"]; +proc toNimblePath(s: string, isStdlib: bool): string = + const stdPrefix = "std/" + const pkgPrefix = "pkg/" + if isStdlib: + let sub = "lib/" + var start = s.find(sub) + if start < 0: + doAssert false + else: + start += sub.len + let base = s[start..^1] + + if base.startsWith("system") or base.startsWith("std"): + result = base + else: + for dir in stdlibDirs: + if base.startsWith(dir): + return stdPrefix & base.splitFile.name + + result = stdPrefix & base + else: + var sub = getEnv("NIMBLE_DIR") + if sub.len == 0: + sub = ".nimble/pkgs/" + else: + sub.add "/pkgs/" + var start = s.find(sub) + if start < 0: + result = s + else: + start += sub.len + start += skipUntil(s, '/', start) + start += 1 + result = pkgPrefix & s[start..^1] + +proc addDependency(c: PPassContext, g: PGen, b: Backend, n: PNode) = + doAssert n.kind == nkSym, $n.kind + + let path = splitFile(toProjPath(g.config, n.sym.position.FileIndex)) + let modulePath = splitFile(toProjPath(g.config, g.module.position.FileIndex)) + let parent = nativeToUnixPath(modulePath.dir / modulePath.name).toNimblePath(belongsToStdlib(g.graph, g.module)) + let child = nativeToUnixPath(path.dir / path.name).toNimblePath(belongsToStdlib(g.graph, n.sym)) + addDependencyAux(b, parent, child) + proc addDotDependency(c: PPassContext, n: PNode): PNode = result = n let g = PGen(c) @@ -35,11 +81,9 @@ proc addDotDependency(c: PPassContext, n: PNode): PNode = case n.kind of nkImportStmt: for i in 0.. 0 and not ret.startsWith ".." -const stdlibDirs = [ +const stdlibDirs* = [ "pure", "core", "arch", "pure/collections", "pure/concurrency", From af140966ea5b58ea248459acb74bf84537deb423 Mon Sep 17 00:00:00 2001 From: metagn <10591326+metagn@users.noreply.github.com> Date: Wed, 13 Jul 2022 00:24:17 +0300 Subject: [PATCH 1696/2472] Add #19406 changes to changelog (#20011) --- changelog.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/changelog.md b/changelog.md index 7498a9567..68b133040 100644 --- a/changelog.md +++ b/changelog.md @@ -66,6 +66,12 @@ becomes an alias for `addr`. ## Language changes +- [Case statement macros](manual.html#macros-case-statement-macros) are no longer experimental, + meaning you no longer need to enable the experimental switch `caseStmtMacros` to use them. +- Templates now accept [macro pragmas](https://nim-lang.github.io/Nim/manual.html#userminusdefined-pragmas-macro-pragmas). +- Macro pragmas for var/let/const sections have been redesigned in a way that works + similarly to routine macro pragmas. The new behavior is documented in the + [experimental manual](https://nim-lang.github.io/Nim/manual_experimental.html#extended-macro-pragmas). - Pragma macros on type definitions can now return `nnkTypeSection` nodes as well as `nnkTypeDef`, allowing multiple type definitions to be injected in place of the original type definition. @@ -96,8 +102,6 @@ becomes an alias for `addr`. x, y, z: int Baz = object ``` -- [Case statement macros](manual.html#macros-case-statement-macros) are no longer experimental, - meaning you no longer need to enable the experimental switch `caseStmtMacros` to use them. - Full command syntax and block arguments i.e. `foo a, b: c` are now allowed for the right-hand side of type definitions in type sections. Previously they would error with "invalid indentation". From 489f6ddfefc4993af97eb2884b6d8173a4ee8b9d Mon Sep 17 00:00:00 2001 From: Daniel Clarke Date: Wed, 13 Jul 2022 22:55:33 +1000 Subject: [PATCH 1697/2472] fix #20012 (#20013) * replace gcc asm with __asm__ and add a test * update test case to specify gcc or clang and not cpp Co-authored-by: daniel --- compiler/extccomp.nim | 2 +- tests/compiler/tasm.nim | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 tests/compiler/tasm.nim diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index f1dc01f44..23c43cb67 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -86,7 +86,7 @@ compiler gcc: linkLibCmd: " -l$1", debug: "", pic: "-fPIC", - asmStmtFrmt: "asm($1);$n", + asmStmtFrmt: "__asm__($1);$n", structStmtFmt: "$1 $3 $2 ", # struct|union [packed] $name produceAsm: gnuAsmListing, cppXsupport: "-std=gnu++14 -funsigned-char", diff --git a/tests/compiler/tasm.nim b/tests/compiler/tasm.nim new file mode 100644 index 000000000..9f60231e0 --- /dev/null +++ b/tests/compiler/tasm.nim @@ -0,0 +1,15 @@ +proc testAsm() = + let src = 41 + var dst = 0 + + asm """ + mov %1, %0\n\t + add $1, %0 + : "=r" (`dst`) + : "r" (`src`)""" + + doAssert dst == 42 + +when defined(gcc) or defined(clang) and not defined(cpp): + {.passc: "-std=c99".} + testAsm() \ No newline at end of file From 73ee34f56fc267e6b8b29d12b22f969f7f8a7788 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Wed, 13 Jul 2022 22:37:31 +0800 Subject: [PATCH 1698/2472] use pull_request_target (#20020) --- .github/workflows/ci_publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_publish.yml b/.github/workflows/ci_publish.yml index 90a3438a7..a56ff9a7e 100644 --- a/.github/workflows/ci_publish.yml +++ b/.github/workflows/ci_publish.yml @@ -1,7 +1,7 @@ name: Tracking orc-booting compiler memory usage on: - pull_request: + pull_request_target: types: [closed] From 93211a2bddf8e6a806c3fd4e9322ca21f0a2a9c4 Mon Sep 17 00:00:00 2001 From: silent-observer Date: Thu, 14 Jul 2022 13:03:16 +0300 Subject: [PATCH 1699/2472] Add sink and lent annotations to the critbits module (#20021) * Add sink and lent to critbits * Remove lent for pairs I guess lent doesn't work well inside tuples * Remove lent from template in critbits Apparently this also doesn't work, because some checks failed --- lib/pure/collections/critbits.nim | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/pure/collections/critbits.nim b/lib/pure/collections/critbits.nim index e12984995..dca60b37b 100644 --- a/lib/pure/collections/critbits.nim +++ b/lib/pure/collections/critbits.nim @@ -197,7 +197,7 @@ proc missingOrExcl*[T](c: var CritBitTree[T], key: string): bool = discard exclImpl(c, key) result = c.count == oldCount -proc containsOrIncl*[T](c: var CritBitTree[T], key: string, val: T): bool = +proc containsOrIncl*[T](c: var CritBitTree[T], key: string, val: sink T): bool = ## Returns true if `c` contains the given `key`. If the key does not exist, ## `c[key] = val` is performed. ## @@ -270,7 +270,7 @@ proc incl*(c: var CritBitTree[void], key: string) = discard rawInsert(c, key) -proc incl*[T](c: var CritBitTree[T], key: string, val: T) = +proc incl*[T](c: var CritBitTree[T], key: string, val: sink T) = ## Inserts `key` with value `val` into `c`. ## ## **See also:** @@ -284,7 +284,7 @@ proc incl*[T](c: var CritBitTree[T], key: string, val: T) = var n = rawInsert(c, key) n.val = val -proc `[]=`*[T](c: var CritBitTree[T], key: string, val: T) = +proc `[]=`*[T](c: var CritBitTree[T], key: string, val: sink T) = ## Alias for `incl <#incl,CritBitTree[T],string,T>`_. ## ## **See also:** @@ -300,7 +300,7 @@ template get[T](c: CritBitTree[T], key: string): T = n.val -func `[]`*[T](c: CritBitTree[T], key: string): T {.inline.} = +func `[]`*[T](c: CritBitTree[T], key: string): lent T {.inline.} = ## Retrieves the value at `c[key]`. If `key` is not in `t`, the ## `KeyError` exception is raised. One can check with `hasKey` whether ## the key exists. @@ -342,7 +342,7 @@ iterator keys*[T](c: CritBitTree[T]): string = for x in leaves(c.root): yield x.key -iterator values*[T](c: CritBitTree[T]): T = +iterator values*[T](c: CritBitTree[T]): lent T = ## Yields all values of `c` in the lexicographical order of the ## corresponding keys. ## @@ -415,7 +415,7 @@ iterator keysWithPrefix*[T](c: CritBitTree[T], prefix: string): string = let top = allprefixedAux(c, prefix) for x in leaves(top): yield x.key -iterator valuesWithPrefix*[T](c: CritBitTree[T], prefix: string): T = +iterator valuesWithPrefix*[T](c: CritBitTree[T], prefix: string): lent T = ## Yields all values of `c` starting with `prefix` of the ## corresponding keys. ## @@ -518,7 +518,7 @@ func commonPrefixLen*[T](c: CritBitTree[T]): int {.inline, since((1, 3)).} = else: c.root.byte else: 0 -proc toCritBitTree*[T](pairs: openArray[(string, T)]): CritBitTree[T] {.since: (1, 3).} = +proc toCritBitTree*[T](pairs: sink openArray[(string, T)]): CritBitTree[T] {.since: (1, 3).} = ## Creates a new `CritBitTree` that contains the given `pairs`. runnableExamples: doAssert {"a": "0", "b": "1", "c": "2"}.toCritBitTree is CritBitTree[string] @@ -526,7 +526,7 @@ proc toCritBitTree*[T](pairs: openArray[(string, T)]): CritBitTree[T] {.since: ( for item in pairs: result.incl item[0], item[1] -proc toCritBitTree*(items: openArray[string]): CritBitTree[void] {.since: (1, 3).} = +proc toCritBitTree*(items: sink openArray[string]): CritBitTree[void] {.since: (1, 3).} = ## Creates a new `CritBitTree` that contains the given `items`. runnableExamples: doAssert ["a", "b", "c"].toCritBitTree is CritBitTree[void] From 10c8e2037db9ec3dc53fdaf1cc27e837e204af76 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Thu, 14 Jul 2022 18:42:56 +0800 Subject: [PATCH 1700/2472] fixes #20015; document `shallowCopy` does a deep copy with ARC/ORC (#20025) --- lib/system.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/system.nim b/lib/system.nim index 4080fee06..6e8519cf6 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -473,6 +473,8 @@ proc shallowCopy*[T](x: var T, y: T) {.noSideEffect, magic: "ShallowCopy".} ## Be careful with the changed semantics though! ## There is a reason why the default assignment does a deep copy of sequences ## and strings. + ## + ## .. warning:: `shallowCopy` does a deep copy with ARC/ORC. # :array|openArray|string|seq|cstring|tuple proc `[]`*[I: Ordinal;T](a: T; i: I): T {. From 800cb006e74f8c52bde69cfc1eb9e55fcc439633 Mon Sep 17 00:00:00 2001 From: quantimnot <54247259+quantimnot@users.noreply.github.com> Date: Thu, 14 Jul 2022 08:20:40 -0400 Subject: [PATCH 1701/2472] Change `styleCheck` to ignore foreign packages (#19822) * Change `styleCheck` to ignore foreign packages * Symbols from foreign packages are now ignored. * Fixed `styleCheck` violations in `compiler` package. * Added symbol ownership to custom annotation pragmas. * Minor refactors to cleanup style check callsites. * Minor internal documentation of reasons why a symbol isn't checked. Style violations were fixed in the compiler after thet were exposed by the changes. The compiler wouldn't compile otherwise. Symbol ownership for custom pragma annotations is needed for checking the annotation's style. A NPE was raised otherwise. Fixes #10201 See also nim-lang/RFCs#456 * Fix a misunderstanding about excluding field style checks I had refactored the callsites of `styleCheckUse` to apply the DRY principle, but I misunderstood the field access handling in a template as a general case. This corrects it. * Fix some `styleCheck` violations in `compiler/evalffi` The violations were exposed in CI when the compiler was built with libffi. * Removed some uneeded transitionary code * Add changelog entry Co-authored-by: quantimnot --- changelog.md | 2 ++ compiler/evalffi.nim | 8 ++--- compiler/ic/bitabs.nim | 8 ++--- compiler/linter.nim | 67 +++++++++++++++++++++++++----------------- compiler/msgs.nim | 4 +-- compiler/pragmas.nim | 6 ++-- compiler/semexprs.nim | 2 +- compiler/semgnrc.nim | 2 +- compiler/semmagic.nim | 2 +- compiler/semstmts.nim | 14 ++++----- compiler/semtempl.nim | 12 ++++---- compiler/semtypes.nim | 8 ++--- compiler/suggest.nim | 3 +- compiler/vm.nim | 8 ++--- compiler/vmops.nim | 64 ++++++++++++++++++++-------------------- compiler/wordrecg.nim | 14 ++++----- 16 files changed, 118 insertions(+), 106 deletions(-) diff --git a/changelog.md b/changelog.md index 68b133040..47479fc5c 100644 --- a/changelog.md +++ b/changelog.md @@ -114,6 +114,8 @@ becomes an alias for `addr`. - `nim` can now compile version 1.4.0 as follows: `nim c --lib:lib --stylecheck:off compiler/nim`, without requiring `-d:nimVersion140` which is now a noop. +- `--styleCheck` now only applies to the current package. + ## Tool changes diff --git a/compiler/evalffi.nim b/compiler/evalffi.nim index 5004011ed..d1d88a1fa 100644 --- a/compiler/evalffi.nim +++ b/compiler/evalffi.nim @@ -110,8 +110,8 @@ proc mapCallConv(conf: ConfigRef, cc: TCallingConvention, info: TLineInfo): TABI else: globalError(conf, info, "cannot map calling convention to FFI") -template rd(T, p: untyped): untyped = (cast[ptr T](p))[] -template wr(T, p, v: untyped): untyped = (cast[ptr T](p))[] = v +template rd(typ, p: untyped): untyped = (cast[ptr typ](p))[] +template wr(typ, p, v: untyped): untyped = (cast[ptr typ](p))[] = v template `+!`(x, y: untyped): untyped = cast[pointer](cast[ByteAddress](x) + y) @@ -177,8 +177,8 @@ const maxPackDepth = 20 var packRecCheck = 0 proc pack(conf: ConfigRef, v: PNode, typ: PType, res: pointer) = - template awr(T, v: untyped): untyped = - wr(T, res, v) + template awr(typ, v: untyped): untyped = + wr(typ, res, v) case typ.kind of tyBool: awr(bool, v.intVal != 0) diff --git a/compiler/ic/bitabs.nim b/compiler/ic/bitabs.nim index ae673b574..8adab8388 100644 --- a/compiler/ic/bitabs.nim +++ b/compiler/ic/bitabs.nim @@ -93,13 +93,13 @@ proc getOrIncl*[T](t: var BiTable[T]; v: T): LitId = t.vals.add v -proc `[]`*[T](t: var BiTable[T]; LitId: LitId): var T {.inline.} = - let idx = idToIdx LitId +proc `[]`*[T](t: var BiTable[T]; litId: LitId): var T {.inline.} = + let idx = idToIdx litId assert idx < t.vals.len result = t.vals[idx] -proc `[]`*[T](t: BiTable[T]; LitId: LitId): lent T {.inline.} = - let idx = idToIdx LitId +proc `[]`*[T](t: BiTable[T]; litId: LitId): lent T {.inline.} = + let idx = idToIdx litId assert idx < t.vals.len result = t.vals[idx] diff --git a/compiler/linter.nim b/compiler/linter.nim index 5fb646051..2c0ad4d6f 100644 --- a/compiler/linter.nim +++ b/compiler/linter.nim @@ -12,7 +12,8 @@ import std/strutils from std/sugar import dup -import options, ast, msgs, idents, lineinfos, wordrecg, astmsgs +import options, ast, msgs, idents, lineinfos, wordrecg, astmsgs, semdata, packages +export packages const Letters* = {'a'..'z', 'A'..'Z', '0'..'9', '\x80'..'\xFF', '_'} @@ -85,24 +86,32 @@ proc differ*(line: string, a, b: int, x: string): string = result = y proc nep1CheckDefImpl(conf: ConfigRef; info: TLineInfo; s: PSym; k: TSymKind) = - # operators stay as they are: - if k in {skResult, skTemp} or s.name.s[0] notin Letters: return - if k in {skType, skGenericParam} and sfAnon in s.flags: return - if s.typ != nil and s.typ.kind == tyTypeDesc: return - if {sfImportc, sfExportc} * s.flags != {}: return - if optStyleCheck notin s.options: return let beau = beautifyName(s.name.s, k) if s.name.s != beau: lintReport(conf, info, beau, s.name.s) -template styleCheckDef*(conf: ConfigRef; info: TLineInfo; s: PSym; k: TSymKind) = - if {optStyleHint, optStyleError} * conf.globalOptions != {} and optStyleUsages notin conf.globalOptions: - nep1CheckDefImpl(conf, info, s, k) +template styleCheckDef*(ctx: PContext; info: TLineInfo; sym: PSym; k: TSymKind) = + ## Check symbol definitions adhere to NEP1 style rules. + if optStyleCheck in ctx.config.options and # ignore if styleChecks are off + hintName in ctx.config.notes and # ignore if name checks are not requested + ctx.config.belongsToProjectPackage(ctx.module) and # ignore foreign packages + optStyleUsages notin ctx.config.globalOptions and # ignore if requested to only check name usage + sym.kind != skResult and # ignore `result` + sym.kind != skTemp and # ignore temporary variables created by the compiler + sym.name.s[0] in Letters and # ignore operators TODO: what about unicode symbols??? + k notin {skType, skGenericParam} and # ignore types and generic params + (sym.typ == nil or sym.typ.kind != tyTypeDesc) and # ignore `typedesc` + {sfImportc, sfExportc} * sym.flags == {} and # ignore FFI + sfAnon notin sym.flags: # ignore if created by compiler + nep1CheckDefImpl(ctx.config, info, sym, k) -template styleCheckDef*(conf: ConfigRef; info: TLineInfo; s: PSym) = - styleCheckDef(conf, info, s, s.kind) -template styleCheckDef*(conf: ConfigRef; s: PSym) = - styleCheckDef(conf, s.info, s, s.kind) +template styleCheckDef*(ctx: PContext; info: TLineInfo; s: PSym) = + ## Check symbol definitions adhere to NEP1 style rules. + styleCheckDef(ctx, info, s, s.kind) + +template styleCheckDef*(ctx: PContext; s: PSym) = + ## Check symbol definitions adhere to NEP1 style rules. + styleCheckDef(ctx, s.info, s, s.kind) proc differs(conf: ConfigRef; info: TLineInfo; newName: string): string = let line = sourceLine(conf, info) @@ -116,23 +125,27 @@ proc differs(conf: ConfigRef; info: TLineInfo; newName: string): string = let last = first+identLen(line, first)-1 result = differ(line, first, last, newName) -proc styleCheckUse*(conf: ConfigRef; info: TLineInfo; s: PSym) = - if info.fileIndex.int < 0: return - # we simply convert it to what it looks like in the definition - # for consistency - - # operators stay as they are: - if s.kind == skTemp or s.name.s[0] notin Letters or sfAnon in s.flags: - return - +proc styleCheckUseImpl(conf: ConfigRef; info: TLineInfo; s: PSym) = let newName = s.name.s let badName = differs(conf, info, newName) if badName.len > 0: - # special rules for historical reasons - let forceHint = badName == "nnkArgList" and newName == "nnkArglist" or badName == "nnkArglist" and newName == "nnkArgList" - lintReport(conf, info, newName, badName, forceHint = forceHint, extraMsg = "".dup(addDeclaredLoc(conf, s))) + lintReport(conf, info, newName, badName, "".dup(addDeclaredLoc(conf, s))) -proc checkPragmaUse*(conf: ConfigRef; info: TLineInfo; w: TSpecialWord; pragmaName: string) = +template styleCheckUse*(ctx: PContext; info: TLineInfo; sym: PSym) = + ## Check symbol uses match their definition's style. + if {optStyleHint, optStyleError} * ctx.config.globalOptions != {} and # ignore if styleChecks are off + hintName in ctx.config.notes and # ignore if name checks are not requested + ctx.config.belongsToProjectPackage(ctx.module) and # ignore foreign packages + sym.kind != skTemp and # ignore temporary variables created by the compiler + sym.name.s[0] in Letters and # ignore operators TODO: what about unicode symbols??? + sfAnon notin sym.flags: # ignore temporary variables created by the compiler + styleCheckUseImpl(ctx.config, info, sym) + +proc checkPragmaUseImpl(conf: ConfigRef; info: TLineInfo; w: TSpecialWord; pragmaName: string) = let wanted = $w if pragmaName != wanted: lintReport(conf, info, wanted, pragmaName) + +template checkPragmaUse*(conf: ConfigRef; info: TLineInfo; w: TSpecialWord; pragmaName: string) = + if {optStyleHint, optStyleError} * conf.globalOptions != {}: + checkPragmaUseImpl(conf, info, w, pragmaName) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 371a886f4..6d770e72f 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -622,9 +622,9 @@ template internalAssert*(conf: ConfigRef, e: bool) = let arg = info2.toFileLineCol internalErrorImpl(conf, unknownLineInfo, arg, info2) -template lintReport*(conf: ConfigRef; info: TLineInfo, beau, got: string, forceHint = false, extraMsg = "") = +template lintReport*(conf: ConfigRef; info: TLineInfo, beau, got: string, extraMsg = "") = let m = "'$1' should be: '$2'$3" % [got, beau, extraMsg] - let msg = if optStyleError in conf.globalOptions and not forceHint: errGenerated else: hintName + let msg = if optStyleError in conf.globalOptions: errGenerated else: hintName liMessage(conf, info, msg, m, doNothing, instLoc()) proc quotedFilename*(conf: ConfigRef; i: TLineInfo): Rope = diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 2262e441b..1487a871d 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -829,8 +829,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, let ident = considerQuotedIdent(c, key) var userPragma = strTableGet(c.userPragmas, ident) if userPragma != nil: - if {optStyleHint, optStyleError} * c.config.globalOptions != {}: - styleCheckUse(c.config, key.info, userPragma) + styleCheckUse(c, key.info, userPragma) # number of pragmas increase/decrease with user pragma expansion inc c.instCounter @@ -844,8 +843,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, else: let k = whichKeyword(ident) if k in validPragmas: - if {optStyleHint, optStyleError} * c.config.globalOptions != {}: - checkPragmaUse(c.config, key.info, k, ident.s) + checkPragmaUse(c.config, key.info, k, ident.s) case k of wExportc, wExportCpp: makeExternExport(c, sym, getOptionalStr(c, it, "$1"), it.info) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index fed9cae21..1bb4f49ce 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -2599,7 +2599,7 @@ proc semBlock(c: PContext, n: PNode; flags: TExprFlags): PNode = labl.owner = c.p.owner n[0] = newSymNode(labl, n[0].info) suggestSym(c.graph, n[0].info, labl, c.graph.usageSym) - styleCheckDef(c.config, labl) + styleCheckDef(c, labl) onDef(n[0].info, labl) n[1] = semExpr(c, n[1], flags) n.typ = n[1].typ diff --git a/compiler/semgnrc.nim b/compiler/semgnrc.nim index e249d88e8..ca0b05fa2 100644 --- a/compiler/semgnrc.nim +++ b/compiler/semgnrc.nim @@ -179,7 +179,7 @@ proc fuzzyLookup(c: PContext, n: PNode, flags: TSemGenericFlags, proc addTempDecl(c: PContext; n: PNode; kind: TSymKind) = let s = newSymS(skUnknown, getIdentNode(c, n), c) addPrelimDecl(c, s) - styleCheckDef(c.config, n.info, s, kind) + styleCheckDef(c, n.info, s, kind) onDef(n.info, s) proc semGenericStmt(c: PContext, n: PNode, diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim index ed1826fd4..d3e12670b 100644 --- a/compiler/semmagic.nim +++ b/compiler/semmagic.nim @@ -428,7 +428,7 @@ proc semQuantifier(c: PContext; n: PNode): PNode = let op = considerQuotedIdent(c, it[0]) if op.id == ord(wIn): let v = newSymS(skForVar, it[1], c) - styleCheckDef(c.config, v) + styleCheckDef(c, v) onDef(it[1].info, v) let domain = semExprWithType(c, it[2], {efWantIterator}) v.typ = domain.typ diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 0516d08ce..03c30b6e6 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -375,7 +375,7 @@ proc semUsing(c: PContext; n: PNode): PNode = let typ = semTypeNode(c, a[^2], nil) for j in 0.. 0: @@ -271,8 +271,8 @@ proc semTemplSymbol(c: PContext, n: PNode, s: PSym; isField: bool): PNode = when defined(nimsuggest): suggestSym(c.graph, n.info, s, c.graph.usageSym, false) # field access (dot expr) will be handled by builtinFieldAccess - if not isField and {optStyleHint, optStyleError} * c.config.globalOptions != {}: - styleCheckUse(c.config, n.info, s) + if not isField: + styleCheckUse(c, n.info, s) proc semRoutineInTemplName(c: var TemplCtx, n: PNode): PNode = result = n @@ -297,7 +297,7 @@ proc semRoutineInTemplBody(c: var TemplCtx, n: PNode, k: TSymKind): PNode = var s = newGenSym(k, ident, c) s.ast = n addPrelimDecl(c.c, s) - styleCheckDef(c.c.config, n.info, s) + styleCheckDef(c.c, n.info, s) onDef(n.info, s) n[namePos] = newSymNode(s, n[namePos].info) else: @@ -431,7 +431,7 @@ proc semTemplBody(c: var TemplCtx, n: PNode): PNode = # labels are always 'gensym'ed: let s = newGenSym(skLabel, n[0], c) addPrelimDecl(c.c, s) - styleCheckDef(c.c.config, s) + styleCheckDef(c.c, s) onDef(n[0].info, s) n[0] = newSymNode(s, n[0].info) n[1] = semTemplBody(c, n[1]) @@ -624,7 +624,7 @@ proc semTemplateDef(c: PContext, n: PNode): PNode = s.owner.name.s == "vm" and s.name.s == "stackTrace": incl(s.flags, sfCallsite) - styleCheckDef(c.config, s) + styleCheckDef(c, s) onDef(n[namePos].info, s) # check parameter list: #s.scope = c.currentScope diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index b4f385fe6..584da1305 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -139,7 +139,7 @@ proc semEnum(c: PContext, n: PNode, prev: PType): PType = e.flags.incl {sfUsed, sfExported} result.n.add symNode - styleCheckDef(c.config, e) + styleCheckDef(c, e) onDef(e.info, e) if sfGenSym notin e.flags: if not isPure: @@ -476,7 +476,7 @@ proc semTuple(c: PContext, n: PNode, prev: PType): PType = else: result.n.add newSymNode(field) addSonSkipIntLit(result, typ, c.idgen) - styleCheckDef(c.config, a[j].info, field) + styleCheckDef(c, a[j].info, field) onDef(field.info, field) if result.n.len == 0: result.n = nil if isTupleRecursive(result): @@ -808,7 +808,7 @@ proc semRecordNodeAux(c: PContext, n: PNode, check: var IntSet, pos: var int, localError(c.config, info, "attempt to redefine: '" & f.name.s & "'") if a.kind == nkEmpty: father.add newSymNode(f) else: a.add newSymNode(f) - styleCheckDef(c.config, f) + styleCheckDef(c, f) onDef(f.info, f) if a.kind != nkEmpty: father.add a of nkSym: @@ -1315,7 +1315,7 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode, result.n.add newSymNode(arg) rawAddSon(result, finalType) addParamOrResult(c, arg, kind) - styleCheckDef(c.config, a[j].info, arg) + styleCheckDef(c, a[j].info, arg) onDef(a[j].info, arg) if {optNimV1Emulation, optNimV12Emulation} * c.config.globalOptions == {}: a[j] = newSymNode(arg) diff --git a/compiler/suggest.nim b/compiler/suggest.nim index 38751fcc7..4b48ef5bb 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -598,8 +598,7 @@ proc markUsed(c: PContext; info: TLineInfo; s: PSym) = if sfError in s.flags: userError(conf, info, s) when defined(nimsuggest): suggestSym(c.graph, info, s, c.graph.usageSym, false) - if {optStyleHint, optStyleError} * conf.globalOptions != {}: - styleCheckUse(conf, info, s) + styleCheckUse(c, info, s) markOwnerModuleAsUsed(c, s) proc safeSemExpr*(c: PContext, n: PNode): PNode = diff --git a/compiler/vm.nim b/compiler/vm.nim index e681bbc96..3cb699482 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -119,13 +119,13 @@ template move(a, b: untyped) {.dirty.} = system.shallowCopy(a, b) proc derefPtrToReg(address: BiggestInt, typ: PType, r: var TFullReg, isAssign: bool): bool = # nim bug: `isAssign: static bool` doesn't work, giving odd compiler error - template fun(field, T, rkind) = + template fun(field, typ, rkind) = if isAssign: - cast[ptr T](address)[] = T(r.field) + cast[ptr typ](address)[] = typ(r.field) else: r.ensureKind(rkind) - let val = cast[ptr T](address)[] - when T is SomeInteger | char: + let val = cast[ptr typ](address)[] + when typ is SomeInteger | char: r.field = BiggestInt(val) else: r.field = val diff --git a/compiler/vmops.nim b/compiler/vmops.nim index 04b753ef6..69599ece3 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -56,13 +56,13 @@ template ioop(op) {.dirty.} = template macrosop(op) {.dirty.} = registerCallback(c, "stdlib.macros." & astToStr(op), `op Wrapper`) -template wrap1f_math(op) {.dirty.} = +template wrap1fMath(op) {.dirty.} = proc `op Wrapper`(a: VmArgs) {.nimcall.} = doAssert a.numArgs == 1 setResult(a, op(getFloat(a, 0))) mathop op -template wrap2f_math(op) {.dirty.} = +template wrap2fMath(op) {.dirty.} = proc `op Wrapper`(a: VmArgs) {.nimcall.} = setResult(a, op(getFloat(a, 0), getFloat(a, 1))) mathop op @@ -172,40 +172,40 @@ proc registerAdditionalOps*(c: PCtx) = proc getProjectPathWrapper(a: VmArgs) = setResult a, c.config.projectPath.string - wrap1f_math(sqrt) - wrap1f_math(cbrt) - wrap1f_math(ln) - wrap1f_math(log10) - wrap1f_math(log2) - wrap1f_math(exp) - wrap1f_math(arccos) - wrap1f_math(arcsin) - wrap1f_math(arctan) - wrap1f_math(arcsinh) - wrap1f_math(arccosh) - wrap1f_math(arctanh) - wrap2f_math(arctan2) - wrap1f_math(cos) - wrap1f_math(cosh) - wrap2f_math(hypot) - wrap1f_math(sinh) - wrap1f_math(sin) - wrap1f_math(tan) - wrap1f_math(tanh) - wrap2f_math(pow) - wrap1f_math(trunc) - wrap1f_math(floor) - wrap1f_math(ceil) - wrap1f_math(erf) - wrap1f_math(erfc) - wrap1f_math(gamma) - wrap1f_math(lgamma) + wrap1fMath(sqrt) + wrap1fMath(cbrt) + wrap1fMath(ln) + wrap1fMath(log10) + wrap1fMath(log2) + wrap1fMath(exp) + wrap1fMath(arccos) + wrap1fMath(arcsin) + wrap1fMath(arctan) + wrap1fMath(arcsinh) + wrap1fMath(arccosh) + wrap1fMath(arctanh) + wrap2fMath(arctan2) + wrap1fMath(cos) + wrap1fMath(cosh) + wrap2fMath(hypot) + wrap1fMath(sinh) + wrap1fMath(sin) + wrap1fMath(tan) + wrap1fMath(tanh) + wrap2fMath(pow) + wrap1fMath(trunc) + wrap1fMath(floor) + wrap1fMath(ceil) + wrap1fMath(erf) + wrap1fMath(erfc) + wrap1fMath(gamma) + wrap1fMath(lgamma) when declared(copySign): - wrap2f_math(copySign) + wrap2fMath(copySign) when declared(signbit): - wrap1f_math(signbit) + wrap1fMath(signbit) registerCallback c, "stdlib.math.round", proc (a: VmArgs) {.nimcall.} = let n = a.numArgs diff --git a/compiler/wordrecg.nim b/compiler/wordrecg.nim index c819f4306..cf2f768b1 100644 --- a/compiler/wordrecg.nim +++ b/compiler/wordrecg.nim @@ -89,23 +89,23 @@ type wLiftLocals = "liftlocals", wEnforceNoRaises = "enforceNoRaises", wAuto = "auto", wBool = "bool", wCatch = "catch", wChar = "char", - wClass = "class", wCompl = "compl", wConst_cast = "const_cast", wDefault = "default", - wDelete = "delete", wDouble = "double", wDynamic_cast = "dynamic_cast", + wClass = "class", wCompl = "compl", wConstCast = "const_cast", wDefault = "default", + wDelete = "delete", wDouble = "double", wDynamicCast = "dynamic_cast", wExplicit = "explicit", wExtern = "extern", wFalse = "false", wFloat = "float", wFriend = "friend", wGoto = "goto", wInt = "int", wLong = "long", wMutable = "mutable", wNamespace = "namespace", wNew = "new", wOperator = "operator", wPrivate = "private", wProtected = "protected", wPublic = "public", wRegister = "register", - wReinterpret_cast = "reinterpret_cast", wRestrict = "restrict", wShort = "short", - wSigned = "signed", wSizeof = "sizeof", wStatic_cast = "static_cast", wStruct = "struct", + wReinterpretCast = "reinterpret_cast", wRestrict = "restrict", wShort = "short", + wSigned = "signed", wSizeof = "sizeof", wStaticCast = "static_cast", wStruct = "struct", wSwitch = "switch", wThis = "this", wThrow = "throw", wTrue = "true", wTypedef = "typedef", wTypeid = "typeid", wTypeof = "typeof", wTypename = "typename", wUnion = "union", wPacked = "packed", wUnsigned = "unsigned", wVirtual = "virtual", - wVoid = "void", wVolatile = "volatile", wWchar_t = "wchar_t", + wVoid = "void", wVolatile = "volatile", wWchar = "wchar_t", wAlignas = "alignas", wAlignof = "alignof", wConstexpr = "constexpr", wDecltype = "decltype", wNullptr = "nullptr", wNoexcept = "noexcept", - wThread_local = "thread_local", wStatic_assert = "static_assert", - wChar16_t = "char16_t", wChar32_t = "char32_t", + wThreadLocal = "thread_local", wStaticAssert = "static_assert", + wChar16 = "char16_t", wChar32 = "char32_t", wStdIn = "stdin", wStdOut = "stdout", wStdErr = "stderr", From 286fcef68ef6f0e3a20ab5b25307c9b4705ce54d Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Fri, 15 Jul 2022 15:42:54 +0800 Subject: [PATCH 1702/2472] [Orc] fixes "streams.readDataStr segafaults" when accepting a string literal (#20019) [backport] fixes streams.readDataStr accept a string literal --- lib/pure/streams.nim | 5 +++++ tests/stdlib/tstreams.nim | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim index f58273ee8..50b3085d2 100644 --- a/lib/pure/streams.nim +++ b/lib/pure/streams.nim @@ -1197,6 +1197,11 @@ else: # after 1.3 or JS not defined proc ssReadDataStr(s: Stream, buffer: var string, slice: Slice[int]): int = var s = StringStream(s) + when nimvm: + discard + else: + when declared(prepareMutation): + prepareMutation(buffer) # buffer might potentially be a CoW literal with ARC result = min(slice.b + 1 - slice.a, s.data.len - s.pos) if result > 0: jsOrVmBlock: diff --git a/tests/stdlib/tstreams.nim b/tests/stdlib/tstreams.nim index d9857926e..cc1343651 100644 --- a/tests/stdlib/tstreams.nim +++ b/tests/stdlib/tstreams.nim @@ -85,3 +85,13 @@ block: static: # Ensure streams it doesnt break with nimscript on arc/orc #19716 let s = newStringStream("a") doAssert s.data == "a" + +template main = + var strm = newStringStream("abcde") + var buffer = "12345" + doAssert strm.readDataStr(buffer, 0..3) == 4 + doAssert buffer == "abcd5" + strm.close() + +static: main() +main() From f35c9cf73ddb3150ab6dbe449db4975866ee8a11 Mon Sep 17 00:00:00 2001 From: metagn <10591326+metagn@users.noreply.github.com> Date: Fri, 15 Jul 2022 13:37:08 +0300 Subject: [PATCH 1703/2472] fix #20002 (#20004) While this fix seems innocent, this unlocks the hidden behavior of method calls not being able to call gensym'ed routines inside templates. --- compiler/semtempl.nim | 2 +- tests/template/tinnerouterproc.nim | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 tests/template/tinnerouterproc.nim diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim index a599edb9c..1f76aff75 100644 --- a/compiler/semtempl.nim +++ b/compiler/semtempl.nim @@ -78,7 +78,7 @@ proc symChoice(c: PContext, n: PNode, s: PSym, r: TSymChoiceRule; result = newNodeIT(kind, info, newTypeS(tyNone, c)) a = initOverloadIter(o, c, n) while a != nil: - if a.kind != skModule and (not isField or sfGenSym notin s.flags): + if a.kind != skModule and (not isField or sfGenSym notin a.flags): incl(a.flags, sfUsed) markOwnerModuleAsUsed(c, a) result.add newSymNode(a, info) diff --git a/tests/template/tinnerouterproc.nim b/tests/template/tinnerouterproc.nim new file mode 100644 index 000000000..1f15fb13e --- /dev/null +++ b/tests/template/tinnerouterproc.nim @@ -0,0 +1,8 @@ +block: # #20002 + proc bar(x: int): int = 10 + template foo = + proc bar(x: int): int {.gensym.} = x + 2 + doAssert bar(3) == 5 + discard 3.bar # evaluates to 10 but only check if it compiles for now + block: + foo() From 417b90a7e5b88bfc0ad1bfbbc81a3205c99e128e Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Fri, 15 Jul 2022 20:27:54 +0300 Subject: [PATCH 1704/2472] Improve Markdown code blocks & start moving docs to Markdown style (#19954) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - add additional parameters parsing (other implementations will just ignore them). E.g. if in RST we have: .. code:: nim :test: "nim c $1" ... then in Markdown that will be: ```nim test="nim c $1" ... ``` - implement Markdown interpretation of additional indentation which is less than 4 spaces (>=4 spaces is a code block but it's not implemented yet). RST interpretes it as quoted block, for Markdown it's just normal paragraphs. - add separate `md2html` and `md2tex` commands. This is to separate Markdown behavior in cases when it diverges w.r.t. RST significantly — most conspicously like in the case of additional indentation above, and also currently the contradicting inline rule of Markdown is also turned on only in `md2html` and `md2tex`. **Rationale:** mixing Markdown and RST arbitrarily is a way to nowhere, we need to provide a way to fix the particular behavior. Note that still all commands have **both** Markdown and RST features **enabled**. In this PR `*.nim` files can be processed only in Markdown mode, while `md2html` is for `*.md` files and `rst2html` for `*.rst` files. - rename `*.rst` files to `.*md` as our current default behavior is already Markdown-ish - convert code blocks in `docgen.rst` to Markdown style as an example. Other code blocks will be converted in the follow-up PRs - fix indentation inside Markdown code blocks — additional indentation is preserved there - allow more than 3 backticks open/close blocks (tildas \~ are still not allowed to avoid conflict with RST adornment headings) see also https://github.com/nim-lang/RFCs/issues/355 - better error messages - (other) fix a bug that admonitions cannot be used in sandbox mode; fix annoying warning on line 2711 --- compiler/commands.nim | 5 +- compiler/docgen.nim | 33 ++-- compiler/lineinfos.nim | 2 + compiler/main.nim | 13 +- compiler/nim.nim | 2 +- compiler/options.nim | 2 + doc/{apis.rst => apis.md} | 0 doc/{backends.rst => backends.md} | 3 +- doc/{contributing.rst => contributing.md} | 2 +- doc/{destructors.rst => destructors.md} | 0 doc/{docgen.rst => docgen.md} | 63 ++++---- doc/{docs.rst => docs.md} | 0 doc/{docstyle.rst => docstyle.md} | 0 doc/{drnim.rst => drnim.md} | 0 doc/{estp.rst => estp.md} | 0 doc/{filters.rst => filters.md} | 0 doc/{hcr.rst => hcr.md} | 0 doc/{idetools.rst => idetools.md} | 0 doc/{intern.rst => intern.md} | 6 +- doc/{koch.rst => koch.md} | 0 doc/{lib.rst => lib.md} | 0 doc/{manual.rst => manual.md} | 8 +- .../{var_t_return.rst => var_t_return.md} | 0 ...xperimental.rst => manual_experimental.md} | 2 +- ...st => manual_experimental_strictnotnil.md} | 0 doc/{mm.rst => mm.md} | 2 +- doc/{nep1.rst => nep1.md} | 0 doc/{nimc.rst => nimc.md} | 6 +- doc/{nimfix.rst => nimfix.md} | 0 doc/{nimgrep.rst => nimgrep.md} | 2 +- doc/{niminst.rst => niminst.md} | 0 doc/{nims.rst => nims.md} | 0 doc/{nimsuggest.rst => nimsuggest.md} | 0 doc/{overview.rst => overview.md} | 2 +- doc/{packaging.rst => packaging.md} | 0 doc/{refc.rst => refc.md} | 0 doc/{testament.rst => testament.md} | 0 doc/{tools.rst => tools.md} | 0 doc/{tut1.rst => tut1.md} | 0 doc/{tut2.rst => tut2.md} | 2 +- doc/{tut3.rst => tut3.md} | 2 +- lib/packages/docutils/highlite.nim | 4 +- lib/packages/docutils/rst.nim | 151 ++++++++++++++---- tests/stdlib/thighlite.nim | 6 + tests/stdlib/trst.nim | 117 +++++++++++++- tests/stdlib/trstgen.nim | 16 +- tools/kochdocs.nim | 16 +- 47 files changed, 341 insertions(+), 126 deletions(-) rename doc/{apis.rst => apis.md} (100%) rename doc/{backends.rst => backends.md} (99%) rename doc/{contributing.rst => contributing.md} (99%) rename doc/{destructors.rst => destructors.md} (100%) rename doc/{docgen.rst => docgen.md} (98%) rename doc/{docs.rst => docs.md} (100%) rename doc/{docstyle.rst => docstyle.md} (100%) rename doc/{drnim.rst => drnim.md} (100%) rename doc/{estp.rst => estp.md} (100%) rename doc/{filters.rst => filters.md} (100%) rename doc/{hcr.rst => hcr.md} (100%) rename doc/{idetools.rst => idetools.md} (100%) rename doc/{intern.rst => intern.md} (99%) rename doc/{koch.rst => koch.md} (100%) rename doc/{lib.rst => lib.md} (100%) rename doc/{manual.rst => manual.md} (99%) rename doc/manual/{var_t_return.rst => var_t_return.md} (100%) rename doc/{manual_experimental.rst => manual_experimental.md} (99%) rename doc/{manual_experimental_strictnotnil.rst => manual_experimental_strictnotnil.md} (100%) rename doc/{mm.rst => mm.md} (98%) rename doc/{nep1.rst => nep1.md} (100%) rename doc/{nimc.rst => nimc.md} (99%) rename doc/{nimfix.rst => nimfix.md} (100%) rename doc/{nimgrep.rst => nimgrep.md} (99%) rename doc/{niminst.rst => niminst.md} (100%) rename doc/{nims.rst => nims.md} (100%) rename doc/{nimsuggest.rst => nimsuggest.md} (100%) rename doc/{overview.rst => overview.md} (86%) rename doc/{packaging.rst => packaging.md} (100%) rename doc/{refc.rst => refc.md} (100%) rename doc/{testament.rst => testament.md} (100%) rename doc/{tools.rst => tools.md} (100%) rename doc/{tut1.rst => tut1.md} (100%) rename doc/{tut2.rst => tut2.md} (99%) rename doc/{tut3.rst => tut3.md} (99%) diff --git a/compiler/commands.nim b/compiler/commands.nim index 31e637aba..b849f503d 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -449,6 +449,8 @@ proc parseCommand*(command: string): Command = of "doc2", "doc": cmdDoc of "doc2tex": cmdDoc2tex of "rst2html": cmdRst2html + of "md2tex": cmdMd2tex + of "md2html": cmdMd2html of "rst2tex": cmdRst2tex of "jsondoc0": cmdJsondoc0 of "jsondoc2", "jsondoc": cmdJsondoc @@ -480,7 +482,8 @@ proc setCommandEarly*(conf: ConfigRef, command: string) = # command early customizations # must be handled here to honor subsequent `--hint:x:on|off` case conf.cmd - of cmdRst2html, cmdRst2tex: # xxx see whether to add others: cmdGendepend, etc. + of cmdRst2html, cmdRst2tex, cmdMd2html, cmdMd2tex: + # xxx see whether to add others: cmdGendepend, etc. conf.foreignPackageNotes = {hintSuccessX} else: conf.foreignPackageNotes = foreignPackageNotesDefault diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 390f44f2e..ed5fe06ef 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -88,7 +88,7 @@ type jEntriesFinal: JsonNode # final JSON after RST pass 2 and rendering types: TStrTable sharedState: PRstSharedState - isPureRst: bool + standaloneDoc: bool conf*: ConfigRef cache*: IdentCache exampleCounter: int @@ -230,6 +230,7 @@ template declareClosures = case msgKind of meCannotOpenFile: k = errCannotOpenFile of meExpected: k = errXExpected + of meMissingClosing: k = errRstMissingClosing of meGridTableNotImplemented: k = errRstGridTableNotImplemented of meMarkdownIllformedTable: k = errRstMarkdownIllformedTable of meIllformedTable: k = errRstIllformedTable @@ -276,16 +277,18 @@ proc isLatexCmd(conf: ConfigRef): bool = conf.cmd in {cmdRst2tex, cmdDoc2tex} proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, outExt: string = HtmlExt, module: PSym = nil, - isPureRst = false): PDoc = + standaloneDoc = false, preferMarkdown = true): PDoc = declareClosures() new(result) result.module = module result.conf = conf result.cache = cache result.outDir = conf.outDir.string - result.isPureRst = isPureRst - var options= {roSupportRawDirective, roSupportMarkdown, roPreferMarkdown, roSandboxDisabled} - if not isPureRst: options.incl roNimFile + result.standaloneDoc = standaloneDoc + var options= {roSupportRawDirective, roSupportMarkdown, roSandboxDisabled} + if preferMarkdown: + options.incl roPreferMarkdown + if not standaloneDoc: options.incl roNimFile result.sharedState = newRstSharedState( options, filename.string, docgenFindFile, compilerMsgHandler) @@ -333,7 +336,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, # Make sure the destination directory exists createDir(outp.splitFile.dir) # Include the current file if we're parsing a nim file - let importStmt = if d.isPureRst: "" else: "import \"$1\"\n" % [d.filename.replace("\\", "/")] + let importStmt = if d.standaloneDoc: "" else: "import \"$1\"\n" % [d.filename.replace("\\", "/")] writeFile(outp, importStmt & content) proc interpSnippetCmd(cmd: string): string = @@ -1512,7 +1515,7 @@ proc genOutFile(d: PDoc, groupedToc = false): string = "\\\\\\vspace{0.5em}\\large $1", [esc(d.target, d.meta[metaSubtitle])]) var groupsection = getConfigVar(d.conf, "doc.body_toc_groupsection") - let bodyname = if d.hasToc and not d.isPureRst and not d.conf.isLatexCmd: + let bodyname = if d.hasToc and not d.standaloneDoc and not d.conf.isLatexCmd: groupsection.setLen 0 "doc.body_toc_group" elif d.hasToc: "doc.body_toc" @@ -1626,9 +1629,11 @@ proc commandDoc*(cache: IdentCache, conf: ConfigRef) = generateIndex(d) proc commandRstAux(cache: IdentCache, conf: ConfigRef; - filename: AbsoluteFile, outExt: string) = + filename: AbsoluteFile, outExt: string, + preferMarkdown: bool) = var filen = addFileExt(filename, "txt") - var d = newDocumentor(filen, cache, conf, outExt, isPureRst = true) + var d = newDocumentor(filen, cache, conf, outExt, standaloneDoc = true, + preferMarkdown = preferMarkdown) let rst = parseRst(readFile(filen.string), line=LineRstInit, column=ColRstInit, conf, d.sharedState) @@ -1637,11 +1642,13 @@ proc commandRstAux(cache: IdentCache, conf: ConfigRef; writeOutput(d) generateIndex(d) -proc commandRst2Html*(cache: IdentCache, conf: ConfigRef) = - commandRstAux(cache, conf, conf.projectFull, HtmlExt) +proc commandRst2Html*(cache: IdentCache, conf: ConfigRef, + preferMarkdown=false) = + commandRstAux(cache, conf, conf.projectFull, HtmlExt, preferMarkdown) -proc commandRst2TeX*(cache: IdentCache, conf: ConfigRef) = - commandRstAux(cache, conf, conf.projectFull, TexExt) +proc commandRst2TeX*(cache: IdentCache, conf: ConfigRef, + preferMarkdown=false) = + commandRstAux(cache, conf, conf.projectFull, TexExt, preferMarkdown) proc commandJson*(cache: IdentCache, conf: ConfigRef) = ## implementation of a deprecated jsondoc0 command diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index 105de1636..071316f15 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -32,6 +32,7 @@ type # non-fatal errors errIllFormedAstX, errCannotOpenFile, errXExpected, + errRstMissingClosing, errRstGridTableNotImplemented, errRstMarkdownIllformedTable, errRstIllformedTable, @@ -105,6 +106,7 @@ const errIllFormedAstX: "illformed AST: $1", errCannotOpenFile: "cannot open '$1'", errXExpected: "'$1' expected", + errRstMissingClosing: "$1", errRstGridTableNotImplemented: "grid table is not implemented", errRstMarkdownIllformedTable: "illformed delimiter row of a markdown table", errRstIllformedTable: "Illformed table: $1", diff --git a/compiler/main.nim b/compiler/main.nim index a4425e510..0354bec9c 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -276,7 +276,8 @@ proc mainCommand*(graph: ModuleGraph) = var ret = if optUseNimcache in conf.globalOptions: getNimcacheDir(conf) else: conf.projectPath doAssert ret.string.isAbsolute # `AbsoluteDir` is not a real guarantee - if conf.cmd in cmdDocLike + {cmdRst2html, cmdRst2tex}: ret = ret / htmldocsDir + if conf.cmd in cmdDocLike + {cmdRst2html, cmdRst2tex, cmdMd2html, cmdMd2tex}: + ret = ret / htmldocsDir conf.outDir = ret ## process all commands @@ -302,7 +303,7 @@ proc mainCommand*(graph: ModuleGraph) = commandDoc2(graph, HtmlExt) if optGenIndex in conf.globalOptions and optWholeProject in conf.globalOptions: commandBuildIndex(conf, $conf.outDir) - of cmdRst2html: + of cmdRst2html, cmdMd2html: # XXX: why are warnings disabled by default for rst2html and rst2tex? for warn in rstWarnings: conf.setNoteDefaults(warn, true) @@ -311,16 +312,16 @@ proc mainCommand*(graph: ModuleGraph) = conf.quitOrRaise "compiler wasn't built with documentation generator" else: loadConfigs(DocConfig, cache, conf, graph.idgen) - commandRst2Html(cache, conf) - of cmdRst2tex, cmdDoc2tex: + commandRst2Html(cache, conf, preferMarkdown = (conf.cmd == cmdMd2html)) + of cmdRst2tex, cmdMd2tex, cmdDoc2tex: for warn in rstWarnings: conf.setNoteDefaults(warn, true) when defined(leanCompiler): conf.quitOrRaise "compiler wasn't built with documentation generator" else: - if conf.cmd == cmdRst2tex: + if conf.cmd in {cmdRst2tex, cmdMd2tex}: loadConfigs(DocTexConfig, cache, conf, graph.idgen) - commandRst2TeX(cache, conf) + commandRst2TeX(cache, conf, preferMarkdown = (conf.cmd == cmdMd2tex)) else: docLikeCmd commandDoc2(graph, TexExt) of cmdJsondoc0: docLikeCmd commandJson(cache, conf) diff --git a/compiler/nim.nim b/compiler/nim.nim index bfb07ba20..48472507d 100644 --- a/compiler/nim.nim +++ b/compiler/nim.nim @@ -122,7 +122,7 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) = # `The parameter is incorrect` let cmd = cmdPrefix & output.quoteShell & ' ' & conf.arguments execExternalProgram(conf, cmd.strip(leading=false,trailing=true)) - of cmdDocLike, cmdRst2html, cmdRst2tex: # bugfix(cmdRst2tex was missing) + of cmdDocLike, cmdRst2html, cmdRst2tex, cmdMd2html, cmdMd2tex: # bugfix(cmdRst2tex was missing) if conf.arguments.len > 0: # reserved for future use rawMessage(conf, errGenerated, "'$1 cannot handle arguments" % [$conf.cmd]) diff --git a/compiler/options.nim b/compiler/options.nim index 1ba157587..d567927cb 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -153,6 +153,8 @@ type cmdDoc2tex # convert .nim doc comments to LaTeX cmdRst2html # convert a reStructuredText file to HTML cmdRst2tex # convert a reStructuredText file to TeX + cmdMd2html # convert a Markdown file to HTML + cmdMd2tex # convert a Markdown file to TeX cmdJsondoc0 cmdJsondoc cmdCtags diff --git a/doc/apis.rst b/doc/apis.md similarity index 100% rename from doc/apis.rst rename to doc/apis.md diff --git a/doc/backends.rst b/doc/backends.md similarity index 99% rename from doc/backends.rst rename to doc/backends.md index 65dd8a6f2..a135e78b0 100644 --- a/doc/backends.rst +++ b/doc/backends.md @@ -10,7 +10,8 @@ .. no syntax highlighting here by default: .. contents:: - "Heresy grows from idleness." -- Unknown. + +> "Heresy grows from idleness." -- Unknown. Introduction diff --git a/doc/contributing.rst b/doc/contributing.md similarity index 99% rename from doc/contributing.rst rename to doc/contributing.md index 0ca0d0cbc..51d1d5065 100644 --- a/doc/contributing.rst +++ b/doc/contributing.md @@ -581,7 +581,7 @@ Code reviews -.. include:: docstyle.rst +.. include:: docstyle.md Evolving the stdlib diff --git a/doc/destructors.rst b/doc/destructors.md similarity index 100% rename from doc/destructors.rst rename to doc/destructors.md diff --git a/doc/docgen.rst b/doc/docgen.md similarity index 98% rename from doc/docgen.rst rename to doc/docgen.md index 48166b2c5..5f5c4ecc0 100644 --- a/doc/docgen.rst +++ b/doc/docgen.md @@ -35,14 +35,13 @@ Quick start Generate HTML documentation for a file: -.. code:: cmd - + ```cmd nim doc .nim + ``` Generate HTML documentation for a whole project: -.. code:: cmd - + ```cmd # delete any htmldocs/*.idx file before starting nim doc --project --index:on --git.url: --git.commit: --outdir:htmldocs .nim # this will generate html files, a theindex.html index, css and js under `htmldocs` @@ -54,7 +53,7 @@ Generate HTML documentation for a whole project: # or `$nimcache/htmldocs` with `--usenimcache` which avoids clobbering your sources; # and likewise without `--project`. # Adding `-r` will open in a browser directly. - + ``` Documentation Comments ---------------------- @@ -120,8 +119,8 @@ Example of Nim file input The following examples will generate documentation for this sample *Nim* module, aptly named ``doc/docgen_sample.nim``: -.. code:: nim - :file: docgen_sample.nim + ```nim file=docgen_sample.nim + ``` All the below commands save their output to ``htmldocs`` directory relative to the directory of file; @@ -137,9 +136,9 @@ optionally, an index file. The `doc`:option: command: -.. code:: cmd - + ```cmd nim doc docgen_sample.nim + ``` Partial Output:: ... @@ -159,8 +158,7 @@ HTML -> PDF conversion). The `doc2tex`:option: command: -.. code:: cmd - + ```cmd nim doc2tex docgen_sample.nim cd htmldocs xelatex docgen_sample.tex @@ -169,6 +167,7 @@ The `doc2tex`:option: command: # large documents) to get all labels generated. # That depends on this warning in the end of `xelatex` output: # LaTeX Warning: Label(s) may have changed. Rerun to get cross-references right. + ``` The output is ``docgen_sample.pdf``. @@ -183,9 +182,9 @@ Note that this tool is built off of the `doc`:option: command The `jsondoc`:option: command: -.. code:: cmd - + ```cmd nim jsondoc docgen_sample.nim + ``` Output:: { @@ -209,9 +208,9 @@ renamed to `jsondoc0`:option:. The `jsondoc0`:option: command: -.. code:: cmd - + ```cmd nim jsondoc0 docgen_sample.nim + ``` Output:: [ @@ -249,9 +248,9 @@ the anchor [*]_ of Nim symbol that corresponds to link text. If you have a constant: -.. code:: Nim - + ```Nim const pi* = 3.14 + ``` then it should be referenced in one of the 2 forms: @@ -262,9 +261,9 @@ B. qualified (with symbol kind specification):: For routine kinds there are more options. Consider this definition: -.. code:: Nim - + ```Nim proc foo*(a: int, b: float): string + ``` Generally following syntax is allowed for referencing `foo`: @@ -352,11 +351,11 @@ recognized fine:: (without parameter names, see form A.2 above). E.g. for this signature: - .. code:: Nim - + ```Nim proc binarySearch*[T, K](a: openArray[T]; key: K; cmp: proc (x: T; y: K): int {.closure.}): int ~~ ~~ ~~~~~ + ``` you cannot use names underlined by `~~` so it must be referenced with ``cmp: proc(T, K)``. Hence these forms are valid:: @@ -379,10 +378,10 @@ recognized fine:: .. Note:: A bit special case is operators (as their signature is also defined with `\``): - .. code:: Nim - + ```Nim func `$`(x: MyType): string func `[]`*[T](x: openArray[T]): T + ``` A short form works without additional backticks:: @@ -412,9 +411,9 @@ Related Options Project switch -------------- -.. code:: cmd - + ```cmd nim doc --project filename.nim + ``` This will recursively generate documentation of all Nim modules imported into the input module that belong to the Nimble package that ``filename.nim`` @@ -425,9 +424,9 @@ also be generated. Index switch ------------ -.. code:: cmd - + ```cmd nim doc --index:on filename.nim + ``` This will generate an index of all the exported symbols in the input Nim module, and put it into a neighboring file with the extension of ``.idx``. The @@ -443,9 +442,9 @@ file. See source switch ----------------- -.. code:: cmd - + ```cmd nim doc --git.url: filename.nim + ``` With the `git.url`:option: switch the *See source* hyperlink will appear below each documented item in your source code pointing to the implementation of that @@ -490,9 +489,9 @@ supports highlighting of a few other languages supported by the Usage: -.. code:: cmd - + ```cmd nim rst2html docgen.rst + ``` Output:: You're reading it! @@ -528,7 +527,7 @@ HTML file, most browsers will go to the first one. To differentiate the rest, you will need to use the complex name. A complex name for a callable type is made up of several parts: - (**plain symbol**)(**.type**),(**first param**)?(**,param type**)\* + (**plain symbol**)(**.type**),(**first param**)?(**,param type**)\* The first thing to note is that all callable types have at least a comma, even if they don't have any parameters. If there are parameters, they are diff --git a/doc/docs.rst b/doc/docs.md similarity index 100% rename from doc/docs.rst rename to doc/docs.md diff --git a/doc/docstyle.rst b/doc/docstyle.md similarity index 100% rename from doc/docstyle.rst rename to doc/docstyle.md diff --git a/doc/drnim.rst b/doc/drnim.md similarity index 100% rename from doc/drnim.rst rename to doc/drnim.md diff --git a/doc/estp.rst b/doc/estp.md similarity index 100% rename from doc/estp.rst rename to doc/estp.md diff --git a/doc/filters.rst b/doc/filters.md similarity index 100% rename from doc/filters.rst rename to doc/filters.md diff --git a/doc/hcr.rst b/doc/hcr.md similarity index 100% rename from doc/hcr.rst rename to doc/hcr.md diff --git a/doc/idetools.rst b/doc/idetools.md similarity index 100% rename from doc/idetools.rst rename to doc/idetools.md diff --git a/doc/intern.rst b/doc/intern.md similarity index 99% rename from doc/intern.rst rename to doc/intern.md index 9103c694c..0fd995582 100644 --- a/doc/intern.rst +++ b/doc/intern.md @@ -10,7 +10,7 @@ .. include:: rstcommon.rst .. contents:: - "Abstraction is layering ignorance on top of reality." -- Richard Gabriel +> "Abstraction is layering ignorance on top of reality." -- Richard Gabriel Directory structure @@ -276,8 +276,8 @@ and `exitingDebugSection()`:nim:. #. Compile the temp compiler with `--debugger:native -d:nimDebugUtils`:option: #. Set your desired breakpoints or watchpoints. #. Configure your debugger: - * GDB: execute `source tools/compiler.gdb` at startup - * LLDB execute `command source tools/compiler.lldb` at startup + * GDB: execute `source tools/compiler.gdb` at startup + * LLDB execute `command source tools/compiler.lldb` at startup #. Use one of the scoping helpers like so: .. code-block:: nim diff --git a/doc/koch.rst b/doc/koch.md similarity index 100% rename from doc/koch.rst rename to doc/koch.md diff --git a/doc/lib.rst b/doc/lib.md similarity index 100% rename from doc/lib.rst rename to doc/lib.md diff --git a/doc/manual.rst b/doc/manual.md similarity index 99% rename from doc/manual.rst rename to doc/manual.md index 571379a87..503b9538b 100644 --- a/doc/manual.rst +++ b/doc/manual.md @@ -10,9 +10,9 @@ Nim Manual .. contents:: - "Complexity" seems to be a lot like "energy": you can transfer it from the - end-user to one/some of the other players, but the total amount seems to remain - pretty much constant for a given task. -- Ran +> "Complexity" seems to be a lot like "energy": you can transfer it from the +> end-user to one/some of the other players, but the total amount seems to remain +> pretty much constant for a given task. -- Ran About this document @@ -4025,7 +4025,7 @@ In the standard library every name of a routine that returns a `var` type starts with the prefix `m` per convention. -.. include:: manual/var_t_return.rst +.. include:: manual/var_t_return.md Future directions ~~~~~~~~~~~~~~~~~ diff --git a/doc/manual/var_t_return.rst b/doc/manual/var_t_return.md similarity index 100% rename from doc/manual/var_t_return.rst rename to doc/manual/var_t_return.md diff --git a/doc/manual_experimental.rst b/doc/manual_experimental.md similarity index 99% rename from doc/manual_experimental.rst rename to doc/manual_experimental.md index 3089755cb..3968163c5 100644 --- a/doc/manual_experimental.rst +++ b/doc/manual_experimental.md @@ -505,7 +505,7 @@ The compiler ensures that every code path initializes variables which contain non-nilable pointers. The details of this analysis are still to be specified here. -.. include:: manual_experimental_strictnotnil.rst +.. include:: manual_experimental_strictnotnil.md Aliasing restrictions in parameter passing diff --git a/doc/manual_experimental_strictnotnil.rst b/doc/manual_experimental_strictnotnil.md similarity index 100% rename from doc/manual_experimental_strictnotnil.rst rename to doc/manual_experimental_strictnotnil.md diff --git a/doc/mm.rst b/doc/mm.md similarity index 98% rename from doc/mm.rst rename to doc/mm.md index b6941a901..09b235228 100644 --- a/doc/mm.rst +++ b/doc/mm.md @@ -11,7 +11,7 @@ Nim's Memory Management .. - "The road to hell is paved with good intentions." +> "The road to hell is paved with good intentions." Multi-paradigm Memory Management Strategies diff --git a/doc/nep1.rst b/doc/nep1.md similarity index 100% rename from doc/nep1.rst rename to doc/nep1.md diff --git a/doc/nimc.rst b/doc/nimc.md similarity index 99% rename from doc/nimc.rst rename to doc/nimc.md index aa6650491..28a917d92 100644 --- a/doc/nimc.rst +++ b/doc/nimc.md @@ -11,9 +11,9 @@ .. - "Look at you, hacker. A pathetic creature of meat and bone, panting and - sweating as you run through my corridors. How can you challenge a perfect, - immortal machine?" +> "Look at you, hacker. A pathetic creature of meat and bone, panting and +> sweating as you run through my corridors. How can you challenge a perfect, +> immortal machine?" Introduction diff --git a/doc/nimfix.rst b/doc/nimfix.md similarity index 100% rename from doc/nimfix.rst rename to doc/nimfix.md diff --git a/doc/nimgrep.rst b/doc/nimgrep.md similarity index 99% rename from doc/nimgrep.rst rename to doc/nimgrep.md index 6088a4c45..ff2bf3a8d 100644 --- a/doc/nimgrep.rst +++ b/doc/nimgrep.md @@ -54,7 +54,7 @@ All examples below use default PCRE Regex patterns: nimgrep --excludeDir:'^\.git$' --excludeDir:'^\.hg$' --excludeDir:'^\.svn$' # short: --ed:'^\.git$' --ed:'^\.hg$' --ed:'^\.svn$' -+ To search only in paths containing the `tests` sub-directory recursively:: ++ To search only in paths containing the `tests` sub-directory recursively: .. code:: cmd nimgrep --recursive --includeDir:'(^|/)tests($|/)' diff --git a/doc/niminst.rst b/doc/niminst.md similarity index 100% rename from doc/niminst.rst rename to doc/niminst.md diff --git a/doc/nims.rst b/doc/nims.md similarity index 100% rename from doc/nims.rst rename to doc/nims.md diff --git a/doc/nimsuggest.rst b/doc/nimsuggest.md similarity index 100% rename from doc/nimsuggest.rst rename to doc/nimsuggest.md diff --git a/doc/overview.rst b/doc/overview.md similarity index 86% rename from doc/overview.rst rename to doc/overview.md index e01520d7c..b21eb1e68 100644 --- a/doc/overview.rst +++ b/doc/overview.md @@ -5,5 +5,5 @@ Nim Documentation Overview :Author: Andreas Rumpf :Version: |nimversion| -.. include:: docs.rst +.. include:: docs.md diff --git a/doc/packaging.rst b/doc/packaging.md similarity index 100% rename from doc/packaging.rst rename to doc/packaging.md diff --git a/doc/refc.rst b/doc/refc.md similarity index 100% rename from doc/refc.rst rename to doc/refc.md diff --git a/doc/testament.rst b/doc/testament.md similarity index 100% rename from doc/testament.rst rename to doc/testament.md diff --git a/doc/tools.rst b/doc/tools.md similarity index 100% rename from doc/tools.rst rename to doc/tools.md diff --git a/doc/tut1.rst b/doc/tut1.md similarity index 100% rename from doc/tut1.rst rename to doc/tut1.md diff --git a/doc/tut2.rst b/doc/tut2.md similarity index 99% rename from doc/tut2.rst rename to doc/tut2.md index 725f68dd5..d37d6a16a 100644 --- a/doc/tut2.rst +++ b/doc/tut2.md @@ -13,7 +13,7 @@ Nim Tutorial (Part II) Introduction ============ - "Repetition renders the ridiculous reasonable." -- Norman Wildberger +> "Repetition renders the ridiculous reasonable." -- Norman Wildberger This document is a tutorial for the advanced constructs of the *Nim* programming language. **Note that this document is somewhat obsolete as the** diff --git a/doc/tut3.rst b/doc/tut3.md similarity index 99% rename from doc/tut3.rst rename to doc/tut3.md index c2c956107..2fcfd4220 100644 --- a/doc/tut3.rst +++ b/doc/tut3.md @@ -13,7 +13,7 @@ Nim Tutorial (Part III) Introduction ============ - "With Great Power Comes Great Responsibility." -- Spider Man's Uncle +> "With Great Power Comes Great Responsibility." -- Spider Man's Uncle This document is a tutorial about Nim's macro system. A macro is a function that is executed at compile-time and transforms diff --git a/lib/packages/docutils/highlite.nim b/lib/packages/docutils/highlite.nim index d36b2c877..3af94ab21 100644 --- a/lib/packages/docutils/highlite.nim +++ b/lib/packages/docutils/highlite.nim @@ -125,9 +125,7 @@ proc initGeneralTokenizer*(g: var GeneralTokenizer, buf: cstring) = g.length = 0 g.state = low(TokenClass) g.lang = low(SourceLanguage) - var pos = 0 # skip initial whitespace: - while g.buf[pos] in {' ', '\t'..'\r'}: inc(pos) - g.pos = pos + g.pos = 0 proc initGeneralTokenizer*(g: var GeneralTokenizer, buf: string) = initGeneralTokenizer(g, cstring(buf)) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index cd5f26278..1721674c8 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -8,20 +8,23 @@ # ## ================================== -## rst +## packages/docutils/rst ## ================================== ## ## ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ## Nim-flavored reStructuredText and Markdown ## ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ## -## This module implements a `reStructuredText`:idx: (RST) parser. +## This module implements a `reStructuredText`:idx: (RST) and +## `Markdown`:idx: parser. ## A large subset is implemented with some limitations_ and ## `Nim-specific features`_. -## A few `extra features`_ of the `Markdown`:idx: syntax are -## also supported. +## Both Markdown and RST are mark-up languages whose goal is to +## typeset texts with complex structure, formatting and references +## using simple plaintext representation. ## -## Nim can output the result to HTML [#html]_ or Latex [#latex]_. +## This module is also embedded into Nim compiler; the compiler can output +## the result to HTML [#html]_ or Latex [#latex]_. ## ## .. [#html] commands `nim doc`:cmd: for ``*.nim`` files and ## `nim rst2html`:cmd: for ``*.rst`` files @@ -29,11 +32,13 @@ ## .. [#latex] commands `nim doc2tex`:cmd: for ``*.nim`` and ## `nim rst2tex`:cmd: for ``*.rst``. ## -## If you are new to RST please consider reading the following: +## If you are new to Markdown/RST please consider reading the following: ## -## 1) a short `quick introduction`_ -## 2) an `RST reference`_: a comprehensive cheatsheet for RST -## 3) a more formal 50-page `RST specification`_. +## 1) `Markdown Basic Syntax`_ +## 2) a long specification of Markdown: `CommonMark Spec`_ +## 3) a short `quick introduction`_ to RST +## 4) an `RST reference`_: a comprehensive cheatsheet for RST +## 5) a more formal 50-page `RST specification`_. ## ## Features ## -------- @@ -120,7 +125,13 @@ ## ## * emoji / smiley symbols ## * Markdown tables -## * Markdown code blocks +## * Markdown code blocks. For them the same additional arguments as for RST +## code blocks can be provided (e.g. `test` or `number-lines`) but with +## a one-line syntax like this:: +## +## ```nim test number-lines=10 +## echo "ok" +## ``` ## * Markdown links ## * Markdown headlines ## * Markdown block quotes @@ -211,6 +222,8 @@ ## See `packages/docutils/rstgen module `_ to know how to ## generate HTML or Latex strings to embed them into your documents. ## +## .. _Markdown Basic Syntax: https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax +## .. _CommonMark Spec: https://spec.commonmark.org/0.30 ## .. _quick introduction: https://docutils.sourceforge.io/docs/user/rst/quickstart.html ## .. _RST reference: https://docutils.sourceforge.io/docs/user/rst/quickref.html ## .. _RST specification: https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html @@ -253,6 +266,7 @@ type MsgKind* = enum ## the possible messages meCannotOpenFile = "cannot open '$1'", meExpected = "'$1' expected", + meMissingClosing = "$1", meGridTableNotImplemented = "grid table is not implemented", meMarkdownIllformedTable = "illformed delimiter row of a Markdown table", meIllformedTable = "Illformed table: $1", @@ -323,7 +337,10 @@ const ":geek:": "icon_e_geek", ":ugeek:": "icon_e_ugeek" } - SandboxDirAllowlist = ["image", "code", "code-block"] + SandboxDirAllowlist = [ + "image", "code", "code-block", "admonition", "attention", "caution", + "container", "contents", "danger", "default-role", "error", "figure", + "hint", "important", "index", "note", "role", "tip", "title", "warning"] type TokType = enum @@ -1616,35 +1633,89 @@ proc parseUntil(p: var RstParser, father: PRstNode, postfix: string, inc p.idx else: rstMessage(p, meExpected, postfix, line, col) +proc parseMarkdownCodeblockFields(p: var RstParser): PRstNode = + ## Parses additional (after language string) code block parameters + ## in a format *suggested* in the `CommonMark Spec`_ with handling of `"`. + if currentTok(p).kind == tkIndent: + result = nil + else: + result = newRstNode(rnFieldList) + while currentTok(p).kind != tkIndent: + if currentTok(p).kind == tkWhite: + inc p.idx + else: + let field = newRstNode(rnField) + var fieldName = "" + while currentTok(p).kind notin {tkWhite, tkIndent, tkEof} and + currentTok(p).symbol != "=": + fieldName.add currentTok(p).symbol + inc p.idx + field.add(newRstNode(rnFieldName, @[newLeaf(fieldName)])) + if currentTok(p).kind == tkWhite: inc p.idx + let fieldBody = newRstNode(rnFieldBody) + if currentTok(p).symbol == "=": + inc p.idx + if currentTok(p).kind == tkWhite: inc p.idx + var fieldValue = "" + if currentTok(p).symbol == "\"": + while true: + fieldValue.add currentTok(p).symbol + inc p.idx + if currentTok(p).kind == tkEof: + rstMessage(p, meExpected, "\"") + elif currentTok(p).symbol == "\"": + fieldValue.add "\"" + inc p.idx + break + else: + while currentTok(p).kind notin {tkWhite, tkIndent, tkEof}: + fieldValue.add currentTok(p).symbol + inc p.idx + fieldBody.add newLeaf(fieldValue) + field.add(fieldBody) + result.add(field) + proc parseMarkdownCodeblock(p: var RstParser): PRstNode = result = newRstNodeA(p, rnCodeBlock) + let line = curLine(p) + let baseCol = currentTok(p).col + let baseSym = currentTok(p).symbol # usually just ``` + inc p.idx result.info = lineInfo(p) var args = newRstNode(rnDirArg) + var fields: PRstNode = nil if currentTok(p).kind == tkWord: args.add(newLeaf(p)) inc p.idx + fields = parseMarkdownCodeblockFields(p) else: args = nil var n = newLeaf("") while true: - case currentTok(p).kind - of tkEof: - rstMessage(p, meExpected, "```") + if currentTok(p).kind == tkEof: + rstMessage(p, meMissingClosing, + "$1 (started at line $2)" % [baseSym, $line]) break - of tkPunct, tkAdornment: - if currentTok(p).symbol == "```": - inc p.idx - break - else: - n.text.add(currentTok(p).symbol) - inc p.idx + elif nextTok(p).kind in {tkPunct, tkAdornment} and + nextTok(p).symbol[0] == baseSym[0] and + nextTok(p).symbol.len >= baseSym.len: + inc p.idx, 2 + break + elif currentTok(p).kind == tkIndent: + n.text.add "\n" + if currentTok(p).ival > baseCol: + n.text.add " ".repeat(currentTok(p).ival - baseCol) + elif currentTok(p).ival < baseCol: + rstMessage(p, mwRstStyle, + "unexpected de-indentation in Markdown code block") + inc p.idx else: n.text.add(currentTok(p).symbol) inc p.idx var lb = newRstNode(rnLiteralBlock) lb.add(n) result.add(args) - result.add(PRstNode(nil)) + result.add(fields) result.add(lb) proc parseMarkdownLink(p: var RstParser; father: PRstNode): bool = @@ -1730,6 +1801,12 @@ proc parseFootnoteName(p: var RstParser, reference: bool): PRstNode = inc i p.idx = i +proc isMarkdownCodeBlock(p: RstParser): bool = + result = (roSupportMarkdown in p.s.options and + currentTok(p).kind in {tkPunct, tkAdornment} and + currentTok(p).symbol[0] == '`' and # tilde ~ is not supported + currentTok(p).symbol.len >= 3) + proc parseInline(p: var RstParser, father: PRstNode) = var n: PRstNode # to be used in `if` condition let saveIdx = p.idx @@ -1755,8 +1832,7 @@ proc parseInline(p: var RstParser, father: PRstNode) = addAnchorRst(p, name = linkName(n), refn = refn, reset = true, anchorType=manualInlineAnchor) father.add(n) - elif roSupportMarkdown in p.s.options and currentTok(p).symbol == "```": - inc p.idx + elif isMarkdownCodeBlock(p): father.add(parseMarkdownCodeblock(p)) elif isInlineMarkupStart(p, "``"): var n = newRstNode(rnInlineLiteral) @@ -1816,8 +1892,7 @@ proc parseInline(p: var RstParser, father: PRstNode) = return parseWordOrRef(p, father) of tkAdornment, tkOther, tkWhite: - if roSupportMarkdown in p.s.options and currentTok(p).symbol == "```": - inc p.idx + if isMarkdownCodeBlock(p): father.add(parseMarkdownCodeblock(p)) return if roSupportSmilies in p.s.options: @@ -2194,7 +2269,7 @@ proc findPipe(p: RstParser, start: int): bool = proc whichSection(p: RstParser): RstNodeKind = if currentTok(p).kind in {tkAdornment, tkPunct}: # for punctuation sequences that can be both tkAdornment and tkPunct - if roSupportMarkdown in p.s.options and currentTok(p).symbol == "```": + if isMarkdownCodeBlock(p): return rnCodeBlock elif currentTok(p).symbol == "::": return rnLiteralBlock @@ -2633,7 +2708,9 @@ proc parseSimpleTable(p: var RstParser): PRstNode = # fix rnTableDataCell -> rnTableHeaderCell for previous table rows: for nRow in 0 ..< result.sons.len: for nCell in 0 ..< result.sons[nRow].len: - result.sons[nRow].sons[nCell].kind = rnTableHeaderCell + template cell: PRstNode = result.sons[nRow].sons[nCell] + cell = PRstNode(kind: rnTableHeaderCell, sons: cell.sons, + span: cell.span, anchor: cell.anchor) if currentTok(p).kind == tkEof: break let tabRow = parseSimpleTableRow(p, cols, colChar) result.add tabRow @@ -2892,11 +2969,19 @@ proc parseSection(p: var RstParser, result: PRstNode) = if currInd(p) == currentTok(p).ival: inc p.idx elif currentTok(p).ival > currInd(p): - pushInd(p, currentTok(p).ival) - var a = newRstNodeA(p, rnBlockQuote) - parseSection(p, a) - result.add(a) - popInd(p) + if roPreferMarkdown in p.s.options: # Markdown => normal paragraphs + if currentTok(p).ival - currInd(p) >= 4: + rstMessage(p, mwRstStyle, + "Markdown indented code not implemented") + pushInd(p, currentTok(p).ival) + parseSection(p, result) + popInd(p) + else: # RST mode => block quotes + pushInd(p, currentTok(p).ival) + var a = newRstNodeA(p, rnBlockQuote) + parseSection(p, a) + result.add(a) + popInd(p) else: while currentTok(p).kind != tkEof and nextTok(p).kind == tkIndent: inc p.idx # skip blank lines diff --git a/tests/stdlib/thighlite.nim b/tests/stdlib/thighlite.nim index 4113a6a80..5134215c1 100644 --- a/tests/stdlib/thighlite.nim +++ b/tests/stdlib/thighlite.nim @@ -12,6 +12,12 @@ block: # Nim tokenizing @[("\"\"\"ok1\\nok2\\nok3\"\"\"", gtLongStringLit) ]) + test "whitespace at beginning of line is preserved": + check(" discard 1".tokenize(langNim) == + @[(" ", gtWhitespace), ("discard", gtKeyword), (" ", gtWhitespace), + ("1", gtDecNumber) + ]) + block: # Cmd (shell) tokenizing test "cmd with dollar and output": check( diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index 24bc03027..823697336 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -24,8 +24,11 @@ import unittest, strutils import std/private/miscdollars import os +const preferMarkdown = {roPreferMarkdown, roSupportMarkdown, roNimFile, roSandboxDisabled} +const preferRst = {roSupportMarkdown, roNimFile, roSandboxDisabled} + proc toAst(input: string, - rstOptions: RstParseOptions = {roPreferMarkdown, roSupportMarkdown, roNimFile, roSandboxDisabled}, + rstOptions: RstParseOptions = preferMarkdown, error: ref string = nil, warnings: ref seq[string] = nil): string = ## If `error` is nil then no errors should be generated. @@ -451,7 +454,7 @@ suite "RST parsing": > - y > > Paragraph. - """.toAst == dedent""" + """.toAst(rstOptions = preferRst) == dedent""" rnMarkdownBlockQuote rnMarkdownBlockQuoteItem quotationDepth=1 rnInner @@ -468,6 +471,93 @@ suite "RST parsing": rnLeaf '.' """) + test "Markdown code blocks with more > 3 backticks": + check(dedent""" + ```` + let a = 1 + ``` + ````""".toAst == + dedent""" + rnCodeBlock + [nil] + [nil] + rnLiteralBlock + rnLeaf ' + let a = 1 + ```' + """) + + test "Markdown code blocks with Nim-specific arguments": + check(dedent""" + ```nim number-lines=1 test + let a = 1 + ```""".toAst == + dedent""" + rnCodeBlock + rnDirArg + rnLeaf 'nim' + rnFieldList + rnField + rnFieldName + rnLeaf 'number-lines' + rnFieldBody + rnLeaf '1' + rnField + rnFieldName + rnLeaf 'test' + rnFieldBody + rnLiteralBlock + rnLeaf ' + let a = 1' + """) + + check(dedent""" + ```nim test = "nim c $1" number-lines = 1 + let a = 1 + ```""".toAst == + dedent""" + rnCodeBlock + rnDirArg + rnLeaf 'nim' + rnFieldList + rnField + rnFieldName + rnLeaf 'test' + rnFieldBody + rnLeaf '"nim c $1"' + rnField + rnFieldName + rnLeaf 'number-lines' + rnFieldBody + rnLeaf '1' + rnLiteralBlock + rnLeaf ' + let a = 1' + """) + + test "additional indentation < 4 spaces is handled fine": + check(dedent""" + Indentation + + ```nim + let a = 1 + ```""".toAst == + dedent""" + rnInner + rnParagraph + rnLeaf 'Indentation' + rnParagraph + rnCodeBlock + rnDirArg + rnLeaf 'nim' + [nil] + rnLiteralBlock + rnLeaf ' + let a = 1' + """) + # | | + # | \ indentation of exactly two spaces before 'let a = 1' + test "option list has priority over definition list": check(dedent""" --defusages @@ -562,7 +652,7 @@ suite "RST parsing": notAcomment1 notAcomment2 - someParagraph""".toAst == + someParagraph""".toAst(rstOptions = preferRst) == dedent""" rnInner rnBlockQuote @@ -574,6 +664,25 @@ suite "RST parsing": rnLeaf 'someParagraph' """) + test "check that additional line right after .. ends comment (Markdown mode)": + # in Markdown small indentation does not matter so this should + # just be split to 2 paragraphs. + check(dedent""" + .. + + notAcomment1 + notAcomment2 + someParagraph""".toAst == + dedent""" + rnInner + rnInner + rnLeaf 'notAcomment1' + rnLeaf ' ' + rnLeaf 'notAcomment2' + rnParagraph + rnLeaf 'someParagraph' + """) + test "but blank lines after 2nd non-empty line don't end the comment": check(dedent""" .. @@ -592,7 +701,7 @@ suite "RST parsing": .. - someBlockQuote""".toAst == + someBlockQuote""".toAst(rstOptions = preferRst) == dedent""" rnInner rnAdmonition adType=note diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index d91b5615e..7fae4ba8b 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -9,8 +9,13 @@ import ../../lib/packages/docutils/rst import unittest, strutils, strtabs import std/private/miscdollars +const + NoSandboxOpts = {roPreferMarkdown, roSupportMarkdown, roNimFile, roSandboxDisabled} + preferMarkdown = {roPreferMarkdown, roSupportMarkdown, roNimFile} + preferRst = {roSupportMarkdown, roNimFile} + proc toHtml(input: string, - rstOptions: RstParseOptions = {roPreferMarkdown, roSupportMarkdown, roNimFile}, + rstOptions: RstParseOptions = preferMarkdown, error: ref string = nil, warnings: ref seq[string] = nil): string = ## If `error` is nil then no errors should be generated. @@ -47,9 +52,6 @@ proc optionListLabel(opt: string): string = opt & "
            " -const - NoSandboxOpts = {roPreferMarkdown, roSupportMarkdown, roNimFile, roSandboxDisabled} - suite "YAML syntax highlighting": test "Basics": @@ -1180,7 +1182,7 @@ Test1 "input(8, 4) Warning: language 'anotherLang' not supported" ]) check(output == "
            anything
            " & - "

            \nsomeCode\n

            ") + "

            \nsomeCode

            ") test "RST admonitions": # check that all admonitions are implemented @@ -1321,7 +1323,7 @@ Test1 That was a transition. """ let output1 = input1.toHtml( - NoSandboxOpts + preferRst ) doAssert "

            text""" & "\n

        Authors:Andreas Rumpf, Zahary Karadjov
        Version:|nimversion|
        ") test "Field list: body after newline": - let output = dedent """ + let output = dedent""" :field: text1""".toHtml check " Date: Fri, 15 Jul 2022 20:56:05 +0300 Subject: [PATCH 1705/2472] Implement type command (#19944) * Implement type command - this will be mapped to textDocument/typeDefinition in LSP protocol. It will be very useful for `nim` in particular because typically most of the time the type is inferred. * Update nimsuggest/nimsuggest.nim Co-authored-by: Andreas Rumpf --- compiler/options.nim | 4 +++- nimsuggest/nimsuggest.nim | 12 +++++++++- nimsuggest/tester.nim | 2 +- nimsuggest/tests/tv3_typeDefinition.nim | 32 +++++++++++++++++++++++++ 4 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 nimsuggest/tests/tv3_typeDefinition.nim diff --git a/compiler/options.nim b/compiler/options.nim index d567927cb..9031e86c1 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -192,7 +192,7 @@ type IdeCmd* = enum ideNone, ideSug, ideCon, ideDef, ideUse, ideDus, ideChk, ideChkFile, ideMod, ideHighlight, ideOutline, ideKnown, ideMsg, ideProject, ideGlobalSymbols, - ideRecompile, ideChanged + ideRecompile, ideChanged, ideType Feature* = enum ## experimental features; DO NOT RENAME THESE! implicitDeref, @@ -1006,6 +1006,7 @@ proc parseIdeCmd*(s: string): IdeCmd = of "globalSymbols": ideGlobalSymbols of "recompile": ideRecompile of "changed": ideChanged + of "type": ideType else: ideNone proc `$`*(c: IdeCmd): string = @@ -1027,6 +1028,7 @@ proc `$`*(c: IdeCmd): string = of ideGlobalSymbols: "globalSymbols" of ideRecompile: "recompile" of ideChanged: "changed" + of ideType: "type" proc floatInt64Align*(conf: ConfigRef): int16 = ## Returns either 4 or 8 depending on reasons. diff --git a/nimsuggest/nimsuggest.nim b/nimsuggest/nimsuggest.nim index a18b2f960..6f35be925 100644 --- a/nimsuggest/nimsuggest.nim +++ b/nimsuggest/nimsuggest.nim @@ -457,6 +457,7 @@ proc execCmd(cmd: string; graph: ModuleGraph; cachedMsgs: CachedMsgs) = of "globalsymbols": conf.ideCmd = ideGlobalSymbols of "chkfile": conf.ideCmd = ideChkFile of "recompile": conf.ideCmd = ideRecompile + of "type": conf.ideCmd = ideType else: err() var dirtyfile = "" var orig = "" @@ -786,7 +787,7 @@ proc executeNoHooksV3(cmd: IdeCmd, file: AbsoluteFile, dirtyfile: AbsoluteFile, graph.unmarkAllDirty() # these commands require partially compiled project - elif cmd in {ideSug, ideOutline, ideHighlight, ideDef, ideChkFile} and + elif cmd in {ideSug, ideOutline, ideHighlight, ideDef, ideChkFile, ideType} and (graph.needsCompilation(fileIndex) or cmd == ideSug): # for ideSug use v2 implementation if cmd == ideSug: @@ -802,6 +803,15 @@ proc executeNoHooksV3(cmd: IdeCmd, file: AbsoluteFile, dirtyfile: AbsoluteFile, let (sym, info) = graph.findSymData(file, line, col) if sym != nil: graph.suggestResult(sym, sym.info) + of ideType: + let (sym, _) = graph.findSymData(file, line, col) + if sym != nil: + let typeSym = sym.typ.sym + if typeSym != nil: + graph.suggestResult(typeSym, typeSym.info, ideType) + elif sym.typ.len != 0: + let genericType = sym.typ[0].sym + graph.suggestResult(genericType, genericType.info, ideType) of ideUse, ideDus: let symbol = graph.findSymData(file, line, col).sym if symbol != nil: diff --git a/nimsuggest/tester.nim b/nimsuggest/tester.nim index fea0a8d45..6e068e067 100644 --- a/nimsuggest/tester.nim +++ b/nimsuggest/tester.nim @@ -282,7 +282,7 @@ proc runEpcTest(filename: string): int = socket.sendEpcStr(req) let sx = parseSexp(socket.recvEpc()) if not req.startsWith("mod "): - let answer = sexpToAnswer(sx) + let answer = if sx[2].kind == SNil: "" else: sexpToAnswer(sx) doReport(filename, answer, resp, report) socket.sendEpcStr "return arg" diff --git a/nimsuggest/tests/tv3_typeDefinition.nim b/nimsuggest/tests/tv3_typeDefinition.nim new file mode 100644 index 000000000..f86d12cc6 --- /dev/null +++ b/nimsuggest/tests/tv3_typeDefinition.nim @@ -0,0 +1,32 @@ +# tests v3 + +type + Foo* = ref object of RootObj + bar*: string + +proc test(ff: Foo) = + echo f#[!]#f.bar + +type + Fo#[!]#o2* = ref object of RootObj + +type + FooGeneric[T] = ref object of RootObj + bar*: T + +let fooGeneric = FooGeneric[string]() +echo fo#[!]#oGeneric.bar + +# bad type +echo unde#[!]#fined + +discard """ +$nimsuggest --v3 --tester $file +>type $1 +type skType tv3_typeDefinition.Foo Foo $file 4 2 "" 100 +>type $2 +type skType tv3_typeDefinition.Foo2 Foo2 $file 11 2 "" 100 +>type $3 +type skType tv3_typeDefinition.FooGeneric FooGeneric $file 14 2 "" 100 +>type $4 +""" From b0b9a3e5fade002b8f117f7ffb4d8fb93686401d Mon Sep 17 00:00:00 2001 From: Ivan Yonchovski Date: Fri, 15 Jul 2022 20:56:33 +0300 Subject: [PATCH 1706/2472] Use module actual file instead of PSym.info (#19956) After this you can do goto module from module import --- compiler/suggest.nim | 30 ++++++++++++++++++------------ nimsuggest/tests/tv3_import.nim | 7 +++++++ 2 files changed, 25 insertions(+), 12 deletions(-) create mode 100644 nimsuggest/tests/tv3_import.nim diff --git a/compiler/suggest.nim b/compiler/suggest.nim index 4b48ef5bb..7e7d876fb 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -157,19 +157,25 @@ proc symToSuggest*(g: ModuleGraph; s: PSym, isLocal: bool, section: IdeCmd, info result.forth = "" when defined(nimsuggest) and not defined(noDocgen) and not defined(leanCompiler): result.doc = extractDocComment(g, s) - let infox = - if useSuppliedInfo or section in {ideUse, ideHighlight, ideOutline}: - info - else: - s.info - result.filePath = toFullPath(g.config, infox) - result.line = toLinenumber(infox) - result.column = toColumn(infox) + if s.kind == skModule and s.ast.len != 0 and section != ideHighlight: + result.filePath = toFullPath(g.config, s.ast[0].info) + result.line = 1 + result.column = 0 + result.tokenLen = 0 + else: + let infox = + if useSuppliedInfo or section in {ideUse, ideHighlight, ideOutline}: + info + else: + s.info + result.filePath = toFullPath(g.config, infox) + result.line = toLinenumber(infox) + result.column = toColumn(infox) + result.tokenLen = if section != ideHighlight: + s.name.s.len + else: + getTokenLenFromSource(g.config, s.name.s, infox) result.version = g.config.suggestVersion - result.tokenLen = if section != ideHighlight: - s.name.s.len - else: - getTokenLenFromSource(g.config, s.name.s, infox) proc `$`*(suggest: Suggest): string = result = $suggest.section diff --git a/nimsuggest/tests/tv3_import.nim b/nimsuggest/tests/tv3_import.nim new file mode 100644 index 000000000..3c128f85b --- /dev/null +++ b/nimsuggest/tests/tv3_import.nim @@ -0,0 +1,7 @@ +import tv#[!]#3 + +discard """ +$nimsuggest --v3 --tester $file +>def $1 +def skModule tv3 */tv3.nim 1 0 "" 100 +""" From 60dd38c50262774cadb1d365e649696380c0595e Mon Sep 17 00:00:00 2001 From: David Krause Date: Sat, 16 Jul 2022 11:59:58 +0200 Subject: [PATCH 1707/2472] make AsyncSocket.getPeerAddr work ; fix #15022 (#20038) Signed-off-by: David Krause --- lib/pure/asyncdispatch.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 0d406b271..ed786cefb 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -1733,6 +1733,8 @@ when defined(windows) or defined(nimdoc): proc (fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) = if not retFuture.finished: if errcode == OSErrorCode(-1): + const SO_UPDATE_CONNECT_CONTEXT = 0x7010 + socket.SocketHandle.setSockOptInt(SOL_SOCKET, SO_UPDATE_CONNECT_CONTEXT, 1) # 15022 retFuture.complete() else: retFuture.fail(newException(OSError, osErrorMsg(errcode))) From cf78c02b70ed105a173cce956ffab90bd57468a1 Mon Sep 17 00:00:00 2001 From: Amjad Ben Hedhili Date: Sat, 16 Jul 2022 21:51:27 +0100 Subject: [PATCH 1708/2472] Make `random.rand` work with `Ordinal` (#20043) * Make `random.rand` work with `Ordinal` * Add changelog entry * It's fine to cast to char --- changelog.md | 1 + lib/pure/random.nim | 15 +++++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/changelog.md b/changelog.md index 47479fc5c..c1dc6e27d 100644 --- a/changelog.md +++ b/changelog.md @@ -39,6 +39,7 @@ becomes an alias for `addr`. - `std/smtp` sends `ehlo` first. If the mail server does not understand, it sends `helo` as a fallback. - Changed mimedb to use an `OrderedTable` instead of `OrderedTableRef`, to use it in a const. - `strutils.find` now use and default to `last=-1` for whole string searches, making limiting it to just the first char (`last=0`) valid. +- `random.rand` now works with `Ordinal`s. [//]: # "Additions:" - Added `IsoWeekRange`, a range type to represent the number of weeks in an ISO week-based year. diff --git a/lib/pure/random.nim b/lib/pure/random.nim index b8aeb86e0..832d54e3d 100644 --- a/lib/pure/random.nim +++ b/lib/pure/random.nim @@ -353,8 +353,8 @@ proc rand*[T: Ordinal or SomeFloat](x: HSlice[T, T]): T = result = rand(state, x) -proc rand*[T: SomeInteger](t: typedesc[T]): T = - ## Returns a random integer in the range `low(T)..high(T)`. +proc rand*[T: Ordinal](t: typedesc[T]): T = + ## Returns a random Ordinal in the range `low(T)..high(T)`. ## ## If `randomize <#randomize>`_ has not been called, the sequence of random ## numbers returned from this proc will always be the same. @@ -368,13 +368,16 @@ proc rand*[T: SomeInteger](t: typedesc[T]): T = ## that accepts a slice runnableExamples: randomize(567) + type E = enum a, b, c, d if false: # implementation defined - assert rand(int8) == -42 - assert rand(uint32) == 578980729'u32 - assert rand(range[1..16]) == 11 + assert rand(E) in a..d + assert rand(char) in low(char)..high(char) + assert rand(int8) in low(int8)..high(int8) + assert rand(uint32) in low(uint32)..high(uint32) + assert rand(range[1..16]) in 1..16 # pending csources >= 1.4.0 or fixing https://github.com/timotheecour/Nim/issues/251#issuecomment-831599772, # use `runnableExamples("-r:off")` instead of `if false` - when T is range: + when T is range or T is enum or T is bool: result = rand(state, low(T)..high(T)) else: result = cast[T](state.next) From 094d86f99783f1366394fa444277c417dcbd1af3 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Sat, 16 Jul 2022 22:56:54 +0200 Subject: [PATCH 1709/2472] testament: use full test name in skips [backport] (#19937) testament: use full test name in skips --- testament/testament.nim | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/testament/testament.nim b/testament/testament.nim index d2b0e7fb4..98fc3c2aa 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -258,6 +258,15 @@ Tests failed and allowed to fail: $3 / $1
        Tests skipped: $4 / $1
        """ % [$x.total, $x.passed, $x.failedButAllowed, $x.skipped] +proc testName(test: TTest, target: TTarget, extraOptions: string, allowFailure: bool): string = + var name = test.name.replace(DirSep, '/') + name.add ' ' & $target + if allowFailure: + name.add " (allowed to fail) " + if test.options.len > 0: name.add ' ' & test.options + if extraOptions.len > 0: name.add ' ' & extraOptions + name.strip() + proc addResult(r: var TResults, test: TTest, target: TTarget, extraOptions, expected, given: string, successOrig: TResultEnum, allowFailure = false, givenSpec: ptr TSpec = nil) = @@ -265,13 +274,7 @@ proc addResult(r: var TResults, test: TTest, target: TTarget, # instead of having to pass individual fields, or abusing existing ones like expected vs given. # test.name is easier to find than test.name.extractFilename # A bit hacky but simple and works with tests/testament/tshould_not_work.nim - var name = test.name.replace(DirSep, '/') - name.add ' ' & $target - if allowFailure: - name.add " (allowed to fail) " - if test.options.len > 0: name.add ' ' & test.options - if extraOptions.len > 0: name.add ' ' & extraOptions - + let name = testName(test, target, extraOptions, allowFailure) let duration = epochTime() - test.startTime let success = if test.spec.timeout > 0.0 and duration > test.spec.timeout: reTimeout else: successOrig @@ -470,6 +473,9 @@ proc equalModuloLastNewline(a, b: string): bool = proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, target: TTarget, extraOptions: string, nimcache: string) = test.startTime = epochTime() + if testName(test, target, extraOptions, false) in skips: + test.spec.err = reDisabled + if test.spec.err in {reDisabled, reJoined}: r.addResult(test, target, extraOptions, "", "", test.spec.err) inc(r.skipped) From c43a377057c56d5213eb298feba883363f2b2a26 Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Sun, 17 Jul 2022 07:41:18 +1000 Subject: [PATCH 1710/2472] Make imports/exports not be a dropdown in sidebar (#19907) * Don't make a section be a dropdown if it has no child links * - Cleaned up code - Updated tests * Document what the 'if' is checking --- compiler/docgen.nim | 17 ++++++++++++++--- config/nimdoc.cfg | 8 ++++++++ nimdoc/testproject/expected/testproject.html | 7 +------ 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index ed5fe06ef..d728c535f 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -1474,10 +1474,21 @@ proc genSection(d: PDoc, kind: TSymKind, groupedToc = false) = for item in d.tocSimple[kind].sorted(cmp): d.toc2[kind].add item.content - d.toc[kind] = getConfigVar(d.conf, "doc.section.toc") % [ - "sectionid", $ord(kind), "sectionTitle", title, - "sectionTitleID", $(ord(kind) + 50), "content", d.toc2[kind]] + let sectionValues = @[ + "sectionID", $ord(kind), "sectionTitleID", $(ord(kind) + 50), + "sectionTitle", title + ] + # Check if the toc has any children + if d.toc2[kind] != "": + # Use the dropdown version instead and store the children in the dropdown + d.toc[kind] = getConfigVar(d.conf, "doc.section.toc") % (sectionValues & @[ + "content", d.toc2[kind] + ]) + else: + # Just have the link + d.toc[kind] = getConfigVar(d.conf, "doc.section.toc_item") % sectionValues + proc relLink(outDir: AbsoluteDir, destFile: AbsoluteFile, linkto: RelativeFile): string = $relativeTo(outDir / linkto, destFile.splitFile().dir, '/') diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index 3b0afe8d6..7aee50d8c 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -16,6 +16,14 @@ doc.section = """ """ +# Just a single item in the TOC (e.g. imports, exports) +doc.section.toc_item = """ +
      • + $sectionTitle +
        • +""" + +# This is a section (e.g. procs, types) in the TOC which gets turned into a drop down doc.section.toc = """
      • diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html index 24dfdc13f..5fb9e697c 100644 --- a/nimdoc/testproject/expected/testproject.html +++ b/nimdoc/testproject/expected/testproject.html @@ -51,12 +51,7 @@
          baz @@ -539,7 +540,15 @@
          -
          +
          + 
          proc bar(f: FooBuzz) {....raises: [], tags: [], forbids: [].}
          +
          + + + +
          +
          +
          proc bar[T](a, b: T): T
          diff --git a/nimdoc/testproject/expected/testproject.idx b/nimdoc/testproject/expected/testproject.idx index 4d0e63c0c..4d0172313 100644 --- a/nimdoc/testproject/expected/testproject.idx +++ b/nimdoc/testproject/expected/testproject.idx @@ -7,6 +7,7 @@ bar testproject.html#bar,T,T testproject: bar[T](a, b: T): T baz testproject.html#baz,T,T testproject: baz[T](a, b: T): T buzz testproject.html#buzz,T,T testproject: buzz[T](a, b: T): T FooBuzz testproject.html#FooBuzz testproject: FooBuzz +bar testproject.html#bar testproject: bar(f: FooBuzz) aVariable testproject.html#aVariable testproject: aVariable A testproject.html#A testproject: A B testproject.html#B testproject: B diff --git a/nimdoc/testproject/expected/theindex.html b/nimdoc/testproject/expected/theindex.html index 402541540..9f73040f2 100644 --- a/nimdoc/testproject/expected/theindex.html +++ b/nimdoc/testproject/expected/theindex.html @@ -78,6 +78,8 @@
        bar:
        • testproject: bar(f: FooBuzz)
          • +
        • testproject: bar[T](a, b: T): T
        • testproject: bar(): untyped
          • diff --git a/nimdoc/testproject/testproject.nim b/nimdoc/testproject/testproject.nim index 57960c69f..d236552ac 100644 --- a/nimdoc/testproject/testproject.nim +++ b/nimdoc/testproject/testproject.nim @@ -42,6 +42,11 @@ proc buzz*[T](a, b: T): T {.deprecated: "since v0.20".} = type FooBuzz* {.deprecated: "FooBuzz msg".} = int +using f: FooBuzz + +proc bar*(f) = # `f` should be expanded to `f: FooBuzz` + discard + import std/macros var aVariable*: array[1, int] From 28bed059aadf808fe56eaf2b36274bdf6baec053 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 12 Dec 2022 11:46:51 +0100 Subject: [PATCH 2268/2472] closes #20808 (#21077) --- tests/effects/tfuncs_cannot_mutate3.nim | 35 +++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 tests/effects/tfuncs_cannot_mutate3.nim diff --git a/tests/effects/tfuncs_cannot_mutate3.nim b/tests/effects/tfuncs_cannot_mutate3.nim new file mode 100644 index 000000000..029152029 --- /dev/null +++ b/tests/effects/tfuncs_cannot_mutate3.nim @@ -0,0 +1,35 @@ +discard """ + errormsg: "cannot mutate location kid.parent within a strict func" + line: 16 +""" + +{.experimental: "strictFuncs".} + +type + Node = ref object + name: string + kids: seq[Node] + parent: Node + +func initParents(tree: Node) = + for kid in tree.kids: + kid.parent = tree + initParents(kid) + +proc process(intro: Node): Node = + var tree = Node(name: "root", kids: @[ + intro, + Node(name: "one", kids: @[ + Node(name: "two"), + Node(name: "three"), + ]), + Node(name: "four"), + ]) + initParents(tree) + +proc main() = + var intro = Node(name: "intro") + var tree = process(intro) + echo intro.parent.name + +main() From 19923aa8b7bb93f687b71942b2ab3de39cba2f5a Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Mon, 12 Dec 2022 21:14:24 +0800 Subject: [PATCH 2269/2472] work around chronos (#21079) * work around chronos chronos has tests against the devel branch, it will be eventually fine. * Apply suggestions from code review * Update testament/important_packages.nim --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 75283edbc..d0c87919b 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -52,7 +52,7 @@ pkg "cascade" pkg "cello", url = "https://github.com/nim-lang/cello", useHead = true pkg "chroma" pkg "chronicles", "nimble install -y stew@#head; nim c -o:chr -r chronicles.nim" -pkg "chronos", "nimble install -y bearssl@#head stew@#head httputils@#head; nim c -r -d:release tests/testall" +pkg "chronos", "nim c -r -d:release tests/testall", url = "https://github.com/nim-lang/nim-chronos" pkg "cligen", "nim c --path:. -r cligen.nim" pkg "combparser", "nimble test --gc:orc" pkg "compactdict" From fd7da7da629f9f569503220b656047fb2f8da125 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Mon, 12 Dec 2022 21:57:27 +0800 Subject: [PATCH 2270/2472] add a changelog for nimble (#21078) --- changelog.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/changelog.md b/changelog.md index 8efb7200c..5762d18ad 100644 --- a/changelog.md +++ b/changelog.md @@ -292,6 +292,4 @@ ## Tool changes -- Nim now supports Nimble version 0.14 which added support for lock-files. This is done by - a simple configuration change setting that you can do yourself too. In `$nim/config/nim.cfg` - replace `pkgs` by `pkgs2`. +- Nim now ships Nimble version 0.14 which added support for lock-files. Libraries are stored in `$nimbleDir/pkgs2` (it was `$nimbleDir/pkgs`). From 584cfa6af81303caeaffe4084472167ad47216fe Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Mon, 12 Dec 2022 22:59:09 +0800 Subject: [PATCH 2271/2472] remove implicitDeref feature enum (#21075) remove implicitDeref feature --- compiler/options.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/compiler/options.nim b/compiler/options.nim index a94405e05..7e4f6898f 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -196,7 +196,6 @@ type ideRecompile, ideChanged, ideType, ideDeclaration Feature* = enum ## experimental features; DO NOT RENAME THESE! - implicitDeref, # deadcode; remains here for backwards compatibility dotOperators, callOperator, parallel, @@ -457,7 +456,7 @@ when false: fn(globalOptions) fn(selectedGC) -const oldExperimentalFeatures* = {implicitDeref, dotOperators, callOperator, parallel} +const oldExperimentalFeatures* = {dotOperators, callOperator, parallel} const ChecksOptions* = {optObjCheck, optFieldCheck, optRangeCheck, From 438863601047cc974d831c6c0dd5533e4e3f0dc5 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Mon, 12 Dec 2022 23:14:44 +0800 Subject: [PATCH 2272/2472] fixes #21043; fixes a named exception in the infixAs expression which generate an implicit uninitialized let statement (#21081) * fixes #21043; fixes a named exception in the infixAs expression which generate an implicit uninitialized let statement * Update compiler/sempass2.nim Co-authored-by: Andreas Rumpf Co-authored-by: Andreas Rumpf --- compiler/sempass2.nim | 4 ++++ tests/init/tlet.nim | 23 ++++++++++++++++++++++- tests/iter/tyieldintry.nim | 1 + 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index 4520609e3..f13d8d8d4 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -491,6 +491,10 @@ proc trackTryStmt(tracked: PEffects, n: PNode) = let b = n[i] if b.kind == nkExceptBranch: setLen(tracked.init, oldState) + for j in 0.. Date: Tue, 13 Dec 2022 08:19:02 +1100 Subject: [PATCH 2273/2472] Nimsuggest now defines the backends symbol (#21083) * Add testcase * Define the backend symbol * Remove unneeded whitespace --- nimsuggest/nimsuggest.nim | 7 +++++-- nimsuggest/tests/t20440.nim | 7 +++++++ nimsuggest/tests/t20440.nims | 1 + 3 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 nimsuggest/tests/t20440.nim create mode 100644 nimsuggest/tests/t20440.nims diff --git a/nimsuggest/nimsuggest.nim b/nimsuggest/nimsuggest.nim index c4798e88c..c5e015ce9 100644 --- a/nimsuggest/nimsuggest.nim +++ b/nimsuggest/nimsuggest.nim @@ -27,7 +27,7 @@ import compiler / [options, commands, modules, sem, passes, passaux, msgs, sigmatch, ast, idents, modulegraphs, prefixmatches, lineinfos, cmdlinehelper, - pathutils] + pathutils, condsyms] when defined(nimPreviewSlimSystem): import std/typedthreads @@ -553,6 +553,7 @@ proc mainCommand(graph: ModuleGraph) = registerPass graph, verbosePass registerPass graph, semPass conf.setCmd cmdIdeTools + defineSymbol(conf.symbols, $conf.backend) wantMainModule(conf) if not fileExists(conf.projectFull): @@ -952,10 +953,12 @@ else: proc mockCommand(graph: ModuleGraph) = retval = graph let conf = graph.config + conf.setCmd cmdIdeTools + defineSymbol(conf.symbols, $conf.backend) clearPasses(graph) registerPass graph, verbosePass registerPass graph, semPass - conf.setCmd cmdIdeTools + wantMainModule(conf) if not fileExists(conf.projectFull): diff --git a/nimsuggest/tests/t20440.nim b/nimsuggest/tests/t20440.nim new file mode 100644 index 000000000..0456aa074 --- /dev/null +++ b/nimsuggest/tests/t20440.nim @@ -0,0 +1,7 @@ +when not defined(js): + {.fatal: "Crash".} +echo 4 + +discard """ +$nimsuggest --v3 --tester $file +""" diff --git a/nimsuggest/tests/t20440.nims b/nimsuggest/tests/t20440.nims new file mode 100644 index 000000000..1336be3d4 --- /dev/null +++ b/nimsuggest/tests/t20440.nims @@ -0,0 +1 @@ +switch("backend", "js") From d84f64d2e7a28d69c9a5b22b8434d1fb2f00ebf2 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Tue, 13 Dec 2022 09:52:35 +0800 Subject: [PATCH 2274/2472] add CI for version-2-0 branch (#21084) --- .github/workflows/ci_packages.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci_packages.yml b/.github/workflows/ci_packages.yml index 93f5bfe19..0bec4cc21 100644 --- a/.github/workflows/ci_packages.yml +++ b/.github/workflows/ci_packages.yml @@ -4,6 +4,7 @@ on: push: branches: - 'devel' + - 'version-2-0' - 'version-1-6' - 'version-1-2' From 1fefb8e92a62c647028d0f00ebd9fc04da169789 Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Tue, 13 Dec 2022 15:21:51 +1100 Subject: [PATCH 2275/2472] Refactor JS sourcemap generator (#21053) * Parse the new line format * Fix pattern Didn't have space after the path * Remove duplicate sources Sources would sometimes double up with a new line at the end * Remove unused variable * Refactor sourcemap.nim Removes the multiple translations needed, now goes from single high level type to the final SourceMap Adds documentation for procs * Line numbers line up properly now Files aren't linking correctly though * Files now link up correctly Lines are sometimes off but overall seems pretty good Just need to implement parser * Add column info to output Add sourceMappingURL to rope directly to prevent copy * Properly handle columns * Remove debug lines * Add testcase * Finish testcase * Use the outdir folder instead of the folder the test is in to find the sourcemap Co-authored-by: ringabout <43030857+ringabout@users.noreply.github.com> --- compiler/jsgen.nim | 8 +- compiler/sourcemap.nim | 540 ++++++++++++++-------------------------- tests/js/tsourcemap.nim | 96 +++++++ 3 files changed, 283 insertions(+), 361 deletions(-) create mode 100644 tests/js/tsourcemap.nim diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index bcabc42ed..42a3dcf31 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -724,8 +724,9 @@ proc hasFrameInfo(p: PProc): bool = ((p.prc == nil) or not (sfPure in p.prc.flags)) proc lineDir(config: ConfigRef, info: TLineInfo, line: int): Rope = - ropes.`%`("/* line $2 \"$1\" */$n", - [rope(toFullPath(config, info)), rope(line)]) + "/* line $2:$3 \"$1\" */$n" % [ + rope(toFullPath(config, info)), rope(line), rope(info.toColumn) + ] proc genLineDir(p: PProc, n: PNode) = let line = toLinenumber(n.info) @@ -2885,7 +2886,8 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode = # Generate an optional source map. if optSourcemap in m.config.globalOptions: var map: SourceMap - (code, map) = genSourceMap($(code), outFile.string) + map = genSourceMap($code, outFile.string) + code &= "\n//# sourceMappingURL=$#.map" % [outFile.string] writeFile(outFile.string & ".map", $(%map)) # Check if the generated JS code matches the output file, or else # write it to the file. diff --git a/compiler/sourcemap.nim b/compiler/sourcemap.nim index b87de75f3..2245250ac 100644 --- a/compiler/sourcemap.nim +++ b/compiler/sourcemap.nim @@ -1,383 +1,207 @@ -import os, strformat, strutils, tables, sets, ropes, json, algorithm +import std/[strutils, strscans, parseutils, assertions] type - SourceNode* = ref object - line*: int - column*: int - source*: string - name*: string - children*: seq[Child] + Segment = object + ## Segment refers to a block of something in the JS output. + ## This could be a token or an entire line + original: int # Column in the Nim source + generated: int # Column in the generated JS + name: int # Index into names list (-1 for no name) - C = enum cSourceNode, cSourceString + Mapping = object + ## Mapping refers to a line in the JS output. + ## It is made up of segments which refer to the tokens in the line + case inSource: bool # Whether the line in JS has Nim equivilant + of true: + file: int # Index into files list + line: int # 0 indexed line of code in the Nim source + segments: seq[Segment] + else: discard - Child* = ref object - case kind*: C: - of cSourceNode: - node*: SourceNode - of cSourceString: - s*: string + SourceInfo = object + mappings: seq[Mapping] + names, files: seq[string] - SourceMap* = ref object + SourceMap* = object version*: int sources*: seq[string] names*: seq[string] mappings*: string file*: string - # sourceRoot*: string - # sourcesContent*: string - SourceMapGenerator = ref object - file: string - sourceRoot: string - skipValidation: bool - sources: seq[string] - names: seq[string] - mappings: seq[Mapping] +func addSegment(info: var SourceInfo, original, generated: int, name: string = "") {.raises: [].} = + ## Adds a new segment into the current line + assert info.mappings.len > 0, "No lines have been added yet" + var segment = Segment(original: original, generated: generated, name: -1) + if name != "": + # Make name be index into names list + segment.name = info.names.find(name) + if segment.name == -1: + segment.name = info.names.len + info.names &= name - Mapping* = ref object - source*: string - original*: tuple[line: int, column: int] - generated*: tuple[line: int, column: int] - name*: string - noSource*: bool - noName*: bool + assert info.mappings[^1].inSource, "Current line isn't in Nim source" + info.mappings[^1].segments &= segment +func newLine(info: var SourceInfo) {.raises: [].} = + ## Add new mapping which doesn't appear in the Nim source + info.mappings &= Mapping(inSource: false) -proc child*(s: string): Child = - Child(kind: cSourceString, s: s) - - -proc child*(node: SourceNode): Child = - Child(kind: cSourceNode, node: node) - - -proc newSourceNode(line: int, column: int, path: string, node: SourceNode, name: string = ""): SourceNode = - SourceNode(line: line, column: column, source: path, name: name, children: @[child(node)]) - - -proc newSourceNode(line: int, column: int, path: string, s: string, name: string = ""): SourceNode = - SourceNode(line: line, column: column, source: path, name: name, children: @[child(s)]) - - -proc newSourceNode(line: int, column: int, path: string, children: seq[Child], name: string = ""): SourceNode = - SourceNode(line: line, column: column, source: path, name: name, children: children) - - - - -# debugging - - -proc text*(sourceNode: SourceNode, depth: int): string = - let empty = " " - result = &"{repeat(empty, depth)}SourceNode({sourceNode.source}:{sourceNode.line}:{sourceNode.column}):\n" - for child in sourceNode.children: - if child.kind == cSourceString: - result.add(&"{repeat(empty, depth + 1)}{child.s}\n") - else: - result.add(child.node.text(depth + 1)) - - -proc `$`*(sourceNode: SourceNode): string = text(sourceNode, 0) +func newLine(info: var SourceInfo, file: string, line: int) {.raises: [].} = + ## Starts a new line in the mappings. Call addSegment after this to add + ## segments into the line + var mapping = Mapping(inSource: true, line: line) + # Set file to file position. Add in if needed + mapping.file = info.files.find(file) + if mapping.file == -1: + mapping.file = info.files.len + info.files &= file + info.mappings &= mapping # base64_VLQ +func encode*(values: seq[int]): string {.raises: [].} = + ## Encodes a series of integers into a VLQ base64 encoded string + # References: + # - https://www.lucidchart.com/techblog/2019/08/22/decode-encoding-base64-vlqs-source-maps/ + # - https://github.com/rails/sprockets/blob/main/guides/source_maps.md#source-map-file + const + alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" + shift = 5 + continueBit = 1 shl 5 + mask = continueBit - 1 + for val in values: + # Sign is stored in first bit + var newVal = abs(val) shl 1 + if val < 0: + newVal = newVal or 1 + # Now comes the variable length part + # This is how we are able to store large numbers + while true: + # We only encode 5 bits. + var masked = newVal and mask + newVal = newVal shr shift + # If there is still something left + # then signify with the continue bit that the + # decoder should keep decoding + if newVal > 0: + masked = masked or continueBit + result &= alphabet[masked] + # If the value is zero then we have nothing left to encode + if newVal == 0: + break +iterator tokenize*(line: string): (int, string) = + ## Goes through a line and splits it into Nim identifiers and + ## normal JS code. This allows us to map mangled names back to Nim names. + ## Yields (column, name). Doesn't yield anything but identifiers. + ## See mangleName in compiler/jsgen.nim for how name mangling is done + var + col = 0 + token = "" + while col < line.len: + var + token: string + name: string + # First we find the next identifier + col += line.skipWhitespace(col) + col += line.skipUntil(IdentStartChars, col) + let identStart = col + col += line.parseIdent(token, col) + # Idents will either be originalName_randomInt or HEXhexCode_randomInt + if token.startsWith("HEX"): + var hex: int + # 3 = "HEX".len and we only want to parse the two integers after it + discard token[3 ..< 5].parseHex(hex) + name = $chr(hex) + elif not token.endsWith("_Idx"): # Ignore address indexes + # It might be in the form originalName_randomInt + let lastUnderscore = token.rfind('_') + if lastUnderscore != -1: + name = token[0.. 0: - var e = n and 31 - n = n shr 5 - if n > 0: - e = e or 32 - - result.add(integers[e]) - z += 1 - - -type TokenState = enum Normal, String, Ident, Mangled - -iterator tokenize*(line: string): (bool, string) = - # result = @[] - var state = Normal - var token = "" - var isMangled = false - for z, ch in line: - if ch.isAlphaAscii: - if state == Normal: - state = Ident - if token.len > 0: - yield (isMangled, token) - token = $ch - isMangled = false - else: - token.add(ch) - elif ch == '_': - if state == Ident: - state = Mangled - isMangled = true - token.add($ch) - elif ch != '"' and not ch.isAlphaNumeric: - if state in {Ident, Mangled}: - state = Normal - if token.len > 0: - yield (isMangled, token) - token = $ch - isMangled = false - else: - token.add($ch) - elif ch == '"': - if state != String: - state = String - if token.len > 0: - yield (isMangled, token) - token = $ch - isMangled = false - else: - state = Normal - token.add($ch) - if token.len > 0: - yield (isMangled, token) - isMangled = false - token = "" +func parse*(source: string): SourceInfo = + ## Parses the JS output for embedded line info + ## So it can convert those into a series of mappings + var + skipFirstLine = true + currColumn = 0 + currLine = 0 + currFile = "" + # Add each line as a node into the output + for line in source.splitLines(): + var + lineNumber: int + linePath: string + column: int + if line.strip().scanf("/* line $i:$i \"$+\" */", lineNumber, column, linePath): + # When we reach the first line mappinsegmentg then we can assume + # we can map the rest of the JS lines to Nim lines + currColumn = column # Column is already zero indexed + currLine = lineNumber - 1 + currFile = linePath + # Lines are zero indexed + result.newLine(currFile, currLine) + # Skip whitespace to find the starting column + result.addSegment(currColumn, line.skipWhitespace()) + elif currFile != "": + result.newLine(currFile, currLine) + # There mightn't be any tokens so add a starting segment + result.addSegment(currColumn, line.skipWhitespace()) + for jsColumn, token in line.tokenize: + result.addSegment(currColumn, jsColumn, token) else: - token.add($ch) - if token.len > 0: - yield (isMangled, token) + result.newLine() -proc parse*(source: string, path: string): SourceNode = - let lines = source.splitLines() - var lastLocation: SourceNode = nil - result = newSourceNode(0, 0, path, @[]) - - # we just use one single parent and add all nim lines - # as its children, I guess in typical codegen - # that happens recursively on ast level - # we also don't have column info, but I doubt more one nim lines can compile to one js - # maybe in macros? +func toSourceMap*(info: SourceInfo, file: string): SourceMap {.raises: [].} = + ## Convert from high level SourceInfo into the required SourceMap object + # Add basic info + result.version = 3 + result.file = file + result.sources = info.files + result.names = info.names + # Convert nodes into mappings. + # Mappings are split into blocks where each block referes to a line in the outputted JS. + # Blocks can be seperated into statements which refere to tokens on the line. + # Since the mappings depend on previous values we need to + # keep track of previous file, name, etc + var + prevFile = 0 + prevLine = 0 + prevName = 0 + prevNimCol = 0 - for i, originalLine in lines: - let line = originalLine.strip - if line.len == 0: - continue - - # this shouldn't be a problem: - # jsgen doesn't generate comments - # and if you emit // line you probably know what you're doing - if line.startsWith("// line"): - if result.children.len > 0: - result.children[^1].node.children.add(child(line & "\n")) - let pos = line.find(" ", 8) - let lineNumber = line[8 .. pos - 1].parseInt - let linePath = line[pos + 2 .. ^2] # quotes - - lastLocation = newSourceNode( - lineNumber, - 0, - linePath, - @[]) - result.children.add(child(lastLocation)) - else: - var last: SourceNode - for token in line.tokenize(): - var name = "" - if token[0]: - name = token[1].split('_', 1)[0] - - - if result.children.len > 0: - result.children[^1].node.children.add( - child( - newSourceNode( - result.children[^1].node.line, - 0, - result.children[^1].node.source, - token[1], - name))) - last = result.children[^1].node.children[^1].node - else: - result.children.add( - child( - newSourceNode(i + 1, 0, path, token[1], name))) - last = result.children[^1].node - let nl = "\n" - if not last.isNil: - last.source.add(nl) + for mapping in info.mappings: + # We know need to encode segments with the following fields + # All these fields are relative to their previous values + # - 0: Column in generated code + # - 1: Index of Nim file in source list + # - 2: Line in Nim source + # - 3: Column in Nim source + # - 4: Index in names list + if mapping.inSource: + # JS Column is special in that it is reset after every line + var prevJSCol = 0 + for segment in mapping.segments: + var values = @[segment.generated - prevJSCol, mapping.file - prevFile, mapping.line - prevLine, segment.original - prevNimCol] + # Add name field if needed + if segment.name != -1: + values &= segment.name - prevName + prevName = segment.name + prevJSCol = segment.generated + prevNimCol = segment.original + prevFile = mapping.file + prevLine = mapping.line + result.mappings &= encode(values) & "," + # Remove trailing , + if mapping.segments.len > 0: + result.mappings.setLen(result.mappings.len - 1) -proc cmp(a: Mapping, b: Mapping): int = - var c = cmp(a.generated, b.generated) - if c != 0: - return c + result.mappings &= ";" - c = cmp(a.source, b.source) - if c != 0: - return c - - c = cmp(a.original, b.original) - if c != 0: - return c - - return cmp(a.name, b.name) - - -proc index*[T](elements: seq[T], element: T): int = - for z in 0 ..< elements.len: - if elements[z] == element: - return z - return -1 - - -proc serializeMappings(map: SourceMapGenerator, mappings: seq[Mapping]): string = - var previous = Mapping(generated: (line: 1, column: 0), original: (line: 0, column: 0), name: "", source: "") - var previousSourceId = 0 - var previousNameId = 0 - var next = "" - var nameId = 0 - var sourceId = 0 - result = "" - - for z, mapping in mappings: - next = "" - - if mapping.generated.line != previous.generated.line: - previous.generated.column = 0 - - while mapping.generated.line != previous.generated.line: - next.add(";") - previous.generated.line += 1 - - else: - if z > 0: - if cmp(mapping, mappings[z - 1]) == 0: - continue - next.add(",") - - next.add(encode(mapping.generated.column - previous.generated.column)) - previous.generated.column = mapping.generated.column - - if not mapping.noSource and mapping.source.len > 0: - sourceId = map.sources.index(mapping.source) - next.add(encode(sourceId - previousSourceId)) - previousSourceId = sourceId - next.add(encode(mapping.original.line - 1 - previous.original.line)) - previous.original.line = mapping.original.line - 1 - next.add(encode(mapping.original.column - previous.original.column)) - previous.original.column = mapping.original.column - - if not mapping.noName and mapping.name.len > 0: - nameId = map.names.index(mapping.name) - next.add(encode(nameId - previousNameId)) - previousNameId = nameId - - result.add(next) - - -proc gen*(map: SourceMapGenerator): SourceMap = - var mappings = map.mappings.sorted do (a: Mapping, b: Mapping) -> int: - cmp(a, b) - result = SourceMap( - file: map.file, - version: 3, - sources: map.sources[0..^1], - names: map.names[0..^1], - mappings: map.serializeMappings(mappings)) - - - -proc addMapping*(map: SourceMapGenerator, mapping: Mapping) = - if not mapping.noSource and mapping.source notin map.sources: - map.sources.add(mapping.source) - - if not mapping.noName and mapping.name.len > 0 and mapping.name notin map.names: - map.names.add(mapping.name) - - # echo "map ", mapping.source, " ", mapping.original, " ", mapping.generated, " ", mapping.name - map.mappings.add(mapping) - - -proc walk*(node: SourceNode, fn: proc(line: string, original: SourceNode)) = - for child in node.children: - if child.kind == cSourceString and child.s.len > 0: - fn(child.s, node) - else: - child.node.walk(fn) - - -proc toSourceMap*(node: SourceNode, file: string): SourceMapGenerator = - var map = SourceMapGenerator(file: file, sources: @[], names: @[], mappings: @[]) - - var generated = (line: 1, column: 0) - var sourceMappingActive = false - var lastOriginal = SourceNode(source: "", line: -1, column: 0, name: "", children: @[]) - - node.walk do (line: string, original: SourceNode): - if original.source.endsWith(".js"): - # ignore it - discard - else: - if original.line != -1: - if lastOriginal.source != original.source or - lastOriginal.line != original.line or - lastOriginal.column != original.column or - lastOriginal.name != original.name: - map.addMapping( - Mapping( - source: original.source, - original: (line: original.line, column: original.column), - generated: (line: generated.line, column: generated.column), - name: original.name)) - - lastOriginal = SourceNode( - source: original.source, - line: original.line, - column: original.column, - name: original.name, - children: lastOriginal.children) - sourceMappingActive = true - elif sourceMappingActive: - map.addMapping( - Mapping( - noSource: true, - noName: true, - generated: (line: generated.line, column: generated.column), - original: (line: -1, column: -1))) - lastOriginal.line = -1 - sourceMappingActive = false - - for z in 0 ..< line.len: - if line[z] in Newlines: - generated.line += 1 - generated.column = 0 - - if z == line.len - 1: - lastOriginal.line = -1 - sourceMappingActive = false - elif sourceMappingActive: - map.addMapping( - Mapping( - source: original.source, - original: (line: original.line, column: original.column), - generated: (line: generated.line, column: generated.column), - name: original.name)) - else: - generated.column += 1 - - map - - -proc genSourceMap*(source: string, outFile: string): (Rope, SourceMap) = - let node = parse(source, outFile) - let map = node.toSourceMap(file = outFile) - ((&"{source}\n//# sourceMappingURL={outFile}.map").rope, map.gen) +proc genSourceMap*(source: string, outFile: string): SourceMap = + let node = parse(source) + result = node.toSourceMap(outFile) diff --git a/tests/js/tsourcemap.nim b/tests/js/tsourcemap.nim new file mode 100644 index 000000000..ff6f6122f --- /dev/null +++ b/tests/js/tsourcemap.nim @@ -0,0 +1,96 @@ +discard """ + action: "run" + target: "js" + cmd: "nim js -r -d:nodejs $options --sourceMap:on $file" +""" +import std/[os, json, strutils, sequtils, algorithm, assertions, paths, compilesettings] + +# Implements a very basic sourcemap parser and then runs it on itself. +# Allows to check for basic problems such as bad counts and lines missing (e.g. issue #21052) + +type + SourceMap = object + version: int + sources: seq[string] + names: seq[string] + mappings: string + file: string + + Line = object + line, column: int + file: string + +const + flag = 1 shl 5 + signBit = 0b1 + fourBits = 0b1111 + fiveBits = 0b11111 + mask = (1 shl 5) - 1 + alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" + +var b64Table: seq[int] = 0.repeat(max(alphabet.mapIt(it.ord)) + 1) +for i, b in alphabet.pairs: + b64Table[b.ord] = i + +# From https://github.com/juancarlospaco/nodejs/blob/main/src/nodejs/jsfs.nim +proc importFs*() {.importjs: "var fs = require(\"fs\");".} +proc readFileSync*(path: cstring): cstring {.importjs: "(fs.$1(#).toString())".} +importFS() +# Read in needed files +let + jsFileName = string(querySetting(outDir).Path / "tsourcemap.js".Path) + mapFileName = jsFileName & ".map" + + data = parseJson($mapFileName.cstring.readFileSync()).to(SourceMap) + jsFile = $readFileSync(jsFileName.cstring) + +proc decodeVLQ(inp: string): seq[int] = + var + shift, value: int + for v in inp.mapIt(b64Table[it.ord]): + value += (v and mask) shl shift + if (v and flag) > 0: + shift += 5 + continue + result &= (value shr 1) * (if (value and 1) > 0: -1 else: 1) + shift = 0 + value = 0 + + +# Keep track of state +var + line = 0 + source = 0 + name = 0 + column = 0 + jsLine = 1 + lines: seq[Line] + +for gline in data.mappings.split(';'): + jsLine += 1 + var jsColumn = 0 + for item in gline.strip().split(','): + let value = item.decodeVLQ() + doAssert value.len in [0, 1, 4, 5] + if value.len == 0: + continue + jsColumn += value[0] + if value.len >= 4: + source += value[1] + line += value[2] + column += value[3] + lines &= Line(line: line, column: column, file: data.sources[source]) + +let jsLines = jsFile.splitLines().len +# There needs to be a mapping for every line in the JS +# If there isn't then the JS lines wont match up with Nim lines. +# Except we don't care about the final line since that doesn't need to line up +doAssert data.mappings.count(';') == jsLines - 1 + +# Check we can find this file somewhere in the source map +var foundSelf = false +for line in lines: + if "tsourcemap.nim" in line.file: + foundSelf = true + doAssert line.line in 0.. Date: Tue, 13 Dec 2022 18:43:37 +1100 Subject: [PATCH 2276/2472] JS backend properly extends string with `setLen` (#21087) * Add test case * Extend string with '0' when setting length to be longer --- compiler/jsgen.nim | 4 +++- tests/js/t20235.nim | 11 +++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 tests/js/t20235.nim diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 42a3dcf31..d8fb6d57d 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -2209,7 +2209,9 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) = if optOverflowCheck notin p.options: binaryExpr(p, n, r, "", "$1 -= $2") else: binaryExpr(p, n, r, "subInt", "$1 = subInt($3, $2)", true) of mSetLengthStr: - binaryExpr(p, n, r, "mnewString", "($1.length = $2)") + binaryExpr(p, n, r, "mnewString", + """if ($1.length < $2) { for (var i = $3.length; i < $4; ++i) $3.push(0); } + else {$3.length = $4; }""") of mSetLengthSeq: var x, y: TCompRes gen(p, n[1], x) diff --git a/tests/js/t20235.nim b/tests/js/t20235.nim new file mode 100644 index 000000000..3a69c2bd6 --- /dev/null +++ b/tests/js/t20235.nim @@ -0,0 +1,11 @@ +discard """ + action: "run" + output: "0 4" +""" + +proc main = + var s = "" + s.setLen(4) + echo s[0].ord, " ", s.len + +main() From e4aadcf1c1b39a5fef55ce9f164b3251c44ca000 Mon Sep 17 00:00:00 2001 From: Xavier Noria Date: Tue, 13 Dec 2022 08:47:01 +0100 Subject: [PATCH 2277/2472] Document that system:pop() may raise IndexDefect (#21070) * Document system:pop() may raise IndexDefect * Add backticks to KeyError --- lib/pure/collections/sets.nim | 6 +++--- lib/system.nim | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/pure/collections/sets.nim b/lib/pure/collections/sets.nim index 09648ea65..11e324923 100644 --- a/lib/pure/collections/sets.nim +++ b/lib/pure/collections/sets.nim @@ -347,7 +347,7 @@ proc missingOrExcl*[A](s: var HashSet[A], key: A): bool = proc pop*[A](s: var HashSet[A]): A = ## Removes and returns an arbitrary element from the set `s`. ## - ## Raises KeyError if the set `s` is empty. + ## Raises `KeyError` if the set `s` is empty. ## ## See also: ## * `clear proc <#clear,HashSet[A]>`_ @@ -425,7 +425,7 @@ proc intersection*[A](s1, s2: HashSet[A]): HashSet[A] = assert c == toHashSet(["b"]) result = initHashSet[A](max(min(s1.data.len, s2.data.len), 2)) - + # iterate over the elements of the smaller set if s1.data.len < s2.data.len: for item in s1: @@ -433,7 +433,7 @@ proc intersection*[A](s1, s2: HashSet[A]): HashSet[A] = else: for item in s2: if item in s1: incl(result, item) - + proc difference*[A](s1, s2: HashSet[A]): HashSet[A] = ## Returns the difference of the sets `s1` and `s2`. diff --git a/lib/system.nim b/lib/system.nim index d0f78bfbd..65055f145 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1664,6 +1664,8 @@ proc contains*[T](a: openArray[T], item: T): bool {.inline.}= proc pop*[T](s: var seq[T]): T {.inline, noSideEffect.} = ## Returns the last item of `s` and decreases `s.len` by one. This treats ## `s` as a stack and implements the common *pop* operation. + ## + ## Raises `IndexDefect` if `s` is empty. runnableExamples: var a = @[1, 3, 5, 7] let b = pop(a) From 2564b5c938602e4f8820d2ed2b778c6d4f1d0cd5 Mon Sep 17 00:00:00 2001 From: Aditya Siram Date: Tue, 13 Dec 2022 08:57:07 -0600 Subject: [PATCH 2278/2472] Fix #20416. Enable the recursion limit for ref/ptr types. (#21092) --- compiler/semtypinst.nim | 18 ++++++++++++++---- tests/generics/tgenerics_issues.nim | 20 ++++++++++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim index 945667a81..0a0ac1704 100644 --- a/compiler/semtypinst.nim +++ b/compiler/semtypinst.nim @@ -499,10 +499,20 @@ proc propagateFieldFlags(t: PType, n: PNode) = proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType = template bailout = - if t.sym != nil and sfGeneratedType in t.sym.flags: - # Only consider the recursion limit if the symbol is a type with generic - # parameters that have not been explicitly supplied, typechecking should - # terminate when generic parameters are explicitly supplied. + if (t.sym == nil) or (t.sym != nil and sfGeneratedType in t.sym.flags): + # In the first case 't.sym' can be 'nil' if the type is a ref/ptr, see + # issue https://github.com/nim-lang/Nim/issues/20416 for more details. + # Fortunately for us this works for now because partial ref/ptr types are + # not allowed in object construction, eg. + # type + # Container[T] = ... + # O = object + # val: ref Container + # + # In the second case only consider the recursion limit if the symbol is a + # type with generic parameters that have not been explicitly supplied, + # typechecking should terminate when generic parameters are explicitly + # supplied. if cl.recursionLimit > 100: # bail out, see bug #2509. But note this caching is in general wrong, # look at this example where TwoVectors should not share the generic diff --git a/tests/generics/tgenerics_issues.nim b/tests/generics/tgenerics_issues.nim index db7a16569..3068a22f2 100644 --- a/tests/generics/tgenerics_issues.nim +++ b/tests/generics/tgenerics_issues.nim @@ -872,3 +872,23 @@ block: # Ensure no segfault from constraint a = Regex[int]() b = Regex[bool]() c = MyOtherType[seq[int]]() + +block: # https://github.com/nim-lang/Nim/issues/20416 + type + Item[T] = object + link:ptr Item[T] + data:T + + KVSeq[A,B] = seq[(A,B)] + + MyTable[A,B] = object + data: KVSeq[A,B] + + Container[T] = object + a: MyTable[int,ref Item[T]] + + proc p1(sg:Container) = discard # Make sure that a non parameterized 'Container' argument still compiles + + proc p2[T](sg:Container[T]) = discard + var v : Container[int] + p2(v) From 9a50033d5b09ec3263a53bd2bb4182a8a50e6f4d Mon Sep 17 00:00:00 2001 From: metagn Date: Tue, 13 Dec 2022 23:20:55 +0300 Subject: [PATCH 2279/2472] generic `define` pragma + string alias (#20979) * generic `define` pragma + string alias * clean * add tests and document * remove char/float, minimize changelog --- changelog.md | 30 +++++++++- compiler/ast.nim | 2 +- compiler/condsyms.nim | 2 + compiler/pragmas.nim | 23 +++++--- compiler/semfold.nim | 109 ++++++++++++++++++++++++++++--------- doc/manual.md | 18 ++++++ doc/manual_experimental.md | 23 ++++++++ tests/misc/tdefine.nim | 64 ++++++++++++++++------ 8 files changed, 218 insertions(+), 53 deletions(-) diff --git a/changelog.md b/changelog.md index 5762d18ad..0181d03d2 100644 --- a/changelog.md +++ b/changelog.md @@ -221,9 +221,33 @@ - Full command syntax and block arguments i.e. `foo a, b: c` are now allowed for the right-hand side of type definitions in type sections. Previously they would error with "invalid indentation". -- `defined` now accepts identifiers separated by dots, i.e. `defined(a.b.c)`. - In the command line, this is defined as `-d:a.b.c`. Older versions can - use accents as in ``defined(`a.b.c`)`` to access such defines. + +- Compile-time define changes: + - `defined` now accepts identifiers separated by dots, i.e. `defined(a.b.c)`. + In the command line, this is defined as `-d:a.b.c`. Older versions can + use accents as in ``defined(`a.b.c`)`` to access such defines. + - [Define pragmas for constants](https://nim-lang.github.io/Nim/manual.html#implementation-specific-pragmas-compileminustime-define-pragmas) + now support a string argument for qualified define names. + + ```nim + # -d:package.FooBar=42 + const FooBar {.intdefine: "package.FooBar".}: int = 5 + echo FooBar # 42 + ``` + + This was added to help disambiguate similar define names for different packages. + In older versions, this could only be achieved with something like the following: + + ```nim + const FooBar = block: + const `package.FooBar` {.intdefine.}: int = 5 + `package.FooBar` + ``` + - A generic `define` pragma for constants has been added that interprets + the value of the define based on the type of the constant value. + See the [experimental manual](https://nim-lang.github.io/Nim/manual_experimental.html#generic-define-pragma) + for a list of supported types. + - [Macro pragmas](https://nim-lang.github.io/Nim/manual.html#userminusdefined-pragmas-macro-pragmas) changes: - Templates now accept macro pragmas. - Macro pragmas for var/let/const sections have been redesigned in a way that works diff --git a/compiler/ast.nim b/compiler/ast.nim index 198d21bea..c85a94860 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -712,7 +712,7 @@ type mEqIdent, mEqNimrodNode, mSameNodeType, mGetImpl, mNGenSym, mNHint, mNWarning, mNError, mInstantiationInfo, mGetTypeInfo, mGetTypeInfoV2, - mNimvm, mIntDefine, mStrDefine, mBoolDefine, mRunnableExamples, + mNimvm, mIntDefine, mStrDefine, mBoolDefine, mGenericDefine, mRunnableExamples, mException, mBuiltinType, mSymOwner, mUncheckedArray, mGetImplTransf, mSymIsInstantiationOf, mNodeId, mPrivateAccess, mZeroDefault diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index 182e49bf8..41b4c83aa 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -149,3 +149,5 @@ proc initDefines*(symbols: StringTableRef) = defineSymbol("nimHasOutParams") defineSymbol("nimHasSystemRaisesDefect") defineSymbol("nimHasWarnUnnamedBreak") + defineSymbol("nimHasGenericDefine") + defineSymbol("nimHasDefineAliases") diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index b60331b29..993e6edce 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -81,7 +81,8 @@ const wGuard, wGoto, wCursor, wNoalias, wAlign} constPragmas* = declPragmas + {wHeader, wMagic, wGensym, wInject, - wIntDefine, wStrDefine, wBoolDefine, wCompilerProc, wCore} + wIntDefine, wStrDefine, wBoolDefine, wDefine, + wCompilerProc, wCore} paramPragmas* = {wNoalias, wInject, wGensym} letPragmas* = varPragmas procTypePragmas* = {FirstCallConv..LastCallConv, wVarargs, wNoSideEffect, @@ -476,8 +477,16 @@ proc processPop(c: PContext, n: PNode) = when defined(debugOptions): echo c.config $ n.info, " POP config is now ", c.config.options -proc processDefine(c: PContext, n: PNode) = - if (n.kind in nkPragmaCallKinds and n.len == 2) and (n[1].kind == nkIdent): +proc processDefineConst(c: PContext, n: PNode, sym: PSym, kind: TMagic) = + sym.magic = kind + if n.kind in nkPragmaCallKinds and n.len == 2: + # could also use TLib + n[1] = getStrLitNode(c, n) + +proc processDefine(c: PContext, n: PNode, sym: PSym) = + if sym != nil and sym.kind == skConst: + processDefineConst(c, n, sym, mGenericDefine) + elif (n.kind in nkPragmaCallKinds and n.len == 2) and (n[1].kind == nkIdent): defineSymbol(c.config.symbols, n[1].ident.s) else: invalidPragma(c, n) @@ -1066,7 +1075,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, recordPragma(c, it, "error", s) localError(c.config, it.info, errUser, s) of wFatal: fatal(c.config, it.info, expectStrLit(c, it)) - of wDefine: processDefine(c, it) + of wDefine: processDefine(c, it, sym) of wUndef: processUndef(c, it) of wCompile: processCompile(c, it) of wLink: processLink(c, it) @@ -1213,11 +1222,11 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, noVal(c, it) sym.flags.incl sfBase of wIntDefine: - sym.magic = mIntDefine + processDefineConst(c, n, sym, mIntDefine) of wStrDefine: - sym.magic = mStrDefine + processDefineConst(c, n, sym, mStrDefine) of wBoolDefine: - sym.magic = mBoolDefine + processDefineConst(c, n, sym, mBoolDefine) of wUsed: noVal(c, it) if sym == nil: invalidPragma(c, it) diff --git a/compiler/semfold.nim b/compiler/semfold.nim index 9e45e3b2d..7c57eb370 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -13,7 +13,7 @@ import strutils, options, ast, trees, nimsets, platform, math, msgs, idents, renderer, types, - commands, magicsys, modulegraphs, strtabs, lineinfos + commands, magicsys, modulegraphs, strtabs, lineinfos, wordrecg from system/memory import nimCStrLen @@ -371,6 +371,11 @@ proc rangeCheck(n: PNode, value: Int128; g: ModuleGraph) = localError(g.config, n.info, "cannot convert " & $value & " to " & typeToString(n.typ)) +proc floatRangeCheck(n: PNode, value: BiggestFloat; g: ModuleGraph) = + if value < firstFloat(n.typ) or value > lastFloat(n.typ): + localError(g.config, n.info, "cannot convert " & $value & + " to " & typeToString(n.typ)) + proc foldConv(n, a: PNode; idgen: IdGenerator; g: ModuleGraph; check = false): PNode = let dstTyp = skipTypes(n.typ, abstractRange - {tyTypeDesc}) let srcTyp = skipTypes(a.typ, abstractRange - {tyTypeDesc}) @@ -490,6 +495,81 @@ proc newSymNodeTypeDesc*(s: PSym; idgen: IdGenerator; info: TLineInfo): PNode = else: result.typ = s.typ +proc foldDefine(m, s: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode = + var name = s.name.s + let prag = extractPragma(s) + if prag != nil: + for it in prag: + if it.kind in nkPragmaCallKinds and it.len == 2 and it[0].kind == nkIdent: + let word = whichKeyword(it[0].ident) + if word in {wStrDefine, wIntDefine, wBoolDefine, wDefine}: + # should be processed in pragmas.nim already + if it[1].kind in {nkStrLit, nkRStrLit, nkTripleStrLit}: + name = it[1].strVal + if isDefined(g.config, name): + let str = g.config.symbols[name] + case s.magic + of mIntDefine: + try: + result = newIntNodeT(toInt128(str.parseInt), n, idgen, g) + except ValueError: + localError(g.config, s.info, + "{.intdefine.} const was set to an invalid integer: '" & + str & "'") + of mStrDefine: + result = newStrNodeT(str, n, g) + of mBoolDefine: + try: + result = newIntNodeT(toInt128(str.parseBool.int), n, idgen, g) + except ValueError: + localError(g.config, s.info, + "{.booldefine.} const was set to an invalid bool: '" & + str & "'") + of mGenericDefine: + let rawTyp = s.typ + # pretend we don't support distinct types + let typ = rawTyp.skipTypes(abstractVarRange-{tyDistinct}) + try: + template intNode(value): PNode = + let val = toInt128(value) + rangeCheck(n, val, g) + newIntNodeT(val, n, idgen, g) + case typ.kind + of tyString, tyCstring: + result = newStrNodeT(str, n, g) + of tyInt..tyInt64: + result = intNode(str.parseBiggestInt) + of tyUInt..tyUInt64: + result = intNode(str.parseBiggestUInt) + of tyBool: + result = intNode(str.parseBool.int) + of tyEnum: + # compile time parseEnum + let ident = getIdent(g.cache, str) + for e in typ.n: + if e.kind != nkSym: internalError(g.config, "foldDefine for enum") + let es = e.sym + let match = + if es.ast.isNil: + es.name.id == ident.id + else: + es.ast.strVal == str + if match: + result = intNode(es.position) + break + if result.isNil: + raise newException(ValueError, "invalid enum value: " & str) + else: + localError(g.config, s.info, "unsupported type $1 for define '$2'" % + [name, typeToString(rawTyp)]) + except ValueError as e: + localError(g.config, s.info, + "could not process define '$1' of type $2; $3" % + [name, typeToString(rawTyp), e.msg]) + else: result = copyTree(s.astdef) # unreachable + else: + result = copyTree(s.astdef) + proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode = result = nil case n.kind @@ -509,31 +589,8 @@ proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode of mBuildOS: result = newStrNodeT(toLowerAscii(platform.OS[g.config.target.hostOS].name), n, g) of mBuildCPU: result = newStrNodeT(platform.CPU[g.config.target.hostCPU].name.toLowerAscii, n, g) of mAppType: result = getAppType(n, g) - of mIntDefine: - if isDefined(g.config, s.name.s): - try: - result = newIntNodeT(toInt128(g.config.symbols[s.name.s].parseInt), n, idgen, g) - except ValueError: - localError(g.config, s.info, - "{.intdefine.} const was set to an invalid integer: '" & - g.config.symbols[s.name.s] & "'") - else: - result = copyTree(s.astdef) - of mStrDefine: - if isDefined(g.config, s.name.s): - result = newStrNodeT(g.config.symbols[s.name.s], n, g) - else: - result = copyTree(s.astdef) - of mBoolDefine: - if isDefined(g.config, s.name.s): - try: - result = newIntNodeT(toInt128(g.config.symbols[s.name.s].parseBool.int), n, idgen, g) - except ValueError: - localError(g.config, s.info, - "{.booldefine.} const was set to an invalid bool: '" & - g.config.symbols[s.name.s] & "'") - else: - result = copyTree(s.astdef) + of mIntDefine, mStrDefine, mBoolDefine, mGenericDefine: + result = foldDefine(m, s, n, idgen, g) else: result = copyTree(s.astdef) of skProc, skFunc, skMethod: diff --git a/doc/manual.md b/doc/manual.md index bd2cf8568..0a295fdb5 100644 --- a/doc/manual.md +++ b/doc/manual.md @@ -8049,6 +8049,24 @@ used. To see if a value was provided, `defined(FooBar)` can be used. The syntax `-d:flag`:option: is actually just a shortcut for `-d:flag=true`:option:. +These pragmas also accept an optional string argument for qualified +define names. + + ```nim + const FooBar {.intdefine: "package.FooBar".}: int = 5 + echo FooBar + ``` + + ```cmd + nim c -d:package.FooBar=42 foobar.nim + ``` + +This helps disambiguate define names in different packages. + +See also the [generic `define` pragma](manual_experimental.html#generic-define-pragma) +for a version of these pragmas that detects the type of the define based on +the constant value. + User-defined pragmas ==================== diff --git a/doc/manual_experimental.md b/doc/manual_experimental.md index 6f5906019..2a56c1354 100644 --- a/doc/manual_experimental.md +++ b/doc/manual_experimental.md @@ -65,6 +65,29 @@ However, a `void` type cannot be inferred in generic code: The `void` type is only valid for parameters and return types; other symbols cannot have the type `void`. +Generic `define` pragma +======================= + +Aside the [typed define pragmas for constants](manual.html#implementation-specific-pragmas-compileminustime-define-pragmas), +there is a generic `{.define.}` pragma that interprets the value of the define +based on the type of the constant value. + + ```nim + const foo {.define: "package.foo".} = 123 + const bar {.define: "package.bar".} = false + ``` + + ```cmd + nim c -d:package.foo=456 -d:package.bar foobar.nim + ``` + +The following types are supported: + +* `string` and `cstring` +* Signed and unsigned integer types +* `bool` +* Enums + Top-down type inference ======================= diff --git a/tests/misc/tdefine.nim b/tests/misc/tdefine.nim index c4d11c941..f3fa4711f 100644 --- a/tests/misc/tdefine.nim +++ b/tests/misc/tdefine.nim @@ -1,12 +1,23 @@ discard """ joinable: false -cmd: "nim c -d:booldef -d:booldef2=false -d:intdef=2 -d:strdef=foobar -d:namespaced.define=false -d:double.namespaced.define -r $file" +cmd: "nim c $options -d:booldef -d:booldef2=false -d:intdef=2 -d:strdef=foobar -d:namespaced.define=false -d:double.namespaced.define -r $file" +matrix: "; -d:useGenericDefine" """ -const booldef {.booldefine.} = false -const booldef2 {.booldefine.} = true -const intdef {.intdefine.} = 0 -const strdef {.strdefine.} = "" +when defined(useGenericDefine): + {.pragma: booldefine2, define.} + {.pragma: intdefine2, define.} + {.pragma: strdefine2, define.} +else: + + {.pragma: booldefine2, booldefine.} + {.pragma: intdefine2, intdefine.} + {.pragma: strdefine2, strdefine.} + +const booldef {.booldefine2.} = false +const booldef2 {.booldefine2.} = true +const intdef {.intdefine2.} = 0 +const strdef {.strdefine2.} = "" doAssert defined(booldef) doAssert defined(booldef2) @@ -17,17 +28,28 @@ doAssert not booldef2 doAssert intdef == 2 doAssert strdef == "foobar" +when defined(useGenericDefine): + block: + const uintdef {.define: "intdef".}: uint = 17 + doAssert intdef == int(uintdef) + const cstrdef {.define: "strdef".}: cstring = "not strdef" + doAssert $cstrdef == strdef + type FooBar = enum foo, bar, foobar + const enumdef {.define: "strdef".} = foo + doAssert $enumdef == strdef + doAssert enumdef == foobar + # Intentionally not defined from command line -const booldef3 {.booldefine.} = true -const intdef2 {.intdefine.} = 1 -const strdef2 {.strdefine.} = "abc" +const booldef3 {.booldefine2.} = true +const intdef2 {.intdefine2.} = 1 +const strdef2 {.strdefine2.} = "abc" type T = object - when booldef3: - field1: int - when intdef2 == 1: - field2: int - when strdef2 == "abc": - field3: int + when booldef3: + field1: int + when intdef2 == 1: + field2: int + when strdef2 == "abc": + field3: int doAssert not defined(booldef3) doAssert not defined(intdef2) @@ -35,11 +57,21 @@ doAssert not defined(strdef2) discard T(field1: 1, field2: 2, field3: 3) doAssert defined(namespaced.define) -const `namespaced.define` {.booldefine.} = true +const `namespaced.define` {.booldefine2.} = true doAssert not `namespaced.define` +when defined(useGenericDefine): + const aliasToNamespacedDefine {.define: "namespaced.define".} = not `namespaced.define` +else: + const aliasToNamespacedDefine {.booldefine: "namespaced.define".} = not `namespaced.define` +doAssert aliasToNamespacedDefine == `namespaced.define` doAssert defined(double.namespaced.define) -const `double.namespaced.define` {.booldefine.} = false +const `double.namespaced.define` {.booldefine2.} = false doAssert `double.namespaced.define` +when defined(useGenericDefine): + const aliasToDoubleNamespacedDefine {.define: "double.namespaced.define".} = not `double.namespaced.define` +else: + const aliasToDoubleNamespacedDefine {.booldefine: "double.namespaced.define".} = not `double.namespaced.define` +doAssert aliasToDoubleNamespacedDefine == `double.namespaced.define` doAssert not defined(namespaced.butnotdefined) From 91ce8c385d4ccbaab8048cf0393b01cd72282272 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 15 Dec 2022 13:45:36 +0800 Subject: [PATCH 2280/2472] fix #19580; add warning for bare except: clause (#21099) * fix #19580; add warning for bare except: clause * fixes some easy ones * Update doc/manual.md * fixes docs * Update changelog.md * addition * Apply suggestions from code review Co-authored-by: Jacek Sieka * Update doc/tut2.md Co-authored-by: Jacek Sieka --- changelog.md | 2 ++ compiler/condsyms.nim | 1 + compiler/docgen.nim | 8 ++++---- compiler/extccomp.nim | 4 ++-- compiler/lineinfos.nim | 2 ++ compiler/nim.cfg | 4 ++++ compiler/semstmts.nim | 5 ++++- compiler/types.nim | 12 ++++++++++++ doc/manual.md | 20 +++++++++----------- doc/tut2.md | 4 ++-- lib/packages/docutils/rst.nim | 6 +++--- lib/pure/htmlparser.nim | 4 ++-- lib/std/assertions.nim | 2 ++ tests/nimdoc/trunnableexamples.nim | 2 +- 14 files changed, 50 insertions(+), 26 deletions(-) diff --git a/changelog.md b/changelog.md index 0181d03d2..00bdb50c2 100644 --- a/changelog.md +++ b/changelog.md @@ -133,6 +133,8 @@ `foo` had type `proc ()` were assumed by the compiler to mean `foo(a, b, proc () = ...)`. This behavior is now deprecated. Use `foo(a, b) do (): ...` or `foo(a, b, proc () = ...)` instead. +- If no exception or any exception deriving from Exception but not Defect or CatchableError given in except, a `warnBareExcept` warning will be triggered. + ## Standard library additions and changes [//]: # "Changes:" diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index 41b4c83aa..9d081ef0a 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -151,3 +151,4 @@ proc initDefines*(symbols: StringTableRef) = defineSymbol("nimHasWarnUnnamedBreak") defineSymbol("nimHasGenericDefine") defineSymbol("nimHasDefineAliases") + defineSymbol("nimHasWarnBareExcept") diff --git a/compiler/docgen.nim b/compiler/docgen.nim index fc37bd596..f8a804cec 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -1746,7 +1746,7 @@ proc commandJson*(cache: IdentCache, conf: ConfigRef) = let filename = getOutFile(conf, RelativeFile conf.projectName, JsonExt) try: writeFile(filename, content) - except: + except IOError: rawMessage(conf, errCannotOpenFile, filename.string) proc commandTags*(cache: IdentCache, conf: ConfigRef) = @@ -1768,7 +1768,7 @@ proc commandTags*(cache: IdentCache, conf: ConfigRef) = let filename = getOutFile(conf, RelativeFile conf.projectName, TagsExt) try: writeFile(filename, content) - except: + except IOError: rawMessage(conf, errCannotOpenFile, filename.string) proc commandBuildIndex*(conf: ConfigRef, dir: string, outFile = RelativeFile"") = @@ -1789,7 +1789,7 @@ proc commandBuildIndex*(conf: ConfigRef, dir: string, outFile = RelativeFile"") try: writeFile(filename, code) - except: + except IOError: rawMessage(conf, errCannotOpenFile, filename.string) proc commandBuildIndexJson*(conf: ConfigRef, dir: string, outFile = RelativeFile"") = @@ -1803,5 +1803,5 @@ proc commandBuildIndexJson*(conf: ConfigRef, dir: string, outFile = RelativeFile try: writeFile(filename, $body) - except: + except IOError: rawMessage(conf, errCannotOpenFile, filename.string) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 256e02b22..66de46556 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -528,7 +528,7 @@ proc ccHasSaneOverflow*(conf: ConfigRef): bool = var exe = getConfigVar(conf, conf.cCompiler, ".exe") if exe.len == 0: exe = CC[conf.cCompiler].compilerExe # NOTE: should we need the full version, use -dumpfullversion - let (s, exitCode) = try: execCmdEx(exe & " -dumpversion") except: ("", 1) + let (s, exitCode) = try: execCmdEx(exe & " -dumpversion") except IOError, OSError, ValueError: ("", 1) if exitCode == 0: var major: int discard parseInt(s, major) @@ -1018,7 +1018,7 @@ proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; jsonFile: Absolute proc runJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile) = var bcache: BuildCache try: bcache.fromJson(jsonFile.string.parseFile) - except: + except ValueError, KeyError, JsonKindError: let e = getCurrentException() conf.quitOrRaise "\ncaught exception:\n$#\nstacktrace:\n$#error evaluating JSON file: $#" % [e.msg, e.getStackTrace(), jsonFile.string] diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index ab9c902c3..cb60d8949 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -86,6 +86,7 @@ type warnImplicitTemplateRedefinition = "ImplicitTemplateRedefinition", warnUnnamedBreak = "UnnamedBreak", warnStmtListLambda = "StmtListLambda", + warnBareExcept = "BareExcept", warnUser = "User", # hints hintSuccess = "Success", hintSuccessX = "SuccessX", @@ -185,6 +186,7 @@ const warnImplicitTemplateRedefinition: "template '$1' is implicitly redefined; this is deprecated, add an explicit .redefine pragma", warnUnnamedBreak: "Using an unnamed break in a block is deprecated; Use a named block with a named break instead", warnStmtListLambda: "statement list expression assumed to be anonymous proc; this is deprecated, use `do (): ...` or `proc () = ...` instead", + warnBareExcept: "$1", warnUser: "$1", hintSuccess: "operation successful: $#", # keep in sync with `testament.isSuccess` diff --git a/compiler/nim.cfg b/compiler/nim.cfg index 20ce3810f..96b47b0e6 100644 --- a/compiler/nim.cfg +++ b/compiler/nim.cfg @@ -38,3 +38,7 @@ define:useStdoutAsStdmsg @if nimHasWarnUnnamedBreak: warningAserror[UnnamedBreak]:on @end + +@if nimHasWarnBareExcept: + warningAserror[BareExcept]:on +@end diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index d19d75611..a7a02d7fa 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -210,6 +210,8 @@ proc semTry(c: PContext, n: PNode; flags: TExprFlags; expectedType: PType = nil) isImported = true elif not isException(typ): localError(c.config, typeNode.info, errExprCannotBeRaised) + elif not isDefectOrCatchableError(typ): + message(c.config, a.info, warnBareExcept, "catch a more precise Exception deriving from CatchableError or Defect.") if containsOrIncl(check, typ.id): localError(c.config, typeNode.info, errExceptionAlreadyHandled) @@ -251,7 +253,8 @@ proc semTry(c: PContext, n: PNode; flags: TExprFlags; expectedType: PType = nil) elif a.len == 1: # count number of ``except: body`` blocks inc catchAllExcepts - + message(c.config, a.info, warnBareExcept, + "The bare except clause is deprecated; use `except CatchableError:` instead") else: # support ``except KeyError, ValueError, ... : body`` if catchAllExcepts > 0: diff --git a/compiler/types.nim b/compiler/types.nim index 4bbbaf13c..457568e32 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -1721,6 +1721,18 @@ proc isDefectException*(t: PType): bool = t = skipTypes(t[0], abstractPtrs) return false +proc isDefectOrCatchableError*(t: PType): bool = + var t = t.skipTypes(abstractPtrs) + while t.kind == tyObject: + if t.sym != nil and t.sym.owner != nil and + sfSystemModule in t.sym.owner.flags and + (t.sym.name.s == "Defect" or + t.sym.name.s == "CatchableError"): + return true + if t[0] == nil: break + t = skipTypes(t[0], abstractPtrs) + return false + proc isSinkTypeForParam*(t: PType): bool = # a parameter like 'seq[owned T]' must not be used only once, but its # elements must, so we detect this case here: diff --git a/doc/manual.md b/doc/manual.md index 0a295fdb5..779e99186 100644 --- a/doc/manual.md +++ b/doc/manual.md @@ -4773,8 +4773,8 @@ Example: echo "overflow!" except ValueError, IOError: echo "catch multiple exceptions!" - except: - echo "Unknown exception!" + except CatchableError: + echo "Catchable exception!" finally: close(f) ``` @@ -4786,9 +4786,6 @@ listed in an `except` clause, the corresponding statements are executed. The statements following the `except` clauses are called `exception handlers`:idx:. -The empty `except`:idx: clause is executed if there is an exception that is -not listed otherwise. It is similar to an `else` clause in `if` statements. - If there is a `finally`:idx: clause, it is always executed after the exception handlers. @@ -4806,11 +4803,11 @@ Try can also be used as an expression; the type of the `try` branch then needs to fit the types of `except` branches, but the type of the `finally` branch always has to be `void`: - ```nim + ```nim test from std/strutils import parseInt let x = try: parseInt("133a") - except: -1 + except ValueError: -1 finally: echo "hi" ``` @@ -4818,8 +4815,9 @@ branch always has to be `void`: To prevent confusing code there is a parsing limitation; if the `try` follows a `(` it has to be written as a one liner: - ```nim - let x = (try: parseInt("133a") except: -1) + ```nim test + from std/strutils import parseInt + let x = (try: parseInt("133a") except ValueError: -1) ``` @@ -4867,7 +4865,7 @@ error message from `e`, and for such situations, it is enough to use ```nim try: # ... - except: + except CatchableError: echo getCurrentExceptionMsg() ``` @@ -5055,7 +5053,7 @@ An empty `raises` list (`raises: []`) means that no exception may be raised: try: unsafeCall() result = true - except: + except CatchableError: result = false ``` diff --git a/doc/tut2.md b/doc/tut2.md index 3c858c64e..4b9049082 100644 --- a/doc/tut2.md +++ b/doc/tut2.md @@ -393,7 +393,7 @@ The `try` statement handles exceptions: echo "could not convert string to integer" except IOError: echo "IO error!" - except: + except CatchableError: echo "Unknown exception!" # reraise the unknown exception: raise @@ -425,7 +425,7 @@ module. Example: ```nim try: doSomethingHere() - except: + except CatchableError: let e = getCurrentException() msg = getCurrentExceptionMsg() diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index f81be7a50..e9f76a26f 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -1610,7 +1610,7 @@ proc getFootnoteType(label: PRstNode): (FootnoteType, int) = elif label.len == 1 and label.sons[0].kind == rnLeaf: try: result = (fnManualNumber, parseInt(label.sons[0].text)) - except: + except ValueError: result = (fnCitation, -1) else: result = (fnCitation, -1) @@ -2899,13 +2899,13 @@ proc parseEnumList(p: var RstParser): PRstNode = let enumerator = p.tok[p.idx + 1 + wildIndex[w]].symbol # check that it's in sequence: enumerator == next(prevEnum) if "n" in wildcards[w]: # arabic numeral - let prevEnumI = try: parseInt(prevEnum) except: 1 + let prevEnumI = try: parseInt(prevEnum) except ValueError: 1 if enumerator in autoEnums: if prevAE != "" and enumerator != prevAE: break prevAE = enumerator curEnum = prevEnumI + 1 - else: curEnum = (try: parseInt(enumerator) except: 1) + else: curEnum = (try: parseInt(enumerator) except ValueError: 1) if curEnum - prevEnumI != 1: break prevEnum = enumerator diff --git a/lib/pure/htmlparser.nim b/lib/pure/htmlparser.nim index 6b1300f11..24eab3abb 100644 --- a/lib/pure/htmlparser.nim +++ b/lib/pure/htmlparser.nim @@ -394,10 +394,10 @@ proc entityToRune*(entity: string): Rune = case entity[1] of '0'..'9': try: runeValue = parseInt(entity[1..^1]) - except: discard + except ValueError: discard of 'x', 'X': # not case sensitive here try: runeValue = parseHexInt(entity[2..^1]) - except: discard + except ValueError: discard else: discard # other entities are not defined with prefix `#` if runeValue notin 0..0x10FFFF: runeValue = 0 # only return legal values return Rune(runeValue) diff --git a/lib/std/assertions.nim b/lib/std/assertions.nim index 03bab1b1b..5623ff8ef 100644 --- a/lib/std/assertions.nim +++ b/lib/std/assertions.nim @@ -98,6 +98,7 @@ template doAssertRaises*(exception: typedesc, code: untyped) = const begin = "expected raising '" & astToStr(exception) & "', instead" const msgEnd = " by: " & astToStr(code) template raisedForeign {.gensym.} = raiseAssert(begin & " raised foreign exception" & msgEnd) + {.warning[BareExcept]:off.} when Exception is exception: try: if true: @@ -116,5 +117,6 @@ template doAssertRaises*(exception: typedesc, code: untyped) = mixin `$` # alternatively, we could define $cstring in this module raiseAssert(begin & " raised '" & $e.name & "'" & msgEnd) except: raisedForeign() + {.warning[BareExcept]:on.} if wrong: raiseAssert(begin & " nothing was raised" & msgEnd) diff --git a/tests/nimdoc/trunnableexamples.nim b/tests/nimdoc/trunnableexamples.nim index ac7a0e26f..c88d8fa3f 100644 --- a/tests/nimdoc/trunnableexamples.nim +++ b/tests/nimdoc/trunnableexamples.nim @@ -65,7 +65,7 @@ when true: # issue #12746 runnableExamples: try: discard - except: + except CatchableError: # just the general except will work discard From d88f46df388604c1d21a2a95482c3255dc5884b6 Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Fri, 16 Dec 2022 02:05:34 +1100 Subject: [PATCH 2281/2472] Make async stacktraces less verbose (#21091) * Name iterators something human readable Remove intermediate async procs from stacktraces Clean async traceback message from reraises message * Remove unused import/variable * Fix failing tests Don't add {.stackTrace: off.} to anonymous procs (They already don't appear in stacktrace) * Fix failing tests in pragma category Now check that the nim is a routine type first so we don't run into any assertion defects * Hide stack trace pragma in docs and update doc tests User doesn't need to know if something won't appear so this more becomes verbose noise If this is a bad idea we can always add a `when defined(nimdoc)` switch so we don't add {.stackTrace: off.} to the Future[T] returning proc for docs --- compiler/renderer.nim | 4 +- lib/pure/asyncfutures.nim | 16 ++----- lib/pure/asyncmacro.nim | 16 ++++--- nimdoc/testproject/expected/testproject.html | 11 ++--- tests/async/tasync_gcunsafe.nim | 2 +- tests/async/tasync_traceback.nim | 47 ++++---------------- 6 files changed, 33 insertions(+), 63 deletions(-) diff --git a/compiler/renderer.nim b/compiler/renderer.nim index ad74cf2fe..cd0d83eb0 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -619,7 +619,9 @@ proc isHideable(config: ConfigRef, n: PNode): bool = # xxx compare `ident` directly with `getIdent(cache, wRaises)`, but # this requires a `cache`. case n.kind - of nkExprColonExpr: result = n[0].kind == nkIdent and n[0].ident.s in ["raises", "tags", "extern", "deprecated", "forbids"] + of nkExprColonExpr: + result = n[0].kind == nkIdent and + n[0].ident.s.nimIdentNormalize in ["raises", "tags", "extern", "deprecated", "forbids", "stacktrace"] of nkIdent: result = n.ident.s in ["gcsafe", "deprecated"] else: result = false diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index 035b6182d..edf5f7277 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -329,29 +329,21 @@ proc `$`*(stackTraceEntries: seq[StackTraceEntry]): string = if leftLen > longestLeft: longestLeft = leftLen - var indent = 2 # Format the entries. for entry in entries: let (filename, procname) = getFilenameProcname(entry) - if procname == "": - if entry.line == reraisedFromBegin: - result.add(spaces(indent) & "#[\n") - indent.inc(2) - elif entry.line == reraisedFromEnd: - indent.dec(2) - result.add(spaces(indent) & "]#\n") - continue + if procname == "" and entry.line == reraisedFromBegin: + break let left = "$#($#)" % [filename, $entry.line] - result.add((spaces(indent) & "$#$# $#\n") % [ + result.add((spaces(2) & "$# $#\n") % [ left, - spaces(longestLeft - left.len + 2), procname ]) let hint = getHint(entry) if hint.len > 0: - result.add(spaces(indent+2) & "## " & hint & "\n") + result.add(spaces(4) & "## " & hint & "\n") proc injectStacktrace[T](future: Future[T]) = when not defined(release): diff --git a/lib/pure/asyncmacro.nim b/lib/pure/asyncmacro.nim index c4f6fd80e..d80a47101 100644 --- a/lib/pure/asyncmacro.nim +++ b/lib/pure/asyncmacro.nim @@ -20,9 +20,8 @@ proc newCallWithLineInfo(fromNode: NimNode; theProc: NimNode, args: varargs[NimN template createCb(retFutureSym, iteratorNameSym, strName, identName, futureVarCompletions: untyped) = bind finished - var nameIterVar = iteratorNameSym - proc identName {.closure.} = + proc identName {.closure, stackTrace: off.} = try: if not nameIterVar.finished: var next = nameIterVar() @@ -151,6 +150,10 @@ proc asyncSingleProc(prc: NimNode): NimNode = result[0][0] = quote do: Future[void] return result + if prc.kind in RoutineNodes and prc.name.kind != nnkEmpty: + # Only non anonymous functions need/can have stack trace disabled + prc.addPragma(nnkExprColonExpr.newTree(ident"stackTrace", ident"off")) + if prc.kind notin {nnkProcDef, nnkLambda, nnkMethodDef, nnkDo}: error("Cannot transform this node kind into an async proc." & " proc/method definition or lambda node expected.", prc) @@ -209,7 +212,7 @@ proc asyncSingleProc(prc: NimNode): NimNode = # -> {.pop.} # -> # -> complete(retFuture, result) - var iteratorNameSym = genSym(nskIterator, $prcName & "Iter") + var iteratorNameSym = genSym(nskIterator, $prcName & " (Async)") var procBody = prc.body.processBody(retFutureSym, futureVarIdents) # don't do anything with forward bodies (empty) if procBody.kind != nnkEmpty: @@ -251,9 +254,10 @@ proc asyncSingleProc(prc: NimNode): NimNode = # friendlier stack traces: var cbName = genSym(nskProc, prcName & NimAsyncContinueSuffix) var procCb = getAst createCb(retFutureSym, iteratorNameSym, - newStrLitNode(prcName), - cbName, - createFutureVarCompletions(futureVarIdents, nil)) + newStrLitNode(prcName), + cbName, + createFutureVarCompletions(futureVarIdents, nil) + ) outerProcBody.add procCb # -> return retFuture diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html index b68b721ff..7d10d7900 100644 --- a/nimdoc/testproject/expected/testproject.html +++ b/nimdoc/testproject/expected/testproject.html @@ -503,7 +503,8 @@
          - 
          proc asyncFun1(): Future[int] {....raises: [Exception, ValueError],
+  
          proc asyncFun1(): Future[int] {....stackTrace: false,
+                                raises: [Exception, ValueError],
                                 tags: [RootEffect], forbids: [].}
          
   
          
     
@@ -515,8 +516,8 @@
          - 
          proc asyncFun2(): owned(Future[void]) {....raises: [Exception], tags: [RootEffect],
-                                        forbids: [].}
          + 
          proc asyncFun2(): owned(Future[void]) {....stackTrace: false, raises: [Exception],
+                                        tags: [RootEffect], forbids: [].}
          @@ -527,8 +528,8 @@
          - 
          proc asyncFun3(): owned(Future[void]) {....raises: [Exception], tags: [RootEffect],
-                                        forbids: [].}
          + 
          proc asyncFun3(): owned(Future[void]) {....stackTrace: false, raises: [Exception],
+                                        tags: [RootEffect], forbids: [].}
          diff --git a/tests/async/tasync_gcunsafe.nim b/tests/async/tasync_gcunsafe.nim index 00c92b109..f3e6bc691 100644 --- a/tests/async/tasync_gcunsafe.nim +++ b/tests/async/tasync_gcunsafe.nim @@ -1,5 +1,5 @@ discard """ - errormsg: "'anotherGCSafeAsyncProcIter' is not GC-safe as it calls 'asyncGCUnsafeProc'" + errormsg: "'anotherGCSafeAsyncProc (Async)' is not GC-safe as it calls 'asyncGCUnsafeProc'" cmd: "nim c --threads:on $file" file: "asyncmacro.nim" """ diff --git a/tests/async/tasync_traceback.nim b/tests/async/tasync_traceback.nim index ec67d34f9..98f71b192 100644 --- a/tests/async/tasync_traceback.nim +++ b/tests/async/tasync_traceback.nim @@ -67,51 +67,22 @@ import re const expected = """ b failure Async traceback: - tasync_traceback\.nim\(\d+?\)\s+?tasync_traceback - asyncmacro\.nim\(\d+?\)\s+?a - asyncmacro\.nim\(\d+?\)\s+?aNimAsyncContinue - ## Resumes an async procedure - tasync_traceback\.nim\(\d+?\)\s+?aIter - asyncmacro\.nim\(\d+?\)\s+?b - asyncmacro\.nim\(\d+?\)\s+?bNimAsyncContinue - ## Resumes an async procedure - tasync_traceback\.nim\(\d+?\)\s+?bIter - #\[ - tasync_traceback\.nim\(\d+?\)\s+?tasync_traceback - asyncmacro\.nim\(\d+?\)\s+?a - asyncmacro\.nim\(\d+?\)\s+?aNimAsyncContinue - ## Resumes an async procedure - asyncmacro\.nim\(\d+?\)\s+?aIter - asyncfutures\.nim\(\d+?\)\s+?read - \]# + tasync_traceback\.nim\(\d+?\) tasync_traceback + tasync_traceback\.nim\(\d+?\) a \(Async\) + tasync_traceback\.nim\(\d+?\) b \(Async\) Exception message: b failure bar failure Async traceback: - tasync_traceback\.nim\(\d+?\)\s+?tasync_traceback - asyncdispatch\.nim\(\d+?\)\s+?waitFor - asyncdispatch\.nim\(\d+?\)\s+?poll + tasync_traceback\.nim\(\d+?\) tasync_traceback + asyncdispatch\.nim\(\d+?\) waitFor + asyncdispatch\.nim\(\d+?\) poll ## Processes asynchronous completion events - asyncdispatch\.nim\(\d+?\)\s+?runOnce - asyncdispatch\.nim\(\d+?\)\s+?processPendingCallbacks + asyncdispatch\.nim\(\d+?\) runOnce + asyncdispatch\.nim\(\d+?\) processPendingCallbacks ## Executes pending callbacks - asyncmacro\.nim\(\d+?\)\s+?barNimAsyncContinue - ## Resumes an async procedure - tasync_traceback\.nim\(\d+?\)\s+?barIter - #\[ - tasync_traceback\.nim\(\d+?\)\s+?tasync_traceback - asyncdispatch\.nim\(\d+?\)\s+?waitFor - asyncdispatch\.nim\(\d+?\)\s+?poll - ## Processes asynchronous completion events - asyncdispatch\.nim\(\d+?\)\s+?runOnce - asyncdispatch\.nim\(\d+?\)\s+?processPendingCallbacks - ## Executes pending callbacks - asyncmacro\.nim\(\d+?\)\s+?fooNimAsyncContinue - ## Resumes an async procedure - asyncmacro\.nim\(\d+?\)\s+?fooIter - asyncfutures\.nim\(\d+?\)\s+?read - \]# + tasync_traceback\.nim\(\d+?\) bar \(Async\) Exception message: bar failure """ From 8054be6e529db9e65a18e4c4a37ec54f64f9121c Mon Sep 17 00:00:00 2001 From: Bung Date: Thu, 15 Dec 2022 23:12:17 +0800 Subject: [PATCH 2282/2472] fix #20588 (#21104) --- compiler/semexprs.nim | 2 ++ tests/arc/t20588.nim | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 tests/arc/t20588.nim diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 2efa1259a..8ac8f4750 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -319,6 +319,8 @@ proc semConv(c: PContext, n: PNode; expectedType: PType = nil): PNode = return evaluated else: targetType = targetType.base + of tyAnything, tyUntyped, tyTyped: + localError(c.config, n.info, "illegal type conversion to '$1'" % typeToString(targetType)) else: discard maybeLiftType(targetType, c, n[0].info) diff --git a/tests/arc/t20588.nim b/tests/arc/t20588.nim new file mode 100644 index 000000000..d747c656d --- /dev/null +++ b/tests/arc/t20588.nim @@ -0,0 +1,22 @@ +discard """ + cmd: "nim check --warnings:off --hints:off $file" + errormsg: "" + nimout: ''' +t20588.nim(20, 12) Error: illegal type conversion to 'auto' +t20588.nim(21, 14) Error: illegal type conversion to 'typed' +t20588.nim(22, 16) Error: illegal type conversion to 'untyped' +''' +""" + + + + + + + + + + +discard 0.0.auto +discard typed("abc") +discard untyped(4) From d00477dffb2c62732b87a440c61706de5b480f48 Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Fri, 16 Dec 2022 05:22:45 +1100 Subject: [PATCH 2283/2472] Check file exists in `{.compile.}` pragma (#21105) * Add test * Check file exists before adding it into compilation * Make error message look like other error messages i.e. following the format `error msg: file` --- compiler/pragmas.nim | 7 +++++-- tests/pragmas/tcompile_missing_file.nim | 5 +++++ 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 tests/pragmas/tcompile_missing_file.nim diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 993e6edce..92e3d1922 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -513,8 +513,11 @@ proc processCompile(c: PContext, n: PNode) = var cf = Cfile(nimname: splitFile(src).name, cname: src, obj: dest, flags: {CfileFlag.External}, customArgs: customArgs) - extccomp.addExternalFileToCompile(c.config, cf) - recordPragma(c, it, "compile", src.string, dest.string, customArgs) + if not fileExists(src): + localError(c.config, n.info, "cannot find: " & src.string) + else: + extccomp.addExternalFileToCompile(c.config, cf) + recordPragma(c, it, "compile", src.string, dest.string, customArgs) proc getStrLit(c: PContext, n: PNode; i: int): string = n[i] = c.semConstExpr(c, n[i]) diff --git a/tests/pragmas/tcompile_missing_file.nim b/tests/pragmas/tcompile_missing_file.nim new file mode 100644 index 000000000..fd90bd57d --- /dev/null +++ b/tests/pragmas/tcompile_missing_file.nim @@ -0,0 +1,5 @@ +discard """ + joinable: false + errormsg: "cannot find: noexist.c" +""" +{.compile: "noexist.c".} From 78b68421da0a678deb0c17cd03345d62410dbeac Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 16 Dec 2022 11:36:52 +0800 Subject: [PATCH 2284/2472] fixes chronicles (#21114) --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index d0c87919b..c134be6f9 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -51,7 +51,7 @@ pkg "c2nim", "nim c testsuite/tester.nim" pkg "cascade" pkg "cello", url = "https://github.com/nim-lang/cello", useHead = true pkg "chroma" -pkg "chronicles", "nimble install -y stew@#head; nim c -o:chr -r chronicles.nim" +pkg "chronicles", "nim c -o:chr -r chronicles.nim", url = "https://github.com/nim-lang/nim-chronicles" pkg "chronos", "nim c -r -d:release tests/testall", url = "https://github.com/nim-lang/nim-chronos" pkg "cligen", "nim c --path:. -r cligen.nim" pkg "combparser", "nimble test --gc:orc" From 644318ad9cbf079f33d8d7b6c5ead4a4429adb3e Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 16 Dec 2022 14:58:40 +0800 Subject: [PATCH 2285/2472] the devel branch now uses csources_v2 (#21108) * the devel branch now uses csources_v2 * fixes hash * bump csources_v2 --- .gitignore | 1 + config/build_config.txt | 6 +++--- nim.nimble | 2 +- readme.md | 4 ++-- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index d228b498c..d0c4558df 100644 --- a/.gitignore +++ b/.gitignore @@ -67,6 +67,7 @@ testament.db /csources /csources_v1 +/csources_v2 /dist/ # /lib/fusion # fusion is now unbundled; `git status` should reveal if it's there so users can act on it diff --git a/config/build_config.txt b/config/build_config.txt index 15f44bb57..3ae618f00 100644 --- a/config/build_config.txt +++ b/config/build_config.txt @@ -1,5 +1,5 @@ nim_comment="key-value pairs for windows/posix bootstrapping build scripts" -nim_csourcesDir=csources_v1 -nim_csourcesUrl=https://github.com/nim-lang/csources_v1.git +nim_csourcesDir=csources_v2 +nim_csourcesUrl=https://github.com/nim-lang/csources_v2.git nim_csourcesBranch=master -nim_csourcesHash=561b417c65791cd8356b5f73620914ceff845d10 +nim_csourcesHash=ffd834c20915dc04c78eb0d3b2162832e7ec9c13 diff --git a/nim.nimble b/nim.nimble index a629c0e04..5d1be49b1 100644 --- a/nim.nimble +++ b/nim.nimble @@ -5,7 +5,7 @@ license = "MIT" bin = @["compiler/nim", "nimsuggest/nimsuggest"] skipFiles = @["azure-pipelines.yml" , "build_all.bat" , "build_all.sh" , "build_nimble.bat" , "build_nimble.sh" , "changelog.md" , "koch.nim.cfg" , "nimblemeta.json" , "readme.md" , "security.md" ] -skipDirs = @["build" , "changelogs" , "ci" , "csources_v1" , "drnim" , "nimdoc", "testament"] +skipDirs = @["build" , "changelogs" , "ci" , "csources_v2" , "drnim" , "nimdoc", "testament"] before install: when defined(windows): diff --git a/readme.md b/readme.md index 3d32bcf87..a0f47eb41 100644 --- a/readme.md +++ b/readme.md @@ -47,7 +47,7 @@ Compiling the Nim compiler is quite straightforward if you follow these steps: First, the C source of an older version of the Nim compiler is needed to bootstrap the latest version because the Nim compiler itself is written in the Nim programming language. Those C sources are available within the -[``nim-lang/csources_v1``][csources-v1-repo] repository. +[``nim-lang/csources_v2``][csources-v2-repo] repository. Next, to build from source you will need: @@ -223,7 +223,7 @@ Copyright © 2006-2022 Andreas Rumpf, all rights reserved. [nimble-repo]: https://github.com/nim-lang/nimble [nimsuggest-repo]: https://github.com/nim-lang/nimsuggest [csources-repo-deprecated]: https://github.com/nim-lang/csources -[csources-v1-repo]: https://github.com/nim-lang/csources_v1 +[csources-v2-repo]: https://github.com/nim-lang/csources_v2 [badge-nim-irc]: https://img.shields.io/badge/chat-on_irc-blue.svg?style=flat-square [badge-nim-discord]: https://img.shields.io/discord/371759389889003530?color=blue&label=discord&logo=discord&logoColor=gold&style=flat-square [badge-nim-gitter]: https://img.shields.io/badge/chat-on_gitter-blue.svg?style=flat-square From 8a3b76b28733a87e5cffa3f4b7b99d76f13ee2ba Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 16 Dec 2022 14:59:18 +0800 Subject: [PATCH 2286/2472] csource building prefers `bin/nim` (#21115) --- tools/niminst/niminst.nim | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tools/niminst/niminst.nim b/tools/niminst/niminst.nim index cd2e5a481..bc763a56f 100644 --- a/tools/niminst/niminst.nim +++ b/tools/niminst/niminst.nim @@ -515,6 +515,17 @@ template gatherFiles(fun, libpath, outDir) = # commenting out for now, see discussion in https://github.com/nim-lang/Nim/pull/13413 # copySrc(libpath / "lib/wrappers/linenoise/linenoise.h") +proc exe(f: string): string = + result = addFileExt(f, ExeExt) + when defined(windows): + result = result.replace('/','\\') + +proc findNim(): string = + let nim = "nim".exe + result = quoteShell("bin" / nim) + if not fileExists(result): + result = "nim" + proc srcdist(c: var ConfigData) = let cCodeDir = getOutputDir(c) / "c_code" if not dirExists(cCodeDir): createDir(cCodeDir) @@ -533,10 +544,10 @@ proc srcdist(c: var ConfigData) = var dir = getOutputDir(c) / buildDir(osA, cpuA) if dirExists(dir): removeDir(dir) createDir(dir) - var cmd = ("nim compile -f --symbolfiles:off --compileonly " & + var cmd = ("$# compile -f --symbolfiles:off --compileonly " & "--gen_mapping --cc:gcc --skipUserCfg" & " --os:$# --cpu:$# $# $#") % - [osname, cpuname, c.nimArgs, c.mainfile] + [findNim(), osname, cpuname, c.nimArgs, c.mainfile] echo(cmd) if execShellCmd(cmd) != 0: quit("Error: call to nim compiler failed") From a9bd78d579168ba50f42d3bc418cd7de70a226f9 Mon Sep 17 00:00:00 2001 From: Bung Date: Fri, 16 Dec 2022 15:01:15 +0800 Subject: [PATCH 2287/2472] fix #12122 (#21096) --- compiler/astalgo.nim | 2 +- compiler/evalffi.nim | 8 ++-- lib/core/typeinfo.nim | 4 +- lib/pure/coro.nim | 4 +- lib/pure/memfiles.nim | 8 ++-- lib/pure/oids.nim | 4 +- lib/pure/strutils.nim | 4 +- lib/std/envvars.nim | 2 +- lib/std/private/strimpl.nim | 4 +- lib/std/syncio.nim | 2 +- lib/system/alloc.nim | 70 ++++++++++++++++----------------- lib/system/assign.nim | 28 ++++++------- lib/system/channels_builtin.nim | 12 +++--- lib/system/ctypes.nim | 2 +- lib/system/deepcopy.nim | 16 ++++---- lib/system/gc.nim | 24 +++++------ lib/system/gc2.nim | 22 +++++------ lib/system/gc_common.nim | 24 +++++------ lib/system/gc_ms.nim | 18 ++++----- lib/system/osalloc.nim | 6 +-- lib/system/repr.nim | 10 ++--- lib/system/sysstr.nim | 4 +- tests/parallel/tguard1.nim | 2 +- 23 files changed, 140 insertions(+), 140 deletions(-) diff --git a/compiler/astalgo.nim b/compiler/astalgo.nim index 28c38129e..e9ed1497a 100644 --- a/compiler/astalgo.nim +++ b/compiler/astalgo.nim @@ -333,7 +333,7 @@ proc typeToYamlAux(conf: ConfigRef; n: PType, marker: var IntSet, indent: int, sonsRope = rope("null") elif containsOrIncl(marker, n.id): sonsRope = "\"$1 @$2\"" % [rope($n.kind), rope( - strutils.toHex(cast[ByteAddress](n), sizeof(n) * 2))] + strutils.toHex(cast[int](n), sizeof(n) * 2))] else: if n.len > 0: sonsRope = rope("[") diff --git a/compiler/evalffi.nim b/compiler/evalffi.nim index 37edef86c..0577619b8 100644 --- a/compiler/evalffi.nim +++ b/compiler/evalffi.nim @@ -77,7 +77,7 @@ proc importcSymbol*(conf: ConfigRef, sym: PSym): PNode = theAddr = dllhandle.symAddr(name.cstring) if theAddr.isNil: globalError(conf, sym.info, "cannot import symbol: " & name & " from " & libPathMsg) - result.intVal = cast[ByteAddress](theAddr) + result.intVal = cast[int](theAddr) proc mapType(conf: ConfigRef, t: ast.PType): ptr libffi.Type = if t == nil: return addr libffi.type_void @@ -113,7 +113,7 @@ proc mapCallConv(conf: ConfigRef, cc: TCallingConvention, info: TLineInfo): TABI template rd(typ, p: untyped): untyped = (cast[ptr typ](p))[] template wr(typ, p, v: untyped): untyped = (cast[ptr typ](p))[] = v template `+!`(x, y: untyped): untyped = - cast[pointer](cast[ByteAddress](x) + y) + cast[pointer](cast[int](x) + y) proc packSize(conf: ConfigRef, v: PNode, typ: PType): int = ## computes the size of the blob @@ -369,13 +369,13 @@ proc unpack(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode = # in their unboxed representation so nothing it to be unpacked: result = n else: - awi(nkPtrLit, cast[ByteAddress](p)) + awi(nkPtrLit, cast[int](p)) of tyPtr, tyRef, tyVar, tyLent: let p = rd(pointer, x) if p.isNil: setNil() elif n == nil or n.kind == nkPtrLit: - awi(nkPtrLit, cast[ByteAddress](p)) + awi(nkPtrLit, cast[int](p)) elif n != nil and n.len == 1: internalAssert(conf, n.kind == nkRefTy) n[0] = unpack(conf, p, typ.lastSon, n[0]) diff --git a/lib/core/typeinfo.nim b/lib/core/typeinfo.nim index 89b1deb31..5075c8458 100644 --- a/lib/core/typeinfo.nim +++ b/lib/core/typeinfo.nim @@ -132,12 +132,12 @@ else: proc prepareSeqAdd(len: int; p: pointer; addlen, elemSize, elemAlign: int): pointer {. importCompilerProc.} -template `+!!`(a, b): untyped = cast[pointer](cast[ByteAddress](a) + b) +template `+!!`(a, b): untyped = cast[pointer](cast[int](a) + b) proc getDiscriminant(aa: pointer, n: ptr TNimNode): int = assert(n.kind == nkCase) var d: int - let a = cast[ByteAddress](aa) + let a = cast[int](aa) case n.typ.size of 1: d = int(cast[ptr uint8](a +% n.offset)[]) of 2: d = int(cast[ptr uint16](a +% n.offset)[]) diff --git a/lib/pure/coro.nim b/lib/pure/coro.nim index 47dfdd791..3b9bba59b 100644 --- a/lib/pure/coro.nim +++ b/lib/pure/coro.nim @@ -281,8 +281,8 @@ proc start*(c: proc(), stacksize: int = defaultStackSize): CoroutineRef {.discar (proc(p: pointer) {.stdcall.} = runCurrentTask()), nil) else: coro = cast[CoroutinePtr](alloc0(sizeof(Coroutine) + stacksize)) - coro.stack.top = cast[pointer](cast[ByteAddress](coro) + sizeof(Coroutine)) - coro.stack.bottom = cast[pointer](cast[ByteAddress](coro.stack.top) + stacksize) + coro.stack.top = cast[pointer](cast[int](coro) + sizeof(Coroutine)) + coro.stack.bottom = cast[pointer](cast[int](coro.stack.top) + stacksize) when coroBackend == CORO_BACKEND_UCONTEXT: discard getcontext(coro.execContext) coro.execContext.uc_stack.ss_sp = coro.stack.top diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim index 48c07b149..9ff8bcd74 100644 --- a/lib/pure/memfiles.nim +++ b/lib/pure/memfiles.nim @@ -495,8 +495,8 @@ proc mmsSetPosition(s: Stream, pos: int) = proc mmsGetPosition(s: Stream): int = MemMapFileStream(s).pos proc mmsPeekData(s: Stream, buffer: pointer, bufLen: int): int = - let startAddress = cast[ByteAddress](MemMapFileStream(s).mf.mem) - let p = cast[ByteAddress](MemMapFileStream(s).pos) + let startAddress = cast[int](MemMapFileStream(s).mf.mem) + let p = cast[int](MemMapFileStream(s).pos) let l = min(bufLen, MemMapFileStream(s).mf.size - p) moveMem(buffer, cast[pointer](startAddress + p), l) result = l @@ -511,8 +511,8 @@ proc mmsWriteData(s: Stream, buffer: pointer, bufLen: int) = let size = MemMapFileStream(s).mf.size if MemMapFileStream(s).pos + bufLen > size: raise newEIO("cannot write to stream") - let p = cast[ByteAddress](MemMapFileStream(s).mf.mem) + - cast[ByteAddress](MemMapFileStream(s).pos) + let p = cast[int](MemMapFileStream(s).mf.mem) + + cast[int](MemMapFileStream(s).pos) moveMem(cast[pointer](p), buffer, bufLen) inc(MemMapFileStream(s).pos, bufLen) diff --git a/lib/pure/oids.nim b/lib/pure/oids.nim index e6e5e6e56..ad8eeefd7 100644 --- a/lib/pure/oids.nim +++ b/lib/pure/oids.nim @@ -44,7 +44,7 @@ proc hexbyte*(hex: char): int {.inline.} = proc parseOid*(str: cstring): Oid = ## Parses an OID. - var bytes = cast[cstring](cast[pointer](cast[ByteAddress](addr(result.time)) + 4)) + var bytes = cast[cstring](cast[pointer](cast[int](addr(result.time)) + 4)) var i = 0 while i < 12: bytes[i] = chr((hexbyte(str[2 * i]) shl 4) or hexbyte(str[2 * i + 1])) @@ -57,7 +57,7 @@ proc `$`*(oid: Oid): string = result.setLen 24 var o = oid - var bytes = cast[cstring](cast[pointer](cast[ByteAddress](addr(o)) + 4)) + var bytes = cast[cstring](cast[pointer](cast[int](addr(o)) + 4)) var i = 0 while i < 12: let b = bytes[i].ord diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 0367ea1e5..f628cfc2d 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -1914,7 +1914,7 @@ func find*(s: string, sub: char, start: Natural = 0, last = -1): int {.rtl, if length > 0: let found = c_memchr(s[start].unsafeAddr, sub, cast[csize_t](length)) if not found.isNil: - return cast[ByteAddress](found) -% cast[ByteAddress](s.cstring) + return cast[int](found) -% cast[int](s.cstring) else: findImpl() @@ -1970,7 +1970,7 @@ func find*(s, sub: string, start: Natural = 0, last = -1): int {.rtl, if last < 0 and start < s.len and subLen != 0: let found = memmem(s[start].unsafeAddr, csize_t(s.len - start), sub.cstring, csize_t(subLen)) result = if not found.isNil: - cast[ByteAddress](found) -% cast[ByteAddress](s.cstring) + cast[int](found) -% cast[int](s.cstring) else: -1 else: diff --git a/lib/std/envvars.nim b/lib/std/envvars.nim index 46595a3cf..817a155ec 100644 --- a/lib/std/envvars.nim +++ b/lib/std/envvars.nim @@ -183,7 +183,7 @@ when not defined(nimscript): let kv = $e let p = find(kv, '=') yield (substr(kv, 0, p-1), substr(kv, p+1)) - e = cast[typ](cast[ByteAddress](eend)+size) + e = cast[typ](cast[int](eend)+size) if typeof(zero)(eend[1]) == zero: break discard free_fun(env) impl(getEnvironmentStringsW, WideCString, 2, 0, freeEnvironmentStringsW) diff --git a/lib/std/private/strimpl.nim b/lib/std/private/strimpl.nim index 6a38cbfd2..f8c9236a5 100644 --- a/lib/std/private/strimpl.nim +++ b/lib/std/private/strimpl.nim @@ -93,7 +93,7 @@ func find*(s: cstring, sub: char, start: Natural = 0, last = 0): int = if L > 0: let found = c_memchr(s[start].unsafeAddr, sub, cast[csize_t](L)) if not found.isNil: - return cast[ByteAddress](found) -% cast[ByteAddress](s) + return cast[int](found) -% cast[int](s) return -1 func find*(s, sub: cstring, start: Natural = 0, last = 0): int = @@ -108,6 +108,6 @@ func find*(s, sub: cstring, start: Natural = 0, last = 0): int = if last == 0 and s.len > start: let found = c_strstr(cast[cstring](s[start].unsafeAddr), sub) if not found.isNil: - result = cast[ByteAddress](found) -% cast[ByteAddress](s) + result = cast[int](found) -% cast[int](s) else: result = -1 diff --git a/lib/std/syncio.nim b/lib/std/syncio.nim index 2b5ea6a3c..46f113450 100644 --- a/lib/std/syncio.nim +++ b/lib/std/syncio.nim @@ -480,7 +480,7 @@ proc readLine*(f: File, line: var string): bool {.tags: [ReadIOEffect], let m = c_memchr(addr line[pos], '\L'.ord, cast[csize_t](sp)) if m != nil: # \l found: Could be our own or the one by fgets, in any case, we're done - var last = cast[ByteAddress](m) - cast[ByteAddress](addr line[0]) + var last = cast[int](m) - cast[int](addr line[0]) if last > 0 and line[last-1] == '\c': line.setLen(last-1) return last > 1 or fgetsSuccess diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index a7db3704b..5d1548977 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -290,7 +290,7 @@ proc llAlloc(a: var MemRegion, size: int): pointer = a.llmem.size = PageSize - sizeof(LLChunk) a.llmem.acc = sizeof(LLChunk) a.llmem.next = old - result = cast[pointer](cast[ByteAddress](a.llmem) + a.llmem.acc) + result = cast[pointer](cast[int](a.llmem) + a.llmem.acc) dec(a.llmem.size, size) inc(a.llmem.acc, size) zeroMem(result, size) @@ -422,7 +422,7 @@ iterator allObjects(m: var MemRegion): pointer {.inline.} = var c = cast[PSmallChunk](c) let size = c.size - var a = cast[ByteAddress](addr(c.data)) + var a = cast[int](addr(c.data)) let limit = a + c.acc while a <% limit: yield cast[pointer](a) @@ -441,13 +441,13 @@ when not defined(gcDestructors): # ------------- chunk management ---------------------------------------------- proc pageIndex(c: PChunk): int {.inline.} = - result = cast[ByteAddress](c) shr PageShift + result = cast[int](c) shr PageShift proc pageIndex(p: pointer): int {.inline.} = - result = cast[ByteAddress](p) shr PageShift + result = cast[int](p) shr PageShift proc pageAddr(p: pointer): PChunk {.inline.} = - result = cast[PChunk](cast[ByteAddress](p) and not PageMask) + result = cast[PChunk](cast[int](p) and not PageMask) #sysAssert(Contains(allocator.chunkStarts, pageIndex(result))) when false: @@ -495,13 +495,13 @@ proc requestOsChunks(a: var MemRegion, size: int): PBigChunk = when defined(memtracker): trackLocation(addr result.size, sizeof(int)) - sysAssert((cast[ByteAddress](result) and PageMask) == 0, "requestOsChunks 1") + sysAssert((cast[int](result) and PageMask) == 0, "requestOsChunks 1") #zeroMem(result, size) result.next = nil result.prev = nil result.size = size # update next.prevSize: - var nxt = cast[ByteAddress](result) +% size + var nxt = cast[int](result) +% size sysAssert((nxt and PageMask) == 0, "requestOsChunks 2") var next = cast[PChunk](nxt) if pageIndex(next) in a.chunkStarts: @@ -509,7 +509,7 @@ proc requestOsChunks(a: var MemRegion, size: int): PBigChunk = next.prevSize = size or (next.prevSize and 1) # set result.prevSize: var lastSize = if a.lastSize != 0: a.lastSize else: PageSize - var prv = cast[ByteAddress](result) -% lastSize + var prv = cast[int](result) -% lastSize sysAssert((nxt and PageMask) == 0, "requestOsChunks 3") var prev = cast[PChunk](prv) if pageIndex(prev) in a.chunkStarts and prev.size == lastSize: @@ -555,13 +555,13 @@ proc listRemove[T](head: var T, c: T) {.inline.} = proc updatePrevSize(a: var MemRegion, c: PBigChunk, prevSize: int) {.inline.} = - var ri = cast[PChunk](cast[ByteAddress](c) +% c.size) - sysAssert((cast[ByteAddress](ri) and PageMask) == 0, "updatePrevSize") + var ri = cast[PChunk](cast[int](c) +% c.size) + sysAssert((cast[int](ri) and PageMask) == 0, "updatePrevSize") if isAccessible(a, ri): ri.prevSize = prevSize or (ri.prevSize and 1) proc splitChunk2(a: var MemRegion, c: PBigChunk, size: int): PBigChunk = - result = cast[PBigChunk](cast[ByteAddress](c) +% size) + result = cast[PBigChunk](cast[int](c) +% size) result.size = c.size - size track("result.size", addr result.size, sizeof(int)) when not defined(nimOptimizedSplitChunk): @@ -590,8 +590,8 @@ proc freeBigChunk(a: var MemRegion, c: PBigChunk) = when coalescLeft: let prevSize = c.prevSize if prevSize != 0: - var le = cast[PChunk](cast[ByteAddress](c) -% prevSize) - sysAssert((cast[ByteAddress](le) and PageMask) == 0, "freeBigChunk 4") + var le = cast[PChunk](cast[int](c) -% prevSize) + sysAssert((cast[int](le) and PageMask) == 0, "freeBigChunk 4") if isAccessible(a, le) and chunkUnused(le): sysAssert(not isSmallChunk(le), "freeBigChunk 5") if not isSmallChunk(le) and le.size < MaxBigChunkSize: @@ -607,8 +607,8 @@ proc freeBigChunk(a: var MemRegion, c: PBigChunk) = addChunkToMatrix(a, c) c = rest when coalescRight: - var ri = cast[PChunk](cast[ByteAddress](c) +% c.size) - sysAssert((cast[ByteAddress](ri) and PageMask) == 0, "freeBigChunk 2") + var ri = cast[PChunk](cast[int](c) +% c.size) + sysAssert((cast[int](ri) and PageMask) == 0, "freeBigChunk 2") if isAccessible(a, ri) and chunkUnused(ri): sysAssert(not isSmallChunk(ri), "freeBigChunk 3") if not isSmallChunk(ri) and c.size < MaxBigChunkSize: @@ -669,7 +669,7 @@ proc getHugeChunk(a: var MemRegion; size: int): PBigChunk = incCurrMem(a, size) # XXX add this to the heap links. But also remove it from it later. when false: a.addHeapLink(result, size) - sysAssert((cast[ByteAddress](result) and PageMask) == 0, "getHugeChunk") + sysAssert((cast[int](result) and PageMask) == 0, "getHugeChunk") result.next = nil result.prev = nil result.size = size @@ -838,7 +838,7 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = c.prev = nil listAdd(a.freeSmallChunks[s], c) result = addr(c.data) - sysAssert((cast[ByteAddress](result) and (MemAlign-1)) == 0, "rawAlloc 4") + sysAssert((cast[int](result) and (MemAlign-1)) == 0, "rawAlloc 4") else: sysAssert(allocInv(a), "rawAlloc: begin c != nil") sysAssert c.next != c, "rawAlloc 5" @@ -856,7 +856,7 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = if c.freeList == nil: sysAssert(c.acc + smallChunkOverhead() + size <= SmallChunkSize, "rawAlloc 7") - result = cast[pointer](cast[ByteAddress](addr(c.data)) +% c.acc) + result = cast[pointer](cast[int](addr(c.data)) +% c.acc) inc(c.acc, size) else: result = c.freeList @@ -864,14 +864,14 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = sysAssert(c.freeList.zeroField == 0, "rawAlloc 8") c.freeList = c.freeList.next dec(c.free, size) - sysAssert((cast[ByteAddress](result) and (MemAlign-1)) == 0, "rawAlloc 9") + sysAssert((cast[int](result) and (MemAlign-1)) == 0, "rawAlloc 9") sysAssert(allocInv(a), "rawAlloc: end c != nil") sysAssert(allocInv(a), "rawAlloc: before c.free < size") if c.free < size: sysAssert(allocInv(a), "rawAlloc: before listRemove test") listRemove(a.freeSmallChunks[s], c) sysAssert(allocInv(a), "rawAlloc: end listRemove test") - sysAssert(((cast[ByteAddress](result) and PageMask) - smallChunkOverhead()) %% + sysAssert(((cast[int](result) and PageMask) - smallChunkOverhead()) %% size == 0, "rawAlloc 21") sysAssert(allocInv(a), "rawAlloc: end small size") inc a.occ, size @@ -893,11 +893,11 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = sysAssert c.prev == nil, "rawAlloc 10" sysAssert c.next == nil, "rawAlloc 11" result = addr(c.data) - sysAssert((cast[ByteAddress](c) and (MemAlign-1)) == 0, "rawAlloc 13") - sysAssert((cast[ByteAddress](c) and PageMask) == 0, "rawAlloc: Not aligned on a page boundary") + sysAssert((cast[int](c) and (MemAlign-1)) == 0, "rawAlloc 13") + sysAssert((cast[int](c) and PageMask) == 0, "rawAlloc: Not aligned on a page boundary") when not defined(gcDestructors): if a.root == nil: a.root = getBottom(a) - add(a, a.root, cast[ByteAddress](result), cast[ByteAddress](result)+%size) + add(a, a.root, cast[int](result), cast[int](result)+%size) inc a.occ, c.size trackSize(c.size) sysAssert(isAccessible(a, result), "rawAlloc 14") @@ -927,10 +927,10 @@ proc rawDealloc(a: var MemRegion, p: pointer) = dec a.occ, s untrackSize(s) sysAssert a.occ >= 0, "rawDealloc: negative occupied memory (case A)" - sysAssert(((cast[ByteAddress](p) and PageMask) - smallChunkOverhead()) %% + sysAssert(((cast[int](p) and PageMask) - smallChunkOverhead()) %% s == 0, "rawDealloc 3") when not defined(gcDestructors): - #echo("setting to nil: ", $cast[ByteAddress](addr(f.zeroField))) + #echo("setting to nil: ", $cast[int](addr(f.zeroField))) sysAssert(f.zeroField != 0, "rawDealloc 1") f.zeroField = 0 f.next = c.freeList @@ -953,7 +953,7 @@ proc rawDealloc(a: var MemRegion, p: pointer) = else: when defined(gcDestructors): addToSharedFreeList(c, f) - sysAssert(((cast[ByteAddress](p) and PageMask) - smallChunkOverhead()) %% + sysAssert(((cast[int](p) and PageMask) - smallChunkOverhead()) %% s == 0, "rawDealloc 2") else: # set to 0xff to check for usage after free bugs: @@ -975,7 +975,7 @@ when not defined(gcDestructors): if not chunkUnused(c): if isSmallChunk(c): var c = cast[PSmallChunk](c) - var offset = (cast[ByteAddress](p) and (PageSize-1)) -% + var offset = (cast[int](p) and (PageSize-1)) -% smallChunkOverhead() result = (c.acc >% offset) and (offset %% c.size == 0) and (cast[ptr FreeCell](p).zeroField >% 1) @@ -993,12 +993,12 @@ when not defined(gcDestructors): if not chunkUnused(c): if isSmallChunk(c): var c = cast[PSmallChunk](c) - var offset = (cast[ByteAddress](p) and (PageSize-1)) -% + var offset = (cast[int](p) and (PageSize-1)) -% smallChunkOverhead() if c.acc >% offset: - sysAssert(cast[ByteAddress](addr(c.data)) +% offset == - cast[ByteAddress](p), "offset is not what you think it is") - var d = cast[ptr FreeCell](cast[ByteAddress](addr(c.data)) +% + sysAssert(cast[int](addr(c.data)) +% offset == + cast[int](p), "offset is not what you think it is") + var d = cast[ptr FreeCell](cast[int](addr(c.data)) +% offset -% (offset %% c.size)) if d.zeroField >% 1: result = d @@ -1025,7 +1025,7 @@ when not defined(gcDestructors): proc ptrSize(p: pointer): int = when not defined(gcDestructors): - var x = cast[pointer](cast[ByteAddress](p) -% sizeof(FreeCell)) + var x = cast[pointer](cast[int](p) -% sizeof(FreeCell)) var c = pageAddr(p) sysAssert(not chunkUnused(c), "ptrSize") result = c.size -% sizeof(FreeCell) @@ -1043,7 +1043,7 @@ proc alloc(allocator: var MemRegion, size: Natural): pointer {.gcsafe.} = result = rawAlloc(allocator, size+sizeof(FreeCell)) cast[ptr FreeCell](result).zeroField = 1 # mark it as used sysAssert(not isAllocatedPtr(allocator, result), "alloc") - result = cast[pointer](cast[ByteAddress](result) +% sizeof(FreeCell)) + result = cast[pointer](cast[int](result) +% sizeof(FreeCell)) track("alloc", result, size) else: result = rawAlloc(allocator, size) @@ -1055,7 +1055,7 @@ proc alloc0(allocator: var MemRegion, size: Natural): pointer = proc dealloc(allocator: var MemRegion, p: pointer) = when not defined(gcDestructors): sysAssert(p != nil, "dealloc: p is nil") - var x = cast[pointer](cast[ByteAddress](p) -% sizeof(FreeCell)) + var x = cast[pointer](cast[int](p) -% sizeof(FreeCell)) sysAssert(x != nil, "dealloc: x is nil") sysAssert(isAccessible(allocator, x), "is not accessible") sysAssert(cast[ptr FreeCell](x).zeroField == 1, "dealloc: object header corrupted") @@ -1116,7 +1116,7 @@ template instantiateForRegion(allocator: untyped) {.dirty.} = result = interiorAllocatedPtr(allocator, p) proc isAllocatedPtr*(p: pointer): bool = - let p = cast[pointer](cast[ByteAddress](p)-%ByteAddress(sizeof(Cell))) + let p = cast[pointer](cast[int](p)-%ByteAddress(sizeof(Cell))) result = isAllocatedPtr(allocator, p) proc deallocOsPages = deallocOsPages(allocator) diff --git a/lib/system/assign.nim b/lib/system/assign.nim index 20b854107..42587929c 100644 --- a/lib/system/assign.nim +++ b/lib/system/assign.nim @@ -15,8 +15,8 @@ proc genericAssignAux(dest, src: pointer, mt: PNimType, shallow: bool) {.benign. proc genericAssignAux(dest, src: pointer, n: ptr TNimNode, shallow: bool) {.benign.} = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) case n.kind of nkSlot: genericAssignAux(cast[pointer](d +% n.offset), @@ -56,8 +56,8 @@ template deepSeqAssignImpl(operation, additionalArg) {.dirty.} = proc genericAssignAux(dest, src: pointer, mt: PNimType, shallow: bool) = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) sysAssert(mt != nil, "genericAssignAux 2") case mt.kind of tyString: @@ -89,17 +89,17 @@ proc genericAssignAux(dest, src: pointer, mt: PNimType, shallow: bool) = var ss = nimNewSeqOfCap(mt, seq.len) cast[PGenericSeq](ss).len = seq.len unsureAsgnRef(x, ss) - var dst = cast[ByteAddress](cast[PPointer](dest)[]) + var dst = cast[int](cast[PPointer](dest)[]) copyMem(cast[pointer](dst +% align(GenericSeqSize, mt.base.align)), - cast[pointer](cast[ByteAddress](s2) +% align(GenericSeqSize, mt.base.align)), + cast[pointer](cast[int](s2) +% align(GenericSeqSize, mt.base.align)), seq.len *% mt.base.size) else: unsureAsgnRef(x, newSeq(mt, seq.len)) - var dst = cast[ByteAddress](cast[PPointer](dest)[]) + var dst = cast[int](cast[PPointer](dest)[]) for i in 0..seq.len-1: genericAssignAux( cast[pointer](dst +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size ), - cast[pointer](cast[ByteAddress](s2) +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size ), + cast[pointer](cast[int](s2) +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size ), mt.base, shallow) of tyObject: var it = mt.base @@ -181,15 +181,15 @@ proc genericSeqAssign(dest, src: pointer, mt: PNimType) {.compilerproc.} = proc genericAssignOpenArray(dest, src: pointer, len: int, mt: PNimType) {.compilerproc.} = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) for i in 0..len-1: genericAssign(cast[pointer](d +% i *% mt.base.size), cast[pointer](s +% i *% mt.base.size), mt.base) proc objectInit(dest: pointer, typ: PNimType) {.compilerproc, benign.} proc objectInitAux(dest: pointer, n: ptr TNimNode) {.benign.} = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) case n.kind of nkNone: sysAssert(false, "objectInitAux") of nkSlot: objectInit(cast[pointer](d +% n.offset), n.typ) @@ -203,7 +203,7 @@ proc objectInitAux(dest: pointer, n: ptr TNimNode) {.benign.} = proc objectInit(dest: pointer, typ: PNimType) = # the generic init proc that takes care of initialization of complex # objects on the stack or heap - var d = cast[ByteAddress](dest) + var d = cast[int](dest) case typ.kind of tyObject: # iterate over any structural type @@ -226,7 +226,7 @@ proc objectInit(dest: pointer, typ: PNimType) = proc genericReset(dest: pointer, mt: PNimType) {.compilerproc, benign.} proc genericResetAux(dest: pointer, n: ptr TNimNode) = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) case n.kind of nkNone: sysAssert(false, "genericResetAux") of nkSlot: genericReset(cast[pointer](d +% n.offset), n.typ) @@ -238,7 +238,7 @@ proc genericResetAux(dest: pointer, n: ptr TNimNode) = zeroMem(cast[pointer](d +% n.offset), n.typ.size) proc genericReset(dest: pointer, mt: PNimType) = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) sysAssert(mt != nil, "genericReset 2") case mt.kind of tyRef: diff --git a/lib/system/channels_builtin.nim b/lib/system/channels_builtin.nim index fbe3f0e98..e04f2a0db 100644 --- a/lib/system/channels_builtin.nim +++ b/lib/system/channels_builtin.nim @@ -185,8 +185,8 @@ when not usesDestructors: proc storeAux(dest, src: pointer, n: ptr TNimNode, t: PRawChannel, mode: LoadStoreMode) {.benign.} = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) case n.kind of nkSlot: storeAux(cast[pointer](d +% n.offset), cast[pointer](s +% n.offset), n.typ, t, mode) @@ -205,8 +205,8 @@ when not usesDestructors: cast[pointer](cast[int](p) +% x) var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) sysAssert(mt != nil, "mt == nil") case mt.kind of tyString: @@ -245,14 +245,14 @@ when not usesDestructors: x[] = alloc0(t.region, align(GenericSeqSize, mt.base.align) +% seq.len *% mt.base.size) else: unsureAsgnRef(x, newSeq(mt, seq.len)) - var dst = cast[ByteAddress](cast[PPointer](dest)[]) + var dst = cast[int](cast[PPointer](dest)[]) var dstseq = cast[PGenericSeq](dst) dstseq.len = seq.len dstseq.reserved = seq.len for i in 0..seq.len-1: storeAux( cast[pointer](dst +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size), - cast[pointer](cast[ByteAddress](s2) +% align(GenericSeqSize, mt.base.align) +% + cast[pointer](cast[int](s2) +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size), mt.base, t, mode) if mode != mStore: dealloc(t.region, s2) diff --git a/lib/system/ctypes.nim b/lib/system/ctypes.nim index 6ba28ed49..f6a341477 100644 --- a/lib/system/ctypes.nim +++ b/lib/system/ctypes.nim @@ -67,7 +67,7 @@ type # these work for most platforms: ## This is the same as the type `unsigned long long` in *C*. type - ByteAddress* = int + ByteAddress* {.deprecated: "use `uint`".} = int ## is the signed integer type that should be used for converting ## pointers to integer addresses for readability. diff --git a/lib/system/deepcopy.nim b/lib/system/deepcopy.nim index 1f30b8427..72d35f518 100644 --- a/lib/system/deepcopy.nim +++ b/lib/system/deepcopy.nim @@ -61,8 +61,8 @@ proc genericDeepCopyAux(dest, src: pointer, mt: PNimType; proc genericDeepCopyAux(dest, src: pointer, n: ptr TNimNode; tab: var PtrTable) {.benign.} = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) case n.kind of nkSlot: genericDeepCopyAux(cast[pointer](d +% n.offset), @@ -85,8 +85,8 @@ proc genericDeepCopyAux(dest, src: pointer, n: ptr TNimNode; proc genericDeepCopyAux(dest, src: pointer, mt: PNimType; tab: var PtrTable) = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) sysAssert(mt != nil, "genericDeepCopyAux 2") case mt.kind of tyString: @@ -113,11 +113,11 @@ proc genericDeepCopyAux(dest, src: pointer, mt: PNimType; tab: var PtrTable) = return sysAssert(dest != nil, "genericDeepCopyAux 3") unsureAsgnRef(x, newSeq(mt, seq.len)) - var dst = cast[ByteAddress](cast[PPointer](dest)[]) + var dst = cast[int](cast[PPointer](dest)[]) for i in 0..seq.len-1: genericDeepCopyAux( cast[pointer](dst +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size), - cast[pointer](cast[ByteAddress](s2) +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size), + cast[pointer](cast[int](s2) +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size), mt.base, tab) of tyObject: # we need to copy m_type field for tyObject, as it could be empty for @@ -199,8 +199,8 @@ proc genericSeqDeepCopy(dest, src: pointer, mt: PNimType) {.compilerproc.} = proc genericDeepCopyOpenArray(dest, src: pointer, len: int, mt: PNimType) {.compilerproc.} = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) for i in 0..len-1: genericDeepCopy(cast[pointer](d +% i *% mt.base.size), cast[pointer](s +% i *% mt.base.size), mt.base) diff --git a/lib/system/gc.nim b/lib/system/gc.nim index b36822aad..7db2b6567 100644 --- a/lib/system/gc.nim +++ b/lib/system/gc.nim @@ -170,11 +170,11 @@ proc addZCT(s: var CellSeq, c: PCell) {.noinline.} = proc cellToUsr(cell: PCell): pointer {.inline.} = # convert object (=pointer to refcount) to pointer to userdata - result = cast[pointer](cast[ByteAddress](cell)+%ByteAddress(sizeof(Cell))) + result = cast[pointer](cast[int](cell)+%ByteAddress(sizeof(Cell))) proc usrToCell(usr: pointer): PCell {.inline.} = # convert pointer to userdata to object (=pointer to refcount) - result = cast[PCell](cast[ByteAddress](usr)-%ByteAddress(sizeof(Cell))) + result = cast[PCell](cast[int](usr)-%ByteAddress(sizeof(Cell))) proc extGetCellType(c: pointer): PNimType {.compilerproc.} = # used for code generation concerning debugging @@ -336,7 +336,7 @@ proc cellsetReset(s: var CellSet) = {.push stacktrace:off.} proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) case n.kind of nkSlot: forAllChildrenAux(cast[pointer](d +% n.offset), n.typ, op) of nkList: @@ -356,7 +356,7 @@ proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = of nkNone: sysAssert(false, "forAllSlotsAux") proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) if dest == nil: return # nothing to do if ntfNoRefs notin mt.flags: case mt.kind @@ -382,7 +382,7 @@ proc forAllChildren(cell: PCell, op: WalkOp) = of tyRef: # common case forAllChildrenAux(cellToUsr(cell), cell.typ.base, op) of tySequence: - var d = cast[ByteAddress](cellToUsr(cell)) + var d = cast[int](cellToUsr(cell)) var s = cast[PGenericSeq](d) if s != nil: for i in 0..s.len-1: @@ -457,7 +457,7 @@ proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer = collectCT(gch) var res = cast[PCell](rawAlloc(gch.region, size + sizeof(Cell))) #gcAssert typ.kind in {tyString, tySequence} or size >= typ.base.size, "size too small" - gcAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") + gcAssert((cast[int](res) and (MemAlign-1)) == 0, "newObj: 2") # now it is buffered in the ZCT res.typ = typ setFrameInfo(res) @@ -507,7 +507,7 @@ proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl, noinline.} = var res = cast[PCell](rawAlloc(gch.region, size + sizeof(Cell))) sysAssert(allocInv(gch.region), "newObjRC1 after rawAlloc") - sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") + sysAssert((cast[int](res) and (MemAlign-1)) == 0, "newObj: 2") # now it is buffered in the ZCT res.typ = typ setFrameInfo(res) @@ -549,9 +549,9 @@ proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = var oldsize = align(GenericSeqSize, elemAlign) + cast[PGenericSeq](old).len * elemSize copyMem(res, ol, oldsize + sizeof(Cell)) - zeroMem(cast[pointer](cast[ByteAddress](res) +% oldsize +% sizeof(Cell)), + zeroMem(cast[pointer](cast[int](res) +% oldsize +% sizeof(Cell)), newsize-oldsize) - sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "growObj: 3") + sysAssert((cast[int](res) and (MemAlign-1)) == 0, "growObj: 3") # This can be wrong for intermediate temps that are nevertheless on the # heap because of lambda lifting: #gcAssert(res.refcount shr rcShift <=% 1, "growObj: 4") @@ -683,7 +683,7 @@ proc collectCycles(gch: var GcHeap) {.raises: [].} = proc gcMark(gch: var GcHeap, p: pointer) {.inline.} = # the addresses are not as cells on the stack, so turn them to cells: sysAssert(allocInv(gch.region), "gcMark begin") - var c = cast[ByteAddress](p) + var c = cast[int](p) if c >% PageSize: # fast check: does it look like a cell? var objStart = cast[PCell](interiorAllocatedPtr(gch.region, p)) @@ -848,10 +848,10 @@ when withRealTime: stack.bottomSaved = stack.bottom when stackIncreases: stack.bottom = cast[pointer]( - cast[ByteAddress](stack.pos) - sizeof(pointer) * 6 - stackSize) + cast[int](stack.pos) - sizeof(pointer) * 6 - stackSize) else: stack.bottom = cast[pointer]( - cast[ByteAddress](stack.pos) + sizeof(pointer) * 6 + stackSize) + cast[int](stack.pos) + sizeof(pointer) * 6 + stackSize) GC_step(gch, us, strongAdvice) diff --git a/lib/system/gc2.nim b/lib/system/gc2.nim index 0593b396e..ed046b5fd 100644 --- a/lib/system/gc2.nim +++ b/lib/system/gc2.nim @@ -133,11 +133,11 @@ template gcAssert(cond: bool, msg: string) = proc cellToUsr(cell: PCell): pointer {.inline.} = # convert object (=pointer to refcount) to pointer to userdata - result = cast[pointer](cast[ByteAddress](cell)+%ByteAddress(sizeof(Cell))) + result = cast[pointer](cast[int](cell)+%ByteAddress(sizeof(Cell))) proc usrToCell(usr: pointer): PCell {.inline.} = # convert pointer to userdata to object (=pointer to refcount) - result = cast[PCell](cast[ByteAddress](usr)-%ByteAddress(sizeof(Cell))) + result = cast[PCell](cast[int](usr)-%ByteAddress(sizeof(Cell))) proc extGetCellType(c: pointer): PNimType {.compilerproc.} = # used for code generation concerning debugging @@ -252,7 +252,7 @@ proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerproc.} = dest[] = src proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) case n.kind of nkSlot: forAllChildrenAux(cast[pointer](d +% n.offset), n.typ, op) of nkList: @@ -264,7 +264,7 @@ proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = of nkNone: sysAssert(false, "forAllSlotsAux") proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) if dest == nil: return # nothing to do if ntfNoRefs notin mt.flags: case mt.kind @@ -290,7 +290,7 @@ proc forAllChildren(cell: PCell, op: WalkOp) = of tyRef: # common case forAllChildrenAux(cellToUsr(cell), cell.typ.base, op) of tySequence: - var d = cast[ByteAddress](cellToUsr(cell)) + var d = cast[int](cellToUsr(cell)) var s = cast[PGenericSeq](d) if s != nil: for i in 0..s.len-1: @@ -330,7 +330,7 @@ proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer = gcAssert(typ.kind in {tyRef, tyString, tySequence}, "newObj: 1") collectCT(gch) var res = cast[PCell](rawAlloc(gch.region, size + sizeof(Cell))) - gcAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") + gcAssert((cast[int](res) and (MemAlign-1)) == 0, "newObj: 2") # now it is buffered in the ZCT res.typ = typ when leakDetector and not hasThreadSupport: @@ -388,9 +388,9 @@ proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = var oldsize = align(GenericSeqSize, elemAlign) + cast[PGenericSeq](old).len*elemSize copyMem(res, ol, oldsize + sizeof(Cell)) - zeroMem(cast[pointer](cast[ByteAddress](res)+% oldsize +% sizeof(Cell)), + zeroMem(cast[pointer](cast[int](res)+% oldsize +% sizeof(Cell)), newsize-oldsize) - sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "growObj: 3") + sysAssert((cast[int](res) and (MemAlign-1)) == 0, "growObj: 3") when false: # this is wrong since seqs can be shared via 'shallow': when reallyDealloc: rawDealloc(gch.region, ol) @@ -593,7 +593,7 @@ proc gcMark(gch: var GcHeap, p: pointer) {.inline.} = # the addresses are not as cells on the stack, so turn them to cells: sysAssert(allocInv(gch.region), "gcMark begin") var cell = usrToCell(p) - var c = cast[ByteAddress](cell) + var c = cast[int](cell) if c >% PageSize: # fast check: does it look like a cell? var objStart = cast[PCell](interiorAllocatedPtr(gch.region, cell)) @@ -697,10 +697,10 @@ when withRealTime: stack.bottomSaved = stack.bottom when stackIncreases: stack.bottom = cast[pointer]( - cast[ByteAddress](stack.pos) - sizeof(pointer) * 6 - stackSize) + cast[int](stack.pos) - sizeof(pointer) * 6 - stackSize) else: stack.bottom = cast[pointer]( - cast[ByteAddress](stack.pos) + sizeof(pointer) * 6 + stackSize) + cast[int](stack.pos) + sizeof(pointer) * 6 + stackSize) GC_step(gch, us, strongAdvice) diff --git a/lib/system/gc_common.nim b/lib/system/gc_common.nim index f5f4f164f..54c51e783 100644 --- a/lib/system/gc_common.nim +++ b/lib/system/gc_common.nim @@ -222,9 +222,9 @@ proc stackSize(stack: ptr GcStack): int {.noinline.} = if pos != nil: when stackIncreases: - result = cast[ByteAddress](pos) -% cast[ByteAddress](stack.bottom) + result = cast[int](pos) -% cast[int](stack.bottom) else: - result = cast[ByteAddress](stack.bottom) -% cast[ByteAddress](pos) + result = cast[int](stack.bottom) -% cast[int](pos) else: result = 0 @@ -295,8 +295,8 @@ when not defined(useNimRtl): # the first init must be the one that defines the stack bottom: gch.stack.bottom = theStackBottom elif theStackBottom != gch.stack.bottom: - var a = cast[ByteAddress](theStackBottom) # and not PageMask - PageSize*2 - var b = cast[ByteAddress](gch.stack.bottom) + var a = cast[int](theStackBottom) # and not PageMask - PageSize*2 + var b = cast[int](gch.stack.bottom) #c_fprintf(stdout, "old: %p new: %p;\n",gch.stack.bottom,theStackBottom) when stackIncreases: gch.stack.bottom = cast[pointer](min(a, b)) @@ -312,11 +312,11 @@ when not defined(useNimRtl): proc isOnStack(p: pointer): bool = var stackTop {.volatile, noinit.}: pointer stackTop = addr(stackTop) - var a = cast[ByteAddress](gch.getActiveStack().bottom) - var b = cast[ByteAddress](stackTop) + var a = cast[int](gch.getActiveStack().bottom) + var b = cast[int](stackTop) when not stackIncreases: swap(a, b) - var x = cast[ByteAddress](p) + var x = cast[int](p) result = a <=% x and x <=% b when defined(sparc): # For SPARC architecture. @@ -337,7 +337,7 @@ when defined(sparc): # For SPARC architecture. # Addresses decrease as the stack grows. while sp <= max: gcMark(gch, sp[]) - sp = cast[PPointer](cast[ByteAddress](sp) +% sizeof(pointer)) + sp = cast[PPointer](cast[int](sp) +% sizeof(pointer)) elif defined(ELATE): {.error: "stack marking code is to be written for this architecture".} @@ -354,8 +354,8 @@ elif stackIncreases: template forEachStackSlotAux(gch, gcMark: untyped) {.dirty.} = for stack in gch.stack.items(): - var max = cast[ByteAddress](gch.stack.bottom) - var sp = cast[ByteAddress](addr(registers)) -% sizeof(pointer) + var max = cast[int](gch.stack.bottom) + var sp = cast[int](addr(registers)) -% sizeof(pointer) while sp >=% max: gcMark(gch, cast[PPointer](sp)[]) sp = sp -% sizeof(pointer) @@ -383,8 +383,8 @@ else: gch.getActiveStack().setPosition(addr(registers)) if c_setjmp(registers) == 0'i32: # To fill the C stack with registers. for stack in gch.stack.items(): - var max = cast[ByteAddress](stack.bottom) - var sp = cast[ByteAddress](addr(registers)) + var max = cast[int](stack.bottom) + var sp = cast[int](addr(registers)) when defined(amd64): if stack.isActiveStack(): # words within the jmp_buf structure may not be properly aligned. diff --git a/lib/system/gc_ms.nim b/lib/system/gc_ms.nim index f91b37b94..72d22f08b 100644 --- a/lib/system/gc_ms.nim +++ b/lib/system/gc_ms.nim @@ -94,11 +94,11 @@ template gcAssert(cond: bool, msg: string) = proc cellToUsr(cell: PCell): pointer {.inline.} = # convert object (=pointer to refcount) to pointer to userdata - result = cast[pointer](cast[ByteAddress](cell)+%ByteAddress(sizeof(Cell))) + result = cast[pointer](cast[int](cell)+%ByteAddress(sizeof(Cell))) proc usrToCell(usr: pointer): PCell {.inline.} = # convert pointer to userdata to object (=pointer to refcount) - result = cast[PCell](cast[ByteAddress](usr)-%ByteAddress(sizeof(Cell))) + result = cast[PCell](cast[int](usr)-%ByteAddress(sizeof(Cell))) proc extGetCellType(c: pointer): PNimType {.compilerproc.} = # used for code generation concerning debugging @@ -217,7 +217,7 @@ proc initGC() = gcAssert(gch.gcThreadId >= 0, "invalid computed thread ID") proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) case n.kind of nkSlot: forAllChildrenAux(cast[pointer](d +% n.offset), n.typ, op) of nkList: @@ -229,7 +229,7 @@ proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = of nkNone: sysAssert(false, "forAllSlotsAux") proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) if dest == nil: return # nothing to do if ntfNoRefs notin mt.flags: case mt.kind @@ -255,7 +255,7 @@ proc forAllChildren(cell: PCell, op: WalkOp) = forAllChildrenAux(cellToUsr(cell), cell.typ.base, op) of tySequence: when not defined(nimSeqsV2): - var d = cast[ByteAddress](cellToUsr(cell)) + var d = cast[int](cellToUsr(cell)) var s = cast[PGenericSeq](d) if s != nil: for i in 0..s.len-1: @@ -268,7 +268,7 @@ proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer = gcAssert(typ.kind in {tyRef, tyString, tySequence}, "newObj: 1") collectCT(gch, size + sizeof(Cell)) var res = cast[PCell](rawAlloc(gch.region, size + sizeof(Cell))) - gcAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") + gcAssert((cast[int](res) and (MemAlign-1)) == 0, "newObj: 2") # now it is buffered in the ZCT res.typ = typ when leakDetector and not hasThreadSupport: @@ -336,9 +336,9 @@ when not defined(nimSeqsV2): var oldsize = align(GenericSeqSize, elemAlign) + cast[PGenericSeq](old).len*elemSize copyMem(res, ol, oldsize + sizeof(Cell)) - zeroMem(cast[pointer](cast[ByteAddress](res)+% oldsize +% sizeof(Cell)), + zeroMem(cast[pointer](cast[int](res)+% oldsize +% sizeof(Cell)), newsize-oldsize) - sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "growObj: 3") + sysAssert((cast[int](res) and (MemAlign-1)) == 0, "growObj: 3") when withBitvectors: incl(gch.allocated, res) when useCellIds: inc gch.idGenerator @@ -446,7 +446,7 @@ proc markGlobals(gch: var GcHeap) = proc gcMark(gch: var GcHeap, p: pointer) {.inline.} = # the addresses are not as cells on the stack, so turn them to cells: - var c = cast[ByteAddress](p) + var c = cast[int](p) if c >% PageSize: # fast check: does it look like a cell? var objStart = cast[PCell](interiorAllocatedPtr(gch.region, p)) diff --git a/lib/system/osalloc.nim b/lib/system/osalloc.nim index 4817059be..925e98ed3 100644 --- a/lib/system/osalloc.nim +++ b/lib/system/osalloc.nim @@ -80,12 +80,12 @@ elif defined(emscripten) and not defined(StandaloneHeapSize): let pos = cast[int](result) # Convert pointer to PageSize correct one. - var new_pos = cast[ByteAddress](pos) +% (PageSize - (pos %% PageSize)) + var new_pos = cast[int](pos) +% (PageSize - (pos %% PageSize)) if (new_pos-pos) < sizeof(EmscriptenMMapBlock): new_pos = new_pos +% PageSize result = cast[pointer](new_pos) - var mmapDescrPos = cast[ByteAddress](result) -% sizeof(EmscriptenMMapBlock) + var mmapDescrPos = cast[int](result) -% sizeof(EmscriptenMMapBlock) var mmapDescr = cast[EmscriptenMMapBlock](mmapDescrPos) mmapDescr.realSize = realSize @@ -96,7 +96,7 @@ elif defined(emscripten) and not defined(StandaloneHeapSize): proc osTryAllocPages(size: int): pointer = osAllocPages(size) proc osDeallocPages(p: pointer, size: int) {.inline.} = - var mmapDescrPos = cast[ByteAddress](p) -% sizeof(EmscriptenMMapBlock) + var mmapDescrPos = cast[int](p) -% sizeof(EmscriptenMMapBlock) var mmapDescr = cast[EmscriptenMMapBlock](mmapDescrPos) munmap(mmapDescr.realPointer, mmapDescr.realSize) diff --git a/lib/system/repr.nim b/lib/system/repr.nim index e47220656..6b6f7e340 100644 --- a/lib/system/repr.nim +++ b/lib/system/repr.nim @@ -155,7 +155,7 @@ when not defined(useNimRtl): var bs = typ.base.size for i in 0..typ.size div bs - 1: if i > 0: add result, ", " - reprAux(result, cast[pointer](cast[ByteAddress](p) + i*bs), typ.base, cl) + reprAux(result, cast[pointer](cast[int](p) + i*bs), typ.base, cl) add result, "]" when defined(nimSeqsV2): @@ -183,7 +183,7 @@ when not defined(useNimRtl): var bs = typ.base.size for i in 0..cast[PGenericSeq](p).len-1: if i > 0: add result, ", " - reprAux(result, cast[pointer](cast[ByteAddress](payloadPtr(p)) + align(payloadOffset, typ.align) + i*bs), + reprAux(result, cast[pointer](cast[int](payloadPtr(p)) + align(payloadOffset, typ.align) + i*bs), typ.base, cl) add result, "]" @@ -194,14 +194,14 @@ when not defined(useNimRtl): of nkSlot: add result, $n.name add result, " = " - reprAux(result, cast[pointer](cast[ByteAddress](p) + n.offset), n.typ, cl) + reprAux(result, cast[pointer](cast[int](p) + n.offset), n.typ, cl) of nkList: for i in 0..n.len-1: if i > 0: add result, ",\n" reprRecordAux(result, p, n.sons[i], cl) of nkCase: var m = selectBranch(p, n) - reprAux(result, cast[pointer](cast[ByteAddress](p) + n.offset), n.typ, cl) + reprAux(result, cast[pointer](cast[int](p) + n.offset), n.typ, cl) if m != nil: reprRecordAux(result, p, m, cl) proc reprRecord(result: var string, p: pointer, typ: PNimType, @@ -307,7 +307,7 @@ when not defined(useNimRtl): var bs = elemtyp.size for i in 0..length - 1: if i > 0: add result, ", " - reprAux(result, cast[pointer](cast[ByteAddress](p) + i*bs), elemtyp, cl) + reprAux(result, cast[pointer](cast[int](p) + i*bs), elemtyp, cl) add result, "]" deinitReprClosure(cl) diff --git a/lib/system/sysstr.nim b/lib/system/sysstr.nim index be32652d8..b8dc7101d 100644 --- a/lib/system/sysstr.nim +++ b/lib/system/sysstr.nim @@ -17,10 +17,10 @@ proc dataPointer(a: PGenericSeq, elemAlign: int): pointer = - cast[pointer](cast[ByteAddress](a) +% align(GenericSeqSize, elemAlign)) + cast[pointer](cast[int](a) +% align(GenericSeqSize, elemAlign)) proc dataPointer(a: PGenericSeq, elemAlign, elemSize, index: int): pointer = - cast[pointer](cast[ByteAddress](a) +% align(GenericSeqSize, elemAlign) +% (index*%elemSize)) + cast[pointer](cast[int](a) +% align(GenericSeqSize, elemAlign) +% (index*%elemSize)) proc resize(old: int): int {.inline.} = if old <= 0: result = 4 diff --git a/tests/parallel/tguard1.nim b/tests/parallel/tguard1.nim index b1eb7e7c5..f4c92319b 100644 --- a/tests/parallel/tguard1.nim +++ b/tests/parallel/tguard1.nim @@ -5,7 +5,7 @@ output: "90" when false: template lock(a, b: ptr Lock; body: stmt) = - if cast[ByteAddress](a) < cast[ByteAddress](b): + if cast[int](a) < cast[int](b): pthread_mutex_lock(a) pthread_mutex_lock(b) else: From 84200f28a114ef9744ecf2b6060ef140e350139b Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 16 Dec 2022 16:42:31 +0800 Subject: [PATCH 2288/2472] bump csource_v2 hash (#21118) --- config/build_config.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/build_config.txt b/config/build_config.txt index 3ae618f00..0ea963c39 100644 --- a/config/build_config.txt +++ b/config/build_config.txt @@ -2,4 +2,4 @@ nim_comment="key-value pairs for windows/posix bootstrapping build scripts" nim_csourcesDir=csources_v2 nim_csourcesUrl=https://github.com/nim-lang/csources_v2.git nim_csourcesBranch=master -nim_csourcesHash=ffd834c20915dc04c78eb0d3b2162832e7ec9c13 +nim_csourcesHash=32cf8431bb19f8803352a0ab3dbee5fec35d5ec6 From 296e7f598e58430415369a34f83552915f57f3f5 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 16 Dec 2022 19:49:26 +0800 Subject: [PATCH 2289/2472] Bump the devel version to 1.9.1 (#21117) --- lib/system/compilation.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/system/compilation.nim b/lib/system/compilation.nim index 6109e9874..234e8cafe 100644 --- a/lib/system/compilation.nim +++ b/lib/system/compilation.nim @@ -6,11 +6,11 @@ const ## ``` # see also std/private/since - NimMinor* {.intdefine.}: int = 7 + NimMinor* {.intdefine.}: int = 9 ## is the minor number of Nim's version. ## Odd for devel, even for releases. - NimPatch* {.intdefine.}: int = 3 + NimPatch* {.intdefine.}: int = 1 ## is the patch number of Nim's version. ## Odd for devel, even for releases. From 7fa983f5e6b5f52a20e477e2443c8fdede244290 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sat, 17 Dec 2022 00:21:07 +0800 Subject: [PATCH 2290/2472] Nim now ships nimble with 0.14.1 version (#21120) * Nimble now uses latest patch * Update koch.nim --- koch.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/koch.nim b/koch.nim index 84d0911ac..6b3d327f5 100644 --- a/koch.nim +++ b/koch.nim @@ -10,7 +10,7 @@ # const - NimbleStableCommit = "0777f33d1ddbd505b3aa7b714032125349323ceb" # master + NimbleStableCommit = "afd03bc000f4013c48f52381728327dabfccfc4c" # master # examples of possible values: #head, #ea82b54, 1.2.3 FusionStableHash = "#372ee4313827ef9f2ea388840f7d6b46c2b1b014" HeadHash = "#head" From d83f66a24d990f2e4a135a4a64e1c464702b953d Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sat, 17 Dec 2022 16:10:56 +0800 Subject: [PATCH 2291/2472] fixes #21116; always mangles the param (#21121) * fixes #21116; always mangles the param * idOrSig * fixes tests * Apply suggestions from code review --- compiler/ccgtypes.nim | 6 +----- tests/ccgbugs/t21116.nim | 10 ++++++++++ tests/ccgbugs/tnoalias.nim | 2 +- 3 files changed, 12 insertions(+), 6 deletions(-) create mode 100644 tests/ccgbugs/t21116.nim diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index d7b56de5b..3c9f9d923 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -48,11 +48,9 @@ proc fillBackendName(m: BModule; s: PSym) = writeMangledName(m.ndi, s, m.config) proc fillParamName(m: BModule; s: PSym) = - ## we cannot use 'sigConflicts' here since we have a BModule, not a BProc. - ## Fortunately C's scoping rules are sane enough so that that doesn't - ## cause any trouble. if s.loc.r == "": var res = s.name.s.mangle + res.add idOrSig(s, res, m.sigConflicts) # Take into account if HCR is on because of the following scenario: # if a module gets imported and it has some more importc symbols in it, # some param names might receive the "_0" suffix to distinguish from what @@ -69,8 +67,6 @@ proc fillParamName(m: BModule; s: PSym) = # and a function called in main or proxy uses `socket` as a parameter name. # That would lead to either needing to reload `proxy` or to overwrite the # executable file for the main module, which is running (or both!) -> error. - if m.hcrOn or isKeyword(s.name) or m.g.config.cppDefines.contains(res): - res.add "_0" s.loc.r = res.rope writeMangledName(m.ndi, s, m.config) diff --git a/tests/ccgbugs/t21116.nim b/tests/ccgbugs/t21116.nim new file mode 100644 index 000000000..6418df539 --- /dev/null +++ b/tests/ccgbugs/t21116.nim @@ -0,0 +1,10 @@ +discard """ + target: "c cpp" + disabled: windows +""" +# bug #21116 +import std/os + +proc p(glob: string) = + for _ in walkFiles(glob): discard +p("dir/*") diff --git a/tests/ccgbugs/tnoalias.nim b/tests/ccgbugs/tnoalias.nim index f200992d7..96c3d390b 100644 --- a/tests/ccgbugs/tnoalias.nim +++ b/tests/ccgbugs/tnoalias.nim @@ -1,5 +1,5 @@ discard """ - ccodecheck: "\\i@'NI* NIM_NOALIAS field;' @'NIM_CHAR* NIM_NOALIAS x,' @'void* NIM_NOALIAS q'" + ccodecheck: "\\i@'NI* NIM_NOALIAS field;' @'NIM_CHAR* NIM_NOALIAS x__0qEngDE9aYoYsF8tWnyPacw,' @'void* NIM_NOALIAS q'" """ type From ae4645e8dfec5c7b5f2d8cbdb672b7a0d55dfb37 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sat, 17 Dec 2022 16:28:59 +0800 Subject: [PATCH 2292/2472] add 2.0 changelog (#21107) [backport: 2.0] add v2.0 changelog --- changelog.md | 296 +------------------------------ changelogs/changelog_2_0_0.md | 321 ++++++++++++++++++++++++++++++++++ 2 files changed, 324 insertions(+), 293 deletions(-) create mode 100644 changelogs/changelog_2_0_0.md diff --git a/changelog.md b/changelog.md index 00bdb50c2..0389faad9 100644 --- a/changelog.md +++ b/changelog.md @@ -1,321 +1,31 @@ -# v2.0.0 - yyyy-mm-dd +# v2.2.0 - yyyy-mm-dd ## Changes affecting backward compatibility -- `httpclient.contentLength` default to `-1` if the Content-Length header is not set in the response, it followed Apache HttpClient(Java), http(go) and .Net HttpWebResponse(C#) behavior. Previously it raised `ValueError`. -- `addr` is now available for all addressable locations, - `unsafeAddr` is now deprecated and an alias for `addr`. - -- Certain definitions from the default `system` module have been moved to - the following new modules: - - - `std/syncio` - - `std/assertions` - - `std/formatfloat` - - `std/objectdollar` - - `std/widestrs` - - `std/typedthreads` - - `std/sysatomics` - - In the future, these definitions will be removed from the `system` module, - and their respective modules will have to be imported to use them. - Currently, to make these imports required, the `-d:nimPreviewSlimSystem` option - may be used. - -- Enabling `-d:nimPreviewSlimSystem` also removes the following deprecated - symbols in the `system` module: - - Aliases with `Error` suffix to exception types that have a `Defect` suffix - (see [exceptions](https://nim-lang.org/docs/exceptions.html)): - `ArithmeticError`, `DivByZeroError`, `OverflowError`, - `AccessViolationError`, `AssertionError`, `OutOfMemError`, `IndexError`, - `FieldError`, `RangeError`, `StackOverflowError`, `ReraiseError`, - `ObjectAssignmentError`, `ObjectConversionError`, `FloatingPointError`, - `FloatOverflowError`, `FloatUnderflowError`, `FloatInexactError`, - `DeadThreadError`, `NilAccessError` - - `addQuitProc`, replaced by `exitprocs.addExitProc` - - Legacy unsigned conversion operations: `ze`, `ze64`, `toU8`, `toU16`, `toU32` - - `TaintedString`, formerly a distinct alias to `string` - - `PInt32`, `PInt64`, `PFloat32`, `PFloat64`, aliases to - `ptr int32`, `ptr int64`, `ptr float32`, `ptr float64` - -- Enabling `-d:nimPreviewSlimSystem` removes the import of `channels_builtin` in - in the `system` module. - -- Enabling `-d:nimPreviewCstringConversion`, `ptr char`, `ptr array[N, char]` and `ptr UncheckedArray[N, char]` don't support conversion to cstring anymore. - -- The `gc:v2` option is removed. - -- The `mainmodule` and `m` options are removed. - -- The `threads:on` option is now the default. - -- Optional parameters in combination with `: body` syntax (RFC #405) are now opt-in via - `experimental:flexibleOptionalParams`. - -- Automatic dereferencing (experimental feature) is removed. - -- The `Math.trunc` polyfill for targeting Internet Explorer was - previously included in most JavaScript output files. - Now, it is only included with `-d:nimJsMathTruncPolyfill`. - If you are targeting Internet Explorer, you may choose to enable this option - or define your own `Math.trunc` polyfill using the [`emit` pragma](https://nim-lang.org/docs/manual.html#implementation-specific-pragmas-emit-pragma). - Nim uses `Math.trunc` for the division and modulo operators for integers. - -- `shallowCopy` and `shallow` are removed for ARC/ORC. Use `move` when possible or combine assignment and -`sink` for optimization purposes. - -- The `nimPreviewDotLikeOps` define is going to be removed or deprecated. - -- The `{.this.}` pragma, deprecated since 0.19, has been removed. -- `nil` literals can no longer be directly assigned to variables or fields of `distinct` pointer types. They must be converted instead. - ```nim - type Foo = distinct ptr int - - # Before: - var x: Foo = nil - # After: - var x: Foo = Foo(nil) - ``` -- Removed two type pragma syntaxes deprecated since 0.20, namely - `type Foo = object {.final.}`, and `type Foo {.final.} [T] = object`. - -- `foo a = b` now means `foo(a = b)` rather than `foo(a) = b`. This is consistent - with the existing behavior of `foo a, b = c` meaning `foo(a, b = c)`. - This decision was made with the assumption that the old syntax was used rarely; - if your code used the old syntax, please be aware of this change. - -- [Overloadable enums](https://nim-lang.github.io/Nim/manual.html#overloadable-enum-value-names) and Unicode Operators - are no longer experimental. - -- Removed the `nimIncrSeqV3` define. - -- `macros.getImpl` for `const` symbols now returns the full definition node - (as `nnkConstDef`) rather than the AST of the constant value. - -- Lock levels are deprecated, now a noop. - -- ORC is now the default memory management strategy. Use - `--mm:refc` for a transition period. - -- `strictEffects` are no longer experimental. - Use `legacy:laxEffects` to keep backward compatibility. - -- The `gorge`/`staticExec` calls will now return a descriptive message in the output - if the execution fails for whatever reason. To get back legacy behaviour use `-d:nimLegacyGorgeErrors`. - -- Pointer to `cstring` conversion now triggers a `[PtrToCstringConv]` warning. - This warning will become an error in future versions! Use a `cast` operation - like `cast[cstring](x)` instead. - -- `logging` will default to flushing all log level messages. To get the legacy behaviour of only flushing Error and Fatal messages, use `-d:nimV1LogFlushBehavior`. - -- Object fields now support default values, see https://nim-lang.github.io/Nim/manual.html#types-default-values-for-object-fields for details. - -- Redefining templates with the same signature was previously - allowed to support certain macro code. To do this explicitly, the - `{.redefine.}` pragma has been added. Note that this is only for templates. - Implicit redefinition of templates is now deprecated and will give an error in the future. - -- Using an unnamed break in a block is deprecated. This warning will become an error in future versions! Use a named block with a named break instead. - -- Several Standard libraries are moved to nimble packages, use `nimble` to install them: - - `std/punycode` => `punycode` - - `std/asyncftpclient` => `asyncftpclient` - - `std/smtp` => `smtp` - - `std/db_common` => `db_connector/db_common` - - `std/db_sqlite` => `db_connector/db_sqlite` - - `std/db_mysql` => `db_connector/db_mysql` - - `std/db_postgres` => `db_connector/db_postgres` - - `std/db_odbc` => `db_connector/db_odbc` - -- Previously, calls like `foo(a, b): ...` or `foo(a, b) do: ...` where the final argument of - `foo` had type `proc ()` were assumed by the compiler to mean `foo(a, b, proc () = ...)`. - This behavior is now deprecated. Use `foo(a, b) do (): ...` or `foo(a, b, proc () = ...)` instead. - -- If no exception or any exception deriving from Exception but not Defect or CatchableError given in except, a `warnBareExcept` warning will be triggered. ## Standard library additions and changes [//]: # "Changes:" -- OpenSSL 3 is now supported. -- `macros.parseExpr` and `macros.parseStmt` now accept an optional - filename argument for more informative errors. -- Module `colors` expanded with missing colors from the CSS color standard. - `colPaleVioletRed` and `colMediumPurple` have also been changed to match the CSS color standard. -- Fixed `lists.SinglyLinkedList` being broken after removing the last node ([#19353](https://github.com/nim-lang/Nim/pull/19353)). -- The `md5` module now works at compile time and in JavaScript. -- Changed `mimedb` to use an `OrderedTable` instead of `OrderedTableRef` to support `const` tables. -- `strutils.find` now uses and defaults to `last = -1` for whole string searches, - making limiting it to just the first char (`last = 0`) valid. -- `random.rand` now works with `Ordinal`s. -- Undeprecated `os.isvalidfilename`. -- `std/oids` now uses `int64` to store time internally (before it was int32). -- `std/uri.Uri` dollar `$` improved, precalculates the `string` result length from the `Uri`. -- `std/uri.Uri.isIpv6` is now exported. -- `std/logging.ConsoleLogger` and `FileLogger` now have a `flushThreshold` attribute to set what log message levels are automatically flushed. For Nim v1 use `-d:nimFlushAllLogs` to automatically flush all message levels. Flushing all logs is the default behavior for Nim v2. - - -- `std/net.IpAddress` dollar `$` improved, uses a fixed capacity for the `string` result based from the `IpAddressFamily`. -- `std/jsfetch.newFetchOptions` now has default values for all parameters -- `std/jsformdata` now accepts `Blob` data type. [//]: # "Additions:" -- Added ISO 8601 week date utilities in `times`: - - Added `IsoWeekRange`, a range type for weeks in a week-based year. - - Added `IsoYear`, a distinct type for a week-based year in contrast to a regular year. - - Added a `initDateTime` overload to create a datetime from an ISO week date. - - Added `getIsoWeekAndYear` to get an ISO week number and week-based year from a datetime. - - Added `getIsoWeeksInYear` to return the number of weeks in a week-based year. -- Added new modules which were part of `std/os`: - - Added `std/oserrors` for OS error reporting. Added `std/envvars` for environment variables handling. - - Added `std/paths`, `std/dirs`, `std/files`, `std/symlinks` and `std/appdirs`. - - Added `std/cmdline` for reading command line parameters. -- Added `sep` parameter in `std/uri` to specify the query separator. -- Added bindings to [`Array.shift`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/shift) - and [`queueMicrotask`](https://developer.mozilla.org/en-US/docs/Web/API/queueMicrotask) - in `jscore` for JavaScript targets. -- Added `UppercaseLetters`, `LowercaseLetters`, `PunctuationChars`, `PrintableChars` sets to `std/strutils`. -- Added `complex.sgn` for obtaining the phase of complex numbers. -- Added `insertAdjacentText`, `insertAdjacentElement`, `insertAdjacentHTML`, - `after`, `before`, `closest`, `append`, `hasAttributeNS`, `removeAttributeNS`, - `hasPointerCapture`, `releasePointerCapture`, `requestPointerLock`, - `replaceChildren`, `replaceWith`, `scrollIntoViewIfNeeded`, `setHTML`, - `toggleAttribute`, and `matches` to `std/dom`. -- Added [`jsre.hasIndices`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp/hasIndices) -- Added `capacity` for `string` and `seq` to return the current capacity, see https://github.com/nim-lang/RFCs/issues/460 -- Added `openArray[char]` overloads for `std/parseutils` allowing more code reuse. -- Added `openArray[char]` overloads for `std/unicode` allowing more code reuse. -- Added `safe` parameter to `base64.encodeMime`. + [//]: # "Deprecations:" -- Deprecated `selfExe` for Nimscript. -- Deprecated `std/sums`. -- Deprecated `std/base64.encode` for collections of arbitrary integer element type. - Now only `byte` and `char` are supported. + [//]: # "Removals:" -- Removed deprecated module `parseopt2`. -- Removed deprecated module `sharedstrings`. -- Removed deprecated module `dom_extensions`. -- Removed deprecated module `LockFreeHash`. -- Removed deprecated module `events`. -- Removed deprecated `oids.oidToString`. -- Removed define `nimExperimentalAsyncjsThen` for `std/asyncjs.then` and `std/jsfetch`. -- Removed deprecated `jsre.test` and `jsre.toString`. -- Removed deprecated `math.c_frexp`. -- Removed deprecated `` httpcore.`==` ``. -- Removed deprecated `std/posix.CMSG_SPACE` and `std/posix.CMSG_LEN` that takes wrong argument types. -- Removed deprecated `osproc.poDemon`, symbol with typo. -- Removed deprecated `tables.rightSize`. - - -- Removed deprecated `posix.CLONE_STOPPED`. ## Language changes -- [Tag tracking](https://nim-lang.github.io/Nim/manual.html#effect-system-tag-tracking) supports the definition of forbidden tags by the `.forbids` pragma - which can be used to disable certain effects in proc types. -- [Case statement macros](https://nim-lang.github.io/Nim/manual.html#macros-case-statement-macros) are no longer experimental, - meaning you no longer need to enable the experimental switch `caseStmtMacros` to use them. -- Full command syntax and block arguments i.e. `foo a, b: c` are now allowed - for the right-hand side of type definitions in type sections. Previously - they would error with "invalid indentation". - -- Compile-time define changes: - - `defined` now accepts identifiers separated by dots, i.e. `defined(a.b.c)`. - In the command line, this is defined as `-d:a.b.c`. Older versions can - use accents as in ``defined(`a.b.c`)`` to access such defines. - - [Define pragmas for constants](https://nim-lang.github.io/Nim/manual.html#implementation-specific-pragmas-compileminustime-define-pragmas) - now support a string argument for qualified define names. - - ```nim - # -d:package.FooBar=42 - const FooBar {.intdefine: "package.FooBar".}: int = 5 - echo FooBar # 42 - ``` - - This was added to help disambiguate similar define names for different packages. - In older versions, this could only be achieved with something like the following: - - ```nim - const FooBar = block: - const `package.FooBar` {.intdefine.}: int = 5 - `package.FooBar` - ``` - - A generic `define` pragma for constants has been added that interprets - the value of the define based on the type of the constant value. - See the [experimental manual](https://nim-lang.github.io/Nim/manual_experimental.html#generic-define-pragma) - for a list of supported types. - -- [Macro pragmas](https://nim-lang.github.io/Nim/manual.html#userminusdefined-pragmas-macro-pragmas) changes: - - Templates now accept macro pragmas. - - Macro pragmas for var/let/const sections have been redesigned in a way that works - similarly to routine macro pragmas. The new behavior is documented in the - [experimental manual](https://nim-lang.github.io/Nim/manual_experimental.html#extended-macro-pragmas). - - Pragma macros on type definitions can now return `nnkTypeSection` nodes as well as `nnkTypeDef`, - allowing multiple type definitions to be injected in place of the original type definition. - - ```nim - import macros - macro multiply(amount: static int, s: untyped): untyped = - let name = $s[0].basename - result = newNimNode(nnkTypeSection) - for i in 1 .. amount: - result.add(newTree(nnkTypeDef, ident(name & $i), s[1], s[2])) - type - Foo = object - Bar {.multiply: 3.} = object - x, y, z: int - Baz = object - # becomes - type - Foo = object - Bar1 = object - x, y, z: int - Bar2 = object - x, y, z: int - Bar3 = object - x, y, z: int - Baz = object - ``` - -- A new form of type inference called [top-down inference](https://nim-lang.github.io/Nim/manual_experimental.html#topminusdown-type-inference) - has been implemented for a variety of basic cases. For example, code like the following now compiles: - - ```nim - let foo: seq[(float, byte, cstring)] = @[(1, 2, "abc")] - ``` - -- `cstring` is now accepted as a selector in `case` statements, removing the - need to convert to `string`. On the JS backend, this is translated directly - to a `switch` statement. - -- Nim now supports `out` parameters and ["strict definitions"](https://nim-lang.github.io/Nim/manual_experimental.html#strict-definitions-and-nimout-parameters). -- Nim now offers a [strict mode](https://nim-lang.github.io/Nim/manual_experimental.html#strict-case-objects) for `case objects`. ## Compiler changes -- The `gc` switch has been renamed to `mm` ("memory management") in order to reflect the - reality better. (Nim moved away from all techniques based on "tracing".) -- Defines the `gcRefc` symbol which allows writing specific code for the refc GC. - -- `nim` can now compile version 1.4.0 as follows: `nim c --lib:lib --stylecheck:off compiler/nim`, - without requiring `-d:nimVersion140` which is now a noop. - -- `--styleCheck`, `--hintAsError` and `--warningAsError` now only apply to the current package. - -- The switch `--nimMainPrefix:prefix` has been added to add a prefix to the names of `NimMain` and - related functions produced on the backend. This prevents conflicts with other Nim - static libraries. - -- When compiling for Release the flag `-fno-math-errno` is used for GCC. ## Tool changes -- Nim now ships Nimble version 0.14 which added support for lock-files. Libraries are stored in `$nimbleDir/pkgs2` (it was `$nimbleDir/pkgs`). diff --git a/changelogs/changelog_2_0_0.md b/changelogs/changelog_2_0_0.md new file mode 100644 index 000000000..00bdb50c2 --- /dev/null +++ b/changelogs/changelog_2_0_0.md @@ -0,0 +1,321 @@ +# v2.0.0 - yyyy-mm-dd + + +## Changes affecting backward compatibility +- `httpclient.contentLength` default to `-1` if the Content-Length header is not set in the response, it followed Apache HttpClient(Java), http(go) and .Net HttpWebResponse(C#) behavior. Previously it raised `ValueError`. + +- `addr` is now available for all addressable locations, + `unsafeAddr` is now deprecated and an alias for `addr`. + +- Certain definitions from the default `system` module have been moved to + the following new modules: + + - `std/syncio` + - `std/assertions` + - `std/formatfloat` + - `std/objectdollar` + - `std/widestrs` + - `std/typedthreads` + - `std/sysatomics` + + In the future, these definitions will be removed from the `system` module, + and their respective modules will have to be imported to use them. + Currently, to make these imports required, the `-d:nimPreviewSlimSystem` option + may be used. + +- Enabling `-d:nimPreviewSlimSystem` also removes the following deprecated + symbols in the `system` module: + - Aliases with `Error` suffix to exception types that have a `Defect` suffix + (see [exceptions](https://nim-lang.org/docs/exceptions.html)): + `ArithmeticError`, `DivByZeroError`, `OverflowError`, + `AccessViolationError`, `AssertionError`, `OutOfMemError`, `IndexError`, + `FieldError`, `RangeError`, `StackOverflowError`, `ReraiseError`, + `ObjectAssignmentError`, `ObjectConversionError`, `FloatingPointError`, + `FloatOverflowError`, `FloatUnderflowError`, `FloatInexactError`, + `DeadThreadError`, `NilAccessError` + - `addQuitProc`, replaced by `exitprocs.addExitProc` + - Legacy unsigned conversion operations: `ze`, `ze64`, `toU8`, `toU16`, `toU32` + - `TaintedString`, formerly a distinct alias to `string` + - `PInt32`, `PInt64`, `PFloat32`, `PFloat64`, aliases to + `ptr int32`, `ptr int64`, `ptr float32`, `ptr float64` + +- Enabling `-d:nimPreviewSlimSystem` removes the import of `channels_builtin` in + in the `system` module. + +- Enabling `-d:nimPreviewCstringConversion`, `ptr char`, `ptr array[N, char]` and `ptr UncheckedArray[N, char]` don't support conversion to cstring anymore. + +- The `gc:v2` option is removed. + +- The `mainmodule` and `m` options are removed. + +- The `threads:on` option is now the default. + +- Optional parameters in combination with `: body` syntax (RFC #405) are now opt-in via + `experimental:flexibleOptionalParams`. + +- Automatic dereferencing (experimental feature) is removed. + +- The `Math.trunc` polyfill for targeting Internet Explorer was + previously included in most JavaScript output files. + Now, it is only included with `-d:nimJsMathTruncPolyfill`. + If you are targeting Internet Explorer, you may choose to enable this option + or define your own `Math.trunc` polyfill using the [`emit` pragma](https://nim-lang.org/docs/manual.html#implementation-specific-pragmas-emit-pragma). + Nim uses `Math.trunc` for the division and modulo operators for integers. + +- `shallowCopy` and `shallow` are removed for ARC/ORC. Use `move` when possible or combine assignment and +`sink` for optimization purposes. + +- The `nimPreviewDotLikeOps` define is going to be removed or deprecated. + +- The `{.this.}` pragma, deprecated since 0.19, has been removed. +- `nil` literals can no longer be directly assigned to variables or fields of `distinct` pointer types. They must be converted instead. + ```nim + type Foo = distinct ptr int + + # Before: + var x: Foo = nil + # After: + var x: Foo = Foo(nil) + ``` +- Removed two type pragma syntaxes deprecated since 0.20, namely + `type Foo = object {.final.}`, and `type Foo {.final.} [T] = object`. + +- `foo a = b` now means `foo(a = b)` rather than `foo(a) = b`. This is consistent + with the existing behavior of `foo a, b = c` meaning `foo(a, b = c)`. + This decision was made with the assumption that the old syntax was used rarely; + if your code used the old syntax, please be aware of this change. + +- [Overloadable enums](https://nim-lang.github.io/Nim/manual.html#overloadable-enum-value-names) and Unicode Operators + are no longer experimental. + +- Removed the `nimIncrSeqV3` define. + +- `macros.getImpl` for `const` symbols now returns the full definition node + (as `nnkConstDef`) rather than the AST of the constant value. + +- Lock levels are deprecated, now a noop. + +- ORC is now the default memory management strategy. Use + `--mm:refc` for a transition period. + +- `strictEffects` are no longer experimental. + Use `legacy:laxEffects` to keep backward compatibility. + +- The `gorge`/`staticExec` calls will now return a descriptive message in the output + if the execution fails for whatever reason. To get back legacy behaviour use `-d:nimLegacyGorgeErrors`. + +- Pointer to `cstring` conversion now triggers a `[PtrToCstringConv]` warning. + This warning will become an error in future versions! Use a `cast` operation + like `cast[cstring](x)` instead. + +- `logging` will default to flushing all log level messages. To get the legacy behaviour of only flushing Error and Fatal messages, use `-d:nimV1LogFlushBehavior`. + +- Object fields now support default values, see https://nim-lang.github.io/Nim/manual.html#types-default-values-for-object-fields for details. + +- Redefining templates with the same signature was previously + allowed to support certain macro code. To do this explicitly, the + `{.redefine.}` pragma has been added. Note that this is only for templates. + Implicit redefinition of templates is now deprecated and will give an error in the future. + +- Using an unnamed break in a block is deprecated. This warning will become an error in future versions! Use a named block with a named break instead. + +- Several Standard libraries are moved to nimble packages, use `nimble` to install them: + - `std/punycode` => `punycode` + - `std/asyncftpclient` => `asyncftpclient` + - `std/smtp` => `smtp` + - `std/db_common` => `db_connector/db_common` + - `std/db_sqlite` => `db_connector/db_sqlite` + - `std/db_mysql` => `db_connector/db_mysql` + - `std/db_postgres` => `db_connector/db_postgres` + - `std/db_odbc` => `db_connector/db_odbc` + +- Previously, calls like `foo(a, b): ...` or `foo(a, b) do: ...` where the final argument of + `foo` had type `proc ()` were assumed by the compiler to mean `foo(a, b, proc () = ...)`. + This behavior is now deprecated. Use `foo(a, b) do (): ...` or `foo(a, b, proc () = ...)` instead. + +- If no exception or any exception deriving from Exception but not Defect or CatchableError given in except, a `warnBareExcept` warning will be triggered. + +## Standard library additions and changes + +[//]: # "Changes:" +- OpenSSL 3 is now supported. +- `macros.parseExpr` and `macros.parseStmt` now accept an optional + filename argument for more informative errors. +- Module `colors` expanded with missing colors from the CSS color standard. + `colPaleVioletRed` and `colMediumPurple` have also been changed to match the CSS color standard. +- Fixed `lists.SinglyLinkedList` being broken after removing the last node ([#19353](https://github.com/nim-lang/Nim/pull/19353)). +- The `md5` module now works at compile time and in JavaScript. +- Changed `mimedb` to use an `OrderedTable` instead of `OrderedTableRef` to support `const` tables. +- `strutils.find` now uses and defaults to `last = -1` for whole string searches, + making limiting it to just the first char (`last = 0`) valid. +- `random.rand` now works with `Ordinal`s. +- Undeprecated `os.isvalidfilename`. +- `std/oids` now uses `int64` to store time internally (before it was int32). +- `std/uri.Uri` dollar `$` improved, precalculates the `string` result length from the `Uri`. +- `std/uri.Uri.isIpv6` is now exported. +- `std/logging.ConsoleLogger` and `FileLogger` now have a `flushThreshold` attribute to set what log message levels are automatically flushed. For Nim v1 use `-d:nimFlushAllLogs` to automatically flush all message levels. Flushing all logs is the default behavior for Nim v2. + + +- `std/net.IpAddress` dollar `$` improved, uses a fixed capacity for the `string` result based from the `IpAddressFamily`. +- `std/jsfetch.newFetchOptions` now has default values for all parameters +- `std/jsformdata` now accepts `Blob` data type. + + +[//]: # "Additions:" +- Added ISO 8601 week date utilities in `times`: + - Added `IsoWeekRange`, a range type for weeks in a week-based year. + - Added `IsoYear`, a distinct type for a week-based year in contrast to a regular year. + - Added a `initDateTime` overload to create a datetime from an ISO week date. + - Added `getIsoWeekAndYear` to get an ISO week number and week-based year from a datetime. + - Added `getIsoWeeksInYear` to return the number of weeks in a week-based year. +- Added new modules which were part of `std/os`: + - Added `std/oserrors` for OS error reporting. Added `std/envvars` for environment variables handling. + - Added `std/paths`, `std/dirs`, `std/files`, `std/symlinks` and `std/appdirs`. + - Added `std/cmdline` for reading command line parameters. +- Added `sep` parameter in `std/uri` to specify the query separator. +- Added bindings to [`Array.shift`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/shift) + and [`queueMicrotask`](https://developer.mozilla.org/en-US/docs/Web/API/queueMicrotask) + in `jscore` for JavaScript targets. +- Added `UppercaseLetters`, `LowercaseLetters`, `PunctuationChars`, `PrintableChars` sets to `std/strutils`. +- Added `complex.sgn` for obtaining the phase of complex numbers. +- Added `insertAdjacentText`, `insertAdjacentElement`, `insertAdjacentHTML`, + `after`, `before`, `closest`, `append`, `hasAttributeNS`, `removeAttributeNS`, + `hasPointerCapture`, `releasePointerCapture`, `requestPointerLock`, + `replaceChildren`, `replaceWith`, `scrollIntoViewIfNeeded`, `setHTML`, + `toggleAttribute`, and `matches` to `std/dom`. +- Added [`jsre.hasIndices`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp/hasIndices) +- Added `capacity` for `string` and `seq` to return the current capacity, see https://github.com/nim-lang/RFCs/issues/460 +- Added `openArray[char]` overloads for `std/parseutils` allowing more code reuse. +- Added `openArray[char]` overloads for `std/unicode` allowing more code reuse. +- Added `safe` parameter to `base64.encodeMime`. + +[//]: # "Deprecations:" +- Deprecated `selfExe` for Nimscript. +- Deprecated `std/sums`. +- Deprecated `std/base64.encode` for collections of arbitrary integer element type. + Now only `byte` and `char` are supported. + +[//]: # "Removals:" +- Removed deprecated module `parseopt2`. +- Removed deprecated module `sharedstrings`. +- Removed deprecated module `dom_extensions`. +- Removed deprecated module `LockFreeHash`. +- Removed deprecated module `events`. +- Removed deprecated `oids.oidToString`. +- Removed define `nimExperimentalAsyncjsThen` for `std/asyncjs.then` and `std/jsfetch`. +- Removed deprecated `jsre.test` and `jsre.toString`. +- Removed deprecated `math.c_frexp`. +- Removed deprecated `` httpcore.`==` ``. +- Removed deprecated `std/posix.CMSG_SPACE` and `std/posix.CMSG_LEN` that takes wrong argument types. +- Removed deprecated `osproc.poDemon`, symbol with typo. +- Removed deprecated `tables.rightSize`. + + +- Removed deprecated `posix.CLONE_STOPPED`. + + +## Language changes + +- [Tag tracking](https://nim-lang.github.io/Nim/manual.html#effect-system-tag-tracking) supports the definition of forbidden tags by the `.forbids` pragma + which can be used to disable certain effects in proc types. +- [Case statement macros](https://nim-lang.github.io/Nim/manual.html#macros-case-statement-macros) are no longer experimental, + meaning you no longer need to enable the experimental switch `caseStmtMacros` to use them. +- Full command syntax and block arguments i.e. `foo a, b: c` are now allowed + for the right-hand side of type definitions in type sections. Previously + they would error with "invalid indentation". + +- Compile-time define changes: + - `defined` now accepts identifiers separated by dots, i.e. `defined(a.b.c)`. + In the command line, this is defined as `-d:a.b.c`. Older versions can + use accents as in ``defined(`a.b.c`)`` to access such defines. + - [Define pragmas for constants](https://nim-lang.github.io/Nim/manual.html#implementation-specific-pragmas-compileminustime-define-pragmas) + now support a string argument for qualified define names. + + ```nim + # -d:package.FooBar=42 + const FooBar {.intdefine: "package.FooBar".}: int = 5 + echo FooBar # 42 + ``` + + This was added to help disambiguate similar define names for different packages. + In older versions, this could only be achieved with something like the following: + + ```nim + const FooBar = block: + const `package.FooBar` {.intdefine.}: int = 5 + `package.FooBar` + ``` + - A generic `define` pragma for constants has been added that interprets + the value of the define based on the type of the constant value. + See the [experimental manual](https://nim-lang.github.io/Nim/manual_experimental.html#generic-define-pragma) + for a list of supported types. + +- [Macro pragmas](https://nim-lang.github.io/Nim/manual.html#userminusdefined-pragmas-macro-pragmas) changes: + - Templates now accept macro pragmas. + - Macro pragmas for var/let/const sections have been redesigned in a way that works + similarly to routine macro pragmas. The new behavior is documented in the + [experimental manual](https://nim-lang.github.io/Nim/manual_experimental.html#extended-macro-pragmas). + - Pragma macros on type definitions can now return `nnkTypeSection` nodes as well as `nnkTypeDef`, + allowing multiple type definitions to be injected in place of the original type definition. + + ```nim + import macros + macro multiply(amount: static int, s: untyped): untyped = + let name = $s[0].basename + result = newNimNode(nnkTypeSection) + for i in 1 .. amount: + result.add(newTree(nnkTypeDef, ident(name & $i), s[1], s[2])) + type + Foo = object + Bar {.multiply: 3.} = object + x, y, z: int + Baz = object + # becomes + type + Foo = object + Bar1 = object + x, y, z: int + Bar2 = object + x, y, z: int + Bar3 = object + x, y, z: int + Baz = object + ``` + +- A new form of type inference called [top-down inference](https://nim-lang.github.io/Nim/manual_experimental.html#topminusdown-type-inference) + has been implemented for a variety of basic cases. For example, code like the following now compiles: + + ```nim + let foo: seq[(float, byte, cstring)] = @[(1, 2, "abc")] + ``` + +- `cstring` is now accepted as a selector in `case` statements, removing the + need to convert to `string`. On the JS backend, this is translated directly + to a `switch` statement. + +- Nim now supports `out` parameters and ["strict definitions"](https://nim-lang.github.io/Nim/manual_experimental.html#strict-definitions-and-nimout-parameters). +- Nim now offers a [strict mode](https://nim-lang.github.io/Nim/manual_experimental.html#strict-case-objects) for `case objects`. + + +## Compiler changes + +- The `gc` switch has been renamed to `mm` ("memory management") in order to reflect the + reality better. (Nim moved away from all techniques based on "tracing".) + +- Defines the `gcRefc` symbol which allows writing specific code for the refc GC. + +- `nim` can now compile version 1.4.0 as follows: `nim c --lib:lib --stylecheck:off compiler/nim`, + without requiring `-d:nimVersion140` which is now a noop. + +- `--styleCheck`, `--hintAsError` and `--warningAsError` now only apply to the current package. + +- The switch `--nimMainPrefix:prefix` has been added to add a prefix to the names of `NimMain` and + related functions produced on the backend. This prevents conflicts with other Nim + static libraries. + +- When compiling for Release the flag `-fno-math-errno` is used for GCC. + + +## Tool changes + +- Nim now ships Nimble version 0.14 which added support for lock-files. Libraries are stored in `$nimbleDir/pkgs2` (it was `$nimbleDir/pkgs`). From b8d200683a77487e7ef5d78abd4c1b981709bddd Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sat, 17 Dec 2022 20:44:30 +0800 Subject: [PATCH 2293/2472] remove unused imports (#21126) --- lib/impure/rdstdin.nim | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/impure/rdstdin.nim b/lib/impure/rdstdin.nim index 935c83671..b0c648373 100644 --- a/lib/impure/rdstdin.nim +++ b/lib/impure/rdstdin.nim @@ -22,8 +22,6 @@ runnableExamples("-r:off"): if line.len > 0: echo line echo "exiting" -when defined(nimPreviewSlimSystem): - import std/syncio when defined(windows): when defined(nimPreviewSlimSystem): From 100b304ac2aaa65222d7a9043acd0c534ed484cb Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sat, 17 Dec 2022 21:42:29 +0800 Subject: [PATCH 2294/2472] fixes misplaced changelog entries; pre-existing issue (#21125) * move misplaced changelog entries to the right place * fixes misplaced changelog --- changelogs/changelog.md | 33 --------------------------------- changelogs/changelog_2_0_0.md | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 33 deletions(-) delete mode 100644 changelogs/changelog.md diff --git a/changelogs/changelog.md b/changelogs/changelog.md deleted file mode 100644 index d793c4313..000000000 --- a/changelogs/changelog.md +++ /dev/null @@ -1,33 +0,0 @@ -# v1.xx.x - yyyy-mm-dd - -## Changes affecting backward compatibility - -## Standard library additions and changes - -- `std/sharedlist` and `std/sharedtables` are now deprecated, see RFC [#433](https://github.com/nim-lang/RFCs/issues/433). - -### New compile flag (`-d:nimNoGetRandom`) when building `std/sysrand` to remove dependency on linux `getrandom` syscall - -This compile flag only affects linux builds and is necessary if either compiling on a linux kernel version < 3.17, or if code built will be executing on kernel < 3.17. - -On linux kernels < 3.17 (such as kernel 3.10 in RHEL7 and CentOS7), the `getrandom` syscall was not yet introduced. Without this, the `std/sysrand` module will not build properly, and if code is built on a kernel >= 3.17 without the flag, any usage of the `std/sysrand` module will fail to execute on a kernel < 3.17 (since it attempts to perform a syscall to `getrandom`, which isn't present in the current kernel). A compile flag has been added to force the `std/sysrand` module to use /dev/urandom (available since linux kernel 1.3.30), rather than the `getrandom` syscall. This allows for use of a cryptographically secure PRNG, regardless of kernel support for the `getrandom` syscall. - -When building for RHEL7/CentOS7 for example, the entire build process for nim from a source package would then be: -```sh -$ yum install devtoolset-8 # Install GCC version 8 vs the standard 4.8.5 on RHEL7/CentOS7. Alternatively use -d:nimEmulateOverflowChecks. See issue #13692 for details -$ scl enable devtoolset-8 bash # Run bash shell with default toolchain of gcc 8 -$ sh build.sh # per unix install instructions -$ bin/nim c koch # per unix install instructions -$ ./koch boot -d:release # per unix install instructions -$ ./koch tools -d:nimNoGetRandom # pass the nimNoGetRandom flag to compile std/sysrand without support for getrandom syscall -``` - -This is necessary to pass when building nim on kernel versions < 3.17 in particular to avoid an error of "SYS_getrandom undeclared" during the build process for stdlib (sysrand in particular). - -## Language changes - - -## Compiler changes - - -## Tool changes diff --git a/changelogs/changelog_2_0_0.md b/changelogs/changelog_2_0_0.md index 00bdb50c2..f34336ff7 100644 --- a/changelogs/changelog_2_0_0.md +++ b/changelogs/changelog_2_0_0.md @@ -160,6 +160,25 @@ - `std/jsfetch.newFetchOptions` now has default values for all parameters - `std/jsformdata` now accepts `Blob` data type. +- `std/sharedlist` and `std/sharedtables` are now deprecated, see RFC [#433](https://github.com/nim-lang/RFCs/issues/433). + +- New compile flag (`-d:nimNoGetRandom`) when building `std/sysrand` to remove dependency on linux `getrandom` syscall. + + This compile flag only affects linux builds and is necessary if either compiling on a linux kernel version < 3.17, or if code built will be executing on kernel < 3.17. + + On linux kernels < 3.17 (such as kernel 3.10 in RHEL7 and CentOS7), the `getrandom` syscall was not yet introduced. Without this, the `std/sysrand` module will not build properly, and if code is built on a kernel >= 3.17 without the flag, any usage of the `std/sysrand` module will fail to execute on a kernel < 3.17 (since it attempts to perform a syscall to `getrandom`, which isn't present in the current kernel). A compile flag has been added to force the `std/sysrand` module to use /dev/urandom (available since linux kernel 1.3.30), rather than the `getrandom` syscall. This allows for use of a cryptographically secure PRNG, regardless of kernel support for the `getrandom` syscall. + + When building for RHEL7/CentOS7 for example, the entire build process for nim from a source package would then be: + ```sh + $ yum install devtoolset-8 # Install GCC version 8 vs the standard 4.8.5 on RHEL7/CentOS7. Alternatively use -d:nimEmulateOverflowChecks. See issue #13692 for details + $ scl enable devtoolset-8 bash # Run bash shell with default toolchain of gcc 8 + $ sh build.sh # per unix install instructions + $ bin/nim c koch # per unix install instructions + $ ./koch boot -d:release # per unix install instructions + $ ./koch tools -d:nimNoGetRandom # pass the nimNoGetRandom flag to compile std/sysrand without support for getrandom syscall + ``` + + This is necessary to pass when building nim on kernel versions < 3.17 in particular to avoid an error of "SYS_getrandom undeclared" during the build process for stdlib (sysrand in particular). [//]: # "Additions:" - Added ISO 8601 week date utilities in `times`: From 53eed2be4515a3ae853ae4d2fbd84daa49c1d6d6 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sun, 18 Dec 2022 22:31:13 +0800 Subject: [PATCH 2295/2472] close #11705; add a testcase (#21128) --- tests/template/t11705.nim | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 tests/template/t11705.nim diff --git a/tests/template/t11705.nim b/tests/template/t11705.nim new file mode 100644 index 000000000..65ddc7e6c --- /dev/null +++ b/tests/template/t11705.nim @@ -0,0 +1,17 @@ +type RefObj = ref object + +proc `[]`(val: static[int]) = # works with different name/overload or without static arg + discard + +template noRef*(T: typedesc): typedesc = # works without template indirection + typeof(default(T)[]) + +proc `=destroy`(x: var noRef(RefObj)) = + discard + +proc foo = + var x = new RefObj + doAssert $(x[]) == "()" + +# bug #11705 +foo() From 70c575095e89497c63af5d8b2c3ed7dca802179f Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Mon, 19 Dec 2022 17:56:49 +0800 Subject: [PATCH 2296/2472] add changelog for the new strict function checking algorithm (#21129) add changelog for the new strict function checking --- changelogs/changelog_2_0_0.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelogs/changelog_2_0_0.md b/changelogs/changelog_2_0_0.md index f34336ff7..a899bb929 100644 --- a/changelogs/changelog_2_0_0.md +++ b/changelogs/changelog_2_0_0.md @@ -135,6 +135,8 @@ - If no exception or any exception deriving from Exception but not Defect or CatchableError given in except, a `warnBareExcept` warning will be triggered. +- The experimental strictFuncs feature now disallows a store to the heap via a `ref` or `ptr` indirection. + ## Standard library additions and changes [//]: # "Changes:" From e278a781fc5bfc115326ed6c1873268d51b25303 Mon Sep 17 00:00:00 2001 From: Bung Date: Mon, 19 Dec 2022 19:35:15 +0800 Subject: [PATCH 2297/2472] fix #21109 (#21127) --- compiler/semexprs.nim | 2 ++ tests/misc/t21109.nim | 13 +++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 tests/misc/t21109.nim diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 8ac8f4750..612af482a 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1947,6 +1947,8 @@ proc semYieldVarResult(c: PContext, n: PNode, restype: PType) = tupleConstr[i] = takeImplicitAddr(c, tupleConstr[i], e.kind == tyLent) else: localError(c.config, n[0].info, errXExpected, "tuple constructor") + elif e.kind == tyEmpty: + localError(c.config, n[0].info, errTypeExpected) else: when false: # XXX investigate what we really need here. diff --git a/tests/misc/t21109.nim b/tests/misc/t21109.nim new file mode 100644 index 000000000..0f7980896 --- /dev/null +++ b/tests/misc/t21109.nim @@ -0,0 +1,13 @@ +discard """ + action: reject + errormsg: "type expected" + file: "iterators.nim" +""" + + +template b(j: untyped) = j +template m() = discard + +b: + for t, f in @[]: + m() From 886572a5162beea672d696fb03422e7777f56cfb Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Tue, 20 Dec 2022 15:04:13 +0800 Subject: [PATCH 2298/2472] disable "Warning: gc is deprecated" in compiler (#21137) * disable "Warning: gc is deprecated" in compiler * Apply @tersec 's suggestion --- compiler/vmops.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compiler/vmops.nim b/compiler/vmops.nim index edf457295..d4a91b22d 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -145,6 +145,7 @@ when defined(nimHasInvariant): from std / compilesettings import SingleValueSetting, MultipleValueSetting proc querySettingImpl(conf: ConfigRef, switch: BiggestInt): string = + {.push warning[Deprecated]:off.} case SingleValueSetting(switch) of arguments: result = conf.arguments of outFile: result = conf.outFile.string @@ -162,6 +163,7 @@ when defined(nimHasInvariant): of libPath: result = conf.libpath.string of gc: result = $conf.selectedGC of mm: result = $conf.selectedGC + {.pop.} proc querySettingSeqImpl(conf: ConfigRef, switch: BiggestInt): seq[string] = template copySeq(field: untyped): untyped = From 40b5c4c4c340d44ee724183306716286e40002cb Mon Sep 17 00:00:00 2001 From: Bung Date: Wed, 21 Dec 2022 03:23:48 +0800 Subject: [PATCH 2299/2472] fix #20248;fix #6215;turns into simple CT error (#21141) --- compiler/semtypes.nim | 2 ++ tests/array/t20248.nim | 14 ++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 tests/array/t20248.nim diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index dca0d753b..534861e97 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -328,6 +328,8 @@ proc semRange(c: PContext, n: PNode, prev: PType): PType = proc semArrayIndex(c: PContext, n: PNode): PType = if isRange(n): result = semRangeAux(c, n, nil) + elif n.kind == nkInfix and n[0].kind == nkIdent and n[0].ident.s == "..<": + result = errorType(c) else: let e = semExprWithType(c, n, {efDetermineType}) if e.typ.kind == tyFromExpr: diff --git a/tests/array/t20248.nim b/tests/array/t20248.nim new file mode 100644 index 000000000..66142548b --- /dev/null +++ b/tests/array/t20248.nim @@ -0,0 +1,14 @@ +discard """ +cmd: "nim check --hints:off $file" +errormsg: "ordinal type expected; given: Error Type" +nimout: ''' +t20248.nim(10, 36) Error: ordinal type expected; given: Error Type +t20248.nim(14, 20) Error: ordinal type expected; given: Error Type +''' +""" + +type Vec[N: static[int]] = array[0 ..< N, float] + +var v: Vec[32] + +var stuff: array[0 ..< 16, int] From 0aec095b261bed8b5841beba7ce8cd52de4d54be Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Wed, 21 Dec 2022 03:26:32 +0800 Subject: [PATCH 2300/2472] fixes #19292; fixes #21122; fixes putEnv and setEnv with vcc (#21143) * fixes #19292; fixes 21122; fixes putEnv and setEnv with vcc * add a test --- lib/std/envvars.nim | 8 ++++++-- lib/std/private/win_setenv.nim | 26 +++++++++++++------------- tests/misc/t21443.nim | 6 ++++++ 3 files changed, 25 insertions(+), 15 deletions(-) create mode 100644 tests/misc/t21443.nim diff --git a/lib/std/envvars.nim b/lib/std/envvars.nim index 817a155ec..ce90f66ba 100644 --- a/lib/std/envvars.nim +++ b/lib/std/envvars.nim @@ -63,9 +63,13 @@ when not defined(nimscript): import winlean when defined(nimPreviewSlimSystem): import std/widestrs - proc c_wgetenv(varname: WideCString): WideCString {.importc: "_wgetenv", + + type wchar_t {.importc: "wchar_t", header: "".} = int16 + proc c_wgetenv(varname: ptr wchar_t): ptr wchar_t {.importc: "_wgetenv", header: "".} - proc getEnvImpl(env: cstring): WideCString = c_wgetenv(env.newWideCString) + proc getEnvImpl(env: cstring): WideCString = + let r: WideCString = env.newWideCString + cast[WideCString](c_wgetenv(cast[ptr wchar_t](r))) else: proc c_getenv(env: cstring): cstring {. importc: "getenv", header: "".} diff --git a/lib/std/private/win_setenv.nim b/lib/std/private/win_setenv.nim index 303889a40..66e199dfe 100644 --- a/lib/std/private/win_setenv.nim +++ b/lib/std/private/win_setenv.nim @@ -33,25 +33,25 @@ else: # same as winlean.setEnvironmentVariableA proc c_getenv(varname: cstring): cstring {.importc: "getenv", header: "".} - proc c_wputenv(envstring: WideCString): cint {.importc: "_wputenv", header: "".} - proc c_wgetenv(varname: WideCString): WideCString {.importc: "_wgetenv", header: "".} + proc c_wputenv(envstring: ptr wchar_t): cint {.importc: "_wputenv", header: "".} + proc c_wgetenv(varname: ptr wchar_t): ptr wchar_t {.importc: "_wgetenv", header: "".} var errno {.importc, header: "".}: cint var genviron {.importc: "_environ".}: ptr ptr char # xxx `ptr UncheckedArray[WideCString]` did not work - proc wcstombs(wcstr: ptr char, mbstr: WideCString, count: csize_t): csize_t {.importc, header: "".} + proc wcstombs(wcstr: ptr char, mbstr: ptr wchar_t, count: csize_t): csize_t {.importc, header: "".} # xxx cint vs errno_t? proc setEnvImpl*(name: string, value: string, overwrite: cint): cint = const EINVAL = cint(22) - let wideName = newWideCString(name) - if overwrite == 0 and c_wgetenv(wideName) != nil: + let wideName: WideCString = newWideCString(name) + if overwrite == 0 and c_wgetenv(cast[ptr wchar_t](wideName)) != nil: return 0 if value != "": - let envstring = name & "=" & value - let e = c_wputenv(newWideCString(envstring)) + let envstring: WideCString = newWideCString(name & "=" & value) + let e = c_wputenv(cast[ptr wchar_t](envstring)) if e != 0: errno = EINVAL return -1 @@ -62,19 +62,19 @@ else: SetEnvironmentVariableA doesn't update `_environ`, so we have to do these terrible things. ]# - let envstring = name & "= " - if c_wputenv(newWideCString(envstring)) != 0: + let envstring: WideCString = newWideCString(name & "= ") + if c_wputenv(cast[ptr wchar_t](envstring)) != 0: errno = EINVAL return -1 # Here lies the documentation we blatently ignore to make this work. - var s = c_wgetenv(wideName) + var s = cast[WideCString](c_wgetenv(cast[ptr wchar_t](wideName))) s[0] = Utf16Char('\0') #[ This would result in a double null termination, which normally signifies the end of the environment variable list, so we stick a completely empty environment variable into the list instead. ]# - s = c_wgetenv(wideName) + s = cast[WideCString](c_wgetenv(cast[ptr wchar_t](wideName))) s[1] = Utf16Char('=') #[ If genviron is null, the MBCS environment has not been initialized @@ -88,12 +88,12 @@ else: # in the current codepage. Skip updating MBCS environment in this case. # For some reason, second `wcstombs` can find non-convertible characters # that the first `wcstombs` cannot. - let requiredSizeS = wcstombs(nil, wideName, 0) + let requiredSizeS = wcstombs(nil, cast[ptr wchar_t](wideName), 0) if requiredSizeS != high(csize_t): let requiredSize = requiredSizeS.int var buf = newSeq[char](requiredSize + 1) let buf2 = buf[0].addr - if wcstombs(buf2, wideName, csize_t(requiredSize + 1)) != high(csize_t): + if wcstombs(buf2, cast[ptr wchar_t](wideName), csize_t(requiredSize + 1)) != high(csize_t): var ptrToEnv = c_getenv(cast[cstring](buf2)) ptrToEnv[0] = '\0' ptrToEnv = c_getenv(cast[cstring](buf2)) diff --git a/tests/misc/t21443.nim b/tests/misc/t21443.nim new file mode 100644 index 000000000..70413c5b3 --- /dev/null +++ b/tests/misc/t21443.nim @@ -0,0 +1,6 @@ +import std/envvars + +# bug #19292 +putEnv("NimPutEnvTest", "test") +# bug #21122 +doAssert getEnv("NimPutEnvTest") == "test" From 8a77798dac66a55b42005db8837508a5af0547cf Mon Sep 17 00:00:00 2001 From: Luca Guzzon Date: Tue, 20 Dec 2022 20:29:42 +0100 Subject: [PATCH 2301/2472] ucpu fix-up for arm64 value on macos m1 (#21142) --- tools/niminst/makefile.nimf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/niminst/makefile.nimf b/tools/niminst/makefile.nimf index 2fe89ed69..b392ab472 100644 --- a/tools/niminst/makefile.nimf +++ b/tools/niminst/makefile.nimf @@ -173,6 +173,9 @@ endif ifeq ($(ucpu),aarch64) mycpu = arm64 endif +ifeq ($(ucpu),arm64) + mycpu = arm64 +endif ifeq ($(ucpu),riscv64) mycpu = riscv64 endif From 81d8ea95af0cfaaedca2fd1881199e113e6f5b41 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Wed, 21 Dec 2022 17:18:47 +0800 Subject: [PATCH 2302/2472] bump csource_v2 to include fixes for macos m1 (#21147) --- config/build_config.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/build_config.txt b/config/build_config.txt index 0ea963c39..286e0dc2a 100644 --- a/config/build_config.txt +++ b/config/build_config.txt @@ -2,4 +2,4 @@ nim_comment="key-value pairs for windows/posix bootstrapping build scripts" nim_csourcesDir=csources_v2 nim_csourcesUrl=https://github.com/nim-lang/csources_v2.git nim_csourcesBranch=master -nim_csourcesHash=32cf8431bb19f8803352a0ab3dbee5fec35d5ec6 +nim_csourcesHash=b66c420697c574be18a20dd4720248a715b4287e From d0721eadf8cd6bf9436b5e5c9113c4c7bcfcc770 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 22 Dec 2022 04:46:14 +0800 Subject: [PATCH 2303/2472] fixes #21144; try expression will not match the less indentation except (#21152) fixes #21144; try expression will not match the less indent except --- compiler/parser.nim | 4 +++- tests/parser/ttry.nim | 27 +++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 tests/parser/ttry.nim diff --git a/compiler/parser.nim b/compiler/parser.nim index 6b3cebb7f..00013c218 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -1770,11 +1770,13 @@ proc parseTry(p: var Parser; isExpr: bool): PNode = #| (optInd 'except' optionalExprList colcom stmt)* #| (optInd 'finally' colcom stmt)? result = newNodeP(nkTryStmt, p) + let parentIndent = p.currInd # isExpr getTok(p) colcom(p, result) result.add(parseStmt(p)) var b: PNode = nil - while sameOrNoInd(p) or isExpr: + + while sameOrNoInd(p) or (isExpr and parentIndent <= p.tok.indent): case p.tok.tokType of tkExcept: b = newNodeP(nkExceptBranch, p) diff --git a/tests/parser/ttry.nim b/tests/parser/ttry.nim new file mode 100644 index 000000000..190b0b8dc --- /dev/null +++ b/tests/parser/ttry.nim @@ -0,0 +1,27 @@ +# bug #21144 +block: + try: + let c = try: + 10 + except ValueError as exc: + 10 + except ValueError as exc: + discard + +if true: + block: + let c = try: + 10 + except ValueError as exc: + 10 + except OSError: + 99 + + +try: + let c = try: + 10 + except ValueError as exc: + 10 +except ValueError as exc: + discard \ No newline at end of file From 613829f7a4da5506269fadb3acc78d64ee0cbddf Mon Sep 17 00:00:00 2001 From: Peter Munch-Ellingsen Date: Thu, 22 Dec 2022 04:34:36 +0100 Subject: [PATCH 2304/2472] Implement setLineInfo (#21153) * Implement setLineInfo * Add tests --- compiler/msgs.nim | 7 +++++++ compiler/vm.nim | 14 ++++++++++++-- compiler/vmdef.nim | 3 ++- compiler/vmgen.nim | 14 +++++++++++++- lib/core/macros.nim | 16 ++++++++++++++++ tests/macros/tmacros_various.nim | 23 ++++++++++++++++++++++- 6 files changed, 72 insertions(+), 5 deletions(-) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 2c807806e..8b44c8dc6 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -128,6 +128,13 @@ proc fileInfoIdx*(conf: ConfigRef; filename: AbsoluteFile): FileIndex = var dummy: bool result = fileInfoIdx(conf, filename, dummy) +proc fileInfoIdx*(conf: ConfigRef; filename: RelativeFile; isKnownFile: var bool): FileIndex = + fileInfoIdx(conf, AbsoluteFile expandFilename(filename.string), isKnownFile) + +proc fileInfoIdx*(conf: ConfigRef; filename: RelativeFile): FileIndex = + var dummy: bool + fileInfoIdx(conf, AbsoluteFile expandFilename(filename.string), dummy) + proc newLineInfo*(fileInfoIdx: FileIndex, line, col: int): TLineInfo = result.fileIndex = fileInfoIdx if line < int high(uint16): diff --git a/compiler/vm.nim b/compiler/vm.nim index 53a16bcb0..61261c44c 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -1934,14 +1934,24 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = of 1: # getLine regs[ra].node = newIntNode(nkIntLit, n.info.line.int) of 2: # getColumn - regs[ra].node = newIntNode(nkIntLit, n.info.col) + regs[ra].node = newIntNode(nkIntLit, n.info.col.int) else: internalAssert c.config, false regs[ra].node.info = n.info regs[ra].node.typ = n.typ - of opcNSetLineInfo: + of opcNCopyLineInfo: decodeB(rkNode) regs[ra].node.info = regs[rb].node.info + of opcNSetLineInfoLine: + decodeB(rkNode) + regs[ra].node.info.line = regs[rb].intVal.uint16 + of opcNSetLineInfoColumn: + decodeB(rkNode) + regs[ra].node.info.col = regs[rb].intVal.int16 + of opcNSetLineInfoFile: + decodeB(rkNode) + regs[ra].node.info.fileIndex = + fileInfoIdx(c.config, RelativeFile regs[rb].node.strVal) of opcEqIdent: decodeBC(rkInt) # aliases for shorter and easier to understand code below diff --git a/compiler/vmdef.nim b/compiler/vmdef.nim index 54876f37d..114a49557 100644 --- a/compiler/vmdef.nim +++ b/compiler/vmdef.nim @@ -141,7 +141,8 @@ type opcNError, opcNWarning, opcNHint, - opcNGetLineInfo, opcNSetLineInfo, + opcNGetLineInfo, opcNCopyLineInfo, opcNSetLineInfoLine, + opcNSetLineInfoColumn, opcNSetLineInfoFile opcEqIdent, opcStrToIdent, opcGetImpl, diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 1fdfea7c4..6a2ab1d11 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1335,7 +1335,19 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = of "copyLineInfo": internalAssert c.config, n.len == 3 unused(c, n, dest) - genBinaryStmt(c, n, opcNSetLineInfo) + genBinaryStmt(c, n, opcNCopyLineInfo) + of "setLine": + internalAssert c.config, n.len == 3 + unused(c, n, dest) + genBinaryStmt(c, n, opcNSetLineInfoLine) + of "setColumn": + internalAssert c.config, n.len == 3 + unused(c, n, dest) + genBinaryStmt(c, n, opcNSetLineInfoColumn) + of "setFile": + internalAssert c.config, n.len == 3 + unused(c, n, dest) + genBinaryStmt(c, n, opcNSetLineInfoFile) else: internalAssert c.config, false of mNHint: unused(c, n, dest) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 18a70f20c..138ec47a0 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -533,6 +533,22 @@ proc getFile(arg: NimNode): string {.magic: "NLineInfo", noSideEffect.} proc copyLineInfo*(arg: NimNode, info: NimNode) {.magic: "NLineInfo", noSideEffect.} ## Copy lineinfo from `info`. +proc setLine(arg: NimNode, line: uint16) {.magic: "NLineInfo", noSideEffect.} +proc setColumn(arg: NimNode, column: int16) {.magic: "NLineInfo", noSideEffect.} +proc setFile(arg: NimNode, file: string) {.magic: "NLineInfo", noSideEffect.} + +proc setLineInfo*(arg: NimNode, file: string, line: int, column: int) = + ## Sets the line info on the NimNode. The file needs to exists, but can be a + ## relative path. If you want to attach line info to a block using `quote` + ## you'll need to add the line information after the quote block. + arg.setFile(file) + arg.setLine(line.uint16) + arg.setColumn(column.int16) + +proc setLineInfo*(arg: NimNode, lineInfo: LineInfo) = + ## See `setLineInfo proc<#setLineInfo,NimNode,string,int,int>`_ + setLineInfo(arg, lineInfo.filename, lineInfo.line, lineInfo.column) + proc lineInfoObj*(n: NimNode): LineInfo = ## Returns `LineInfo` of `n`, using absolute path for `filename`. result = LineInfo(filename: n.getFile, line: n.getLine, column: n.getColumn) diff --git a/tests/macros/tmacros_various.nim b/tests/macros/tmacros_various.nim index d702db56a..08be4602c 100644 --- a/tests/macros/tmacros_various.nim +++ b/tests/macros/tmacros_various.nim @@ -91,7 +91,7 @@ block tlexerex: -block tlineinfo: +block tcopylineinfo: # issue #5617, feature request type Test = object @@ -103,6 +103,27 @@ block tlineinfo: var z = mixer(Test) doAssert z +block tsetgetlineinfo: + # issue #21098, feature request + type Test = object + + macro mixer1(n: typed): untyped = + let x = newIdentNode("echo") + var lineInfo = n.lineInfoObj + x.setLineInfo lineInfo + result = newLit(x.lineInfo == n.lineInfo) + + macro mixer2(n: typed): untyped = + let x = newIdentNode("echo") + var lineInfo = n.lineInfoObj + lineInfo.line += 1 + x.setLineInfo lineInfo + result = newLit(x.lineInfo != n.lineInfo) + + doAssert mixer1(Test) + + doAssert mixer2(Test) + block tdebugstmt: From c5a72ebddd88b6d3a2712230d36367f180faa7da Mon Sep 17 00:00:00 2001 From: Bung Date: Thu, 22 Dec 2022 13:16:25 +0800 Subject: [PATCH 2305/2472] fix #16541 (#21148) --- compiler/sigmatch.nim | 2 ++ tests/misc/t16541.nim | 12 ++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 tests/misc/t16541.nim diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 51728b103..61a4395f3 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -2150,6 +2150,8 @@ proc paramTypesMatchAux(m: var TCandidate, f, a: PType, elif arg.sym.kind in {skMacro, skTemplate}: return nil else: + if arg.sym.ast == nil: + return nil let inferred = c.semGenerateInstance(c, arg.sym, m.bindings, arg.info) result = newSymNode(inferred, arg.info) if r == isInferredConvertible: diff --git a/tests/misc/t16541.nim b/tests/misc/t16541.nim new file mode 100644 index 000000000..452327e8f --- /dev/null +++ b/tests/misc/t16541.nim @@ -0,0 +1,12 @@ +discard """ + action: "reject" + +""" + +import strutils, sugar, nre + +proc my_replace*(s: string, r: Regex, by: string | (proc (match: string): string)): string = + nre.replace(s, r, by) + +discard my_replace("abcde", re"[bcd]", match => match.to_upper) == "aBCDe" +discard my_replace("abcde", re"[bcd]", (match: string) => match.to_upper) == "aBCDe" From 70fe360456b77912ec7f5014f7815137fa089fce Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Thu, 22 Dec 2022 18:25:24 +1100 Subject: [PATCH 2306/2472] Use `ErrorColor` when a warning that is turned into an error is raised (#21131) Use ErrorColor when a warning that is turned into an error is raised --- compiler/msgs.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 8b44c8dc6..8e391f2fb 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -541,10 +541,11 @@ proc liMessage*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string, ignoreMsg = not conf.hasWarn(msg) if not ignoreMsg and msg in conf.warningAsErrors: title = ErrorTitle + color = ErrorColor else: title = WarningTitle + color = WarningColor if not ignoreMsg: writeContext(conf, info) - color = WarningColor inc(conf.warnCounter) of hintMin..hintMax: sev = Severity.Hint From 93b59da4902886cd68dd7df1dce09a1b455a06dc Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 22 Dec 2022 15:27:11 +0800 Subject: [PATCH 2307/2472] fixes #20244; fixes castSizes warnings (#21102) * fixes #20244; fixes castSizes warnings * fixes js * fixes js * fixes js * fixes * typo * extend using uint64 * Update lib/std/syncio.nim --- compiler/ccgutils.nim | 2 +- compiler/condsyms.nim | 1 + compiler/nim.cfg | 4 ++++ lib/pure/strutils.nim | 16 ++++++++++++++-- 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/compiler/ccgutils.nim b/compiler/ccgutils.nim index e19fccfa7..d86ebe461 100644 --- a/compiler/ccgutils.nim +++ b/compiler/ccgutils.nim @@ -53,7 +53,7 @@ proc hashString*(conf: ConfigRef; s: string): BiggestInt = a = a + (a shl 3) a = a xor (a shr 11) a = a + (a shl 15) - result = cast[Hash](a) + result = cast[Hash](uint(a)) template getUniqueType*(key: PType): PType = key diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index 9d081ef0a..24c6d82d5 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -146,6 +146,7 @@ proc initDefines*(symbols: StringTableRef) = defineSymbol("nimHasCallsitePragma") defineSymbol("nimHasAmbiguousEnumHint") + defineSymbol("nimHasWarnCastSizes") defineSymbol("nimHasOutParams") defineSymbol("nimHasSystemRaisesDefect") defineSymbol("nimHasWarnUnnamedBreak") diff --git a/compiler/nim.cfg b/compiler/nim.cfg index 96b47b0e6..b9c8fdc08 100644 --- a/compiler/nim.cfg +++ b/compiler/nim.cfg @@ -31,6 +31,10 @@ define:useStdoutAsStdmsg warning[ObservableStores]:off @end +@if nimHasWarnCastSizes: + warning[CastSizes]:on +@end + @if nimHasWarningAsError: warningAsError[GcUnsafe2]:on @end diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index f628cfc2d..dbdd0d6a1 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -944,14 +944,26 @@ func toHex*[T: SomeInteger](x: T, len: Positive): string = doAssert b.toHex(4) == "1001" doAssert toHex(62, 3) == "03E" doAssert toHex(-8, 6) == "FFFFF8" - toHexImpl(cast[BiggestUInt](x), len, x < 0) + when defined(js): + toHexImpl(cast[BiggestUInt](x), len, x < 0) + else: + when T is SomeSignedInt: + toHexImpl(cast[BiggestUInt](BiggestInt(x)), len, x < 0) + else: + toHexImpl(BiggestUInt(x), len, x < 0) func toHex*[T: SomeInteger](x: T): string = ## Shortcut for `toHex(x, T.sizeof * 2)` runnableExamples: doAssert toHex(1984'i64) == "00000000000007C0" doAssert toHex(1984'i16) == "07C0" - toHexImpl(cast[BiggestUInt](x), 2*sizeof(T), x < 0) + when defined(js): + toHexImpl(cast[BiggestUInt](x), 2*sizeof(T), x < 0) + else: + when T is SomeSignedInt: + toHexImpl(cast[BiggestUInt](BiggestInt(x)), 2*sizeof(T), x < 0) + else: + toHexImpl(BiggestUInt(x), 2*sizeof(T), x < 0) func toHex*(s: string): string {.rtl.} = ## Converts a bytes string to its hexadecimal representation. From 7931bdac956cb3ad6734eab91c62920f4e80f919 Mon Sep 17 00:00:00 2001 From: Michael Voronin Date: Thu, 22 Dec 2022 10:32:12 +0300 Subject: [PATCH 2308/2472] Feature/xmltree additions (#20988) * [change] add/insert/delete family of xmltree expanded with several variations. Added replace methods family * [change] Lifted child limitations on insert methods (consulted with @araq) * [tests] add/insert/replace/delete of xmltree XmlNodes tests added --- lib/pure/xmltree.nim | 153 ++++++++++++++++++++++++++++++++++- tests/xml/ttree_add.nim | 51 ++++++++++++ tests/xml/ttree_add1.nim | 53 ++++++++++++ tests/xml/ttree_delete.nim | 47 +++++++++++ tests/xml/ttree_delete1.nim | 48 +++++++++++ tests/xml/ttree_insert.nim | 53 ++++++++++++ tests/xml/ttree_insert1.nim | 51 ++++++++++++ tests/xml/ttree_replace.nim | 46 +++++++++++ tests/xml/ttree_replace1.nim | 53 ++++++++++++ 9 files changed, 552 insertions(+), 3 deletions(-) create mode 100644 tests/xml/ttree_add.nim create mode 100644 tests/xml/ttree_add1.nim create mode 100644 tests/xml/ttree_delete.nim create mode 100644 tests/xml/ttree_delete1.nim create mode 100644 tests/xml/ttree_insert.nim create mode 100644 tests/xml/ttree_insert1.nim create mode 100644 tests/xml/ttree_replace.nim create mode 100644 tests/xml/ttree_replace1.nim diff --git a/lib/pure/xmltree.nim b/lib/pure/xmltree.nim index 6228bd10f..7a2cebe87 100644 --- a/lib/pure/xmltree.nim +++ b/lib/pure/xmltree.nim @@ -31,7 +31,7 @@ runnableExamples: ## * `htmlgen module `_ for html code generator import std/private/since -import macros, strtabs, strutils +import macros, strtabs, strutils, sequtils when defined(nimPreviewSlimSystem): import std/assertions @@ -298,26 +298,60 @@ proc innerText*(n: XmlNode): string = proc add*(father, son: XmlNode) {.inline.} = ## Adds the child `son` to `father`. + ## `father` must be of `xnElement` type ## ## See also: + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ runnableExamples: var f = newElement("myTag") f.add newText("my text") f.add newElement("sonTag") f.add newEntity("my entity") assert $f == "my text&my entity;" + + assert father.k == xnElement add(father.s, son) +proc add*(father: XmlNode, sons: openArray[XmlNode]) {.inline.} = + ## Adds the children `sons` to `father`. + ## `father` must be of `xnElement` type + ## + ## See also: + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ + runnableExamples: + var f = newElement("myTag") + f.add(@[newText("my text"), newElement("sonTag"), newEntity("my entity")]) + assert $f == "my text&my entity;" + + assert father.k == xnElement + add(father.s, sons) + + proc insert*(father, son: XmlNode, index: int) {.inline.} = ## Inserts the child `son` to a given position in `father`. ## - ## `father` and `son` must be of `xnElement` kind. + ## `father` must be of `xnElement` kind. ## ## See also: + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ runnableExamples: var f = newElement("myTag") f.add newElement("first") @@ -327,18 +361,52 @@ proc insert*(father, son: XmlNode, index: int) {.inline.} = """ - assert father.k == xnElement and son.k == xnElement + assert father.k == xnElement if len(father.s) > index: insert(father.s, son, index) else: insert(father.s, son, len(father.s)) +proc insert*(father: XmlNode, sons: openArray[XmlNode], index: int) {.inline.} = + ## Inserts the children openArray[`sons`] to a given position in `father`. + ## + ## `father` must be of `xnElement` kind. + ## + ## See also: + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ + ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert([newElement("second"), newElement("third")], 0) + assert $f == """ + + + +""" + + assert father.k == xnElement + if len(father.s) > index: + insert(father.s, sons, index) + else: + insert(father.s, sons, len(father.s)) + proc delete*(n: XmlNode, i: Natural) = ## Deletes the `i`'th child of `n`. ## ## See also: + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ runnableExamples: var f = newElement("myTag") f.add newElement("first") @@ -351,6 +419,85 @@ proc delete*(n: XmlNode, i: Natural) = assert n.k == xnElement n.s.delete(i) +proc delete*(n: XmlNode, slice: Slice[int]) = + ## Deletes the items `n[slice]` of `n`. + ## + ## See also: + ## * `delete proc <#delete.XmlNode,int>`_ + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert([newElement("second"), newElement("third")], 0) + f.delete(0..1) + assert $f == """ + +""" + + assert n.k == xnElement + n.s.delete(slice) + +proc replace*(n: XmlNode, i: Natural, replacement: openArray[XmlNode]) = + ## Replaces the `i`'th child of `n` with `replacement` openArray. + ## + ## `n` must be of `xnElement` kind. + ## + ## See also: + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ + ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert(newElement("second"), 0) + f.replace(0, @[newElement("third"), newElement("fourth")]) + assert $f == """ + + + +""" + + assert n.k == xnElement + n.s.delete(i) + n.s.insert(replacement, i) + +proc replace*(n: XmlNode, slice: Slice[int], replacement: openArray[XmlNode]) = + ## Deletes the items `n[slice]` of `n`. + ## + ## `n` must be of `xnElement` kind. + ## + ## See also: + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ + ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert([newElement("second"), newElement("fifth")], 0) + f.replace(0..1, @[newElement("third"), newElement("fourth")]) + assert $f == """ + + + +""" + + assert n.k == xnElement + n.s.delete(slice) + n.s.insert(replacement, slice.a) + proc len*(n: XmlNode): int {.inline.} = ## Returns the number of `n`'s children. runnableExamples: diff --git a/tests/xml/ttree_add.nim b/tests/xml/ttree_add.nim new file mode 100644 index 000000000..4c6ef6cf9 --- /dev/null +++ b/tests/xml/ttree_add.nim @@ -0,0 +1,51 @@ +discard """ + output: ''' + +
          Some text in body
          +
          Some more text in body
          + + + +
          Some text
          +
          Some more text
          + + +
          Some text in body
          +
          Some more text in body
          + +           +''' +""" + +# Test xmltree add/insert/delete/replace operations +import xmlparser +import xmltree +var baseDocHead = """ + + +
          Some text
          +
          Some more text
          + +           +""" +var baseDocHeadTree = parseXml(baseDocHead) +var baseDocBody = """ + +
          Some text in body
          +
          Some more text in body
          + +""" +var baseDocBodyTree = parseXml(baseDocBody) + +proc test_add() = + var testDoc = baseDocHeadTree + var newBody = newElement("body") + for item in baseDocBodyTree.items(): + newBody.add(item) + + echo $newBody + + testDoc.add(newBody) + echo $testDoc + +test_add() diff --git a/tests/xml/ttree_add1.nim b/tests/xml/ttree_add1.nim new file mode 100644 index 000000000..30ec83c02 --- /dev/null +++ b/tests/xml/ttree_add1.nim @@ -0,0 +1,53 @@ +discard """ + output: ''' + +
          Some text in body
          +
          Some more text in body
          + + + +
          Some text
          +
          Some more text
          + + +
          Some text in body
          +
          Some more text in body
          + +           +''' +""" + +# Test xmltree add/insert/delete/replace operations +import xmlparser +import xmltree +var baseDocHead = """ + + +
          Some text
          +
          Some more text
          + +           +""" +var baseDocHeadTree = parseXml(baseDocHead) +var baseDocBody = """ + +
          Some text in body
          +
          Some more text in body
          + +""" +var baseDocBodyTree = parseXml(baseDocBody) + +proc test_add() = + var testDoc = baseDocHeadTree + var newBody = newElement("body") + var bodyItems: seq[XmlNode] = @[] + for item in baseDocBodyTree.items(): + bodyItems.add(item) + newBody.add(bodyItems) + + echo $newBody + + testDoc.add(newBody) + echo $testDoc + +test_add() diff --git a/tests/xml/ttree_delete.nim b/tests/xml/ttree_delete.nim new file mode 100644 index 000000000..32b477839 --- /dev/null +++ b/tests/xml/ttree_delete.nim @@ -0,0 +1,47 @@ +discard """ + output: ''' + + +
          Some text
          +
          Some more text
          + + +
          Some text in body
          +
          Some more text in body
          + +           +''' +""" + +# Test xmltree add/insert/delete/replace operations +import xmlparser +import xmltree +let initialDocBase = """ + + +
          Some text
          +
          Some more text
          + + +
          MORE TEXT
          +
          MORE TEXT Some more text
          +           + +
          MORE TEXT
          +
          MORE TEXT Some more text
          +           + +
          Some text in body
          +
          Some more text in body
          + +           +""" +var initialDocBaseTree = parseXml(initialDocBase) + +proc test_delete() = + var testDoc = initialDocBaseTree + + testDoc.delete(1..2) + echo $testDoc + +test_delete() diff --git a/tests/xml/ttree_delete1.nim b/tests/xml/ttree_delete1.nim new file mode 100644 index 000000000..a8442a093 --- /dev/null +++ b/tests/xml/ttree_delete1.nim @@ -0,0 +1,48 @@ +discard """ + output: ''' + + +
          Some text
          +
          Some more text
          + + +
          Some text in body
          +
          Some more text in body
          + +           +''' +""" + +# Test xmltree add/insert/delete/replace operations +import xmlparser +import xmltree +let initialDocBase = """ + + +
          Some text
          +
          Some more text
          + + +
          MORE TEXT
          +
          MORE TEXT Some more text
          +           + +
          MORE TEXT
          +
          MORE TEXT Some more text
          +           + +
          Some text in body
          +
          Some more text in body
          + +           +""" +var initialDocBaseTree = parseXml(initialDocBase) + +proc test_delete() = + var testDoc = initialDocBaseTree + + testDoc.delete(1) + testDoc.delete(1) + echo $testDoc + +test_delete() diff --git a/tests/xml/ttree_insert.nim b/tests/xml/ttree_insert.nim new file mode 100644 index 000000000..b2941395b --- /dev/null +++ b/tests/xml/ttree_insert.nim @@ -0,0 +1,53 @@ +discard """ + output: ''' + +
          Some text in body
          +
          Some more text in body
          + + + +
          Some text
          +
          Some more text
          + + +
          Some text in body
          +
          Some more text in body
          + +           +''' +""" + +# Test xmltree add/insert/delete/replace operations +import xmlparser +import xmltree +var baseDocHead = """ + + +
          Some text
          +
          Some more text
          + +           +""" +var baseDocHeadTree = parseXml(baseDocHead) +var baseDocBody = """ + +
          Some text in body
          +
          Some more text in body
          + +""" +var baseDocBodyTree = parseXml(baseDocBody) + +proc test_insert() = + var testDoc = baseDocHeadTree + var newBody = newElement("body") + var bodyItems: seq[XmlNode] = @[] + for item in baseDocBodyTree.items(): + bodyItems.insert(item, len(bodyItems)) + newBody.insert(bodyItems, 1) + + echo $newBody + + testDoc.insert(newBody, 1) + echo $testDoc + +test_insert() diff --git a/tests/xml/ttree_insert1.nim b/tests/xml/ttree_insert1.nim new file mode 100644 index 000000000..9aa3faf69 --- /dev/null +++ b/tests/xml/ttree_insert1.nim @@ -0,0 +1,51 @@ +discard """ + output: ''' + +
          Some text in body
          +
          Some more text in body
          + + + +
          Some text
          +
          Some more text
          + + +
          Some text in body
          +
          Some more text in body
          + +           +''' +""" + +# Test xmltree add/insert/delete/replace operations +import xmlparser +import xmltree +var baseDocHead = """ + + +
          Some text
          +
          Some more text
          + +           +""" +var baseDocHeadTree = parseXml(baseDocHead) +var baseDocBody = """ + +
          Some text in body
          +
          Some more text in body
          + +""" +var baseDocBodyTree = parseXml(baseDocBody) + +proc test_insert() = + var testDoc = baseDocHeadTree + var newBody = newElement("body") + for item in baseDocBodyTree.items(): + newBody.insert(item, len(newBody)) + + echo $newBody + + testDoc.insert(newBody, 1) + echo $testDoc + +test_insert() diff --git a/tests/xml/ttree_replace.nim b/tests/xml/ttree_replace.nim new file mode 100644 index 000000000..97d2db638 --- /dev/null +++ b/tests/xml/ttree_replace.nim @@ -0,0 +1,46 @@ +discard """ + output: ''' + + +
          Some text
          +
          Some more text
          + + +
          Some text in body
          +
          Some more text in body
          + +           +''' +""" + +# Test xmltree add/insert/delete/replace operations +import xmlparser +import xmltree +var baseDocBody = """ + +
          Some text in body
          +
          Some more text in body
          + +""" +var baseDocBodyTree = parseXml(baseDocBody) +let initialDocBase = """ + + +
          Some text
          +
          Some more text
          + + +
          Some text in body before replace
          +
          Some more text in body before replace
          + +           +""" +var initialDocBaseTree = parseXml(initialDocBase) + +proc test_replace() = + var testDoc = initialDocBaseTree + + testDoc.replace(1, @[baseDocBodyTree]) + echo $testDoc + +test_replace() diff --git a/tests/xml/ttree_replace1.nim b/tests/xml/ttree_replace1.nim new file mode 100644 index 000000000..059ce2085 --- /dev/null +++ b/tests/xml/ttree_replace1.nim @@ -0,0 +1,53 @@ +discard """ + output: ''' + + +
          Some text
          +
          Some more text
          + + +
          Some text in body
          +
          Some more text in body
          + +           +''' +""" + +# Test xmltree add/insert/delete/replace operations +import xmlparser +import xmltree +var baseDocHead = """ + +
          Some text
          +
          Some more text
          + +""" +var baseDocHeadTree = parseXml(baseDocHead) +var baseDocBody = """ + +
          Some text in body
          +
          Some more text in body
          + +""" +var baseDocBodyTree = parseXml(baseDocBody) +let initialDocBase = """ + + +
          Some text before replace
          +
          Some more text before replace
          + + +
          Some text in body before replace
          +
          Some more text in body before replace
          + +           +""" +var initialDocBaseTree = parseXml(initialDocBase) + +proc test_replace() = + var testDoc = initialDocBaseTree + + testDoc.replace(0..1, @[baseDocHeadTree, baseDocBodyTree]) + echo $testDoc + +test_replace() From 314a24472e83eedf058ffa614edf432e8744d0cb Mon Sep 17 00:00:00 2001 From: Peder Bergebakken Sundt Date: Thu, 22 Dec 2022 09:54:17 +0100 Subject: [PATCH 2309/2472] add link to setutils in system module docs (#21130) --- lib/system_overview.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/system_overview.rst b/lib/system_overview.rst index b338845d6..cc0643bf1 100644 --- a/lib/system_overview.rst +++ b/lib/system_overview.rst @@ -99,6 +99,7 @@ Proc Usage =============================== ====================================== **See also:** +* `setutils module `_ for bit set convenience functions * `sets module `_ for hash sets * `intsets module `_ for efficient int sets From 37daed389735ac66f5c950be9ee1c6658b019016 Mon Sep 17 00:00:00 2001 From: metagn Date: Thu, 22 Dec 2022 11:54:41 +0300 Subject: [PATCH 2310/2472] remove misleading slimsystem deprecated warnings (#21156) refs #20967 --- lib/std/formatfloat.nim | 2 -- lib/system.nim | 3 --- 2 files changed, 5 deletions(-) diff --git a/lib/std/formatfloat.nim b/lib/std/formatfloat.nim index 8abbe59cd..b216d1fd0 100644 --- a/lib/std/formatfloat.nim +++ b/lib/std/formatfloat.nim @@ -11,8 +11,6 @@ when defined(nimPreviewSlimSystem): import std/assertions -else: - {.deprecated: "formatfloat is about to move out of system; use `-d:nimPreviewSlimSystem` and import `std/formatfloat`".} proc c_memcpy(a, b: pointer, size: csize_t): pointer {.importc: "memcpy", header: "", discardable.} diff --git a/lib/system.nim b/lib/system.nim index 65055f145..1b6235fb3 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1630,7 +1630,6 @@ template newException*(exceptn: typedesc, message: string; (ref exceptn)(msg: message, parent: parentException) when not defined(nimPreviewSlimSystem): - {.deprecated: "assertions is about to move out of system; use `-d:nimPreviewSlimSystem` and import `std/assertions`".} import std/assertions export assertions @@ -2068,7 +2067,6 @@ when not defined(js): when hostOS != "standalone": include system/threadimpl when not defined(nimPreviewSlimSystem): - {.deprecated: "threads is about to move out of system; use `-d:nimPreviewSlimSystem` and import `std/typedthreads`".} import std/typedthreads export typedthreads @@ -2782,7 +2780,6 @@ when notJSnotNims: releaseSys echoLock when not defined(nimPreviewSlimSystem): - {.deprecated: "io is about to move out of system; use `-d:nimPreviewSlimSystem` and import `std/syncio`".} import std/syncio export syncio From 18c115c8d0b38e6dbb7fae5bdda94bfca1a0ecef Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Fri, 23 Dec 2022 00:44:10 +1100 Subject: [PATCH 2311/2472] Don't repeat suggestions for same symbol (#21140) * Track seen module graphs so symbols from the same module aren't repeated Add test case * Track symbols instead of modules * Don't show duplicate symbols in spell checker Removes the declared location from the message. Since we don't show duplicates anymore it would be a bit misleading if we only show the location for the first declaration of the symbol --- compiler/lookups.nim | 25 ++++++++++++++++++++----- compiler/suggest.nim | 6 +++--- tests/misc/tspellsuggest.nim | 30 +++++++++++++++--------------- tests/misc/tspellsuggest2.nim | 24 ++++++++++++------------ tests/misc/tspellsuggest3.nim | 21 +++++++++++++++++++++ 5 files changed, 71 insertions(+), 35 deletions(-) create mode 100644 tests/misc/tspellsuggest3.nim diff --git a/compiler/lookups.nim b/compiler/lookups.nim index df5a5333e..d50fc6e27 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -15,7 +15,7 @@ when defined(nimPreviewSlimSystem): import intsets, ast, astalgo, idents, semdata, types, msgs, options, - renderer, nimfix/prettybase, lineinfos, modulegraphs, astmsgs + renderer, nimfix/prettybase, lineinfos, modulegraphs, astmsgs, sets proc ensureNoMissingOrUnusedSymbols(c: PContext; scope: PScope) @@ -170,6 +170,7 @@ iterator allSyms*(c: PContext): (PSym, int, bool) = # really iterate over all symbols in all the scopes. This is expensive # and only used by suggest.nim. var isLocal = true + var scopeN = 0 for scope in allScopes(c.currentScope): if scope == c.topLevelScope: isLocal = false @@ -184,6 +185,17 @@ iterator allSyms*(c: PContext): (PSym, int, bool) = assert s != nil yield (s, scopeN, isLocal) +iterator uniqueSyms*(c: PContext): (PSym, int, bool) = + ## Like [allSyms] except only returns unique symbols (Uniqueness determined by line + name) + # Track seen symbols so we don't duplicate them. + # The int is for the symbols name, and line info is + # to be able to tell apart symbols with same name but on different lines + var seen = initHashSet[(TLineInfo, int)]() + for res in allSyms(c): + if not seen.containsOrIncl((res[0].info, res[0].name.id)): + yield res + + proc someSymFromImportTable*(c: PContext; name: PIdent; ambiguous: var bool): PSym = var marked = initIntSet() var symSet = OverloadableSyms @@ -445,12 +457,13 @@ proc fixSpelling(c: PContext, n: PNode, ident: PIdent, result: var string) = let dist = editDistance(name0, sym.name.s.nimIdentNormalize) var msg: string msg.add "\n ($1, $2): '$3'" % [$dist, $depth, sym.name.s] - addDeclaredLoc(msg, c.config, sym) # `msg` needed for deterministic ordering. list.push SpellCandidate(dist: dist, depth: depth, msg: msg, sym: sym) if list.len == 0: return let e0 = list[0] - var count = 0 + var + count = 0 + last: PIdent = nil while true: # pending https://github.com/timotheecour/Nim/issues/373 use more efficient `itemsSorted`. if list.len == 0: break @@ -466,8 +479,10 @@ proc fixSpelling(c: PContext, n: PNode, ident: PIdent, result: var string) = elif count >= c.config.spellSuggestMax: break if count == 0: result.add "\ncandidates (edit distance, scope distance); see '--spellSuggest': " - result.add e.msg - count.inc + if e.sym.name != last: + result.add e.msg + count.inc + last = e.sym.name proc errorUseQualifier(c: PContext; info: TLineInfo; s: PSym; amb: var bool): PSym = var err = "ambiguous identifier: '" & s.name.s & "'" diff --git a/compiler/suggest.nim b/compiler/suggest.nim index 41d39ccab..e0b0fb516 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -289,7 +289,7 @@ proc suggestField(c: PContext, s: PSym; f: PNode; info: TLineInfo; outputs: var s.getQuality, pm, c.inTypeContext > 0, 0)) template wholeSymTab(cond, section: untyped) {.dirty.} = - for (item, scopeN, isLocal) in allSyms(c): + for (item, scopeN, isLocal) in uniqueSyms(c): let it = item var pm: PrefixMatch if cond: @@ -362,7 +362,7 @@ proc suggestOperations(c: PContext, n, f: PNode, typ: PType, outputs: var Sugges proc suggestEverything(c: PContext, n, f: PNode, outputs: var Suggestions) = # do not produce too many symbols: - for (it, scopeN, isLocal) in allSyms(c): + for (it, scopeN, isLocal) in uniqueSyms(c): var pm: PrefixMatch if filterSym(it, f, pm): outputs.add(symToSuggest(c.graph, it, isLocal = isLocal, ideSug, n.info, @@ -680,7 +680,7 @@ proc suggestSentinel*(c: PContext) = inc(c.compilesContextId) var outputs: Suggestions = @[] # suggest everything: - for (it, scopeN, isLocal) in allSyms(c): + for (it, scopeN, isLocal) in uniqueSyms(c): var pm: PrefixMatch if filterSymNoOpr(it, nil, pm): outputs.add(symToSuggest(c.graph, it, isLocal = isLocal, ideSug, diff --git a/tests/misc/tspellsuggest.nim b/tests/misc/tspellsuggest.nim index 033ed0afc..345458bb1 100644 --- a/tests/misc/tspellsuggest.nim +++ b/tests/misc/tspellsuggest.nim @@ -5,21 +5,21 @@ discard """ nimout: ''' tspellsuggest.nim(45, 13) Error: undeclared identifier: 'fooBar' candidates (edit distance, scope distance); see '--spellSuggest': - (1, 0): 'fooBar8' [var declared in tspellsuggest.nim(43, 9)] - (1, 1): 'fooBar7' [var declared in tspellsuggest.nim(41, 7)] - (1, 3): 'fooBar1' [var declared in tspellsuggest.nim(33, 5)] - (1, 3): 'fooBar2' [let declared in tspellsuggest.nim(34, 5)] - (1, 3): 'fooBar3' [const declared in tspellsuggest.nim(35, 7)] - (1, 3): 'fooBar4' [proc declared in tspellsuggest.nim(36, 6)] - (1, 3): 'fooBar5' [template declared in tspellsuggest.nim(37, 10)] - (1, 3): 'fooBar6' [macro declared in tspellsuggest.nim(38, 7)] - (1, 5): 'FooBar' [type declared in mspellsuggest.nim(5, 6)] - (1, 5): 'fooBar4' [proc declared in mspellsuggest.nim(1, 6)] - (1, 5): 'fooBar9' [var declared in mspellsuggest.nim(2, 5)] - (1, 5): 'fooCar' [var declared in mspellsuggest.nim(4, 5)] - (2, 5): 'FooCar' [type declared in mspellsuggest.nim(6, 6)] - (2, 5): 'GooBa' [type declared in mspellsuggest.nim(7, 6)] - (3, 0): 'fooBarBaz' [const declared in tspellsuggest.nim(44, 11)] + (1, 0): 'fooBar8' + (1, 1): 'fooBar7' + (1, 3): 'fooBar1' + (1, 3): 'fooBar2' + (1, 3): 'fooBar3' + (1, 3): 'fooBar4' + (1, 3): 'fooBar5' + (1, 3): 'fooBar6' + (1, 5): 'FooBar' + (1, 5): 'fooBar4' + (1, 5): 'fooBar9' + (1, 5): 'fooCar' + (2, 5): 'FooCar' + (2, 5): 'GooBa' + (3, 0): 'fooBarBaz' ''' """ diff --git a/tests/misc/tspellsuggest2.nim b/tests/misc/tspellsuggest2.nim index 78504c513..bf76cc208 100644 --- a/tests/misc/tspellsuggest2.nim +++ b/tests/misc/tspellsuggest2.nim @@ -5,18 +5,18 @@ discard """ nimout: ''' tspellsuggest2.nim(45, 13) Error: undeclared identifier: 'fooBar' candidates (edit distance, scope distance); see '--spellSuggest': - (1, 0): 'fooBar8' [var declared in tspellsuggest2.nim(43, 9)] - (1, 1): 'fooBar7' [var declared in tspellsuggest2.nim(41, 7)] - (1, 3): 'fooBar1' [var declared in tspellsuggest2.nim(33, 5)] - (1, 3): 'fooBar2' [let declared in tspellsuggest2.nim(34, 5)] - (1, 3): 'fooBar3' [const declared in tspellsuggest2.nim(35, 7)] - (1, 3): 'fooBar4' [proc declared in tspellsuggest2.nim(36, 6)] - (1, 3): 'fooBar5' [template declared in tspellsuggest2.nim(37, 10)] - (1, 3): 'fooBar6' [macro declared in tspellsuggest2.nim(38, 7)] - (1, 5): 'FooBar' [type declared in mspellsuggest.nim(5, 6)] - (1, 5): 'fooBar4' [proc declared in mspellsuggest.nim(1, 6)] - (1, 5): 'fooBar9' [var declared in mspellsuggest.nim(2, 5)] - (1, 5): 'fooCar' [var declared in mspellsuggest.nim(4, 5)] + (1, 0): 'fooBar8' + (1, 1): 'fooBar7' + (1, 3): 'fooBar1' + (1, 3): 'fooBar2' + (1, 3): 'fooBar3' + (1, 3): 'fooBar4' + (1, 3): 'fooBar5' + (1, 3): 'fooBar6' + (1, 5): 'FooBar' + (1, 5): 'fooBar4' + (1, 5): 'fooBar9' + (1, 5): 'fooCar' ''' """ diff --git a/tests/misc/tspellsuggest3.nim b/tests/misc/tspellsuggest3.nim new file mode 100644 index 000000000..9b0b84602 --- /dev/null +++ b/tests/misc/tspellsuggest3.nim @@ -0,0 +1,21 @@ +discard """ + # pending bug #16521 (bug 12) use `matrix` + cmd: "nim c --spellsuggest:4 --hints:off $file" + action: "reject" + nimout: ''' +tspellsuggest3.nim(21, 1) Error: undeclared identifier: 'fooBar' +candidates (edit distance, scope distance); see '--spellSuggest': + (1, 2): 'FooBar' + (1, 2): 'fooBar4' + (1, 2): 'fooBar9' + (1, 2): 'fooCar' +''' +""" + +import ./mspellsuggest +import ./mspellsuggest +import ./mspellsuggest +import ./mspellsuggest + + +fooBar From 3bba2b34fd632a4c01a6eb0e6f13d4e3a20bf104 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 22 Dec 2022 13:43:27 -0700 Subject: [PATCH 2312/2472] fix socket send for string types (#21155) * fix socket send for string types * include windows version Co-authored-by: ringabout <43030857+ringabout@users.noreply.github.com> --- lib/pure/net.nim | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 1dfdbbbe6..be9f2e48c 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -1735,15 +1735,36 @@ proc send*(socket: Socket, data: pointer, size: int): int {. result = send(socket.fd, data, size, int32(MSG_NOSIGNAL)) proc send*(socket: Socket, data: string, - flags = {SocketFlag.SafeDisconn}) {.tags: [WriteIOEffect].} = - ## sends data to a socket. - let sent = send(socket, cstring(data), data.len) - if sent < 0: - let lastError = osLastError() - socketError(socket, lastError = lastError, flags = flags) + flags = {SocketFlag.SafeDisconn}, maxRetries = 100) {.tags: [WriteIOEffect].} = + ## Sends data to a socket. Will try to send all the data by handling interrupts + ## and incomplete writes up to `maxRetries`. + var written = 0 + var attempts = 0 + while data.len - written > 0: + let sent = send(socket, cstring(data), data.len) - if sent != data.len: - raiseOSError(osLastError(), "Could not send all data.") + if sent < 0: + let lastError = osLastError() + let isBlockingErr = + when defined(nimdoc): + false + elif useWinVersion: + lastError.int32 == WSAEINTR or + lastError.int32 == WSAEWOULDBLOCK + else: + lastError.int32 == EINTR or + lastError.int32 == EWOULDBLOCK or + lastError.int32 == EAGAIN + + if not isBlockingErr: + let lastError = osLastError() + socketError(socket, lastError = lastError, flags = flags) + else: + attempts.inc() + if attempts > maxRetries: + raiseOSError(osLastError(), "Could not send all data.") + else: + written.inc(sent) template `&=`*(socket: Socket; data: typed) = ## an alias for 'send'. From 13251c2ac9c7e76fb506eeb686a5e5d19d67d0de Mon Sep 17 00:00:00 2001 From: Bung Date: Fri, 23 Dec 2022 16:47:01 +0800 Subject: [PATCH 2313/2472] fix #12946 Bad C++ codegen on distinct generics C++ types (#21157) --- compiler/ccgtypes.nim | 9 +++++---- tests/cpp/t12946.nim | 4 ++++ 2 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 tests/cpp/t12946.nim diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 3c9f9d923..e7677f2f2 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -826,7 +826,8 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin m.s[cfsTypes].addf("typedef $1 $2[$3];$n", [foo, result, rope(n)]) of tyObject, tyTuple: - if isImportedCppType(t) and origTyp.kind == tyGenericInst: + let tt = origTyp.skipTypes({tyDistinct}) + if isImportedCppType(t) and tt.kind == tyGenericInst: let cppNameAsRope = getTypeName(m, t, sig) let cppName = $cppNameAsRope var i = 0 @@ -849,7 +850,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin result.add cppName.substr(chunkStart, chunkEnd) chunkStart = i - let typeInSlot = resolveStarsInCppType(origTyp, idx + 1, stars) + let typeInSlot = resolveStarsInCppType(tt, idx + 1, stars) addResultType(typeInSlot) else: inc i @@ -858,9 +859,9 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin result.add cppName.substr(chunkStart) else: result = cppNameAsRope & "<" - for i in 1.. 1: result.add(" COMMA ") - addResultType(origTyp[i]) + addResultType(tt[i]) result.add("> ") # always call for sideeffects: assert t.kind != tyTuple diff --git a/tests/cpp/t12946.nim b/tests/cpp/t12946.nim new file mode 100644 index 000000000..cf6bf5cfc --- /dev/null +++ b/tests/cpp/t12946.nim @@ -0,0 +1,4 @@ +import std/atomics +type Futex = distinct Atomic[int32] + +var x: Futex From ca9c74391a2169be0b0e5c7b6705eaf9560a44bd Mon Sep 17 00:00:00 2001 From: Bung Date: Fri, 23 Dec 2022 18:32:03 +0800 Subject: [PATCH 2314/2472] fix #11634 (#21146) --- compiler/semstmts.nim | 9 ++++++--- tests/misc/t11634.nim | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 tests/misc/t11634.nim diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index a7a02d7fa..356c2d63e 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -725,9 +725,12 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = if v.kind == skLet and sfImportc notin v.flags and (strictDefs notin c.features or not isLocalSym(v)): localError(c.config, a.info, errLetNeedsInit) if sfCompileTime in v.flags: - var x = newNodeI(result.kind, v.info) - x.add result[i] - vm.setupCompileTimeVar(c.module, c.idgen, c.graph, x) + if a.kind != nkVarTuple: + var x = newNodeI(result.kind, v.info) + x.add result[i] + vm.setupCompileTimeVar(c.module, c.idgen, c.graph, x) + else: + localError(c.config, a.info, "cannot destructure to compile time variable") if v.flags * {sfGlobal, sfThread} == {sfGlobal}: message(c.config, v.info, hintGlobalVar) if {sfGlobal, sfPure} <= v.flags: diff --git a/tests/misc/t11634.nim b/tests/misc/t11634.nim new file mode 100644 index 000000000..4ecb6a53c --- /dev/null +++ b/tests/misc/t11634.nim @@ -0,0 +1,20 @@ +discard """ + action: reject + nimout: ''' +t11634.nim(20, 7) Error: cannot destructure to compile time variable +''' +""" + +type Foo = ref object + val: int + +proc divmod(a, b: Foo): (Foo, Foo) = + ( + Foo(val: a.val div b.val), + Foo(val: a.val mod b.val) + ) + +block: + let a {.compileTime.} = Foo(val: 2) + let b {.compileTime.} = Foo(val: 3) + let (c11634 {.compileTime.}, d11634 {.compileTime.}) = divmod(a, b) From 0b319fee3de90759987beba289df3c48ab6d3f1a Mon Sep 17 00:00:00 2001 From: Bung Date: Fri, 23 Dec 2022 19:20:25 +0800 Subject: [PATCH 2315/2472] =?UTF-8?q?fix=20#20997=20calling=20system.card[?= =?UTF-8?q?T](x:=20set[T])=20with=20T=20of=20int8=20or=20uint8=20=E2=80=A6?= =?UTF-8?q?=20(#21010)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix #20997 calling system.card[T](x: set[T]) with T of int8 or uint8 uses mismatched C array sizes * fullfil set variant --- compiler/ccgexprs.nim | 2 +- lib/system/sets.nim | 8 +++++--- tests/sets/t20997.nim | 18 ++++++++++++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 tests/sets/t20997.nim diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 2a4335144..d7ef024b1 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -2126,7 +2126,7 @@ proc genSetOp(p: BProc, e: PNode, d: var TLoc, op: TMagic) = of mCard: var a: TLoc initLocExpr(p, e[1], a) - putIntoDest(p, d, e, ropecg(p.module, "#cardSet($1, $2)", [rdCharLoc(a), size])) + putIntoDest(p, d, e, ropecg(p.module, "#cardSet($1, $2)", [addrLoc(p.config, a), size])) of mLtSet, mLeSet: getTemp(p, getSysType(p.module.g.graph, unknownLineInfo, tyInt), i) # our counter initLocExpr(p, e[1], a) diff --git a/lib/system/sets.nim b/lib/system/sets.nim index 103c8d343..e230985e0 100644 --- a/lib/system/sets.nim +++ b/lib/system/sets.nim @@ -10,10 +10,9 @@ # set handling type - NimSet = array[0..4*2048-1, uint8] + NimSet = array[0..8192-1, uint8] - -proc cardSet(s: NimSet, len: int): int {.compilerproc, inline.} = +proc cardSetImpl(s: openArray[uint8], len: int): int {.inline.} = var i = 0 result = 0 when defined(x86) or defined(amd64): @@ -24,3 +23,6 @@ proc cardSet(s: NimSet, len: int): int {.compilerproc, inline.} = while i < len: inc(result, countBits32(uint32(s[i]))) inc(i, 1) + +proc cardSet(s: NimSet, len: int): int {.compilerproc, inline.} = + result = cardSetImpl(s, len) diff --git a/tests/sets/t20997.nim b/tests/sets/t20997.nim new file mode 100644 index 000000000..b320eee1a --- /dev/null +++ b/tests/sets/t20997.nim @@ -0,0 +1,18 @@ +discard """ + joinable: false +""" + +{.passC: "-flto".} +{.passL: "-flto".} + +template f(n: int) = discard card(default(set[range[0 .. (1 shl n) - 1]])) +f( 7) +f( 8) +f( 9) +f(10) +f(11) +f(12) +f(13) +f(14) +f(15) +f(16) From 0ffc322cf615685ae4d528e26ddd13f6ea072684 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 23 Dec 2022 22:31:53 +0800 Subject: [PATCH 2316/2472] add the cpp target (#21164) The issue is related to cpp codegen, the previous test doesn't test cpp backend, which will join into the megatest. --- tests/cpp/t12946.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/cpp/t12946.nim b/tests/cpp/t12946.nim index cf6bf5cfc..79cd56251 100644 --- a/tests/cpp/t12946.nim +++ b/tests/cpp/t12946.nim @@ -1,3 +1,7 @@ +discard """ + targets: "c cpp" +""" + import std/atomics type Futex = distinct Atomic[int32] From ac5a36c0a4be3de6565f2a071c04d1fd8f1b67da Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 23 Dec 2022 23:50:07 +0800 Subject: [PATCH 2317/2472] re and nre now link to regex and tinyre (#21161) --- lib/impure/nre.nim | 8 +++++--- lib/impure/re.nim | 4 ++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/impure/nre.nim b/lib/impure/nre.nim index e9dd49df0..2c1b1deae 100644 --- a/lib/impure/nre.nim +++ b/lib/impure/nre.nim @@ -20,15 +20,17 @@ when defined(js): ## search the internet for a wide variety of third-party documentation and ## tools. ## -## **Note**: If you love `sequtils.toSeq` we have bad news for you. This -## library doesn't work with it due to documented compiler limitations. As -## a workaround, use this: +## .. warning:: If you love `sequtils.toSeq` we have bad news for you. This +## library doesn't work with it due to documented compiler limitations. As +## a workaround, use this: runnableExamples: # either `import std/nre except toSeq` or fully qualify `sequtils.toSeq`: import std/sequtils iterator iota(n: int): int = for i in 0.. Date: Sat, 24 Dec 2022 05:20:51 +1100 Subject: [PATCH 2318/2472] Make search results be relative to documentation root (#21145) Use Nim instead of JS for searching --- tools/dochack/dochack.nim | 48 ++++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/tools/dochack/dochack.nim b/tools/dochack/dochack.nim index 4067d2ed4..0753248bc 100644 --- a/tools/dochack/dochack.nim +++ b/tools/dochack/dochack.nim @@ -1,5 +1,6 @@ import dom import fuzzysearch +import std/[jsfetch, asyncjs] proc setTheme(theme: cstring) {.exportc.} = @@ -252,21 +253,7 @@ proc escapeCString(x: var cstring) = proc dosearch(value: cstring): Element = if db.len == 0: - var stuff: Element - {.emit: """ - var request = new XMLHttpRequest(); - request.open("GET", document.getElementById("indexLink").href, false); - request.send(null); - - var doc = document.implementation.createHTMLDocument("theindex"); - doc.documentElement.innerHTML = request.responseText; - - `stuff` = doc.documentElement; - """.} - db = stuff.getElementsByClass"reference" - contents = @[] - for ahref in db: - contents.add ahref.getAttribute("data-doc-search-tag") + return let ul = tree("UL") result = tree("DIV") result.setClass"search_results" @@ -293,8 +280,28 @@ proc dosearch(value: cstring): Element = result.add tree("B", text"search results") result.add ul -var oldtoc: Element -var timer: Timeout +proc loadIndex() {.async.} = + ## Loads theindex.html to enable searching + let + indexURL = document.getElementById("indexLink").getAttribute("href") + # Get root of project documentation by cutting off theindex.html from index href + rootURL = ($indexURL)[0 ..< ^"theindex.html".len] + var resp = fetch(indexURL).await().text().await() + # Convert into element so we can use DOM functions to parse the html + var indexElem = document.createElement("div") + indexElem.innerHtml = resp + # Add items into the DB/contents + for href in indexElem.getElementsByClass("reference"): + # Make links be relative to project root instead of current page + href.setAttr("href", cstring(rootURL & $href.getAttribute("href"))) + db &= href + contents &= href.getAttribute("data-doc-search-tag") + + +var + oldtoc: Element + timer: Timeout + loadIndexFut: Future[void] = nil proc search*() {.exportc.} = proc wrapper() = @@ -307,7 +314,12 @@ proc search*() {.exportc.} = replaceById("tocRoot", results) elif not oldtoc.isNil: replaceById("tocRoot", oldtoc) - + # Start loading index as soon as user starts typing. + # Will only be loaded the once anyways + if loadIndexFut == nil: + loadIndexFut = loadIndex() + # Run wrapper once loaded so we don't miss the users query + discard loadIndexFut.then(wrapper) if timer != nil: clearTimeout(timer) timer = setTimeout(wrapper, 400) From 9323cb7b2a761543df9d875c05f4963c1a8b050f Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sat, 24 Dec 2022 17:37:32 +0800 Subject: [PATCH 2319/2472] enforce void for nkWhileStmt [backport: 2.0] (#21170) enforce void for nkWhileStmt --- compiler/semstmts.nim | 1 - tests/discard/tdiscardable.nim | 11 +++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 356c2d63e..96883255c 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -105,7 +105,6 @@ proc semWhile(c: PContext, n: PNode; flags: TExprFlags): PNode = result.typ = n[1].typ elif implicitlyDiscardable(n[1]): result[1].typ = c.enforceVoidContext - result.typ = c.enforceVoidContext proc semProc(c: PContext, n: PNode): PNode diff --git a/tests/discard/tdiscardable.nim b/tests/discard/tdiscardable.nim index b13130a13..69cb9f6a1 100644 --- a/tests/discard/tdiscardable.nim +++ b/tests/discard/tdiscardable.nim @@ -99,3 +99,14 @@ block: # bug #13583 let t = test doAssert t() == 12 +block: + proc bar(): string {.discardable.} = + "15" + + proc foo(): int = + while true: + raise newException(ValueError, "check") + 12 + + doAssertRaises(ValueError): + doAssert foo() == 12 From b08c50bb553364eec1e20ab788d990b63f4548e0 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Mon, 26 Dec 2022 13:52:40 +0800 Subject: [PATCH 2320/2472] sync some changes from release notes (#21173) --- changelogs/changelog_2_0_0.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/changelogs/changelog_2_0_0.md b/changelogs/changelog_2_0_0.md index a899bb929..614d97d85 100644 --- a/changelogs/changelog_2_0_0.md +++ b/changelogs/changelog_2_0_0.md @@ -2,7 +2,7 @@ ## Changes affecting backward compatibility -- `httpclient.contentLength` default to `-1` if the Content-Length header is not set in the response, it followed Apache HttpClient(Java), http(go) and .Net HttpWebResponse(C#) behavior. Previously it raised `ValueError`. +- `httpclient.contentLength` default to `-1` if the Content-Length header is not set in the response. It follows Apache HttpClient(Java), http(go) and .Net HttpWebResponse(C#) behavior. Previously it raised `ValueError`. - `addr` is now available for all addressable locations, `unsafeAddr` is now deprecated and an alias for `addr`. @@ -65,7 +65,7 @@ - `shallowCopy` and `shallow` are removed for ARC/ORC. Use `move` when possible or combine assignment and `sink` for optimization purposes. -- The `nimPreviewDotLikeOps` define is going to be removed or deprecated. +- The experimental `nimPreviewDotLikeOps` switch is going to be removed or deprecated because it didn't fullfill its promises. - The `{.this.}` pragma, deprecated since 0.19, has been removed. - `nil` literals can no longer be directly assigned to variables or fields of `distinct` pointer types. They must be converted instead. @@ -110,8 +110,6 @@ - `logging` will default to flushing all log level messages. To get the legacy behaviour of only flushing Error and Fatal messages, use `-d:nimV1LogFlushBehavior`. -- Object fields now support default values, see https://nim-lang.github.io/Nim/manual.html#types-default-values-for-object-fields for details. - - Redefining templates with the same signature was previously allowed to support certain macro code. To do this explicitly, the `{.redefine.}` pragma has been added. Note that this is only for templates. From f7c203fb6c89b5cef83c4f326aeb23ef8c4a2c40 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Mon, 26 Dec 2022 20:20:05 +0800 Subject: [PATCH 2321/2472] remove legacy code (#21134) * remove legacy code * fixes --- compiler/astalgo.nim | 3 - compiler/cgen.nim | 2 - compiler/condsyms.nim | 47 +-- compiler/depends.nim | 3 - compiler/renderer.nim | 7 +- compiler/sem.nim | 3 - compiler/sigmatch.nim | 2 - compiler/vm.nim | 5 - compiler/vmops.nim | 72 +++-- lib/core/macros.nim | 9 +- lib/pure/collections/lists.nim | 3 - lib/pure/hashes.nim | 40 +-- lib/pure/json.nim | 540 ++++++++++++++++---------------- lib/pure/nimprof.nim | 3 +- lib/std/assertions.nim | 2 - lib/std/jsonutils.nim | 18 +- lib/std/private/miscdollars.nim | 16 +- lib/system.nim | 35 +-- lib/system/alloc.nim | 10 +- lib/system/arithm.nim | 425 ------------------------- lib/system/arithmetics.nim | 140 +++------ lib/system/compilation.nim | 40 +-- lib/system/fatal.nim | 5 +- lib/system/inclrtl.nim | 3 +- lib/system/iterators.nim | 2 +- lib/system/jssys.nim | 22 -- lib/system/strmantle.nim | 7 +- 27 files changed, 432 insertions(+), 1032 deletions(-) delete mode 100644 lib/system/arithm.nim diff --git a/compiler/astalgo.nim b/compiler/astalgo.nim index e9ed1497a..4e09fab02 100644 --- a/compiler/astalgo.nim +++ b/compiler/astalgo.nim @@ -20,9 +20,6 @@ import strutils except addf when defined(nimPreviewSlimSystem): import std/assertions -when not defined(nimHasCursor): - {.pragma: cursor.} - proc hashNode*(p: RootRef): Hash proc treeToYaml*(conf: ConfigRef; n: PNode, indent: int = 0, maxRecDepth: int = - 1): Rope # Convert a tree into its YAML representation; this is used by the diff --git a/compiler/cgen.nim b/compiler/cgen.nim index e85cfd773..f6457f1e0 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1929,8 +1929,6 @@ template injectG() {.dirty.} = graph.backend = newModuleList(graph) let g = BModuleList(graph.backend) -when not defined(nimHasSinkInference): - {.pragma: nosinks.} proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} = injectG() diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index 24c6d82d5..c05aaf10b 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -84,10 +84,24 @@ proc initDefines*(symbols: StringTableRef) = defineSymbol("nimHasSignatureHashInMacro") # deadcode defineSymbol("nimHasDefault") # deadcode defineSymbol("nimMacrosSizealignof") # deadcode + defineSymbol("nimNoZeroExtendMagic") # deadcode + defineSymbol("nimMacrosGetNodeId") # deadcode + defineSymbol("nimFixedForwardGeneric") # deadcode + defineSymbol("nimToOpenArrayCString") # deadcode + defineSymbol("nimHasUsed") # deadcode + defineSymbol("nimnomagic64") # deadcode + defineSymbol("nimNewShiftOps") # deadcode + defineSymbol("nimHasCursor") # deadcode + defineSymbol("nimAlignPragma") # deadcode + defineSymbol("nimHasExceptionsQuery") # deadcode + defineSymbol("nimHasIsNamedTuple") # deadcode + defineSymbol("nimHashOrdinalFixed") # deadcode + defineSymbol("nimHasSinkInference") # deadcode + defineSymbol("nimNewIntegerOps") # deadcode + defineSymbol("nimHasInvariant") # deadcode + + - # > 0.20.0 - defineSymbol("nimNoZeroExtendMagic") - defineSymbol("nimMacrosGetNodeId") for f in Feature: defineSymbol("nimHas" & $f) @@ -98,31 +112,18 @@ proc initDefines*(symbols: StringTableRef) = defineSymbol("nimFixedOwned") defineSymbol("nimHasStyleChecks") - defineSymbol("nimToOpenArrayCString") - defineSymbol("nimHasUsed") - defineSymbol("nimFixedForwardGeneric") - defineSymbol("nimnomagic64") - defineSymbol("nimNewShiftOps") - defineSymbol("nimHasCursor") - defineSymbol("nimAlignPragma") - defineSymbol("nimHasExceptionsQuery") - defineSymbol("nimHasIsNamedTuple") - defineSymbol("nimHashOrdinalFixed") when defined(nimHasLibFFI): # Renaming as we can't conflate input vs output define flags; e.g. this # will report the right thing regardless of whether user adds # `-d:nimHasLibFFI` in his user config. - defineSymbol("nimHasLibFFIEnabled") + defineSymbol("nimHasLibFFIEnabled") # deadcode - defineSymbol("nimHasSinkInference") - defineSymbol("nimNewIntegerOps") - defineSymbol("nimHasInvariant") - defineSymbol("nimHasStacktraceMsgs") + defineSymbol("nimHasStacktraceMsgs") # deadcode defineSymbol("nimDoesntTrackDefects") - defineSymbol("nimHasLentIterators") - defineSymbol("nimHasDeclaredMagic") - defineSymbol("nimHasStacktracesModule") + defineSymbol("nimHasLentIterators") # deadcode + defineSymbol("nimHasDeclaredMagic") # deadcode + defineSymbol("nimHasStacktracesModule") # deadcode defineSymbol("nimHasEffectTraitsModule") defineSymbol("nimHasCastPragmaBlocks") defineSymbol("nimHasDeclaredLocs") @@ -133,8 +134,8 @@ proc initDefines*(symbols: StringTableRef) = defineSymbol("nimHasCustomLiterals") defineSymbol("nimHasUnifiedTuple") defineSymbol("nimHasIterable") - defineSymbol("nimHasTypeofVoid") - defineSymbol("nimHasDragonBox") + defineSymbol("nimHasTypeofVoid") # deadcode + defineSymbol("nimHasDragonBox") # deadcode defineSymbol("nimHasHintAll") defineSymbol("nimHasTrace") defineSymbol("nimHasEffectsOf") diff --git a/compiler/depends.nim b/compiler/depends.nim index 93f34dc7c..59cdc0330 100644 --- a/compiler/depends.nim +++ b/compiler/depends.nim @@ -100,9 +100,6 @@ proc generateDot*(graph: ModuleGraph; project: AbsoluteFile) = rope(project.splitFile.name), b.dotGraph], changeFileExt(project, "dot")) -when not defined(nimHasSinkInference): - {.pragma: nosinks.} - proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} = var g: PGen new(g) diff --git a/compiler/renderer.nim b/compiler/renderer.nim index cd0d83eb0..87a5e6580 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -9,10 +9,9 @@ # This module implements the renderer of the standard Nim representation. -when defined(nimHasUsed): - # 'import renderer' is so useful for debugging - # that Nim shouldn't produce a warning for that: - {.used.} +# 'import renderer' is so useful for debugging +# that Nim shouldn't produce a warning for that: +{.used.} import lexer, options, idents, strutils, ast, msgs, lineinfos diff --git a/compiler/sem.nim b/compiler/sem.nim index 1f8e217b3..f5283fa3d 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -405,9 +405,6 @@ proc semExprFlagDispatched(c: PContext, n: PNode, flags: TExprFlags; expectedTyp evaluated = evalAtCompileTime(c, result) if evaluated != nil: return evaluated -when not defined(nimHasSinkInference): - {.pragma: nosinks.} - include hlo, seminst, semcall proc resetSemFlag(n: PNode) = diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 61a4395f3..d05503c00 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -2661,8 +2661,6 @@ proc argtypeMatches*(c: PContext, f, a: PType, fromHlo = false): bool = # pattern templates do not allow for conversions except from int literal res != nil and m.convMatches == 0 and m.intConvMatches in [0, 256] -when not defined(nimHasSinkInference): - {.pragma: nosinks.} proc instTypeBoundOp*(c: PContext; dc: PSym; t: PType; info: TLineInfo; op: TTypeAttachedOp; col: int): PSym {.nosinks.} = diff --git a/compiler/vm.nim b/compiler/vm.nim index 61261c44c..d87fa33ba 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -32,8 +32,6 @@ const when hasFFI: import evalffi -when not defined(nimHasCursor): - {.pragma: cursor.} proc stackTraceAux(c: PCtx; x: PStackFrame; pc: int; recursionLimit=100) = if x != nil: @@ -533,9 +531,6 @@ template maybeHandlePtr(node2: PNode, reg: TFullReg, isAssign2: bool): bool = else: false -when not defined(nimHasSinkInference): - {.pragma: nosinks.} - template takeAddress(reg, source) = reg.nodeAddr = addr source GC_ref source diff --git a/compiler/vmops.nim b/compiler/vmops.nim index d4a91b22d..bb943b8a0 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -141,41 +141,40 @@ proc staticWalkDirImpl(path: string, relative: bool): PNode = for k, f in walkDir(path, relative): result.add toLit((k, f)) -when defined(nimHasInvariant): - from std / compilesettings import SingleValueSetting, MultipleValueSetting +from std / compilesettings import SingleValueSetting, MultipleValueSetting - proc querySettingImpl(conf: ConfigRef, switch: BiggestInt): string = - {.push warning[Deprecated]:off.} - case SingleValueSetting(switch) - of arguments: result = conf.arguments - of outFile: result = conf.outFile.string - of outDir: result = conf.outDir.string - of nimcacheDir: result = conf.getNimcacheDir().string - of projectName: result = conf.projectName - of projectPath: result = conf.projectPath.string - of projectFull: result = conf.projectFull.string - of command: result = conf.command - of commandLine: result = conf.commandLine - of linkOptions: result = conf.linkOptions - of compileOptions: result = conf.compileOptions - of ccompilerPath: result = conf.cCompilerPath - of backend: result = $conf.backend - of libPath: result = conf.libpath.string - of gc: result = $conf.selectedGC - of mm: result = $conf.selectedGC - {.pop.} +proc querySettingImpl(conf: ConfigRef, switch: BiggestInt): string = + {.push warning[Deprecated]:off.} + case SingleValueSetting(switch) + of arguments: result = conf.arguments + of outFile: result = conf.outFile.string + of outDir: result = conf.outDir.string + of nimcacheDir: result = conf.getNimcacheDir().string + of projectName: result = conf.projectName + of projectPath: result = conf.projectPath.string + of projectFull: result = conf.projectFull.string + of command: result = conf.command + of commandLine: result = conf.commandLine + of linkOptions: result = conf.linkOptions + of compileOptions: result = conf.compileOptions + of ccompilerPath: result = conf.cCompilerPath + of backend: result = $conf.backend + of libPath: result = conf.libpath.string + of gc: result = $conf.selectedGC + of mm: result = $conf.selectedGC + {.pop.} - proc querySettingSeqImpl(conf: ConfigRef, switch: BiggestInt): seq[string] = - template copySeq(field: untyped): untyped = - for i in field: result.add i.string +proc querySettingSeqImpl(conf: ConfigRef, switch: BiggestInt): seq[string] = + template copySeq(field: untyped): untyped = + for i in field: result.add i.string - case MultipleValueSetting(switch) - of nimblePaths: copySeq(conf.nimblePaths) - of searchPaths: copySeq(conf.searchPaths) - of lazyPaths: copySeq(conf.lazyPaths) - of commandArgs: result = conf.commandArgs - of cincludes: copySeq(conf.cIncludes) - of clibs: copySeq(conf.cLibs) + case MultipleValueSetting(switch) + of nimblePaths: copySeq(conf.nimblePaths) + of searchPaths: copySeq(conf.searchPaths) + of lazyPaths: copySeq(conf.lazyPaths) + of commandArgs: result = conf.commandArgs + of cincludes: copySeq(conf.cIncludes) + of clibs: copySeq(conf.cLibs) proc stackTrace2(c: PCtx, msg: string, n: PNode) = stackTrace(c, PStackFrame(prc: c.prc.sym, comesFrom: 0, next: nil), c.exceptionInstr, msg, n.info) @@ -257,11 +256,10 @@ proc registerAdditionalOps*(c: PCtx) = systemop getCurrentException registerCallback c, "stdlib.*.staticWalkDir", proc (a: VmArgs) {.nimcall.} = setResult(a, staticWalkDirImpl(getString(a, 0), getBool(a, 1))) - when defined(nimHasInvariant): - registerCallback c, "stdlib.compilesettings.querySetting", proc (a: VmArgs) = - setResult(a, querySettingImpl(c.config, getInt(a, 0))) - registerCallback c, "stdlib.compilesettings.querySettingSeq", proc (a: VmArgs) = - setResult(a, querySettingSeqImpl(c.config, getInt(a, 0))) + registerCallback c, "stdlib.compilesettings.querySetting", proc (a: VmArgs) = + setResult(a, querySettingImpl(c.config, getInt(a, 0))) + registerCallback c, "stdlib.compilesettings.querySettingSeq", proc (a: VmArgs) = + setResult(a, querySettingSeqImpl(c.config, getInt(a, 0))) if defined(nimsuggest) or c.config.cmd == cmdCheck: discard "don't run staticExec for 'nim suggest'" diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 138ec47a0..bcc16038b 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -1519,11 +1519,10 @@ proc boolVal*(n: NimNode): bool {.noSideEffect.} = if n.kind == nnkIntLit: n.intVal != 0 else: n == bindSym"true" # hacky solution for now -when defined(nimMacrosGetNodeId): - proc nodeID*(n: NimNode): int {.magic: "NodeId".} - ## Returns the id of `n`, when the compiler has been compiled - ## with the flag `-d:useNodeids`, otherwise returns `-1`. This - ## proc is for the purpose to debug the compiler only. +proc nodeID*(n: NimNode): int {.magic: "NodeId".} + ## Returns the id of `n`, when the compiler has been compiled + ## with the flag `-d:useNodeids`, otherwise returns `-1`. This + ## proc is for the purpose to debug the compiler only. macro expandMacros*(body: typed): untyped = ## Expands one level of macro - useful for debugging. diff --git a/lib/pure/collections/lists.nim b/lib/pure/collections/lists.nim index 829ec2ccb..a08c62964 100644 --- a/lib/pure/collections/lists.nim +++ b/lib/pure/collections/lists.nim @@ -62,9 +62,6 @@ import std/private/since when defined(nimPreviewSlimSystem): import std/assertions -when not defined(nimHasCursor): - {.pragma: cursor.} - type DoublyLinkedNodeObj*[T] = object ## A node of a doubly linked list. diff --git a/lib/pure/hashes.nim b/lib/pure/hashes.nim index e88210757..09e072812 100644 --- a/lib/pure/hashes.nim +++ b/lib/pure/hashes.nim @@ -380,16 +380,10 @@ proc hash*(x: string): Hash = runnableExamples: doAssert hash("abracadabra") != hash("AbracadabrA") - when not defined(nimToOpenArrayCString): - result = 0 - for c in x: - result = result !& ord(c) - result = !$result + when nimvm: + result = hashVmImpl(x, 0, high(x)) else: - when nimvm: - result = hashVmImpl(x, 0, high(x)) - else: - result = murmurHash(toOpenArrayByte(x, 0, high(x))) + result = murmurHash(toOpenArrayByte(x, 0, high(x))) proc hash*(x: cstring): Hash = ## Efficient hashing of null-terminated strings. @@ -398,22 +392,14 @@ proc hash*(x: cstring): Hash = doAssert hash(cstring"AbracadabrA") == hash("AbracadabrA") doAssert hash(cstring"abracadabra") != hash(cstring"AbracadabrA") - when not defined(nimToOpenArrayCString): - result = 0 - var i = 0 - while x[i] != '\0': - result = result !& ord(x[i]) - inc i - result = !$result + when nimvm: + hashVmImpl(x, 0, high(x)) else: - when nimvm: - hashVmImpl(x, 0, high(x)) + when not defined(js): + murmurHash(toOpenArrayByte(x, 0, x.high)) else: - when not defined(js) and defined(nimToOpenArrayCString): - murmurHash(toOpenArrayByte(x, 0, x.high)) - else: - let xx = $x - murmurHash(toOpenArrayByte(xx, 0, high(xx))) + let xx = $x + murmurHash(toOpenArrayByte(xx, 0, high(xx))) proc hash*(sBuf: string, sPos, ePos: int): Hash = ## Efficient hashing of a string buffer, from starting @@ -424,13 +410,7 @@ proc hash*(sBuf: string, sPos, ePos: int): Hash = var a = "abracadabra" doAssert hash(a, 0, 3) == hash(a, 7, 10) - when not defined(nimToOpenArrayCString): - result = 0 - for i in sPos..ePos: - result = result !& ord(sBuf[i]) - result = !$result - else: - murmurHash(toOpenArrayByte(sBuf, sPos, ePos)) + murmurHash(toOpenArrayByte(sBuf, sPos, ePos)) proc hashIgnoreStyle*(x: string): Hash = ## Efficient hashing of strings; style is ignored. diff --git a/lib/pure/json.nim b/lib/pure/json.nim index d0b1a4051..78d6bd5bd 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -1059,310 +1059,308 @@ template verifyJsonKind(node: JsonNode, kinds: set[JsonNodeKind], ] raise newException(JsonKindError, msg) -when defined(nimFixedForwardGeneric): +macro isRefSkipDistinct*(arg: typed): untyped = + ## internal only, do not use + var impl = getTypeImpl(arg) + if impl.kind == nnkBracketExpr and impl[0].eqIdent("typeDesc"): + impl = getTypeImpl(impl[1]) + while impl.kind == nnkDistinctTy: + impl = getTypeImpl(impl[0]) + result = newLit(impl.kind == nnkRefTy) - macro isRefSkipDistinct*(arg: typed): untyped = - ## internal only, do not use - var impl = getTypeImpl(arg) - if impl.kind == nnkBracketExpr and impl[0].eqIdent("typeDesc"): - impl = getTypeImpl(impl[1]) - while impl.kind == nnkDistinctTy: - impl = getTypeImpl(impl[0]) - result = newLit(impl.kind == nnkRefTy) +# The following forward declarations don't work in older versions of Nim - # The following forward declarations don't work in older versions of Nim +# forward declare all initFromJson - # forward declare all initFromJson +proc initFromJson(dst: var string; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson(dst: var bool; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson(dst: var JsonNode; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T: SomeInteger](dst: var T; jsonNode: JsonNode, jsonPath: var string) +proc initFromJson[T: SomeFloat](dst: var T; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T: enum](dst: var T; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var seq[T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[S, T](dst: var array[S, T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var Table[string, T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var OrderedTable[string, T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var Option[T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T: object|tuple](dst: var T; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson(dst: var string; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson(dst: var bool; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson(dst: var JsonNode; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T: SomeInteger](dst: var T; jsonNode: JsonNode, jsonPath: var string) - proc initFromJson[T: SomeFloat](dst: var T; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T: enum](dst: var T; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var seq[T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[S, T](dst: var array[S, T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var Table[string, T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var OrderedTable[string, T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var Option[T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T: object|tuple](dst: var T; jsonNode: JsonNode; jsonPath: var string) +# initFromJson definitions - # initFromJson definitions +proc initFromJson(dst: var string; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JString, JNull}, jsonPath) + # since strings don't have a nil state anymore, this mapping of + # JNull to the default string is questionable. `none(string)` and + # `some("")` have the same potentional json value `JNull`. + if jsonNode.kind == JNull: + dst = "" + else: + dst = jsonNode.str - proc initFromJson(dst: var string; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JString, JNull}, jsonPath) - # since strings don't have a nil state anymore, this mapping of - # JNull to the default string is questionable. `none(string)` and - # `some("")` have the same potentional json value `JNull`. - if jsonNode.kind == JNull: - dst = "" +proc initFromJson(dst: var bool; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JBool}, jsonPath) + dst = jsonNode.bval + +proc initFromJson(dst: var JsonNode; jsonNode: JsonNode; jsonPath: var string) = + if jsonNode == nil: + raise newException(KeyError, "key not found: " & jsonPath) + dst = jsonNode.copy + +proc initFromJson[T: SomeInteger](dst: var T; jsonNode: JsonNode, jsonPath: var string) = + when T is uint|uint64 or (not defined(js) and int.sizeof == 4): + verifyJsonKind(jsonNode, {JInt, JString}, jsonPath) + case jsonNode.kind + of JString: + let x = parseBiggestUInt(jsonNode.str) + dst = cast[T](x) else: - dst = jsonNode.str + dst = T(jsonNode.num) + else: + verifyJsonKind(jsonNode, {JInt}, jsonPath) + dst = cast[T](jsonNode.num) - proc initFromJson(dst: var bool; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JBool}, jsonPath) - dst = jsonNode.bval - - proc initFromJson(dst: var JsonNode; jsonNode: JsonNode; jsonPath: var string) = - if jsonNode == nil: - raise newException(KeyError, "key not found: " & jsonPath) - dst = jsonNode.copy - - proc initFromJson[T: SomeInteger](dst: var T; jsonNode: JsonNode, jsonPath: var string) = - when T is uint|uint64 or (not defined(js) and int.sizeof == 4): - verifyJsonKind(jsonNode, {JInt, JString}, jsonPath) - case jsonNode.kind - of JString: - let x = parseBiggestUInt(jsonNode.str) - dst = cast[T](x) - else: - dst = T(jsonNode.num) +proc initFromJson[T: SomeFloat](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + if jsonNode.kind == JString: + case jsonNode.str + of "nan": + let b = NaN + dst = T(b) + # dst = NaN # would fail some tests because range conversions would cause CT error + # in some cases; but this is not a hot-spot inside this branch and backend can optimize this. + of "inf": + let b = Inf + dst = T(b) + of "-inf": + let b = -Inf + dst = T(b) + else: raise newException(JsonKindError, "expected 'nan|inf|-inf', got " & jsonNode.str) + else: + verifyJsonKind(jsonNode, {JInt, JFloat}, jsonPath) + if jsonNode.kind == JFloat: + dst = T(jsonNode.fnum) else: - verifyJsonKind(jsonNode, {JInt}, jsonPath) - dst = cast[T](jsonNode.num) + dst = T(jsonNode.num) - proc initFromJson[T: SomeFloat](dst: var T; jsonNode: JsonNode; jsonPath: var string) = - if jsonNode.kind == JString: - case jsonNode.str - of "nan": - let b = NaN - dst = T(b) - # dst = NaN # would fail some tests because range conversions would cause CT error - # in some cases; but this is not a hot-spot inside this branch and backend can optimize this. - of "inf": - let b = Inf - dst = T(b) - of "-inf": - let b = -Inf - dst = T(b) - else: raise newException(JsonKindError, "expected 'nan|inf|-inf', got " & jsonNode.str) +proc initFromJson[T: enum](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JString}, jsonPath) + dst = parseEnum[T](jsonNode.getStr) + +proc initFromJson[T](dst: var seq[T]; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JArray}, jsonPath) + dst.setLen jsonNode.len + let orignalJsonPathLen = jsonPath.len + for i in 0 ..< jsonNode.len: + jsonPath.add '[' + jsonPath.addInt i + jsonPath.add ']' + initFromJson(dst[i], jsonNode[i], jsonPath) + jsonPath.setLen orignalJsonPathLen + +proc initFromJson[S,T](dst: var array[S,T]; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JArray}, jsonPath) + let originalJsonPathLen = jsonPath.len + for i in 0 ..< jsonNode.len: + jsonPath.add '[' + jsonPath.addInt i + jsonPath.add ']' + initFromJson(dst[i.S], jsonNode[i], jsonPath) # `.S` for enum indexed arrays + jsonPath.setLen originalJsonPathLen + +proc initFromJson[T](dst: var Table[string,T]; jsonNode: JsonNode; jsonPath: var string) = + dst = initTable[string, T]() + verifyJsonKind(jsonNode, {JObject}, jsonPath) + let originalJsonPathLen = jsonPath.len + for key in keys(jsonNode.fields): + jsonPath.add '.' + jsonPath.add key + initFromJson(mgetOrPut(dst, key, default(T)), jsonNode[key], jsonPath) + jsonPath.setLen originalJsonPathLen + +proc initFromJson[T](dst: var OrderedTable[string,T]; jsonNode: JsonNode; jsonPath: var string) = + dst = initOrderedTable[string,T]() + verifyJsonKind(jsonNode, {JObject}, jsonPath) + let originalJsonPathLen = jsonPath.len + for key in keys(jsonNode.fields): + jsonPath.add '.' + jsonPath.add key + initFromJson(mgetOrPut(dst, key, default(T)), jsonNode[key], jsonPath) + jsonPath.setLen originalJsonPathLen + +proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JObject, JNull}, jsonPath) + if jsonNode.kind == JNull: + dst = nil + else: + dst = new(T) + initFromJson(dst[], jsonNode, jsonPath) + +proc initFromJson[T](dst: var Option[T]; jsonNode: JsonNode; jsonPath: var string) = + if jsonNode != nil and jsonNode.kind != JNull: + when T is ref: + dst = some(new(T)) else: - verifyJsonKind(jsonNode, {JInt, JFloat}, jsonPath) - if jsonNode.kind == JFloat: - dst = T(jsonNode.fnum) - else: - dst = T(jsonNode.num) + dst = some(default(T)) + initFromJson(dst.get, jsonNode, jsonPath) - proc initFromJson[T: enum](dst: var T; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JString}, jsonPath) - dst = parseEnum[T](jsonNode.getStr) +macro assignDistinctImpl[T: distinct](dst: var T;jsonNode: JsonNode; jsonPath: var string) = + let typInst = getTypeInst(dst) + let typImpl = getTypeImpl(dst) + let baseTyp = typImpl[0] - proc initFromJson[T](dst: var seq[T]; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JArray}, jsonPath) - dst.setLen jsonNode.len - let orignalJsonPathLen = jsonPath.len - for i in 0 ..< jsonNode.len: - jsonPath.add '[' - jsonPath.addInt i - jsonPath.add ']' - initFromJson(dst[i], jsonNode[i], jsonPath) - jsonPath.setLen orignalJsonPathLen - - proc initFromJson[S,T](dst: var array[S,T]; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JArray}, jsonPath) - let originalJsonPathLen = jsonPath.len - for i in 0 ..< jsonNode.len: - jsonPath.add '[' - jsonPath.addInt i - jsonPath.add ']' - initFromJson(dst[i.S], jsonNode[i], jsonPath) # `.S` for enum indexed arrays - jsonPath.setLen originalJsonPathLen - - proc initFromJson[T](dst: var Table[string,T]; jsonNode: JsonNode; jsonPath: var string) = - dst = initTable[string, T]() - verifyJsonKind(jsonNode, {JObject}, jsonPath) - let originalJsonPathLen = jsonPath.len - for key in keys(jsonNode.fields): - jsonPath.add '.' - jsonPath.add key - initFromJson(mgetOrPut(dst, key, default(T)), jsonNode[key], jsonPath) - jsonPath.setLen originalJsonPathLen - - proc initFromJson[T](dst: var OrderedTable[string,T]; jsonNode: JsonNode; jsonPath: var string) = - dst = initOrderedTable[string,T]() - verifyJsonKind(jsonNode, {JObject}, jsonPath) - let originalJsonPathLen = jsonPath.len - for key in keys(jsonNode.fields): - jsonPath.add '.' - jsonPath.add key - initFromJson(mgetOrPut(dst, key, default(T)), jsonNode[key], jsonPath) - jsonPath.setLen originalJsonPathLen - - proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JObject, JNull}, jsonPath) - if jsonNode.kind == JNull: - dst = nil + result = quote do: + when nimvm: + # workaround #12282 + var tmp: `baseTyp` + initFromJson( tmp, `jsonNode`, `jsonPath`) + `dst` = `typInst`(tmp) else: - dst = new(T) - initFromJson(dst[], jsonNode, jsonPath) + initFromJson( `baseTyp`(`dst`), `jsonNode`, `jsonPath`) - proc initFromJson[T](dst: var Option[T]; jsonNode: JsonNode; jsonPath: var string) = - if jsonNode != nil and jsonNode.kind != JNull: - when T is ref: - dst = some(new(T)) - else: - dst = some(default(T)) - initFromJson(dst.get, jsonNode, jsonPath) +proc initFromJson[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + assignDistinctImpl(dst, jsonNode, jsonPath) - macro assignDistinctImpl[T: distinct](dst: var T;jsonNode: JsonNode; jsonPath: var string) = - let typInst = getTypeInst(dst) - let typImpl = getTypeImpl(dst) - let baseTyp = typImpl[0] +proc detectIncompatibleType(typeExpr, lineinfoNode: NimNode) = + if typeExpr.kind == nnkTupleConstr: + error("Use a named tuple instead of: " & typeExpr.repr, lineinfoNode) - result = quote do: +proc foldObjectBody(dst, typeNode, tmpSym, jsonNode, jsonPath, originalJsonPathLen: NimNode) = + case typeNode.kind + of nnkEmpty: + discard + of nnkRecList, nnkTupleTy: + for it in typeNode: + foldObjectBody(dst, it, tmpSym, jsonNode, jsonPath, originalJsonPathLen) + + of nnkIdentDefs: + typeNode.expectLen 3 + let fieldSym = typeNode[0] + let fieldNameLit = newLit(fieldSym.strVal) + let fieldPathLit = newLit("." & fieldSym.strVal) + let fieldType = typeNode[1] + + # Detecting incompatiple tuple types in `assignObjectImpl` only + # would be much cleaner, but the ast for tuple types does not + # contain usable type information. + detectIncompatibleType(fieldType, fieldSym) + + dst.add quote do: + jsonPath.add `fieldPathLit` when nimvm: - # workaround #12282 - var tmp: `baseTyp` - initFromJson( tmp, `jsonNode`, `jsonPath`) - `dst` = `typInst`(tmp) - else: - initFromJson( `baseTyp`(`dst`), `jsonNode`, `jsonPath`) - - proc initFromJson[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) = - assignDistinctImpl(dst, jsonNode, jsonPath) - - proc detectIncompatibleType(typeExpr, lineinfoNode: NimNode) = - if typeExpr.kind == nnkTupleConstr: - error("Use a named tuple instead of: " & typeExpr.repr, lineinfoNode) - - proc foldObjectBody(dst, typeNode, tmpSym, jsonNode, jsonPath, originalJsonPathLen: NimNode) = - case typeNode.kind - of nnkEmpty: - discard - of nnkRecList, nnkTupleTy: - for it in typeNode: - foldObjectBody(dst, it, tmpSym, jsonNode, jsonPath, originalJsonPathLen) - - of nnkIdentDefs: - typeNode.expectLen 3 - let fieldSym = typeNode[0] - let fieldNameLit = newLit(fieldSym.strVal) - let fieldPathLit = newLit("." & fieldSym.strVal) - let fieldType = typeNode[1] - - # Detecting incompatiple tuple types in `assignObjectImpl` only - # would be much cleaner, but the ast for tuple types does not - # contain usable type information. - detectIncompatibleType(fieldType, fieldSym) - - dst.add quote do: - jsonPath.add `fieldPathLit` - when nimvm: - when isRefSkipDistinct(`tmpSym`.`fieldSym`): - # workaround #12489 - var tmp: `fieldType` - initFromJson(tmp, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) - `tmpSym`.`fieldSym` = tmp - else: - initFromJson(`tmpSym`.`fieldSym`, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) + when isRefSkipDistinct(`tmpSym`.`fieldSym`): + # workaround #12489 + var tmp: `fieldType` + initFromJson(tmp, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) + `tmpSym`.`fieldSym` = tmp else: initFromJson(`tmpSym`.`fieldSym`, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) - jsonPath.setLen `originalJsonPathLen` + else: + initFromJson(`tmpSym`.`fieldSym`, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) + jsonPath.setLen `originalJsonPathLen` - of nnkRecCase: - let kindSym = typeNode[0][0] - let kindNameLit = newLit(kindSym.strVal) - let kindPathLit = newLit("." & kindSym.strVal) - let kindType = typeNode[0][1] - let kindOffsetLit = newLit(uint(getOffset(kindSym))) - dst.add quote do: - var kindTmp: `kindType` - jsonPath.add `kindPathLit` - initFromJson(kindTmp, `jsonNode`[`kindNameLit`], `jsonPath`) - jsonPath.setLen `originalJsonPathLen` - when defined js: + of nnkRecCase: + let kindSym = typeNode[0][0] + let kindNameLit = newLit(kindSym.strVal) + let kindPathLit = newLit("." & kindSym.strVal) + let kindType = typeNode[0][1] + let kindOffsetLit = newLit(uint(getOffset(kindSym))) + dst.add quote do: + var kindTmp: `kindType` + jsonPath.add `kindPathLit` + initFromJson(kindTmp, `jsonNode`[`kindNameLit`], `jsonPath`) + jsonPath.setLen `originalJsonPathLen` + when defined js: + `tmpSym`.`kindSym` = kindTmp + else: + when nimvm: `tmpSym`.`kindSym` = kindTmp else: - when nimvm: - `tmpSym`.`kindSym` = kindTmp - else: - # fuck it, assign kind field anyway - ((cast[ptr `kindType`](cast[uint](`tmpSym`.addr) + `kindOffsetLit`))[]) = kindTmp - dst.add nnkCaseStmt.newTree(nnkDotExpr.newTree(tmpSym, kindSym)) - for i in 1 ..< typeNode.len: - foldObjectBody(dst, typeNode[i], tmpSym, jsonNode, jsonPath, originalJsonPathLen) + # fuck it, assign kind field anyway + ((cast[ptr `kindType`](cast[uint](`tmpSym`.addr) + `kindOffsetLit`))[]) = kindTmp + dst.add nnkCaseStmt.newTree(nnkDotExpr.newTree(tmpSym, kindSym)) + for i in 1 ..< typeNode.len: + foldObjectBody(dst, typeNode[i], tmpSym, jsonNode, jsonPath, originalJsonPathLen) - of nnkOfBranch, nnkElse: - let ofBranch = newNimNode(typeNode.kind) - for i in 0 ..< typeNode.len-1: - ofBranch.add copyNimTree(typeNode[i]) - let dstInner = newNimNode(nnkStmtListExpr) - foldObjectBody(dstInner, typeNode[^1], tmpSym, jsonNode, jsonPath, originalJsonPathLen) - # resOuter now contains the inner stmtList - ofBranch.add dstInner - dst[^1].expectKind nnkCaseStmt - dst[^1].add ofBranch + of nnkOfBranch, nnkElse: + let ofBranch = newNimNode(typeNode.kind) + for i in 0 ..< typeNode.len-1: + ofBranch.add copyNimTree(typeNode[i]) + let dstInner = newNimNode(nnkStmtListExpr) + foldObjectBody(dstInner, typeNode[^1], tmpSym, jsonNode, jsonPath, originalJsonPathLen) + # resOuter now contains the inner stmtList + ofBranch.add dstInner + dst[^1].expectKind nnkCaseStmt + dst[^1].add ofBranch - of nnkObjectTy: - typeNode[0].expectKind nnkEmpty - typeNode[1].expectKind {nnkEmpty, nnkOfInherit} - if typeNode[1].kind == nnkOfInherit: - let base = typeNode[1][0] - var impl = getTypeImpl(base) - while impl.kind in {nnkRefTy, nnkPtrTy}: - impl = getTypeImpl(impl[0]) - foldObjectBody(dst, impl, tmpSym, jsonNode, jsonPath, originalJsonPathLen) - let body = typeNode[2] - foldObjectBody(dst, body, tmpSym, jsonNode, jsonPath, originalJsonPathLen) + of nnkObjectTy: + typeNode[0].expectKind nnkEmpty + typeNode[1].expectKind {nnkEmpty, nnkOfInherit} + if typeNode[1].kind == nnkOfInherit: + let base = typeNode[1][0] + var impl = getTypeImpl(base) + while impl.kind in {nnkRefTy, nnkPtrTy}: + impl = getTypeImpl(impl[0]) + foldObjectBody(dst, impl, tmpSym, jsonNode, jsonPath, originalJsonPathLen) + let body = typeNode[2] + foldObjectBody(dst, body, tmpSym, jsonNode, jsonPath, originalJsonPathLen) - else: - error("unhandled kind: " & $typeNode.kind, typeNode) + else: + error("unhandled kind: " & $typeNode.kind, typeNode) - macro assignObjectImpl[T](dst: var T; jsonNode: JsonNode; jsonPath: var string) = - let typeSym = getTypeInst(dst) - let originalJsonPathLen = genSym(nskLet, "originalJsonPathLen") - result = newStmtList() - result.add quote do: - let `originalJsonPathLen` = len(`jsonPath`) - if typeSym.kind in {nnkTupleTy, nnkTupleConstr}: - # both, `dst` and `typeSym` don't have good lineinfo. But nothing - # else is available here. - detectIncompatibleType(typeSym, dst) - foldObjectBody(result, typeSym, dst, jsonNode, jsonPath, originalJsonPathLen) - else: - foldObjectBody(result, typeSym.getTypeImpl, dst, jsonNode, jsonPath, originalJsonPathLen) +macro assignObjectImpl[T](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + let typeSym = getTypeInst(dst) + let originalJsonPathLen = genSym(nskLet, "originalJsonPathLen") + result = newStmtList() + result.add quote do: + let `originalJsonPathLen` = len(`jsonPath`) + if typeSym.kind in {nnkTupleTy, nnkTupleConstr}: + # both, `dst` and `typeSym` don't have good lineinfo. But nothing + # else is available here. + detectIncompatibleType(typeSym, dst) + foldObjectBody(result, typeSym, dst, jsonNode, jsonPath, originalJsonPathLen) + else: + foldObjectBody(result, typeSym.getTypeImpl, dst, jsonNode, jsonPath, originalJsonPathLen) - proc initFromJson[T: object|tuple](dst: var T; jsonNode: JsonNode; jsonPath: var string) = - assignObjectImpl(dst, jsonNode, jsonPath) +proc initFromJson[T: object|tuple](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + assignObjectImpl(dst, jsonNode, jsonPath) - proc to*[T](node: JsonNode, t: typedesc[T]): T = - ## `Unmarshals`:idx: the specified node into the object type specified. - ## - ## Known limitations: - ## - ## * Heterogeneous arrays are not supported. - ## * Sets in object variants are not supported. - ## * Not nil annotations are not supported. - ## - runnableExamples: - let jsonNode = parseJson(""" - { - "person": { - "name": "Nimmer", - "age": 21 - }, - "list": [1, 2, 3, 4] - } - """) +proc to*[T](node: JsonNode, t: typedesc[T]): T = + ## `Unmarshals`:idx: the specified node into the object type specified. + ## + ## Known limitations: + ## + ## * Heterogeneous arrays are not supported. + ## * Sets in object variants are not supported. + ## * Not nil annotations are not supported. + ## + runnableExamples: + let jsonNode = parseJson(""" + { + "person": { + "name": "Nimmer", + "age": 21 + }, + "list": [1, 2, 3, 4] + } + """) - type - Person = object - name: string - age: int + type + Person = object + name: string + age: int - Data = object - person: Person - list: seq[int] + Data = object + person: Person + list: seq[int] - var data = to(jsonNode, Data) - doAssert data.person.name == "Nimmer" - doAssert data.person.age == 21 - doAssert data.list == @[1, 2, 3, 4] + var data = to(jsonNode, Data) + doAssert data.person.name == "Nimmer" + doAssert data.person.age == 21 + doAssert data.list == @[1, 2, 3, 4] - var jsonPath = "" - initFromJson(result, node, jsonPath) + var jsonPath = "" + initFromJson(result, node, jsonPath) when false: import os diff --git a/lib/pure/nimprof.nim b/lib/pure/nimprof.nim index 7f493418e..9849e42db 100644 --- a/lib/pure/nimprof.nim +++ b/lib/pure/nimprof.nim @@ -15,8 +15,7 @@ when not defined(profiler) and not defined(memProfiler): {.error: "Profiling support is turned off! Enable profiling by passing `--profiler:on --stackTrace:on` to the compiler (see the Nim Compiler User Guide for more options).".} -when defined(nimHasUsed): - {.used.} +{.used.} # We don't want to profile the profiling code ... {.push profiler: off.} diff --git a/lib/std/assertions.nim b/lib/std/assertions.nim index 5623ff8ef..229033c58 100644 --- a/lib/std/assertions.nim +++ b/lib/std/assertions.nim @@ -23,8 +23,6 @@ proc `$`(info: InstantiationInfo): string = # --------------------------------------------------------------------------- -when not defined(nimHasSinkInference): - {.pragma: nosinks.} proc raiseAssert*(msg: string) {.noinline, noreturn, nosinks.} = ## Raises an `AssertionDefect` with `msg`. diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim index eec8dea7d..17d08e02e 100644 --- a/lib/std/jsonutils.nim +++ b/lib/std/jsonutils.nim @@ -37,22 +37,8 @@ from typetraits import OrdinalEnum, tupleLen when defined(nimPreviewSlimSystem): import std/assertions -when not defined(nimFixedForwardGeneric): - # xxx remove pending csources_v1 update >= 1.2.0 - proc to[T](node: JsonNode, t: typedesc[T]): T = - when T is string: node.getStr - elif T is bool: node.getBool - else: static: doAssert false, $T # support as needed (only needed during bootstrap) - proc isNamedTuple(T: typedesc): bool = # old implementation - when T isnot tuple: result = false - else: - var t: T - for name, _ in t.fieldPairs: - when name == "Field0": return compiles(t.Field0) - else: return true - return false -else: - proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} + +proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} type Joptions* = object # xxx rename FromJsonOptions diff --git a/lib/std/private/miscdollars.nim b/lib/std/private/miscdollars.nim index 47b788ee9..06fda6fa1 100644 --- a/lib/std/private/miscdollars.nim +++ b/lib/std/private/miscdollars.nim @@ -13,21 +13,7 @@ template toLocation*(result: var string, file: string | cstring, line: int, col: addInt(result, col) result.add ")" -when defined(nimHasIsNamedTuple): - proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} -else: - # for bootstrap; remove after release 1.2 - proc isNamedTuple(T: typedesc): bool = - # Taken from typetraits. - when T isnot tuple: result = false - else: - var t: T - for name, _ in t.fieldPairs: - when name == "Field0": - return compiles(t.Field0) - else: - return true - return false +proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} template tupleObjectDollar*[T: tuple | object](result: var string, x: T) = result = "(" diff --git a/lib/system.nim b/lib/system.nim index 1b6235fb3..e9c9ebfeb 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -85,17 +85,12 @@ when defined(nimHasIterable): type iterable*[T] {.magic: IterableType.} ## Represents an expression that yields `T` -when defined(nimHashOrdinalFixed): - type - Ordinal*[T] {.magic: Ordinal.} ## Generic ordinal type. Includes integer, - ## bool, character, and enumeration types - ## as well as their subtypes. See also - ## `SomeOrdinal`. -else: - # bootstrap < 1.2.0 - type - OrdinalImpl[T] {.magic: Ordinal.} - Ordinal* = OrdinalImpl | uint | uint64 +type + Ordinal*[T] {.magic: Ordinal.} ## Generic ordinal type. Includes integer, + ## bool, character, and enumeration types + ## as well as their subtypes. See also + ## `SomeOrdinal`. + proc `addr`*[T](x: T): ptr T {.magic: "Addr", noSideEffect.} = ## Builtin `addr` operator for taking the address of a memory location. @@ -451,9 +446,7 @@ type ## However, objects that have no ancestor are also allowed. RootRef* = ref RootObj ## Reference to `RootObj`. -const NimStackTraceMsgs = - when defined(nimHasStacktraceMsgs): compileOption("stacktraceMsgs") - else: false +const NimStackTraceMsgs = compileOption("stacktraceMsgs") type RootEffect* {.compilerproc.} = object of RootObj ## \ @@ -2113,10 +2106,7 @@ when notJSnotNims: # we cannot compile this with stack tracing on # as it would recurse endlessly! - when defined(nimNewIntegerOps): - include "system/integerops" - else: - include "system/arithm" + include "system/integerops" {.pop.} @@ -2660,11 +2650,10 @@ when defined(nimconfig): when not defined(js): proc toOpenArray*[T](x: ptr UncheckedArray[T]; first, last: int): openArray[T] {. magic: "Slice".} - when defined(nimToOpenArrayCString): - proc toOpenArray*(x: cstring; first, last: int): openArray[char] {. - magic: "Slice".} - proc toOpenArrayByte*(x: cstring; first, last: int): openArray[byte] {. - magic: "Slice".} + proc toOpenArray*(x: cstring; first, last: int): openArray[char] {. + magic: "Slice".} + proc toOpenArrayByte*(x: cstring; first, last: int): openArray[byte] {. + magic: "Slice".} proc toOpenArray*[T](x: seq[T]; first, last: int): openArray[T] {. magic: "Slice".} diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index 5d1548977..cee70f677 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -95,17 +95,11 @@ type acc: int # accumulator for small object allocation when defined(gcDestructors): sharedFreeList: ptr FreeCell # make no attempt at avoiding false sharing for now for this object field - when defined(nimAlignPragma): - data {.align: MemAlign.}: UncheckedArray[byte] # start of usable memory - else: - data: UncheckedArray[byte] + data {.align: MemAlign.}: UncheckedArray[byte] # start of usable memory BigChunk = object of BaseChunk # not necessarily > PageSize! next, prev: PBigChunk # chunks of the same (or bigger) size - when defined(nimAlignPragma): - data {.align: MemAlign.}: UncheckedArray[byte] # start of usable memory - else: - data: UncheckedArray[byte] + data {.align: MemAlign.}: UncheckedArray[byte] # start of usable memory HeapLinks = object len: int diff --git a/lib/system/arithm.nim b/lib/system/arithm.nim deleted file mode 100644 index 158f40177..000000000 --- a/lib/system/arithm.nim +++ /dev/null @@ -1,425 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - - -# simple integer arithmetic with overflow checking - -proc raiseOverflow {.compilerproc, noinline.} = - # a single proc to reduce code size to a minimum - sysFatal(OverflowDefect, "over- or underflow") - -proc raiseDivByZero {.compilerproc, noinline.} = - sysFatal(DivByZeroDefect, "division by zero") - -when defined(builtinOverflow): - # Builtin compiler functions for improved performance - when sizeof(clong) == 8: - proc addInt64Overflow[T: int64|int](a, b: T, c: var T): bool {. - importc: "__builtin_saddl_overflow", nodecl, nosideeffect.} - - proc subInt64Overflow[T: int64|int](a, b: T, c: var T): bool {. - importc: "__builtin_ssubl_overflow", nodecl, nosideeffect.} - - proc mulInt64Overflow[T: int64|int](a, b: T, c: var T): bool {. - importc: "__builtin_smull_overflow", nodecl, nosideeffect.} - - elif sizeof(clonglong) == 8: - proc addInt64Overflow[T: int64|int](a, b: T, c: var T): bool {. - importc: "__builtin_saddll_overflow", nodecl, nosideeffect.} - - proc subInt64Overflow[T: int64|int](a, b: T, c: var T): bool {. - importc: "__builtin_ssubll_overflow", nodecl, nosideeffect.} - - proc mulInt64Overflow[T: int64|int](a, b: T, c: var T): bool {. - importc: "__builtin_smulll_overflow", nodecl, nosideeffect.} - - when sizeof(int) == 8: - proc addIntOverflow(a, b: int, c: var int): bool {.inline.} = - addInt64Overflow(a, b, c) - - proc subIntOverflow(a, b: int, c: var int): bool {.inline.} = - subInt64Overflow(a, b, c) - - proc mulIntOverflow(a, b: int, c: var int): bool {.inline.} = - mulInt64Overflow(a, b, c) - - elif sizeof(int) == 4 and sizeof(cint) == 4: - proc addIntOverflow(a, b: int, c: var int): bool {. - importc: "__builtin_sadd_overflow", nodecl, nosideeffect.} - - proc subIntOverflow(a, b: int, c: var int): bool {. - importc: "__builtin_ssub_overflow", nodecl, nosideeffect.} - - proc mulIntOverflow(a, b: int, c: var int): bool {. - importc: "__builtin_smul_overflow", nodecl, nosideeffect.} - - proc addInt64(a, b: int64): int64 {.compilerproc, inline.} = - if addInt64Overflow(a, b, result): - raiseOverflow() - - proc subInt64(a, b: int64): int64 {.compilerproc, inline.} = - if subInt64Overflow(a, b, result): - raiseOverflow() - - proc mulInt64(a, b: int64): int64 {.compilerproc, inline.} = - if mulInt64Overflow(a, b, result): - raiseOverflow() -else: - proc addInt64(a, b: int64): int64 {.compilerproc, inline.} = - result = a +% b - if (result xor a) >= int64(0) or (result xor b) >= int64(0): - return result - raiseOverflow() - - proc subInt64(a, b: int64): int64 {.compilerproc, inline.} = - result = a -% b - if (result xor a) >= int64(0) or (result xor not b) >= int64(0): - return result - raiseOverflow() - - # - # This code has been inspired by Python's source code. - # The native int product x*y is either exactly right or *way* off, being - # just the last n bits of the true product, where n is the number of bits - # in an int (the delivered product is the true product plus i*2**n for - # some integer i). - # - # The native float64 product x*y is subject to three - # rounding errors: on a sizeof(int)==8 box, each cast to double can lose - # info, and even on a sizeof(int)==4 box, the multiplication can lose info. - # But, unlike the native int product, it's not in *range* trouble: even - # if sizeof(int)==32 (256-bit ints), the product easily fits in the - # dynamic range of a float64. So the leading 50 (or so) bits of the float64 - # product are correct. - # - # We check these two ways against each other, and declare victory if they're - # approximately the same. Else, because the native int product is the only - # one that can lose catastrophic amounts of information, it's the native int - # product that must have overflowed. - # - proc mulInt64(a, b: int64): int64 {.compilerproc.} = - var - resAsFloat, floatProd: float64 - result = a *% b - floatProd = toBiggestFloat(a) # conversion - floatProd = floatProd * toBiggestFloat(b) - resAsFloat = toBiggestFloat(result) - - # Fast path for normal case: small multiplicands, and no info - # is lost in either method. - if resAsFloat == floatProd: return result - - # Somebody somewhere lost info. Close enough, or way off? Note - # that a != 0 and b != 0 (else resAsFloat == floatProd == 0). - # The difference either is or isn't significant compared to the - # true value (of which floatProd is a good approximation). - - # abs(diff)/abs(prod) <= 1/32 iff - # 32 * abs(diff) <= abs(prod) -- 5 good bits is "close enough" - if 32.0 * abs(resAsFloat - floatProd) <= abs(floatProd): - return result - raiseOverflow() - -proc negInt64(a: int64): int64 {.compilerproc, inline.} = - if a != low(int64): return -a - raiseOverflow() - -proc absInt64(a: int64): int64 {.compilerproc, inline.} = - if a != low(int64): - if a >= 0: return a - else: return -a - raiseOverflow() - -proc divInt64(a, b: int64): int64 {.compilerproc, inline.} = - if b == int64(0): - raiseDivByZero() - if a == low(int64) and b == int64(-1): - raiseOverflow() - return a div b - -proc modInt64(a, b: int64): int64 {.compilerproc, inline.} = - if b == int64(0): - raiseDivByZero() - return a mod b - -proc absInt(a: int): int {.compilerproc, inline.} = - if a != low(int): - if a >= 0: return a - else: return -a - raiseOverflow() - -const - asmVersion = defined(i386) and (defined(vcc) or defined(wcc) or - defined(dmc) or defined(gcc) or defined(llvm_gcc)) - # my Version of Borland C++Builder does not have - # tasm32, which is needed for assembler blocks - # this is why Borland is not included in the 'when' - -when asmVersion and not defined(gcc) and not defined(llvm_gcc): - # assembler optimized versions for compilers that - # have an intel syntax assembler: - proc addInt(a, b: int): int {.compilerproc, asmNoStackFrame.} = - # a in eax, and b in edx - asm """ - mov eax, ecx - add eax, edx - jno theEnd - call `raiseOverflow` - theEnd: - ret - """ - - proc subInt(a, b: int): int {.compilerproc, asmNoStackFrame.} = - asm """ - mov eax, ecx - sub eax, edx - jno theEnd - call `raiseOverflow` - theEnd: - ret - """ - - proc negInt(a: int): int {.compilerproc, asmNoStackFrame.} = - asm """ - mov eax, ecx - neg eax - jno theEnd - call `raiseOverflow` - theEnd: - ret - """ - - proc divInt(a, b: int): int {.compilerproc, asmNoStackFrame.} = - asm """ - test edx, edx - jne L_NOT_ZERO - call `raiseDivByZero` - L_NOT_ZERO: - cmp ecx, 0x80000000 - jne L_DO_DIV - cmp edx, -1 - jne L_DO_DIV - call `raiseOverflow` - L_DO_DIV: - mov eax, ecx - mov ecx, edx - cdq - idiv ecx - ret - """ - - proc modInt(a, b: int): int {.compilerproc, asmNoStackFrame.} = - asm """ - test edx, edx - jne L_NOT_ZERO - call `raiseDivByZero` - L_NOT_ZERO: - cmp ecx, 0x80000000 - jne L_DO_DIV - cmp edx, -1 - jne L_DO_DIV - call `raiseOverflow` - L_DO_DIV: - mov eax, ecx - mov ecx, edx - cdq - idiv ecx - mov eax, edx - ret - """ - - proc mulInt(a, b: int): int {.compilerproc, asmNoStackFrame.} = - asm """ - mov eax, ecx - mov ecx, edx - xor edx, edx - imul ecx - jno theEnd - call `raiseOverflow` - theEnd: - ret - """ - -elif false: # asmVersion and (defined(gcc) or defined(llvm_gcc)): - proc addInt(a, b: int): int {.compilerproc, inline.} = - # don't use a pure proc here! - asm """ - "addl %%ecx, %%eax\n" - "jno 1\n" - "call _raiseOverflow\n" - "1: \n" - :"=a"(`result`) - :"a"(`a`), "c"(`b`) - """ - #".intel_syntax noprefix" - #/* Intel syntax here */ - #".att_syntax" - - proc subInt(a, b: int): int {.compilerproc, inline.} = - asm """ "subl %%ecx,%%eax\n" - "jno 1\n" - "call _raiseOverflow\n" - "1: \n" - :"=a"(`result`) - :"a"(`a`), "c"(`b`) - """ - - proc mulInt(a, b: int): int {.compilerproc, inline.} = - asm """ "xorl %%edx, %%edx\n" - "imull %%ecx\n" - "jno 1\n" - "call _raiseOverflow\n" - "1: \n" - :"=a"(`result`) - :"a"(`a`), "c"(`b`) - :"%edx" - """ - - proc negInt(a: int): int {.compilerproc, inline.} = - asm """ "negl %%eax\n" - "jno 1\n" - "call _raiseOverflow\n" - "1: \n" - :"=a"(`result`) - :"a"(`a`) - """ - - proc divInt(a, b: int): int {.compilerproc, inline.} = - asm """ "xorl %%edx, %%edx\n" - "idivl %%ecx\n" - "jno 1\n" - "call _raiseOverflow\n" - "1: \n" - :"=a"(`result`) - :"a"(`a`), "c"(`b`) - :"%edx" - """ - - proc modInt(a, b: int): int {.compilerproc, inline.} = - asm """ "xorl %%edx, %%edx\n" - "idivl %%ecx\n" - "jno 1\n" - "call _raiseOverflow\n" - "1: \n" - "movl %%edx, %%eax" - :"=a"(`result`) - :"a"(`a`), "c"(`b`) - :"%edx" - """ - -when not declared(addInt) and defined(builtinOverflow): - proc addInt(a, b: int): int {.compilerproc, inline.} = - if addIntOverflow(a, b, result): - raiseOverflow() - -when not declared(subInt) and defined(builtinOverflow): - proc subInt(a, b: int): int {.compilerproc, inline.} = - if subIntOverflow(a, b, result): - raiseOverflow() - -when not declared(mulInt) and defined(builtinOverflow): - proc mulInt(a, b: int): int {.compilerproc, inline.} = - if mulIntOverflow(a, b, result): - raiseOverflow() - -# Platform independent versions of the above (slower!) -when not declared(addInt): - proc addInt(a, b: int): int {.compilerproc, inline.} = - result = a +% b - if (result xor a) >= 0 or (result xor b) >= 0: - return result - raiseOverflow() - -when not declared(subInt): - proc subInt(a, b: int): int {.compilerproc, inline.} = - result = a -% b - if (result xor a) >= 0 or (result xor not b) >= 0: - return result - raiseOverflow() - -when not declared(negInt): - proc negInt(a: int): int {.compilerproc, inline.} = - if a != low(int): return -a - raiseOverflow() - -when not declared(divInt): - proc divInt(a, b: int): int {.compilerproc, inline.} = - if b == 0: - raiseDivByZero() - if a == low(int) and b == -1: - raiseOverflow() - return a div b - -when not declared(modInt): - proc modInt(a, b: int): int {.compilerproc, inline.} = - if b == 0: - raiseDivByZero() - return a mod b - -when not declared(mulInt): - # - # This code has been inspired by Python's source code. - # The native int product x*y is either exactly right or *way* off, being - # just the last n bits of the true product, where n is the number of bits - # in an int (the delivered product is the true product plus i*2**n for - # some integer i). - # - # The native float64 product x*y is subject to three - # rounding errors: on a sizeof(int)==8 box, each cast to double can lose - # info, and even on a sizeof(int)==4 box, the multiplication can lose info. - # But, unlike the native int product, it's not in *range* trouble: even - # if sizeof(int)==32 (256-bit ints), the product easily fits in the - # dynamic range of a float64. So the leading 50 (or so) bits of the float64 - # product are correct. - # - # We check these two ways against each other, and declare victory if - # they're approximately the same. Else, because the native int product is - # the only one that can lose catastrophic amounts of information, it's the - # native int product that must have overflowed. - # - proc mulInt(a, b: int): int {.compilerproc.} = - var - resAsFloat, floatProd: float - - result = a *% b - floatProd = toFloat(a) * toFloat(b) - resAsFloat = toFloat(result) - - # Fast path for normal case: small multiplicands, and no info - # is lost in either method. - if resAsFloat == floatProd: return result - - # Somebody somewhere lost info. Close enough, or way off? Note - # that a != 0 and b != 0 (else resAsFloat == floatProd == 0). - # The difference either is or isn't significant compared to the - # true value (of which floatProd is a good approximation). - - # abs(diff)/abs(prod) <= 1/32 iff - # 32 * abs(diff) <= abs(prod) -- 5 good bits is "close enough" - if 32.0 * abs(resAsFloat - floatProd) <= abs(floatProd): - return result - raiseOverflow() - -# We avoid setting the FPU control word here for compatibility with libraries -# written in other languages. - -proc raiseFloatInvalidOp {.compilerproc, noinline.} = - sysFatal(FloatInvalidOpDefect, "FPU operation caused a NaN result") - -proc nanCheck(x: float64) {.compilerproc, inline.} = - if x != x: raiseFloatInvalidOp() - -proc raiseFloatOverflow(x: float64) {.compilerproc, noinline.} = - if x > 0.0: - sysFatal(FloatOverflowDefect, "FPU operation caused an overflow") - else: - sysFatal(FloatUnderflowDefect, "FPU operations caused an underflow") - -proc infCheck(x: float64) {.compilerproc, inline.} = - if x != 0.0 and x*0.5 == x: raiseFloatOverflow(x) diff --git a/lib/system/arithmetics.nim b/lib/system/arithmetics.nim index d05aaaa5b..5c21e56ef 100644 --- a/lib/system/arithmetics.nim +++ b/lib/system/arithmetics.nim @@ -405,105 +405,57 @@ proc `%%`*(x, y: int32): int32 {.inline.} = cast[int32](cast[uint32](x) mod cast proc `%%`*(x, y: int64): int64 {.inline.} = cast[int64](cast[uint64](x) mod cast[uint64](y)) when not defined(nimPreviewSlimSystem): - when defined(nimNoZeroExtendMagic): - proc ze*(x: int8): int {.deprecated.} = - ## zero extends a smaller integer type to `int`. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int](uint(cast[uint8](x))) + proc ze*(x: int8): int {.deprecated.} = + ## zero extends a smaller integer type to `int`. This treats `x` as + ## unsigned. + ## **Deprecated since version 0.19.9**: Use unsigned integers instead. + cast[int](uint(cast[uint8](x))) - proc ze*(x: int16): int {.deprecated.} = - ## zero extends a smaller integer type to `int`. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int](uint(cast[uint16](x))) + proc ze*(x: int16): int {.deprecated.} = + ## zero extends a smaller integer type to `int`. This treats `x` as + ## unsigned. + ## **Deprecated since version 0.19.9**: Use unsigned integers instead. + cast[int](uint(cast[uint16](x))) - proc ze64*(x: int8): int64 {.deprecated.} = - ## zero extends a smaller integer type to `int64`. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int64](uint64(cast[uint8](x))) + proc ze64*(x: int8): int64 {.deprecated.} = + ## zero extends a smaller integer type to `int64`. This treats `x` as + ## unsigned. + ## **Deprecated since version 0.19.9**: Use unsigned integers instead. + cast[int64](uint64(cast[uint8](x))) - proc ze64*(x: int16): int64 {.deprecated.} = - ## zero extends a smaller integer type to `int64`. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int64](uint64(cast[uint16](x))) + proc ze64*(x: int16): int64 {.deprecated.} = + ## zero extends a smaller integer type to `int64`. This treats `x` as + ## unsigned. + ## **Deprecated since version 0.19.9**: Use unsigned integers instead. + cast[int64](uint64(cast[uint16](x))) - proc ze64*(x: int32): int64 {.deprecated.} = - ## zero extends a smaller integer type to `int64`. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int64](uint64(cast[uint32](x))) + proc ze64*(x: int32): int64 {.deprecated.} = + ## zero extends a smaller integer type to `int64`. This treats `x` as + ## unsigned. + ## **Deprecated since version 0.19.9**: Use unsigned integers instead. + cast[int64](uint64(cast[uint32](x))) - proc ze64*(x: int): int64 {.deprecated.} = - ## zero extends a smaller integer type to `int64`. This treats `x` as - ## unsigned. Does nothing if the size of an `int` is the same as `int64`. - ## (This is the case on 64 bit processors.) - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int64](uint64(cast[uint](x))) + proc ze64*(x: int): int64 {.deprecated.} = + ## zero extends a smaller integer type to `int64`. This treats `x` as + ## unsigned. Does nothing if the size of an `int` is the same as `int64`. + ## (This is the case on 64 bit processors.) + ## **Deprecated since version 0.19.9**: Use unsigned integers instead. + cast[int64](uint64(cast[uint](x))) - proc toU8*(x: int): int8 {.deprecated.} = - ## treats `x` as unsigned and converts it to a byte by taking the last 8 bits - ## from `x`. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int8](x) + proc toU8*(x: int): int8 {.deprecated.} = + ## treats `x` as unsigned and converts it to a byte by taking the last 8 bits + ## from `x`. + ## **Deprecated since version 0.19.9**: Use unsigned integers instead. + cast[int8](x) - proc toU16*(x: int): int16 {.deprecated.} = - ## treats `x` as unsigned and converts it to an `int16` by taking the last - ## 16 bits from `x`. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int16](x) + proc toU16*(x: int): int16 {.deprecated.} = + ## treats `x` as unsigned and converts it to an `int16` by taking the last + ## 16 bits from `x`. + ## **Deprecated since version 0.19.9**: Use unsigned integers instead. + cast[int16](x) - proc toU32*(x: int64): int32 {.deprecated.} = - ## treats `x` as unsigned and converts it to an `int32` by taking the - ## last 32 bits from `x`. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int32](x) - - elif not defined(js): - proc ze*(x: int8): int {.magic: "Ze8ToI", noSideEffect, deprecated.} - ## zero extends a smaller integer type to `int`. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - - proc ze*(x: int16): int {.magic: "Ze16ToI", noSideEffect, deprecated.} - ## zero extends a smaller integer type to `int`. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - - proc ze64*(x: int8): int64 {.magic: "Ze8ToI64", noSideEffect, deprecated.} - ## zero extends a smaller integer type to `int64`. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - - proc ze64*(x: int16): int64 {.magic: "Ze16ToI64", noSideEffect, deprecated.} - ## zero extends a smaller integer type to `int64`. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - - proc ze64*(x: int32): int64 {.magic: "Ze32ToI64", noSideEffect, deprecated.} - ## zero extends a smaller integer type to `int64`. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - - proc ze64*(x: int): int64 {.magic: "ZeIToI64", noSideEffect, deprecated.} - ## zero extends a smaller integer type to `int64`. This treats `x` as - ## unsigned. Does nothing if the size of an `int` is the same as `int64`. - ## (This is the case on 64 bit processors.) - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - - proc toU8*(x: int): int8 {.magic: "ToU8", noSideEffect, deprecated.} - ## treats `x` as unsigned and converts it to a byte by taking the last 8 bits - ## from `x`. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - - proc toU16*(x: int): int16 {.magic: "ToU16", noSideEffect, deprecated.} - ## treats `x` as unsigned and converts it to an `int16` by taking the last - ## 16 bits from `x`. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - - proc toU32*(x: int64): int32 {.magic: "ToU32", noSideEffect, deprecated.} - ## treats `x` as unsigned and converts it to an `int32` by taking the - ## last 32 bits from `x`. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. + proc toU32*(x: int64): int32 {.deprecated.} = + ## treats `x` as unsigned and converts it to an `int32` by taking the + ## last 32 bits from `x`. + ## **Deprecated since version 0.19.9**: Use unsigned integers instead. + cast[int32](x) diff --git a/lib/system/compilation.nim b/lib/system/compilation.nim index 234e8cafe..c04990277 100644 --- a/lib/system/compilation.nim +++ b/lib/system/compilation.nim @@ -51,30 +51,24 @@ proc defined*(x: untyped): bool {.magic: "Defined", noSideEffect, compileTime.} ## * `compileOption <#compileOption,string,string>`_ for enum options ## * `define pragmas `_ -when defined(nimHasDeclaredMagic): - proc declared*(x: untyped): bool {.magic: "Declared", noSideEffect, compileTime.} - ## Special compile-time procedure that checks whether `x` is - ## declared. `x` has to be an identifier or a qualified identifier. - ## - ## This can be used to check whether a library provides a certain - ## feature or not: - ## ``` - ## when not declared(strutils.toUpper): - ## # provide our own toUpper proc here, because strutils is - ## # missing it. - ## ``` - ## - ## See also: - ## * `declaredInScope <#declaredInScope,untyped>`_ -else: - proc declared*(x: untyped): bool {.magic: "Defined", noSideEffect, compileTime.} +proc declared*(x: untyped): bool {.magic: "Declared", noSideEffect, compileTime.} + ## Special compile-time procedure that checks whether `x` is + ## declared. `x` has to be an identifier or a qualified identifier. + ## + ## This can be used to check whether a library provides a certain + ## feature or not: + ## ``` + ## when not declared(strutils.toUpper): + ## # provide our own toUpper proc here, because strutils is + ## # missing it. + ## ``` + ## + ## See also: + ## * `declaredInScope <#declaredInScope,untyped>`_ -when defined(nimHasDeclaredMagic): - proc declaredInScope*(x: untyped): bool {.magic: "DeclaredInScope", noSideEffect, compileTime.} - ## Special compile-time procedure that checks whether `x` is - ## declared in the current scope. `x` has to be an identifier. -else: - proc declaredInScope*(x: untyped): bool {.magic: "DefinedInScope", noSideEffect, compileTime.} +proc declaredInScope*(x: untyped): bool {.magic: "DeclaredInScope", noSideEffect, compileTime.} + ## Special compile-time procedure that checks whether `x` is + ## declared in the current scope. `x` has to be an identifier. proc compiles*(x: untyped): bool {.magic: "Compiles", noSideEffect, compileTime.} = ## Special compile-time procedure that checks whether `x` can be compiled diff --git a/lib/system/fatal.nim b/lib/system/fatal.nim index a55af2dc3..6073ee779 100644 --- a/lib/system/fatal.nim +++ b/lib/system/fatal.nim @@ -9,10 +9,7 @@ {.push profiler: off.} -when defined(nimHasExceptionsQuery): - const gotoBasedExceptions = compileOption("exceptions", "goto") -else: - const gotoBasedExceptions = false +const gotoBasedExceptions = compileOption("exceptions", "goto") when hostOS == "standalone": include "$projectpath/panicoverride" diff --git a/lib/system/inclrtl.nim b/lib/system/inclrtl.nim index 42c85ad26..3bf0b9893 100644 --- a/lib/system/inclrtl.nim +++ b/lib/system/inclrtl.nim @@ -47,5 +47,4 @@ else: {.pragma: benign, gcsafe.} -when defined(nimHasSinkInference): - {.push sinkInference: on.} +{.push sinkInference: on.} diff --git a/lib/system/iterators.nim b/lib/system/iterators.nim index b16a605b7..4bd12680f 100644 --- a/lib/system/iterators.nim +++ b/lib/system/iterators.nim @@ -3,7 +3,7 @@ when defined(nimPreviewSlimSystem): import std/assertions -when defined(nimHasLentIterators) and not defined(nimNoLentIterators): +when not defined(nimNoLentIterators): template lent2(T): untyped = lent T else: template lent2(T): untyped = T diff --git a/lib/system/jssys.nim b/lib/system/jssys.nim index 9dfa80877..4f64403fe 100644 --- a/lib/system/jssys.nim +++ b/lib/system/jssys.nim @@ -503,28 +503,6 @@ proc absInt(a: int): int {.compilerproc.} = proc absInt64(a: int64): int64 {.compilerproc.} = result = if a < 0: a*(-1) else: a -when not defined(nimNoZeroExtendMagic): - proc ze*(a: int): int {.compilerproc.} = - result = a - - proc ze64*(a: int64): int64 {.compilerproc.} = - result = a - - proc toU8*(a: int): int8 {.asmNoStackFrame, compilerproc.} = - asm """ - return `a`; - """ - - proc toU16*(a: int): int16 {.asmNoStackFrame, compilerproc.} = - asm """ - return `a`; - """ - - proc toU32*(a: int64): int32 {.asmNoStackFrame, compilerproc.} = - asm """ - return `a`; - """ - proc nimMin(a, b: int): int {.compilerproc.} = return if a <= b: a else: b proc nimMax(a, b: int): int {.compilerproc.} = return if a >= b: a else: b diff --git a/lib/system/strmantle.nim b/lib/system/strmantle.nim index 0b2578280..f4733ed21 100644 --- a/lib/system/strmantle.nim +++ b/lib/system/strmantle.nim @@ -75,8 +75,8 @@ const 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22] -when defined(nimHasInvariant): - {.push staticBoundChecks: off.} + +{.push staticBoundChecks: off.} proc nimParseBiggestFloat(s: openArray[char], number: var BiggestFloat, ): int {.compilerproc.} = @@ -234,8 +234,7 @@ proc nimParseBiggestFloat(s: openArray[char], number: var BiggestFloat, t[ti-3] = ('0'.ord + absExponent mod 10).char number = c_strtod(cast[cstring](addr t), nil) -when defined(nimHasInvariant): - {.pop.} # staticBoundChecks +{.pop.} # staticBoundChecks proc nimBoolToStr(x: bool): string {.compilerRtl.} = return if x: "true" else: "false" From 9b4516fbcb4ad9c5db1e6f563601855f0e9982a0 Mon Sep 17 00:00:00 2001 From: rockcavera Date: Tue, 27 Dec 2022 01:13:05 -0300 Subject: [PATCH 2322/2472] fix in msg `doAssert()` to update grammar.txt (#21179) --- compiler/parser.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/parser.nim b/compiler/parser.nim index 00013c218..0c8edd2a8 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -42,7 +42,7 @@ when isMainModule or defined(nimTestGrammar): proc checkSameGrammar*() = doAssert sameFileContent(newGrammarText, "doc/grammar.txt"), - "execute 'nim r compiler.nim' to keep grammar.txt up-to-date" + "execute 'nim r compiler/parser.nim' to keep grammar.txt up-to-date" else: writeGrammarFile("doc/grammar.txt") import ".." / tools / grammar_nanny From f232329bb12fc9d93fe1e09769e291eae4e2953d Mon Sep 17 00:00:00 2001 From: Bung Date: Tue, 27 Dec 2022 19:07:15 +0800 Subject: [PATCH 2323/2472] add test case for #20289 (#21180) --- tests/misc/t20289.nim | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 tests/misc/t20289.nim diff --git a/tests/misc/t20289.nim b/tests/misc/t20289.nim new file mode 100644 index 000000000..5a0a269f0 --- /dev/null +++ b/tests/misc/t20289.nim @@ -0,0 +1,15 @@ +discard """ + action: reject +""" + +type E[T] = object + v: T + +template j[T](R: type E[T], x: untyped): R = R(v: x) +template d[T](O: type E, v: T): E[T] = E[T].j(v) + +proc w[T](): E[T] = + template r(k: int): auto = default(T) + E.d r + +discard w[int]() From 82d80e6e2cb09652e15d8ec81621bba734872aaa Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 27 Dec 2022 14:30:26 +0100 Subject: [PATCH 2324/2472] atlas: minor doc improvements (#21183) * closes #20808 * atlas: better docs --- tools/atlas/atlas.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/atlas/atlas.md b/tools/atlas/atlas.md index a36817dc5..ee770cd62 100644 --- a/tools/atlas/atlas.md +++ b/tools/atlas/atlas.md @@ -28,7 +28,7 @@ a `nim.cfg` file for the compiler. For example: ``` The version selection is deterministic, it picks up the *minimum* required -version. Thanks to this design, lock files are not required. +version. Thanks to this design, lock files are much less important. ## Dependencies @@ -75,3 +75,7 @@ then `clone ` is performed. Search the package index `packages.json` for a package that the given terms in its description (or name or list of tags). + +### Install + +Use the .nimble file to setup the project's dependencies. From 56840931355fdf8ce12a59ad085d90b04404f831 Mon Sep 17 00:00:00 2001 From: metagn Date: Tue, 27 Dec 2022 23:24:00 +0300 Subject: [PATCH 2325/2472] fix for bad error message with const in case statement (#21182) * preliminary fix for bad error message with const * add test case * fix tmatrixconcept and tmatrixlib --- compiler/semfold.nim | 4 ++++ tests/errmsgs/tconstinfo.nim | 7 +++++++ 2 files changed, 11 insertions(+) create mode 100644 tests/errmsgs/tconstinfo.nim diff --git a/compiler/semfold.nim b/compiler/semfold.nim index 7c57eb370..7f966c6fd 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -569,6 +569,8 @@ proc foldDefine(m, s: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode else: result = copyTree(s.astdef) # unreachable else: result = copyTree(s.astdef) + if result != nil: + result.info = n.info proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode = result = nil @@ -593,6 +595,8 @@ proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode result = foldDefine(m, s, n, idgen, g) else: result = copyTree(s.astdef) + if result != nil: + result.info = n.info of skProc, skFunc, skMethod: result = n of skParam: diff --git a/tests/errmsgs/tconstinfo.nim b/tests/errmsgs/tconstinfo.nim new file mode 100644 index 000000000..170972b9f --- /dev/null +++ b/tests/errmsgs/tconstinfo.nim @@ -0,0 +1,7 @@ +# https://forum.nim-lang.org/t/9762 + +const foo = "abc" +case 'a' +of foo: echo "should error" #[tt.Error + ^ type mismatch: got but expected 'char']# +else: discard From 9e35631191ad67994bb45cfad4d9452f790534e0 Mon Sep 17 00:00:00 2001 From: Phil Krylov Date: Wed, 28 Dec 2022 03:26:52 +0100 Subject: [PATCH 2326/2472] std/encodings: Fix open() documentation in regard to exception raised (#21187) --- lib/pure/encodings.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/encodings.nim b/lib/pure/encodings.nim index 26b6e1f22..0d7e878e5 100644 --- a/lib/pure/encodings.nim +++ b/lib/pure/encodings.nim @@ -339,7 +339,7 @@ proc getCurrentEncoding*(uiApp = false): string = proc open*(destEncoding = "UTF-8", srcEncoding = "CP1252"): EncodingConverter = ## Opens a converter that can convert from `srcEncoding` to `destEncoding`. - ## Raises `IOError` if it cannot fulfill the request. + ## Raises `EncodingError` if it cannot fulfill the request. when not defined(windows): result = iconvOpen(destEncoding, srcEncoding) if result == nil: From 9efa56a8bbacab2075a4c2fe4c5424d57b6eab46 Mon Sep 17 00:00:00 2001 From: rockcavera Date: Tue, 27 Dec 2022 23:40:17 -0300 Subject: [PATCH 2327/2472] [backport: 2.0] prevents the jsonscript command from exceeding the maximum length of a command line during linking (#21186) --- compiler/extccomp.nim | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 66de46556..6515f6968 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -851,6 +851,17 @@ proc displayProgressCC(conf: ConfigRef, path, compileCmd: string): string = else: result = MsgKindToStr[hintCC] % demangleModuleName(path.splitFile.name) +proc preventLinkCmdMaxCmdLen(conf: ConfigRef, linkCmd: string) = + # Prevent linkcmd from exceeding the maximum command line length. + # Windows's command line limit is about 8K (8191 characters) so C compilers on + # Windows support a feature where the command line can be passed via ``@linkcmd`` + # to them. + const MaxCmdLen = when defined(windows): 8_000 else: 32_000 + if linkCmd.len > MaxCmdLen: + linkViaResponseFile(conf, linkCmd) + else: + execLinkCmd(conf, linkCmd) + proc callCCompiler*(conf: ConfigRef) = var linkCmd: string @@ -927,14 +938,7 @@ proc callCCompiler*(conf: ConfigRef) = linkCmd = getLinkCmd(conf, mainOutput, objfiles, removeStaticFile = true) extraCmds = getExtraCmds(conf, mainOutput) if optCompileOnly notin conf.globalOptions: - const MaxCmdLen = when defined(windows): 8_000 else: 32_000 - if linkCmd.len > MaxCmdLen: - # Windows's command line limit is about 8K (don't laugh...) so C compilers on - # Windows support a feature where the command line can be passed via ``@linkcmd`` - # to them. - linkViaResponseFile(conf, linkCmd) - else: - execLinkCmd(conf, linkCmd) + preventLinkCmdMaxCmdLen(conf, linkCmd) for cmd in extraCmds: execExternalProgram(conf, cmd, hintExecuting) else: @@ -1035,7 +1039,7 @@ proc runJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile) = cmds.add cmd prettyCmds.add displayProgressCC(conf, name, cmd) execCmdsInParallel(conf, cmds, prettyCb) - execLinkCmd(conf, bcache.linkcmd) + preventLinkCmdMaxCmdLen(conf, bcache.linkcmd) for cmd in bcache.extraCmds: execExternalProgram(conf, cmd, hintExecuting) proc genMappingFiles(conf: ConfigRef; list: CfileList): Rope = From 3d5dbf8f96a5106a3b817a45a926303bc5623786 Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Wed, 28 Dec 2022 17:35:11 +1100 Subject: [PATCH 2328/2472] Fix nimsuggest not suggesting fields in when theres static parameters (#21189) Don't check against empty arguments Add test case --- compiler/sigmatch.nim | 2 +- nimsuggest/tests/t21185.nim | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 nimsuggest/tests/t21185.nim diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index d05503c00..68068cd26 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -2055,7 +2055,7 @@ proc paramTypesMatchAux(m: var TCandidate, f, a: PType, a.n == nil and tfGenericTypeParam notin a.flags: return newNodeIT(nkType, argOrig.info, makeTypeFromExpr(c, arg)) - else: + elif arg.kind != nkEmpty: var evaluated = c.semTryConstExpr(c, arg) if evaluated != nil: # Don't build the type in-place because `evaluated` and `arg` may point diff --git a/nimsuggest/tests/t21185.nim b/nimsuggest/tests/t21185.nim new file mode 100644 index 000000000..bf5a0e3cc --- /dev/null +++ b/nimsuggest/tests/t21185.nim @@ -0,0 +1,18 @@ + +# Reduced case of 21185. Issue was first parameter being static +proc foo(x: static[int]) = discard + +type + Person = object + name: string + age: int + +let p = Person() +p.#[!]# + +discard """ +$nimsuggest --tester --v3 --maxresults:2 $file +>sug $1 +sug;;skField;;age;;int;;$file;;8;;4;;"";;100;;None +sug;;skField;;name;;string;;$file;;7;;4;;"";;100;;None +""" From 7a74c2dc3a82488671237555faa95fb38ef31bd5 Mon Sep 17 00:00:00 2001 From: Bung Date: Wed, 28 Dec 2022 15:28:48 +0800 Subject: [PATCH 2329/2472] fix #14667 (#21190) --- compiler/ccgexprs.nim | 4 ++++ compiler/cgendata.nim | 1 + tests/misc/t14667.nim | 12 ++++++++++++ 3 files changed, 17 insertions(+) create mode 100644 tests/misc/t14667.nim diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index d7ef024b1..e2885a294 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -3200,6 +3200,8 @@ proc getDefaultValue(p: BProc; typ: PType; info: TLineInfo; result: var Rope) = result.add "}" of tyTuple: result.add "{" + if p.vccAndC and t.len == 0: + result.add "0" for i in 0.. 0: result.add ", " getDefaultValue(p, t[i], info, result) @@ -3338,6 +3340,8 @@ proc genConstSimpleList(p: BProc, n: PNode; isConst: bool; result: var Rope) = proc genConstTuple(p: BProc, n: PNode; isConst: bool; tup: PType; result: var Rope) = result.add "{" + if p.vccAndC and n.len == 0: + result.add "0" for i in 0.. 0: result.add ",\n" diff --git a/compiler/cgendata.nim b/compiler/cgendata.nim index 2daa8da7c..f75598652 100644 --- a/compiler/cgendata.nim +++ b/compiler/cgendata.nim @@ -172,6 +172,7 @@ type template config*(m: BModule): ConfigRef = m.g.config template config*(p: BProc): ConfigRef = p.module.g.config +template vccAndC*(p: BProc): bool = p.module.config.cCompiler == ccVcc and p.module.config.backend == backendC proc includeHeader*(this: BModule; header: string) = if not this.headerFiles.contains header: diff --git a/tests/misc/t14667.nim b/tests/misc/t14667.nim new file mode 100644 index 000000000..3034e2841 --- /dev/null +++ b/tests/misc/t14667.nim @@ -0,0 +1,12 @@ +discard """ + matrix: "--cc:vcc" + disabled: "linux" + disabled: "bsd" + disabled: "osx" + disabled: "unix" + disabled: "posix" +""" + +type A = tuple +discard () +discard default(A) From 4b63ac4b87c54a1b31d60b7555e2c60d232817c8 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Wed, 28 Dec 2022 23:23:37 +0800 Subject: [PATCH 2330/2472] fixes #21171; dynamic acyclic refs need to use dyn decRef (#21184) * fixes #21171; dyn destructors for acyclic inherited refs * add a test * Update compiler/liftdestructors.nim --- compiler/liftdestructors.nim | 7 ++++ lib/system/orc.nim | 13 ++++++ tests/arc/t21184.nim | 77 ++++++++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+) create mode 100644 tests/arc/t21184.nim diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim index 5174a908f..2b27a58fc 100644 --- a/compiler/liftdestructors.nim +++ b/compiler/liftdestructors.nim @@ -609,6 +609,11 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = createTypeBoundOps(c.g, c.c, elemType, c.info, c.idgen) let isCyclic = c.g.config.selectedGC == gcOrc and types.canFormAcycle(elemType) + let isInheritableAcyclicRef = c.g.config.selectedGC == gcOrc and + (not isPureObject(elemType)) and + tfAcyclic in skipTypes(elemType, abstractInst+{tyOwned}-{tyTypeDesc}).flags + # dynamic Acyclic refs need to use dyn decRef + let tmp = if isCyclic and c.kind in {attachedAsgn, attachedSink}: declareTempOf(c, body, x) @@ -632,6 +637,8 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = cond = callCodegenProc(c.g, "nimDecRefIsLastCyclicStatic", c.info, tmp, typInfo) else: cond = callCodegenProc(c.g, "nimDecRefIsLastCyclicDyn", c.info, tmp) + elif isInheritableAcyclicRef: + cond = callCodegenProc(c.g, "nimDecRefIsLastDyn", c.info, x) else: cond = callCodegenProc(c.g, "nimDecRefIsLast", c.info, x) cond.typ = getSysType(c.g, x.info, tyBool) diff --git a/lib/system/orc.nim b/lib/system/orc.nim index 83b983ee1..a56a0c057 100644 --- a/lib/system/orc.nim +++ b/lib/system/orc.nim @@ -485,6 +485,19 @@ proc nimDecRefIsLastCyclicDyn(p: pointer): bool {.compilerRtl, inl.} = #if cell.color == colPurple: rememberCycle(result, cell, cast[ptr PNimTypeV2](p)[]) +proc nimDecRefIsLastDyn(p: pointer): bool {.compilerRtl, inl.} = + if p != nil: + var cell = head(p) + if (cell.rc and not rcMask) == 0: + result = true + #cprintf("[DESTROY] %p\n", p) + else: + dec cell.rc, rcIncrement + #if cell.color == colPurple: + if result: + if cell.rootIdx > 0: + unregisterCycle(cell) + proc nimDecRefIsLastCyclicStatic(p: pointer; desc: PNimTypeV2): bool {.compilerRtl, inl.} = if p != nil: var cell = head(p) diff --git a/tests/arc/t21184.nim b/tests/arc/t21184.nim new file mode 100644 index 000000000..91d0c42c7 --- /dev/null +++ b/tests/arc/t21184.nim @@ -0,0 +1,77 @@ +discard """ + matrix: "--mm:orc" +""" + +import std/[with] + +type + Node* {.acyclic.} = ref object of RootObj + name: string + data: pointer + children: seq[Node] + TextNode = ref object of Node + text: string + +proc fakeEcho(s: string) = + if s.len < 0: + echo s + +proc newNode[T: Node](parent: Node): T = + new result + result.data = alloc0(250) + parent.children.add(result) + +proc newRootNode(): Node = + new result + result.data = alloc0(250) + +method printNode(node: Node) {.base.} = + fakeEcho node.name + +method printNode(node: TextNode) = + procCall printNode(Node(node)) + fakeEcho node.text + +proc printChildren(node: Node) = + for child in node.children: + child.printNode() + printChildren(child) + +proc free(node: Node) = + for child in node.children: + free(child) + dealloc(node.data) + +template node(parent: Node, body: untyped): untyped = + var node = newNode[Node](parent) + with node: + body + +proc textNode(parent: Node, text: string) = + var node = newNode[TextNode](parent) + node.text = text + +template withRootNode(body: untyped): untyped = + var root = newRootNode() + root.name = "root" + with root: + body + root.printNode() + printChildren(root) + root.free() + +proc doTest() = + withRootNode: + node: + name = "child1" + node: + name = "child2" + node: + name = "child3" + textNode "Hello, world!" + + +# bug #21171 +if isMainModule: + for i in 0..100000: + doTest() From 646932b3f316d6b8e3ceb6b94b9d01232d5cdb33 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 29 Dec 2022 03:41:27 +0800 Subject: [PATCH 2331/2472] fixes warnings when building csources (#21194) * replace `symbolfiles` with `incremental`; fixes warnings when build csources * fixes self conversion warnings --- tools/niminst/niminst.nim | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/niminst/niminst.nim b/tools/niminst/niminst.nim index bc763a56f..81b46653d 100644 --- a/tools/niminst/niminst.nim +++ b/tools/niminst/niminst.nim @@ -168,11 +168,11 @@ proc parseCmdLine(c: var ConfigData) = next(p) var kind = p.kind var key = p.key - var val = p.val.string + var val = p.val case kind of cmdArgument: if c.actions == {}: - for a in split(normalize(key.string), {';', ','}): + for a in split(normalize(key), {';', ','}): case a of "csource": incl(c.actions, actionCSource) of "scripts": incl(c.actions, actionScripts) @@ -183,11 +183,11 @@ proc parseCmdLine(c: var ConfigData) = of "deb": incl(c.actions, actionDeb) else: quit(Usage) else: - c.infile = addFileExt(key.string, "ini") - c.nimArgs = cmdLineRest(p).string + c.infile = addFileExt(key, "ini") + c.nimArgs = cmdLineRest(p) break of cmdLongOption, cmdShortOption: - case normalize(key.string) + case normalize(key) of "help", "h": stdout.write(Usage) quit(0) @@ -544,7 +544,7 @@ proc srcdist(c: var ConfigData) = var dir = getOutputDir(c) / buildDir(osA, cpuA) if dirExists(dir): removeDir(dir) createDir(dir) - var cmd = ("$# compile -f --symbolfiles:off --compileonly " & + var cmd = ("$# compile -f --incremental:off --compileonly " & "--gen_mapping --cc:gcc --skipUserCfg" & " --os:$# --cpu:$# $# $#") % [findNim(), osname, cpuname, c.nimArgs, c.mainfile] From 761c5a0830a18cb0db721d27aff53a07ca30b1c3 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 29 Dec 2022 03:54:15 +0800 Subject: [PATCH 2332/2472] less verbose type mismatch messages (#21191) * less verbose type mismatch messages * Update compiler/types.nim * fixes i386 * fixes i386 --- changelogs/changelog_2_0_0.md | 2 ++ compiler/options.nim | 1 + compiler/semcall.nim | 44 +++++++++++++++++++------ compiler/sigmatch.nim | 40 ++++++++++++---------- tests/config.nims | 1 + tests/errmsgs/tconcisetypemismatch.nim | 23 +++++++++++++ tests/errmsgs/tconcisetypemismatch.nims | 21 ++++++++++++ 7 files changed, 105 insertions(+), 27 deletions(-) create mode 100644 tests/errmsgs/tconcisetypemismatch.nim create mode 100644 tests/errmsgs/tconcisetypemismatch.nims diff --git a/changelogs/changelog_2_0_0.md b/changelogs/changelog_2_0_0.md index 614d97d85..29d906d62 100644 --- a/changelogs/changelog_2_0_0.md +++ b/changelogs/changelog_2_0_0.md @@ -135,6 +135,8 @@ - The experimental strictFuncs feature now disallows a store to the heap via a `ref` or `ptr` indirection. +- - Added the `--legacy:verboseTypeMismatch` switch to get legacy type mismatch error messages. + ## Standard library additions and changes [//]: # "Changes:" diff --git a/compiler/options.nim b/compiler/options.nim index 7e4f6898f..ded084ae4 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -231,6 +231,7 @@ type ## are not anymore. laxEffects ## Lax effects system prior to Nim 2.0. + verboseTypeMismatch SymbolFilesOption* = enum disabledSf, writeOnlySf, readOnlySf, v2Sf, stressTest diff --git a/compiler/semcall.nim b/compiler/semcall.nim index 31b490781..2147a9645 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -10,7 +10,8 @@ ## This module implements semantic checking for calls. # included from sem.nim -from algorithm import sort +from std/algorithm import sort + proc sameMethodDispatcher(a, b: PSym): bool = result = false @@ -192,7 +193,7 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors): # argument in order to remove plenty of candidates. This is # comparable to what C# does and C# is doing fine. var filterOnlyFirst = false - if optShowAllMismatches notin c.config.globalOptions: + if optShowAllMismatches notin c.config.globalOptions and verboseTypeMismatch in c.config.legacyFeatures: for err in errors: if err.firstMismatch.arg > 1: filterOnlyFirst = true @@ -208,6 +209,10 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors): if filterOnlyFirst and err.firstMismatch.arg == 1: inc skipped continue + + if verboseTypeMismatch notin c.config.legacyFeatures: + candidates.add "[" & $err.firstMismatch.arg & "] " + if err.sym.kind in routineKinds and err.sym.ast != nil: candidates.add(renderTree(err.sym.ast, {renderNoBody, renderNoComments, renderNoPragmas})) @@ -217,7 +222,7 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors): candidates.add("\n") let nArg = if err.firstMismatch.arg < n.len: n[err.firstMismatch.arg] else: nil let nameParam = if err.firstMismatch.formal != nil: err.firstMismatch.formal.name.s else: "" - if n.len > 1: + if n.len > 1 and verboseTypeMismatch in c.config.legacyFeatures: candidates.add(" first type mismatch at position: " & $err.firstMismatch.arg) # candidates.add "\n reason: " & $err.firstMismatch.kind # for debugging case err.firstMismatch.kind @@ -274,11 +279,28 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors): const errTypeMismatch = "type mismatch: got <" errButExpected = "but expected one of:" + errExpectedPosition = "Expected one of (first mismatch at position [#]):" errUndeclaredField = "undeclared field: '$1'" errUndeclaredRoutine = "attempting to call undeclared routine: '$1'" errBadRoutine = "attempting to call routine: '$1'$2" errAmbiguousCallXYZ = "ambiguous call; both $1 and $2 match for: $3" +proc describeParamList(c: PContext, n: PNode, startIdx = 1; prefer = preferName): string = + result = "Expression: " & $n + for i in startIdx..') + if candidates != "": + result.add("\n" & errButExpected & "\n" & candidates) + localError(c.config, n.info, result & "\nexpression: " & $n) + proc notFoundError*(c: PContext, n: PNode, errors: CandidateErrors) = # Gives a detailed error message; this is separated from semOverloadedCall, # as semOverloadedCall is already pretty slow (and we need this information @@ -306,13 +328,15 @@ proc notFoundError*(c: PContext, n: PNode, errors: CandidateErrors) = localError(c.config, n.info, "expression '$1' cannot be called" % n[0].renderTree) return - let (prefer, candidates) = presentFailedCandidates(c, n, errors) - var result = errTypeMismatch - result.add(describeArgs(c, n, 1, prefer)) - result.add('>') - if candidates != "": - result.add("\n" & errButExpected & "\n" & candidates) - localError(c.config, n.info, result & "\nexpression: " & $n) + if verboseTypeMismatch in c.config.legacyFeatures: + legacynotFoundError(c, n, errors) + else: + let (prefer, candidates) = presentFailedCandidates(c, n, errors) + var result = "type mismatch\n" + result.add describeParamList(c, n, 1, prefer) + if candidates != "": + result.add("\n" & errExpectedPosition & "\n" & candidates) + localError(c.config, n.info, result) proc bracketNotFoundError(c: PContext; n: PNode) = var errors: CandidateErrors = @[] diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 68068cd26..75e3e5428 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -322,26 +322,32 @@ proc argTypeToString(arg: PNode; prefer: TPreferedDesc): string = else: result = arg.typ.typeToString(prefer) +template describeArgImpl(c: PContext, n: PNode, i: int, startIdx = 1; prefer = preferName) = + var arg = n[i] + if n[i].kind == nkExprEqExpr: + result.add renderTree(n[i][0]) + result.add ": " + if arg.typ.isNil and arg.kind notin {nkStmtList, nkDo}: + # XXX we really need to 'tryExpr' here! + arg = c.semOperand(c, n[i][1]) + n[i].typ = arg.typ + n[i][1] = arg + else: + if arg.typ.isNil and arg.kind notin {nkStmtList, nkDo, nkElse, + nkOfBranch, nkElifBranch, + nkExceptBranch}: + arg = c.semOperand(c, n[i]) + n[i] = arg + if arg.typ != nil and arg.typ.kind == tyError: return + result.add argTypeToString(arg, prefer) + +proc describeArg*(c: PContext, n: PNode, i: int, startIdx = 1; prefer = preferName): string = + describeArgImpl(c, n, i, startIdx, prefer) + proc describeArgs*(c: PContext, n: PNode, startIdx = 1; prefer = preferName): string = result = "" for i in startIdx..= (1,5,1): + # to make it easier to test against older nim versions, (best effort only) + switch("filenames", "legacyRelProj") + switch("spellSuggest", "0") + +# for std/unittest +switch("define", "nimUnittestOutputLevel:PRINT_FAILURES") +switch("define", "nimUnittestColor:off") + +hint("Processing", off) From e602ebeb66986e3de65324c81253513b139c7f0e Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 29 Dec 2022 15:29:40 +0800 Subject: [PATCH 2333/2472] fixes #19997; ships changelogs directory for a release (#21200) fixes #19997; ship changelogs directory for a release --- compiler/installer.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler/installer.ini b/compiler/installer.ini index 094b92a68..1e57455c6 100644 --- a/compiler/installer.ini +++ b/compiler/installer.ini @@ -70,6 +70,7 @@ Files: "nimpretty" Files: "testament" Files: "nimsuggest" Files: "nimsuggest/tests/*.nim" +Files: "changelogs/*.md" [Lib] Files: "lib" From 705da9d452d19536689a32a8d4378bcce2ec320a Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 30 Dec 2022 00:06:57 +0800 Subject: [PATCH 2334/2472] fixes #14444; add `genLineDir` before assignment (#21201) * fixes #14444; add `genLineDir` before raises * add a test case * fixes differently --- compiler/ccgstmts.nim | 1 + tests/errmsgs/t14444.nim | 14 ++++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 tests/errmsgs/t14444.nim diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index fbdb9a347..9d6640fe2 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -1620,6 +1620,7 @@ proc genAsgn(p: BProc, e: PNode, fastAsgn: bool) = initLoc(a, locNone, le, OnUnknown) a.flags.incl(lfEnforceDeref) a.flags.incl(lfPrepareForMutation) + genLineDir(p, le) # it can be a nkBracketExpr, which may raise expr(p, le, a) a.flags.excl(lfPrepareForMutation) if fastAsgn: incl(a.flags, lfNoDeepCopy) diff --git a/tests/errmsgs/t14444.nim b/tests/errmsgs/t14444.nim new file mode 100644 index 000000000..143b4542e --- /dev/null +++ b/tests/errmsgs/t14444.nim @@ -0,0 +1,14 @@ +discard """ + matrix: "--hints:off" + exitcode: "1" + output: ''' +t14444.nim(13) t14444 +fatal.nim(51) sysFatal +Error: unhandled exception: index out of bounds, the container is empty [IndexDefect] +''' +""" + +when true: # bug #14444 + var i: string + i[10] = 'j' + echo i \ No newline at end of file From ebd1c678be5e0acee4fca67e1b7060234821ccf6 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 30 Dec 2022 03:15:08 +0800 Subject: [PATCH 2335/2472] fixes #21195; `std/assertions` continue to use `sysFatal` when `nimPreviewSlimSystem` is not defined (#21196) * fixes #21195; `std/assertions` continue to use `sysFatal` * try includes * make `std/assertions` self-contained * fixes tests * fixes tests --- lib/std/assertions.nim | 9 ++++++++- lib/system.nim | 28 +++------------------------- lib/system/rawquits.nim | 27 +++++++++++++++++++++++++++ tests/assert/panicoverride.nim | 15 +++++++++++++++ tests/assert/t21195.nim | 6 ++++++ tests/assert/tassert_c.nim | 4 ++-- tests/errmsgs/t9768.nim | 4 ++-- 7 files changed, 63 insertions(+), 30 deletions(-) create mode 100644 lib/system/rawquits.nim create mode 100644 tests/assert/panicoverride.nim create mode 100644 tests/assert/t21195.nim diff --git a/lib/std/assertions.nim b/lib/std/assertions.nim index 229033c58..a249d7751 100644 --- a/lib/std/assertions.nim +++ b/lib/std/assertions.nim @@ -7,6 +7,10 @@ # distribution, for details about the copyright. # +when not defined(nimPreviewSlimSystem) and not declared(sysFatal): + include "system/rawquits" + include "system/fatal" + ## This module implements assertion handling. import std/private/miscdollars @@ -26,7 +30,10 @@ proc `$`(info: InstantiationInfo): string = proc raiseAssert*(msg: string) {.noinline, noreturn, nosinks.} = ## Raises an `AssertionDefect` with `msg`. - raise newException(AssertionDefect, msg) + when defined(nimPreviewSlimSystem): + raise newException(AssertionDefect, msg) + else: + sysFatal(AssertionDefect, msg) proc failedAssertImpl*(msg: string) {.raises: [], tags: [].} = ## Raises an `AssertionDefect` with `msg`, but this is hidden diff --git a/lib/system.nim b/lib/system.nim index e9c9ebfeb..d8fafc0ef 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1081,31 +1081,9 @@ proc align(address, alignment: int): int = else: result = (address + (alignment - 1)) and not (alignment - 1) -when defined(nimNoQuit): - proc rawQuit(errorcode: int = QuitSuccess) = discard "ignoring quit" - -elif defined(genode): - import genode/env - - var systemEnv {.exportc: runtimeEnvSym.}: GenodeEnvPtr - - type GenodeEnv* = GenodeEnvPtr - ## Opaque type representing Genode environment. - - proc rawQuit(env: GenodeEnv; errorcode: int) {.magic: "Exit", noreturn, - importcpp: "#->parent().exit(@); Genode::sleep_forever()", header: "".} - - proc rawQuit(errorcode: int = QuitSuccess) {.inline, noreturn.} = - systemEnv.rawQuit(errorcode) - - -elif defined(js) and defined(nodejs) and not defined(nimscript): - proc rawQuit(errorcode: int = QuitSuccess) {.magic: "Exit", - importc: "process.exit", noreturn.} - -else: - proc rawQuit(errorcode: cint) {. - magic: "Exit", importc: "exit", header: "", noreturn.} +include system/rawquits +when defined(genode): + export GenodeEnv template sysAssert(cond: bool, msg: string) = when defined(useSysAssert): diff --git a/lib/system/rawquits.nim b/lib/system/rawquits.nim new file mode 100644 index 000000000..f0ead10c6 --- /dev/null +++ b/lib/system/rawquits.nim @@ -0,0 +1,27 @@ +import system/ctypes + +when defined(nimNoQuit): + proc rawQuit(errorcode: int = QuitSuccess) = discard "ignoring quit" + +elif defined(genode): + import genode/env + + var systemEnv {.exportc: runtimeEnvSym.}: GenodeEnvPtr + + type GenodeEnv = GenodeEnvPtr + ## Opaque type representing Genode environment. + + proc rawQuit(env: GenodeEnv; errorcode: int) {.magic: "Exit", noreturn, + importcpp: "#->parent().exit(@); Genode::sleep_forever()", header: "".} + + proc rawQuit(errorcode: int = QuitSuccess) {.inline, noreturn.} = + systemEnv.rawQuit(errorcode) + + +elif defined(js) and defined(nodejs) and not defined(nimscript): + proc rawQuit(errorcode: int = QuitSuccess) {.magic: "Exit", + importc: "process.exit", noreturn.} + +else: + proc rawQuit(errorcode: cint) {. + magic: "Exit", importc: "exit", header: "", noreturn.} \ No newline at end of file diff --git a/tests/assert/panicoverride.nim b/tests/assert/panicoverride.nim new file mode 100644 index 000000000..53ad64215 --- /dev/null +++ b/tests/assert/panicoverride.nim @@ -0,0 +1,15 @@ +# panicoverride.nim + +proc printf(fmt: cstring) {.varargs, importc, header:"stdio.h".} +proc exit(code: cint) {.importc, header:"stdlib.h".} + +{.push stack_trace: off, profiler:off.} + +proc rawoutput(s: cstring) = + printf("RAW: %s\n", s) + +proc panic(s: cstring) {.noreturn.} = + printf("PANIC: %s\n", s) + exit(0) + +{.pop.} \ No newline at end of file diff --git a/tests/assert/t21195.nim b/tests/assert/t21195.nim new file mode 100644 index 000000000..b690d22a0 --- /dev/null +++ b/tests/assert/t21195.nim @@ -0,0 +1,6 @@ +discard """ + matrix: "--verbosity:0 --os:standalone --mm:none" +""" +# bug #21195 +var n = 11 +assert(n == 12) diff --git a/tests/assert/tassert_c.nim b/tests/assert/tassert_c.nim index 110a9aabf..4d49a6035 100644 --- a/tests/assert/tassert_c.nim +++ b/tests/assert/tassert_c.nim @@ -1,8 +1,8 @@ discard """ - cmd: "nim $target $options --excessiveStackTrace:off $file" + cmd: "nim $target $options -d:nimPreviewSlimSystem --excessiveStackTrace:off $file" output: '''true''' """ - +import std/assertions const expected = """ tassert_c.nim(35) tassert_c tassert_c.nim(34) foo diff --git a/tests/errmsgs/t9768.nim b/tests/errmsgs/t9768.nim index 058d297b3..b5ff58367 100644 --- a/tests/errmsgs/t9768.nim +++ b/tests/errmsgs/t9768.nim @@ -1,14 +1,14 @@ discard """ errormsg: "unhandled exception: t9768.nim(24, 3) `a < 4` [AssertionDefect]" file: "std/assertions.nim" + matrix: "-d:nimPreviewSlimSystem" nimout: ''' stack trace: (most recent call last) t9768.nim(29, 33) main t9768.nim(24, 11) foo1 ''' """ - - +import std/assertions From c598d0b6eccd3133bfca710fc5fcc4576501862c Mon Sep 17 00:00:00 2001 From: Bung Date: Fri, 30 Dec 2022 07:50:12 +0800 Subject: [PATCH 2336/2472] fix #15117 zero size array cause incorrect codegen for VCC compiler (#21197) fix #15117 --- compiler/ccgexprs.nim | 2 ++ tests/array/t15117.nim | 27 +++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 tests/array/t15117.nim diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index e2885a294..844a37997 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -3331,6 +3331,8 @@ proc genConstObjConstr(p: BProc; n: PNode; isConst: bool; result: var Rope) = proc genConstSimpleList(p: BProc, n: PNode; isConst: bool; result: var Rope) = result.add "{" + if p.vccAndC and n.len == 0 and n.typ.kind == tyArray: + getDefaultValue(p, n.typ[1], n.info, result) for i in 0.. 0: result.add ",\n" diff --git a/tests/array/t15117.nim b/tests/array/t15117.nim new file mode 100644 index 000000000..157b04bee --- /dev/null +++ b/tests/array/t15117.nim @@ -0,0 +1,27 @@ +discard """ + matrix: "--cc:vcc" + disabled: "linux" + disabled: "bsd" + disabled: "osx" + disabled: "unix" + disabled: "posix" +""" +{.experimental: "views".} + +let a: array[0, byte] = [] +discard a + +type B = object + a:int +let b: array[0, B] = [] +let c: array[0, ptr B] = [] +let d: array[0, ref B] = [] +discard b +discard c +discard d + +discard default(array[0, B]) + +type + View1 = openArray[byte] +discard default(View1) From e5639408d55c3b9164790bee37580a74a15b666f Mon Sep 17 00:00:00 2001 From: rockcavera Date: Fri, 30 Dec 2022 05:20:32 -0300 Subject: [PATCH 2337/2472] avoiding unnecessary allocation for dollar IpAddress (#21199) * avoiding allocating an unnecessary byte for IPv4 The inet.h file uses 16 as the string in C needs the last null byte https://github.com/torvalds/linux/blob/1b929c02afd37871d5afb9d498426f83432e71c2/include/linux/inet.h#L49 However, strings in Nim do not need this. So one byte is being allocated unnecessary and will never be used. * avoid unnecessary allocation in IPv6 dollar It is currently allocating 48 bytes. However, the Nim implementation for IPv6 will print a maximum of 39 characters. Nim does not implement IPv6 "0000:0000:0000:0000:0000:ffff:255.255.255.255" (45 characters) nor "0000:0000:0000:0000:0000:ffff:255.255.255.255%3" (47 characters). The indication in inet.h for 48 is due to the maximum use of 47 characters of a C string that needs a null byte at the end. So 48. https://github.com/torvalds/linux/blob/1b929c02afd37871d5afb9d498426f83432e71c2/include/linux/inet.h#L50 --- lib/pure/net.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index be9f2e48c..76a39226f 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -1921,7 +1921,7 @@ proc `$`*(address: IpAddress): string = ## Converts an IpAddress into the textual representation case address.family of IpAddressFamily.IPv4: - result = newStringOfCap(16) + result = newStringOfCap(15) result.addInt address.address_v4[0] result.add '.' result.addInt address.address_v4[1] @@ -1930,7 +1930,7 @@ proc `$`*(address: IpAddress): string = result.add '.' result.addInt address.address_v4[3] of IpAddressFamily.IPv6: - result = newStringOfCap(48) + result = newStringOfCap(39) var currentZeroStart = -1 currentZeroCount = 0 From 4032eb4baafcfb4a1cb8fd41a474daac70103856 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sun, 1 Jan 2023 16:51:58 +0800 Subject: [PATCH 2338/2472] fixes #20906; update copyright year [backport 1.6] (#21210) --- compiler/options.nim | 2 +- copying.txt | 2 +- readme.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/options.nim b/compiler/options.nim index ded084ae4..47e8155f8 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -24,7 +24,7 @@ const useEffectSystem* = true useWriteTracking* = false hasFFI* = defined(nimHasLibFFI) - copyrightYear* = "2022" + copyrightYear* = "2023" nimEnableCovariance* = defined(nimEnableCovariance) diff --git a/copying.txt b/copying.txt index 60b6a0217..ae3a162a7 100644 --- a/copying.txt +++ b/copying.txt @@ -1,7 +1,7 @@ ===================================================== Nim -- a Compiler for Nim. https://nim-lang.org/ -Copyright (C) 2006-2022 Andreas Rumpf. All rights reserved. +Copyright (C) 2006-2023 Andreas Rumpf. All rights reserved. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/readme.md b/readme.md index a0f47eb41..325e8fff9 100644 --- a/readme.md +++ b/readme.md @@ -203,7 +203,7 @@ Nim. You are explicitly permitted to develop commercial applications using Nim. Please read the [copying.txt](copying.txt) file for more details. -Copyright © 2006-2022 Andreas Rumpf, all rights reserved. +Copyright © 2006-2023 Andreas Rumpf, all rights reserved. [nim-site]: https://nim-lang.org [nim-forum]: https://forum.nim-lang.org From 5b20f0685c71e68ab08f7a1941f7e160d91e538f Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sun, 1 Jan 2023 17:25:04 +0800 Subject: [PATCH 2339/2472] fixes pragmas reorder (#21205) --- compiler/reorder.nim | 1 + tests/pragmas/tpragmas_reorder.nim | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 tests/pragmas/tpragmas_reorder.nim diff --git a/compiler/reorder.nim b/compiler/reorder.nim index 4fabf9041..4ad3f1219 100644 --- a/compiler/reorder.nim +++ b/compiler/reorder.nim @@ -106,6 +106,7 @@ proc computeDeps(cache: IdentCache; n: PNode, declares, uses: var IntSet; topLev if a.kind == nkExprColonExpr and a[0].kind == nkIdent and a[0].ident.s == "pragma": # user defined pragma decl(a[1]) + for i in 1.. Date: Mon, 2 Jan 2023 08:16:18 +0000 Subject: [PATCH 2340/2472] Update nim.cfg (#21204) tlsEmulation:on under NetBSD-10Beta and NetBSD-current produces an executable which crashes immediately as follows: Core was generated by `koch'. Program terminated with signal SIGSEGV, Segmentation fault. #0 0x000000000047b4c2 in nimZeroMem () (gdb) bt #0 0x000000000047b4c2 in nimZeroMem () #1 0x00000000004897b2 in threadVarAlloc__system_2162 () #2 0x000000000048980e in initThreadVarsEmulation () #3 0x0000000000489848 in PreMain () #4 0x000000000048986a in NimMain () #5 0x00000000004898a9 in main () I can't speak about the other BSDs. --- config/nim.cfg | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config/nim.cfg b/config/nim.cfg index 1601ebe08..f7b7c119a 100644 --- a/config/nim.cfg +++ b/config/nim.cfg @@ -181,10 +181,9 @@ nimblepath="$home/.nimble/pkgs/" gcc.maxerrorsimpl = "-fmax-errors=3" -@if freebsd: +@if freebsd or netbsd: tlsEmulation:off @elif bsd: - # at least NetBSD has problems with thread local storage: tlsEmulation:on @end From cf1b16ef8ba1c9f8411e551d317784c5ffaece0c Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 2 Jan 2023 10:39:17 +0100 Subject: [PATCH 2341/2472] minor code cleanups (#21215) --- lib/posix/posix.nim | 8 ++++---- lib/posix/posix_utils.nim | 6 +++--- lib/std/enumerate.nim | 2 +- lib/std/enumutils.nim | 2 +- lib/std/envvars.nim | 35 +++++++++++++++++------------------ lib/std/oserrors.nim | 15 ++++++--------- 6 files changed, 32 insertions(+), 36 deletions(-) diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index ddf00cbb7..43346608d 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -1140,12 +1140,12 @@ type ## The getrlimit() and setrlimit() system calls get and set resource limits respectively. ## Each resource has an associated soft and hard limit, as defined by the RLimit structure -proc setrlimit*(resource: cint, rlp: var RLimit): cint - {.importc: "setrlimit",header: "".} +proc setrlimit*(resource: cint, rlp: var RLimit): cint {. + importc: "setrlimit", header: "".} ## The setrlimit() system calls sets resource limits. -proc getrlimit*(resource: cint, rlp: var RLimit): cint - {.importc: "getrlimit",header: "".} +proc getrlimit*(resource: cint, rlp: var RLimit): cint {. + importc: "getrlimit", header: "".} ## The getrlimit() system call gets resource limits. when defined(nimHasStyleChecks): diff --git a/lib/posix/posix_utils.nim b/lib/posix/posix_utils.nim index 92fe0940d..7e4a2eeda 100644 --- a/lib/posix/posix_utils.nim +++ b/lib/posix/posix_utils.nim @@ -43,8 +43,8 @@ proc uname*(): Uname = result.machine = charArrayToString u.machine proc fsync*(fd: int) = - ## synchronize a file's buffer cache to the storage device - if fsync(fd.cint) != 0: + ## synchronize a file's buffer cache to the storage device + if fsync(fd.cint) != 0: raise newException(OSError, $strerror(errno)) proc stat*(path: string): Stat = @@ -90,7 +90,7 @@ proc mkstemp*(prefix: string, suffix=""): (string, File) = ## Returns the filename and a file opened in r/w mode. var tmpl = cstring(prefix & "XXXXXX" & suffix) let fd = - if len(suffix)==0: + if len(suffix) == 0: when declared(mkostemp): mkostemp(tmpl, O_CLOEXEC) else: diff --git a/lib/std/enumerate.nim b/lib/std/enumerate.nim index a8f0e1ba7..4f0161b7c 100644 --- a/lib/std/enumerate.nim +++ b/lib/std/enumerate.nim @@ -21,7 +21,7 @@ macro enumerate*(x: ForLoopStmt): untyped {.since: (1, 3).} = ## The default starting count `0` can be manually overridden if needed. runnableExamples: let a = [10, 20, 30] - var b: seq[(int, int)] + var b: seq[(int, int)] = @[] for i, x in enumerate(a): b.add((i, x)) assert b == @[(0, 10), (1, 20), (2, 30)] diff --git a/lib/std/enumutils.nim b/lib/std/enumutils.nim index 9d4ff1bcf..59fb112ed 100644 --- a/lib/std/enumutils.nim +++ b/lib/std/enumutils.nim @@ -112,9 +112,9 @@ const invalidSlot = uint8.high proc genLookup[T: typedesc[HoleyEnum]](_: T): auto = const n = span(T) - var ret: array[n, uint8] var i = 0 assert n <= invalidSlot.int + var ret {.noinit.}: array[n, uint8] for ai in mitems(ret): ai = invalidSlot for ai in items(T): ret[ai.ord - T.low.ord] = uint8(i) diff --git a/lib/std/envvars.nim b/lib/std/envvars.nim index ce90f66ba..ab5c9f06e 100644 --- a/lib/std/envvars.nim +++ b/lib/std/envvars.nim @@ -94,8 +94,10 @@ when not defined(nimscript): assert getEnv("unknownEnv", "doesn't exist") == "doesn't exist" let env = getEnvImpl(key) - if env == nil: return default - result = $env + if env == nil: + result = default + else: + result = $env proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = ## Checks whether the environment variable named `key` exists. @@ -109,7 +111,7 @@ when not defined(nimscript): runnableExamples: assert not existsEnv("unknownEnv") - return getEnvImpl(key) != nil + result = getEnvImpl(key) != nil proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = ## Sets the value of the `environment variable`:idx: named `key` to `val`. @@ -140,7 +142,7 @@ when not defined(nimscript): ## * `envPairs iterator`_ template bail = raiseOSError(osLastError(), key) when defined(windows): - #[ + #[ # https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/putenv-s-wputenv-s?view=msvc-160 > You can remove a variable from the environment by specifying an empty string (that is, "") for value_string note that nil is not legal @@ -177,20 +179,17 @@ when not defined(nimscript): iterator envPairsImpl(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = when defined(windows): - block implBlock: - template impl(get_fun, typ, size, zero, free_fun) = - let env = get_fun() - var e = env - if e == nil: break implBlock - while true: - let eend = strEnd(e) - let kv = $e - let p = find(kv, '=') - yield (substr(kv, 0, p-1), substr(kv, p+1)) - e = cast[typ](cast[int](eend)+size) - if typeof(zero)(eend[1]) == zero: break - discard free_fun(env) - impl(getEnvironmentStringsW, WideCString, 2, 0, freeEnvironmentStringsW) + let env = getEnvironmentStringsW() + var e = env + if e != nil: + while true: + let eend = strEnd(e) + let kv = $e + let p = find(kv, '=') + yield (substr(kv, 0, p-1), substr(kv, p+1)) + e = cast[WideCString](cast[ByteAddress](eend)+2) + if int(eend[1]) == 0: break + discard freeEnvironmentStringsW(env) else: var i = 0 when defined(macosx) and not defined(ios) and not defined(emscripten): diff --git a/lib/std/oserrors.nim b/lib/std/oserrors.nim index 8d06c41da..a641a7f47 100644 --- a/lib/std/oserrors.nim +++ b/lib/std/oserrors.nim @@ -75,17 +75,14 @@ proc newOSError*( ## See also: ## * `osErrorMsg proc`_ ## * `osLastError proc`_ - var e: owned(ref OSError); new(e) - e.errorCode = errorCode.int32 - e.msg = osErrorMsg(errorCode) + result = (ref OSError)(errorCode: errorCode.int32, msg: osErrorMsg(errorCode)) if additionalInfo.len > 0: - if e.msg.len > 0 and e.msg[^1] != '\n': e.msg.add '\n' - e.msg.add "Additional info: " - e.msg.add additionalInfo + if result.msg.len > 0 and result.msg[^1] != '\n': result.msg.add '\n' + result.msg.add "Additional info: " + result.msg.add additionalInfo # don't add trailing `.` etc, which negatively impacts "jump to file" in IDEs. - if e.msg == "": - e.msg = "unknown OS error" - return e + if result.msg == "": + result.msg = "unknown OS error" proc raiseOSError*(errorCode: OSErrorCode, additionalInfo = "") {.noinline.} = ## Raises an `OSError exception `_. From 3b965f463b41eace340393af636e4abcd9c223d6 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Mon, 2 Jan 2023 22:10:26 +0800 Subject: [PATCH 2342/2472] bump csource_v2 to include fix for linking parameter maximum length (#21216) > If accepted, I believe it should be used in csources_v2 and also backport 2.0 ref https://github.com/nim-lang/Nim/pull/21186 --- config/build_config.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/build_config.txt b/config/build_config.txt index 286e0dc2a..66390e695 100644 --- a/config/build_config.txt +++ b/config/build_config.txt @@ -2,4 +2,4 @@ nim_comment="key-value pairs for windows/posix bootstrapping build scripts" nim_csourcesDir=csources_v2 nim_csourcesUrl=https://github.com/nim-lang/csources_v2.git nim_csourcesBranch=master -nim_csourcesHash=b66c420697c574be18a20dd4720248a715b4287e +nim_csourcesHash=86742fb02c6606ab01a532a0085784effb2e753e From 81b7f9108f139bbf237f3e121fdbe9ff32a7193f Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Tue, 3 Jan 2023 15:15:10 +0800 Subject: [PATCH 2343/2472] fixes #21207; reports redefinition error in the definition of enums (#21217) * fixes #21207; reports redefinition in the enums * add a test --- compiler/semtypes.nim | 4 ++-- tests/enum/tredefinition.nim | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 tests/enum/tredefinition.nim diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 534861e97..a8b147eef 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -73,7 +73,7 @@ proc semEnum(c: PContext, n: PNode, prev: PType): PType = rawAddSon(result, base) let isPure = result.sym != nil and sfPure in result.sym.flags var symbols: TStrTable - if isPure: initStrTable(symbols) + initStrTable(symbols) var hasNull = false for i in 1.. Date: Wed, 4 Jan 2023 12:10:44 -0800 Subject: [PATCH 2344/2472] make it printer friendly (#21218) --- doc/nimdoc.css | 9 +++++++++ nimdoc/testproject/expected/nimdoc.out.css | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/doc/nimdoc.css b/doc/nimdoc.css index f924f5a36..1417d9eff 100644 --- a/doc/nimdoc.css +++ b/doc/nimdoc.css @@ -147,6 +147,15 @@ body { box-sizing: border-box; margin-left: 1%; } +@media print { + #global-links, .link-seesrc, .theme-switch-wrapper, #searchInputDiv, .search-groupby { + display:none; + } + .columns { + width:100% !important; + } +} + .column:first-child, .columns:first-child { margin-left: 0; } diff --git a/nimdoc/testproject/expected/nimdoc.out.css b/nimdoc/testproject/expected/nimdoc.out.css index f924f5a36..1417d9eff 100644 --- a/nimdoc/testproject/expected/nimdoc.out.css +++ b/nimdoc/testproject/expected/nimdoc.out.css @@ -147,6 +147,15 @@ body { box-sizing: border-box; margin-left: 1%; } +@media print { + #global-links, .link-seesrc, .theme-switch-wrapper, #searchInputDiv, .search-groupby { + display:none; + } + .columns { + width:100% !important; + } +} + .column:first-child, .columns:first-child { margin-left: 0; } From 2620da9bf9994d09da3d5b31b29c95855fb79a41 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Wed, 4 Jan 2023 23:19:01 +0300 Subject: [PATCH 2345/2472] docgen: implement cross-document links (#20990) * docgen: implement cross-document links Fully implements https://github.com/nim-lang/RFCs/issues/125 Follow-up of: https://github.com/nim-lang/Nim/pull/18642 (for internal links) and https://github.com/nim-lang/Nim/issues/20127. Overview -------- Explicit import-like directive is required, called `.. importdoc::`. (the syntax is % RST, Markdown will use it for a while). Then one can reference any symbols/headings/anchors, as if they were in the local file (but they will be prefixed with a module name or markup document in link text). It's possible to reference anything from anywhere (any direction in `.nim`/`.md`/`.rst` files). See `doc/docgen.md` for full description. Working is based on `.idx` files, hence one needs to generate all `.idx` beforehand. A dedicated option `--index:only` is introduced (and a separate stage for `--index:only` is added to `kochdocs.nim`). Performance note ---------------- Full run for `./koch docs` now takes 185% of the time before this PR. (After: 315 s, before: 170 s on my PC). All the time seems to be spent on `--index:only` run, which takes almost as much (85%) of normal doc run -- it seems that most time is spent on file parsing, turning off HTML generation phase has not helped much. (One could avoid it by specifying list of files that can be referenced and pre-processing only them. But it can become error-prone and I assume that these linke will be **everywhere** in the repository anyway, especially considering https://github.com/nim-lang/RFCs/issues/478. So every `.nim`/`.md` file is processed for `.idx` first). But that's all without significant part of repository converted to cross-module auto links. To estimate impact I checked the time for `doc`ing a few files (after all indexes have been generated), and everywhere difference was **negligible**. E.g. for `lib/std/private/osfiles.nim` that `importdoc`s large `os.idx` and hence should have been a case with relatively large performance impact, but: * After: 0.59 s. * Before: 0.59 s. So Nim compiler works so slow that doc part basically does not matter :-) Testing ------- 1) added `extlinks` test to `nimdoc/` 2) checked that `theindex.html` is still correct 2) fixed broken auto-links for modules that were derived from `os.nim` by adding appropriate ``importdoc`` Implementation note ------------------- Parsing and formating of `.idx` entries is moved into a dedicated `rstidx.nim` module from `rstgen.nim`. `.idx` file format changed: * fields are not escaped in most cases because we need original strings for referencing, not HTML ones (the exception is linkTitle for titles and headings). Escaping happens later -- on the stage of `rstgen` buildIndex, etc. * all lines have fixed number of columns 6 * added discriminator tag as a first column, it always allows distinguish Nim/markup entries, titles/headings, etc. `rstgen` does not rely any more (in most cases) on ad-hoc logic to determine what type each entry is. * there is now always a title entry added at the first line. * add a line number as 6th column * linkTitle (4th) column has a different format: before it was like `module: funcName()`, now it's `proc funcName()`. (This format is also propagated to `theindex.html` and search results, I kept it that way since I like it more though it's discussible.) This column is what used for Nim symbols resolution. * also changed details on column format for headings and titles: "keyword" is original, "linkTitle" is HTML one * fix paths on Windows + more clear code * Update compiler/docgen.nim Co-authored-by: Andreas Rumpf * Handle .md and .nim paths uniformly in findRefFile * handle titles better + more comments * don't allow markup overwrite index title for .nim files Co-authored-by: Andreas Rumpf --- compiler/commands.nim | 8 +- compiler/docgen.nim | 200 ++++++++---- compiler/lineinfos.nim | 2 + compiler/options.nim | 2 + doc/advopt.txt | 4 +- doc/docgen.md | 309 ++++++++++++++++-- doc/markdown_rst.md | 84 +++++ lib/core/macros.nim | 2 + lib/packages/docutils/dochelpers.nim | 27 +- lib/packages/docutils/rst.nim | 263 +++++++++++++-- lib/packages/docutils/rstgen.nim | 200 ++++-------- lib/packages/docutils/rstidx.nim | 138 ++++++++ lib/pure/os.nim | 2 + lib/std/appdirs.nim | 2 + lib/std/private/osappdirs.nim | 2 + lib/std/private/oscommon.nim | 2 + lib/std/private/osdirs.nim | 2 + lib/std/private/osfiles.nim | 3 +- lib/std/private/ospaths2.nim | 2 + lib/std/private/osseps.nim | 2 + lib/std/private/ossymlinks.nim | 1 + lib/std/symlinks.nim | 3 +- nimdoc/extlinks/project/doc/manual.md | 17 + .../extlinks/project/expected/_._/util.html | 104 ++++++ nimdoc/extlinks/project/expected/_._/util.idx | 2 + .../extlinks/project/expected/doc/manual.html | 44 +++ .../extlinks/project/expected/doc/manual.idx | 3 + nimdoc/extlinks/project/expected/main.html | 144 ++++++++ nimdoc/extlinks/project/expected/main.idx | 4 + .../project/expected/sub/submodule.html | 130 ++++++++ .../project/expected/sub/submodule.idx | 3 + .../extlinks/project/expected/theindex.html | 58 ++++ nimdoc/extlinks/project/main.nim | 23 ++ nimdoc/extlinks/project/sub/submodule.nim | 13 + nimdoc/extlinks/util.nim | 2 + .../test_out_index_dot_html/expected/foo.idx | 3 +- .../expected/theindex.html | 2 +- nimdoc/tester.nim | 63 ++++ .../expected/subdir/subdir_b/utils.html | 6 +- .../expected/subdir/subdir_b/utils.idx | 87 ++--- nimdoc/testproject/expected/testproject.idx | 131 ++++---- nimdoc/testproject/expected/theindex.html | 196 +++++------ tests/stdlib/tdochelpers.nim | 16 + tests/stdlib/trst.nim | 2 +- tools/kochdocs.nim | 41 ++- 45 files changed, 1863 insertions(+), 491 deletions(-) create mode 100644 lib/packages/docutils/rstidx.nim create mode 100644 nimdoc/extlinks/project/doc/manual.md create mode 100644 nimdoc/extlinks/project/expected/_._/util.html create mode 100644 nimdoc/extlinks/project/expected/_._/util.idx create mode 100644 nimdoc/extlinks/project/expected/doc/manual.html create mode 100644 nimdoc/extlinks/project/expected/doc/manual.idx create mode 100644 nimdoc/extlinks/project/expected/main.html create mode 100644 nimdoc/extlinks/project/expected/main.idx create mode 100644 nimdoc/extlinks/project/expected/sub/submodule.html create mode 100644 nimdoc/extlinks/project/expected/sub/submodule.idx create mode 100644 nimdoc/extlinks/project/expected/theindex.html create mode 100644 nimdoc/extlinks/project/main.nim create mode 100644 nimdoc/extlinks/project/sub/submodule.nim create mode 100644 nimdoc/extlinks/util.nim diff --git a/compiler/commands.nim b/compiler/commands.nim index a252f505f..73140036a 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -820,7 +820,13 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; if conf != nil: conf.headerFile = arg incl(conf.globalOptions, optGenIndex) of "index": - processOnOffSwitchG(conf, {optGenIndex}, arg, pass, info) + case arg.normalize + of "", "on": conf.globalOptions.incl {optGenIndex} + of "only": conf.globalOptions.incl {optGenIndexOnly, optGenIndex} + of "off": conf.globalOptions.excl {optGenIndex, optGenIndexOnly} + else: localError(conf, info, errOnOrOffExpectedButXFound % arg) + of "noimportdoc": + processOnOffSwitchG(conf, {optNoImportdoc}, arg, pass, info) of "import": expectArg(conf, switch, arg, pass, info) if pass in {passCmd2, passPP}: diff --git a/compiler/docgen.nim b/compiler/docgen.nim index f8a804cec..cab334e7a 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -7,13 +7,17 @@ # distribution, for details about the copyright. # -# This is the documentation generator. Cross-references are generated -# by knowing how the anchors are going to be named. +## This is the Nim documentation generator. Cross-references are generated +## by knowing how the anchors are going to be named. +## +## .. importdoc:: ../docgen.md +## +## For corresponding users' documentation see [Nim DocGen Tools Guide]. import ast, strutils, strtabs, algorithm, sequtils, options, msgs, os, idents, wordrecg, syntaxes, renderer, lexer, - packages/docutils/[rst, rstgen, dochelpers], + packages/docutils/[rst, rstidx, rstgen, dochelpers], json, xmltree, trees, types, typesrenderer, astalgo, lineinfos, intsets, pathutils, tables, nimpaths, renderverbatim, osproc, packages @@ -91,7 +95,7 @@ type jEntriesFinal: JsonNode # final JSON after RST pass 2 and rendering types: TStrTable sharedState: PRstSharedState - standaloneDoc: bool + standaloneDoc: bool # is markup (.rst/.md) document? conf*: ConfigRef cache*: IdentCache exampleCounter: int @@ -225,7 +229,7 @@ proc attachToType(d: PDoc; p: PSym): PSym = if params.len > 0: check(0) for i in 2.."), "itemSymOrIDEnc", symbolOrIdEnc]) - # Ironically for types the complexSymbol is *cleaner* than the plainName - # because it doesn't include object fields or documentation comments. So we - # use the plain one for callable elements, and the complex for the rest. - var linkTitle = changeFileExt(extractFilename(d.filename), "") & ": " - if n.kind in routineDefs: linkTitle.add(xmltree.escape(plainName.strip)) - else: linkTitle.add(xmltree.escape(complexSymbol.strip)) - - setIndexTerm(d[], external, symbolOrId, name, linkTitle, - xmltree.escape(plainDocstring.docstringSummary)) + setIndexTerm(d[], ieNim, htmlFile = external, id = symbolOrId, term = name, + linkTitle = detailedName, + linkDesc = xmltree.escape(plainDocstring.docstringSummary), + line = n.info.line.int) if k == skType and nameNode.kind == nkSym: d.types.strTableAdd nameNode.sym proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonItem = if not isVisible(d, nameNode): return var - name = getName(d, nameNode) + name = getNameEsc(d, nameNode) comm = genRecComment(d, n) r: TSrcGen initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments, renderExpandUsing}) @@ -1337,9 +1369,31 @@ proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) = proc overloadGroupName(s: string, k: TSymKind): string = ## Turns a name like `f` into anchor `f-procs-all` - #s & " " & k.toHumanStr & "s all" s & "-" & k.toHumanStr & "s-all" +proc setIndexTitle(d: PDoc, useMetaTitle: bool) = + let titleKind = if d.standaloneDoc: ieMarkupTitle else: ieNimTitle + let external = AbsoluteFile(d.destFile) + .relativeTo(d.conf.outDir, '/') + .changeFileExt(HtmlExt) + .string + var term, linkTitle: string + if useMetaTitle and d.meta[metaTitle].len != 0: + term = d.meta[metaTitleRaw] + linkTitle = d.meta[metaTitleRaw] + else: + let filename = extractFilename(d.filename) + term = + if d.standaloneDoc: filename # keep .rst/.md extension + else: changeFileExt(filename, "") # rm .nim extension + linkTitle = + if d.standaloneDoc: term # keep .rst/.md extension + else: canonicalImport(d.conf, AbsoluteFile d.filename) + if not d.standaloneDoc: + linkTitle = "module " & linkTitle + setIndexTerm(d[], titleKind, htmlFile = external, id = "", + term = term, linkTitle = linkTitle) + proc finishGenerateDoc*(d: var PDoc) = ## Perform 2nd RST pass for resolution of links/footnotes/headings... # copy file map `filenames` to ``rstgen.nim`` for its warnings @@ -1352,7 +1406,24 @@ proc finishGenerateDoc*(d: var PDoc) = firstRst = fragment.rst break d.hasToc = d.hasToc or d.sharedState.hasToc - preparePass2(d.sharedState, firstRst) + # in --index:only mode we do NOT want to load other .idx, only write ours: + let importdoc = optGenIndexOnly notin d.conf.globalOptions and + optNoImportdoc notin d.conf.globalOptions + preparePass2(d.sharedState, firstRst, importdoc) + + if optGenIndexOnly in d.conf.globalOptions: + # Top-level doc.comments may contain titles and :idx: statements: + for fragment in d.modDescPre: + if fragment.isRst: + traverseForIndex(d[], fragment.rst) + setIndexTitle(d, useMetaTitle = d.standaloneDoc) + # Symbol-associated doc.comments may contain :idx: statements: + for k in TSymKind: + for _, overloadChoices in d.section[k].secItems: + for item in overloadChoices: + for fragment in item.descRst: + if fragment.isRst: + traverseForIndex(d[], fragment.rst) # add anchors to overload groups before RST resolution for k in TSymKind: @@ -1362,14 +1433,25 @@ proc finishGenerateDoc*(d: var PDoc) = let refn = overloadGroupName(plainName, k) let tooltip = "$1 ($2 overloads)" % [ k.toHumanStr & " " & plainName, $overloadChoices.len] - addAnchorNim(d.sharedState, refn, tooltip, + let name = nimIdentBackticksNormalize(plainName) + # save overload group to ``.idx`` + let external = d.destFile.AbsoluteFile.relativeTo(d.conf.outDir, '/'). + changeFileExt(HtmlExt).string + setIndexTerm(d[], ieNimGroup, htmlFile = external, id = refn, + term = name, linkTitle = k.toHumanStr, + linkDesc = "", line = overloadChoices[0].info.line.int) + if optGenIndexOnly in d.conf.globalOptions: continue + addAnchorNim(d.sharedState, external=false, refn, tooltip, LangSymbol(symKind: k.toHumanStr, - name: nimIdentBackticksNormalize(plainName), + name: name, isGroup: true), priority = symbolPriority(k), # select index `0` just to have any meaningful warning: info = overloadChoices[0].info) + if optGenIndexOnly in d.conf.globalOptions: + return + # Finalize fragments of ``.nim`` or ``.rst`` file proc renderItemPre(d: PDoc, fragments: ItemPre, result: var string) = for f in fragments: @@ -1421,6 +1503,9 @@ proc finishGenerateDoc*(d: var PDoc) = d.jEntriesFinal.add entry.json # generates docs + setIndexTitle(d, useMetaTitle = d.standaloneDoc) + completePass2(d.sharedState) + proc add(d: PDoc; j: JsonItem) = if j.json != nil or j.rst != nil: d.jEntriesPre.add j @@ -1467,7 +1552,7 @@ proc generateJson*(d: PDoc, n: PNode, includeComments: bool = true) = else: discard proc genTagsItem(d: PDoc, n, nameNode: PNode, k: TSymKind): string = - result = getName(d, nameNode) & "\n" + result = getNameEsc(d, nameNode) & "\n" proc generateTags*(d: PDoc, n: PNode, r: var string) = case n.kind @@ -1574,13 +1659,7 @@ proc genOutFile(d: PDoc, groupedToc = false): string = # Extract the title. Non API modules generate an entry in the index table. if d.meta[metaTitle].len != 0: title = d.meta[metaTitle] - let external = AbsoluteFile(d.destFile) - .relativeTo(d.conf.outDir, '/') - .changeFileExt(HtmlExt) - .string - setIndexTerm(d[], external, "", title) else: - # Modules get an automatic title for the HTML, but no entry in the index. title = canonicalImport(d.conf, AbsoluteFile d.filename) title = esc(d.target, title) var subtitle = "" @@ -1619,11 +1698,17 @@ proc genOutFile(d: PDoc, groupedToc = false): string = code = content result = code +proc indexFile(d: PDoc): AbsoluteFile = + let dir = d.conf.outDir + result = dir / changeFileExt(presentationPath(d.conf, + AbsoluteFile d.filename), + IndexExt) + let (finalDir, _, _) = result.string.splitFile + createDir(finalDir) + proc generateIndex*(d: PDoc) = if optGenIndex in d.conf.globalOptions: - let dir = d.conf.outDir - createDir(dir) - let dest = dir / changeFileExt(presentationPath(d.conf, AbsoluteFile d.filename), IndexExt) + let dest = indexFile(d) writeIndexFile(d[], dest.string) proc updateOutfile(d: PDoc, outfile: AbsoluteFile) = @@ -1634,6 +1719,9 @@ proc updateOutfile(d: PDoc, outfile: AbsoluteFile) = d.conf.outFile = splitPath(d.conf.outFile.string)[1].RelativeFile proc writeOutput*(d: PDoc, useWarning = false, groupedToc = false) = + if optGenIndexOnly in d.conf.globalOptions: + d.conf.outFile = indexFile(d).relativeTo(d.conf.outDir) # just for display + return runAllExamples(d) var content = genOutFile(d, groupedToc) if optStdout in d.conf.globalOptions: @@ -1772,6 +1860,8 @@ proc commandTags*(cache: IdentCache, conf: ConfigRef) = rawMessage(conf, errCannotOpenFile, filename.string) proc commandBuildIndex*(conf: ConfigRef, dir: string, outFile = RelativeFile"") = + if optGenIndexOnly in conf.globalOptions: + return var content = mergeIndexes(dir) var outFile = outFile diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index cb60d8949..89aff57df 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -57,6 +57,7 @@ type warnRstBrokenLink = "BrokenLink", warnRstLanguageXNotSupported = "LanguageXNotSupported", warnRstFieldXNotSupported = "FieldXNotSupported", + warnRstUnusedImportdoc = "UnusedImportdoc", warnRstStyle = "warnRstStyle", warnCommentXIgnored = "CommentXIgnored", warnTypelessParam = "TypelessParam", @@ -142,6 +143,7 @@ const warnRstBrokenLink: "broken link '$1'", warnRstLanguageXNotSupported: "language '$1' not supported", warnRstFieldXNotSupported: "field '$1' not supported", + warnRstUnusedImportdoc: "importdoc for '$1' is not used", warnRstStyle: "RST style: $1", warnCommentXIgnored: "comment '$1' ignored", warnTypelessParam: "", # deadcode diff --git a/compiler/options.nim b/compiler/options.nim index 47e8155f8..fc970e420 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -78,6 +78,8 @@ type # please make sure we have under 32 options optThreadAnalysis, # thread analysis pass optTlsEmulation, # thread var emulation turned on optGenIndex # generate index file for documentation; + optGenIndexOnly # generate only index file for documentation + optNoImportdoc # disable loading external documentation files optEmbedOrigSrc # embed the original source in the generated code # also: generate header file optIdeDebug # idetools: debug mode diff --git a/doc/advopt.txt b/doc/advopt.txt index 3f439fdab..efff428e8 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -130,7 +130,9 @@ Advanced options: select which memory management to use; default is 'orc' --exceptions:setjmp|cpp|goto|quirky select the exception handling implementation - --index:on|off turn index file generation on|off + --index:on|off|only docgen: turn index file generation? (`only` means + not generate output files like HTML) + --noImportdoc:on|off disable loading documentation ``.idx`` files? --putenv:key=value set an environment variable --NimblePath:PATH add a path for Nimble support --noNimblePath deactivate the Nimble path diff --git a/doc/docgen.md b/doc/docgen.md index e4b040064..70b41f17d 100644 --- a/doc/docgen.md +++ b/doc/docgen.md @@ -9,6 +9,7 @@ .. include:: rstcommon.rst .. contents:: +.. importdoc:: markdown_rst.md, compiler/docgen.nim Introduction ============ @@ -103,6 +104,64 @@ won't influence RST formatting. ## Paragraph. ``` +Structuring output directories +------------------------------ + +Basic directory for output is set by `--outdir:OUTDIR`:option: switch, +by default `OUTDIR` is ``htmldocs`` sub-directory in the directory of +the processed file. + +There are 2 basic options as to how generated HTML output files are stored: + +1) complex hierarchy when docgen-compiling with `--project`:option:, + which follows directory structure of the project itself. + So `nim doc`:cmd: replicates project's directory structure + inside `--outdir:OUTDIR`:option: directory. + `--project`:option: is well suited for projects that have 1 main module. + File name clashes are impossible in this case. + +2) flattened structure, where user-provided script goes through all + needed input files and calls commands like `nim doc`:cmd: + with `--outdir:OUTDIR`:option: switch, thus putting all HTML (and + ``.idx``) files into 1 directory. + + .. Important:: Make sure that you don't have files with same base name + like ``x.nim`` and ``x.md`` in the same package, otherwise you'll + have name conflict for ``x.html``. + + .. Tip:: To structure your output directories and avoid file name + clashes you can split your project into + different *packages* -- parts of your repository that are + docgen-compiled with different `--outdir:OUTDIR`:option: options. + + An example of such strategy is Nim repository itself which has: + + * its stdlib ``.nim`` files from different directories and ``.md`` + documentation from ``doc/`` directory are all docgen-compiled + into `--outdir:web/upload//`:option: directory + * its ``.nim`` files from ``compiler/`` directory are docgen-compiled + into `--outdir:web/upload//compiler/`:option: directory. + Interestingly, it's compiled with complex hierarchy using + `--project`:option: switch. + + Contents of ``web/upload/`` are then deployed into Nim's + Web server. + + This output directory structure allows to work correctly with files like + ``compiler/docgen.nim`` (implementation) and ``doc/docgen.md`` (user + documentation) in 1 repository. + + +Index files +----------- + +Index (``.idx``) files are used for 2 different purposes: + +1. easy cross-referencing between different ``.nim`` and/or ``.md`` / ``.rst`` + files described in [Nim external referencing] +2. creating a whole-project index for searching of symbols and keywords, + see [Buildindex command]. + Document Types ============== @@ -226,13 +285,46 @@ Note that the `jsondoc`:option: command outputs its JSON without pretty-printing while `jsondoc0`:option: outputs pretty-printed JSON. -Referencing Nim symbols: simple documentation links -=================================================== +Simple documentation links +========================== -You can reference Nim identifiers from Nim documentation comments, currently -only inside their ``.nim`` file (or inside a ``.rst`` file included from -a ``.nim``). The point is that such links will be resolved automatically -by `nim doc`:cmd: (or `nim jsondoc`:cmd: or `nim doc2tex`:cmd:). +It's possible to use normal Markdown/RST syntax to *manually* +reference Nim symbols using HTML anchors, however Nim has an *automatic* +facility that makes referencing inside ``.nim`` and ``.md/.rst`` files and +between them easy and seamless. +The point is that such links will be resolved automatically +by `nim doc`:cmd: (or `md2html`:option:, or `jsondoc`:option:, +or `doc2tex`:option:, ...). And, unlike manual links, such automatic +links **check** that their target exists -- a warning is emitted for +any broken link, so you avoid broken links in your project. + +Nim treats both ``.md/.rst`` files and ``.nim`` modules (their doc comment +part) as *documents* uniformly. +Hence all directions of referencing are equally possible having the same syntax: + +1. ``.md/rst`` -> itself (internal). See [Markup local referencing]. +2. ``.md/rst`` -> external ``.md/rst``. See [Markup external referencing]. + To summarize, referencing in `.md`/`.rst` files was already described in + [Nim-flavored Markdown and reStructuredText] + (particularly it described usage of index files for referencing), + while in this document we focus on Nim-specific details. +3. ``.md/rst`` -> external ``.nim``. See [Nim external referencing]. +4. ``.nim`` -> itself (internal). See [Nim local referencing]. +5. ``.nim`` -> external ``.md/rst``. See [Markup external referencing]. +6. ``.nim`` -> external ``.nim``. See [Nim external referencing]. + +To put it shortly, local referencing always works out of the box, +external referencing requires to use ``.. importdoc:: `` +directive to import `file` and to ensure that the corresponding +``.idx`` file was generated. + + +Nim local referencing +--------------------- + +You can reference Nim identifiers from Nim documentation comments +inside their ``.nim`` file (or inside a ``.rst`` file included from +a ``.nim``). This pertains to any exported symbol like `proc`, `const`, `iterator`, etc. Syntax for referencing is basically a normal RST one: addition of underscore `_` to a *link text*. @@ -405,6 +497,143 @@ recognized fine: ... ## Ref. `CopyFlag enum`_ +Nim external referencing +------------------------ + +Just like for [Markup external referencing], which saves markup anchors, +the Nim symbols are also saved in ``.idx`` files, so one needs +to generate them beforehand, and they should be loaded by +an ``.. importdoc::`` directive. Arguments to ``.. importdoc::`` is a +comma-separated list of Nim modules or Markdown/RST documents. + +`--index:only`:option: tells Nim to only generate ``.idx`` file and +do **not** attempt to generate HTML/LaTeX output. +For ``.nim`` modules there are 2 alternatives to work with ``.idx`` files: + +1. using [Project switch] implies generation of ``.idx`` files, + however, if ``importdoc`` is called on upper modules as its arguments, + their ``.idx`` are not yet created. Thus one should generate **all** + required ``.idx`` first: + ```cmd + nim doc --project --index:only
          .nim + nim doc --project
          .nim + ``` +2. or run `nim doc --index:only `:cmd: command for **all** (used) + Nim modules in your project. Then run `nim doc ` on them for + output HTML generation. + + .. Warning:: A mere `nim doc --index:on`:cmd: may fail on an attempt to do + ``importdoc`` from another module (for which ``.idx`` was not yet + generated), that's why `--index:only`:option: shall be used instead. + + For ``.md``/``.rst`` markup documents point 2 is the only option. + +Then, you can freely use something like this in ``your_module.nim``: + + ```nim + ## .. importdoc:: user_manual.md, another_module.nim + + ... + ## Ref. [some section from User Manual]. + + ... + ## Ref. [proc f] + ## (assuming you have a proc `f` in ``another_module``). + ``` + +and compile it by `nim doc`:cmd:. Note that link text will +be automatically prefixed by the module name of symbol, +so you will see something like "Ref. [another_module: proc f](#)" +in the generated output. + +It's also possible to reference a whole module by prefixing or +suffixing full canonical module name with "module": + + Ref. [module subdir/name] or [subdir/name module]. + +Markup documents as a whole can be referenced just by their title +(or by their file name if the title was not set) without any prefix. + +.. Tip:: During development process the stage of ``.idx`` files generation + can be done only *once*, after that you use already generated ``.idx`` + files while working with a document *being developed* (unless you do + incompatible changes to *referenced* documents). + +.. Hint:: After changing a *referenced* document file one may need + to regenerate its corresponding ``.idx`` file to get correct results. + Of course, when referencing *internally* inside any given ``.nim`` file, + it's not needed, one can even immediately use any freshly added anchor + (a document's own ``.idx`` file is not used for resolving its internal links). + +If an ``importdoc`` directive fails to find a ``.idx``, then an error +is emitted. + +In case of such compilation failures please note that: + +* **all** relative paths, given to ``importdoc``, relate to insides of + ``OUTDIR``, and **not** project's directory structure. + +* ``importdoc`` searches for ``.idx`` in `--outdir:OUTDIR`:option: directory + (``htmldocs`` by default) and **not** around original modules, so: + + .. Tip:: look into ``OUTDIR`` to understand what's going on. + +* also keep in mind that ``.html`` and ``.idx`` files should always be + output to the same directory, so check this and, if it's not true, check + that both runs *with* and *without* `--index:only`:option: have all + other options the same. + +To summarize, for 2 basic options of [Structuring output directories] +compilation options are different: + +1) complex hierarchy with `--project`:option: switch. + + As the **original** project's directory structure is replicated in + `OUTDIR`, all passed paths are related to this structure also. + + E.g. if a module ``path1/module.nim`` does + ``.. importdoc:: path2/another.nim`` then docgen tries to load file + ``OUTDIR/path1/path2/another.idx``. + + .. Note:: markup documents are just placed into the specified directory + `OUTDIR`:option: by default (i.e. they are **not** affected by + `--project`:option:), so if you have ``PROJECT/doc/manual.md`` + document and want to use complex hirearchy (with ``doc/``), + compile it with `--docroot`:option:\: + ```cmd + # 1st stage + nim md2html --outdir:OUTDIR --docroot:/absolute/path/to/PROJECT \ + --index:only PROJECT/doc/manual.md + ... + # 2nd stage + nim md2html --outdir:OUTDIR --docroot:/absolute/path/to/PROJECT \ + PROJECT/doc/manual.md + ``` + + Then the output file will be placed as ``OUTDIR/doc/manual.idx``. + So if you have ``PROJECT/path1/module.nim``, then ``manual.md`` can + be referenced as ``../doc/manual.md``. + +2) flattened structure. + + E.g. if a module ``path1/module.nim`` does + ``.. importdoc:: path2/another.nim`` then docgen tries to load + ``OUTDIR/path2/another.idx``, so the path ``path1`` + does not matter and providing ``path2`` can be useful only + in the case it contains another package that was placed there + using `--outdir:OUTDIR/path2`:option:. + + The links' text will be prefixed as ``another: ...`` in both cases. + + .. Warning:: Again, the same `--outdir:OUTDIR`:option: option should + be provided to both `doc --index:only`:option: / + `md2html --index:only`:option: and final generation by + `doc`:option:/`md2html`:option: inside 1 package. + +To temporarily disable ``importdoc``, e.g. if you don't need +correct link resolution at the moment, use a `--noImportdoc`:option: switch +(only warnings about unresolved links will be generated for external references). + Related Options =============== @@ -434,10 +663,21 @@ index file is line-oriented (newlines have to be escaped). Each line represents a tab-separated record of several columns, the first two mandatory, the rest optional. See the [Index (idx) file format] section for details. +.. Note:: `--index`:option: switch only affects creation of ``.idx`` + index files, while user-searchable Index HTML file is created by + `buildIndex`:option: commmand. + +Buildindex command +------------------ + Once index files have been generated for one or more modules, the Nim -compiler command `buildIndex directory` can be run to go over all the index +compiler command `nim buildIndex directory`:cmd: can be run to go over all the index files in the specified directory to generate a [theindex.html](theindex.html) -file. +file: + + ```cmd + nim buildIndex -o:path/to/htmldocs/theindex.html path/to/htmldocs + ``` See source switch ----------------- @@ -568,10 +808,22 @@ references so they can be later concatenated into a big index file with the file format in detail. Index files are line-oriented and tab-separated (newline and tab characters -have to be escaped). Each line represents a record with at least two fields -but can have up to four (additional columns are ignored). The content of these -columns is: +have to be escaped). Each line represents a record with 6 fields. +The content of these columns is: +0. Discriminator tag denoting type of the index entry, allowed values are: + `markupTitle` + : a title for ``.md``/``.rst`` document + `nimTitle` + : a title of ``.nim`` module + `heading` + : heading of sections, can be both in Nim and markup files + `idx` + : terms marked with :idx: role + `nim` + : a Nim symbol + `nimgrp` + : a Nim group for overloadable symbols like `proc`s 1. Mandatory term being indexed. Terms can include quoting according to Nim's rules (e.g. \`^\`). 2. Base filename plus anchor hyperlink (e.g. ``algorithm.html#*,int,SortOrder``). @@ -581,29 +833,20 @@ columns is: not for an API symbol but for a TOC entry. 4. Optional title or description of the hyperlink. Browsers usually display this as a tooltip after hovering a moment over the hyperlink. +5. A line number of file where the entry was defined. -The index generation tools try to differentiate between documentation -generated from ``.nim`` files and documentation generated from ``.txt`` or -``.rst`` files. The former are always closely related to source code and -consist mainly of API entries. The latter are generic documents meant for -human reading. +The index generation tools differentiate between documentation +generated from ``.nim`` files and documentation generated from ``.md`` or +``.rst`` files by tag `nimTitle` or `markupTitle` in the 1st line of +the ``.idx`` file. -To differentiate both types (documents and APIs), the index generator will add -to the index of documents an entry with the title of the document. Since the -title is the topmost element, it will be added with a second field containing -just the filename without any HTML anchor. By convention, this entry without -anchor is the *title entry*, and since entries in the index file are added as -they are scanned, the title entry will be the first line. The title for APIs -is not present because it can be generated concatenating the name of the file -to the word **Module**. - -Normal symbols are added to the index with surrounding whitespaces removed. An -exception to this are the table of content (TOC) entries. TOC entries are added to -the index file with their third column having as much prefix spaces as their -level is in the TOC (at least 1 character). The prefix whitespace helps to -filter TOC entries from API or text symbols. This is important because the -amount of spaces is used to replicate the hierarchy for document TOCs in the -final index, and TOC entries found in ``.nim`` files are discarded. +.. TODO Normal symbols are added to the index with surrounding whitespaces removed. An + exception to this are the table of content (TOC) entries. TOC entries are added to + the index file with their third column having as much prefix spaces as their + level is in the TOC (at least 1 character). The prefix whitespace helps to + filter TOC entries from API or text symbols. This is important because the + amount of spaces is used to replicate the hierarchy for document TOCs in the + final index, and TOC entries found in ``.nim`` files are discarded. Additional resources @@ -615,6 +858,8 @@ Additional resources [Markdown and RST markup languages](markdown_rst.html), which also contains the list of implemented features of these markup languages. +* the implementation is in [module compiler/docgen]. + The output for HTML and LaTeX comes from the ``config/nimdoc.cfg`` and ``config/nimdoc.tex.cfg`` configuration files. You can add and modify these files to your project to change the look of the docgen output. diff --git a/doc/markdown_rst.md b/doc/markdown_rst.md index 5fb3caefc..a374749cb 100644 --- a/doc/markdown_rst.md +++ b/doc/markdown_rst.md @@ -9,6 +9,8 @@ Nim-flavored Markdown and reStructuredText .. include:: rstcommon.rst .. contents:: +.. importdoc:: docgen.md + Both `Markdown`:idx: (md) and `reStructuredText`:idx: (RST) are markup languages whose goal is to typeset texts with complex structure, formatting and references using simple plaintext representation. @@ -110,6 +112,8 @@ Supported standard RST features: Additional Nim-specific features -------------------------------- +* referencing to definitions in external files, see + [Markup external referencing] section * directives: ``code-block`` \[cmp:Sphinx], ``title``, ``index`` \[cmp:Sphinx] * predefined roles @@ -170,6 +174,86 @@ Optional additional features, by default turned on: .. warning:: Using Nim-specific features can cause other RST implementations to fail on your document. +Referencing +=========== + +To be able to copy and share links Nim generates anchors for all +main document elements: + +* headlines (including document title) +* footnotes +* explicitly set anchors: RST internal cross-references and + inline internal targets +* Nim symbols (external referencing), see [Nim DocGen Tools Guide] for details. + +But direct use of those anchors have 2 problems: + +1. the anchors are usually mangled (e.g. spaces substituted to minus + signs, etc). +2. manual usage of anchors is not checked, so it's easy to get broken + links inside your project if e.g. spelling has changed for a heading + or you use a wrong relative path to your document. + +That's why Nim implementation has syntax for using +*original* labels for referencing. +Such referencing can be either local/internal or external: + +* Local referencing (inside any given file) is defined by + RST standard or Pandoc Markdown User guide. +* External (cross-document) referencing is a Nim-specific feature, + though it's not really different from local referencing by its syntax. + +Markup local referencing +------------------------ + +There are 2 syntax option available for referencing to objects +inside any given file, e.g. for headlines: + + Markdown RST + + Some headline Some headline + ============= ============= + + Ref. [Some headline] Ref. `Some headline`_ + + +Markup external referencing +--------------------------- + +The syntax is the same as for local referencing, but the anchors are +saved in ``.idx`` files, so one needs to generate them beforehand, +and they should be loaded by an `.. importdoc::` directive. +E.g. if we want to reference section "Some headline" in ``file1.md`` +from ``file2.md``, then ``file2.md`` may look like: + +``` +.. importdoc:: file1.md + +Ref. [Some headline] +``` + +```cmd +nim md2html --index:only file1.md # creates ``htmldocs/file1.idx`` +nim md2html file2.md # creates ``htmldocs/file2.html`` +``` + +To allow cross-references between any files in any order (especially, if +circular references are present), it's strongly reccommended +to make a run for creating all the indexes first: + +```cmd +nim md2html --index:only file1.md # creates ``htmldocs/file1.idx`` +nim md2html --index:only file2.md # creates ``htmldocs/file2.idx`` +nim md2html file1.md # creates ``htmldocs/file1.html`` +nim md2html file2.md # creates ``htmldocs/file2.html`` +``` + +and then one can freely reference any objects as if these 2 documents +are actually 1 file. + +Other +===== + Idiosyncrasies -------------- diff --git a/lib/core/macros.nim b/lib/core/macros.nim index bcc16038b..beb3aa695 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -23,6 +23,8 @@ when defined(nimPreviewSlimSystem): ## .. include:: ../../doc/astspec.txt +## .. importdoc:: system.nim + # If you look for the implementation of the magic symbol # ``{.magic: "Foo".}``, search for `mFoo` and `opcFoo`. diff --git a/lib/packages/docutils/dochelpers.nim b/lib/packages/docutils/dochelpers.nim index c7f7f73f5..b7cdd16d2 100644 --- a/lib/packages/docutils/dochelpers.nim +++ b/lib/packages/docutils/dochelpers.nim @@ -13,7 +13,7 @@ ## `type LangSymbol`_ in ``rst.nim``, while `match(generated, docLink)`_ ## matches it with `generated`, produced from `PNode` by ``docgen.rst``. -import rstast +import rstast, strutils when defined(nimPreviewSlimSystem): import std/[assertions, syncio] @@ -35,6 +35,12 @@ type ## name-type seq, e.g. for proc outType*: string ## result type, e.g. for proc +proc `$`*(s: LangSymbol): string = # for debug + ("(symkind=$1, symTypeKind=$2, name=$3, generics=$4, isGroup=$5, " & + "parametersProvided=$6, parameters=$7, outType=$8)") % [ + s.symKind, s.symTypeKind , s.name, s.generics, $s.isGroup, + $s.parametersProvided, $s.parameters, s.outType] + func nimIdentBackticksNormalize*(s: string): string = ## Normalizes the string `s` as a Nim identifier. ## @@ -71,6 +77,12 @@ func nimIdentBackticksNormalize*(s: string): string = else: discard # just omit '`' or ' ' if j != s.len: setLen(result, j) +proc langSymbolGroup*(kind: string, name: string): LangSymbol = + if kind notin ["proc", "func", "macro", "method", "iterator", + "template", "converter"]: + raise newException(ValueError, "unknown symbol kind $1" % [kind]) + result = LangSymbol(symKind: kind, name: name, isGroup: true) + proc toLangSymbol*(linkText: PRstNode): LangSymbol = ## Parses `linkText` into a more structured form using a state machine. ## @@ -82,11 +94,14 @@ proc toLangSymbol*(linkText: PRstNode): LangSymbol = ## ## This proc should be kept in sync with the `renderTypes` proc from ## ``compiler/typesrenderer.nim``. - assert linkText.kind in {rnRstRef, rnInner} + template fail(msg: string) = + raise newException(ValueError, msg) + if linkText.kind notin {rnRstRef, rnInner}: + fail("toLangSymbol: wrong input kind " & $linkText.kind) const NimDefs = ["proc", "func", "macro", "method", "iterator", "template", "converter", "const", "type", "var", - "enum", "object", "tuple"] + "enum", "object", "tuple", "module"] template resolveSymKind(x: string) = if x in ["enum", "object", "tuple"]: result.symKind = "type" @@ -109,11 +124,11 @@ proc toLangSymbol*(linkText: PRstNode): LangSymbol = template flushIdent() = if curIdent != "": case state - of inBeginning: doAssert false, "incorrect state inBeginning" + of inBeginning: fail("incorrect state inBeginning") of afterSymKind: resolveSymKind curIdent - of beforeSymbolName: doAssert false, "incorrect state beforeSymbolName" + of beforeSymbolName: fail("incorrect state beforeSymbolName") of atSymbolName: result.name = curIdent.nimIdentBackticksNormalize - of afterSymbolName: doAssert false, "incorrect state afterSymbolName" + of afterSymbolName: fail("incorrect state afterSymbolName") of genericsPar: result.generics = curIdent of parameterName: result.parameters.add (curIdent, "") of parameterType: diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index e9f76a26f..b3b1fb9c0 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -22,7 +22,7 @@ import os, strutils, rstast, dochelpers, std/enumutils, algorithm, lists, sequtils, - std/private/miscdollars, tables, strscans + std/private/miscdollars, tables, strscans, rstidx from highlite import SourceLanguage, getSourceLanguage when defined(nimPreviewSlimSystem): @@ -40,7 +40,7 @@ type roNimFile ## set for Nim files where default interpreted ## text role should be :nim: roSandboxDisabled ## this option enables certain options - ## (e.g. raw, include) + ## (e.g. raw, include, importdoc) ## which are disabled by default as they can ## enable users to read arbitrary data and ## perform XSS if the parser is used in a web @@ -73,11 +73,17 @@ type mwUnsupportedLanguage = "language '$1' not supported", mwUnsupportedField = "field '$1' not supported", mwRstStyle = "RST style: $1", + mwUnusedImportdoc = "importdoc for '$1' is not used", meSandboxedDirective = "disabled directive: '$1'", MsgHandler* = proc (filename: string, line, col: int, msgKind: MsgKind, arg: string) {.closure, gcsafe.} ## what to do in case of an error FindFileHandler* = proc (filename: string): string {.closure, gcsafe.} + FindRefFileHandler* = + proc (targetRelPath: string): + tuple[targetPath: string, linkRelPath: string] {.closure, gcsafe.} + ## returns where .html or .idx file should be found by its relative path; + ## `linkRelPath` is a prefix to be added before a link anchor from such file proc rstnodeToRefname*(n: PRstNode): string proc addNodes*(n: PRstNode): string @@ -333,7 +339,8 @@ type arInternalRst, ## For automatically generated RST anchors (from ## headings, footnotes, inline internal targets): ## case-insensitive, 1-space-significant (by RST spec) - arNim ## For anchors generated by ``docgen.rst``: Nim-style case + arExternalRst, ## For external .nim doc comments or .rst/.md + arNim ## For anchors generated by ``docgen.nim``: Nim-style case ## sensitivity, etc. (see `proc normalizeNimName`_ for details) arHyperlink, ## For links with manually set anchors in ## form `text `_ @@ -349,11 +356,15 @@ type of arInternalRst: anchorType: RstAnchorKind target: PRstNode + of arExternalRst: + anchorTypeExt: RstAnchorKind + refnameExt: string of arNim: tooltip: string # displayed tooltip for Nim-generated anchors langSym: LangSymbol refname: string # A reference name that will be inserted directly # into HTML/Latex. + external: bool AnchorSubstTable = Table[string, seq[AnchorSubst]] # use `seq` to account for duplicate anchors FootnoteType = enum @@ -372,6 +383,12 @@ type RstFileTable* = object filenameToIdx*: Table[string, FileIndex] idxToFilename*: seq[string] + ImportdocInfo = object + used: bool # was this import used? + fromInfo: TLineInfo # place of `.. importdoc::` directive + idxPath: string # full path to ``.idx`` file + linkRelPath: string # prefix before target anchor + title: string # document title obtained from ``.idx`` RstSharedState = object options*: RstParseOptions # parsing options hLevels: LevelMap # hierarchy of heading styles @@ -393,12 +410,17 @@ type footnotes: seq[FootnoteSubst] # correspondence b/w footnote label, # number, order of occurrence msgHandler: MsgHandler # How to handle errors. - findFile: FindFileHandler # How to find files. + findFile: FindFileHandler # How to find files for include. + findRefFile: FindRefFileHandler + # How to find files imported by importdoc. filenames*: RstFileTable # map file name <-> FileIndex (for storing # file names for warnings after 1st stage) currFileIdx*: FileIndex # current index in `filenames` tocPart*: seq[PRstNode] # all the headings of a document hasToc*: bool + idxImports*: Table[string, ImportdocInfo] + # map `importdoc`ed filename -> it's info + nimFileImported*: bool # Was any ``.nim`` module `importdoc`ed ? PRstSharedState* = ref RstSharedState ManualAnchor = object @@ -452,6 +474,9 @@ proc defaultFindFile*(filename: string): string = if fileExists(filename): result = filename else: result = "" +proc defaultFindRefFile*(filename: string): (string, string) = + (filename, "") + proc defaultRole(options: RstParseOptions): string = if roNimFile in options: "nim" else: "literal" @@ -492,12 +517,16 @@ proc getFilename(filenames: RstFileTable, fid: FileIndex): string = $fid.int, $(filenames.len - 1)]) result = filenames.idxToFilename[fid.int] +proc getFilename(s: PRstSharedState, subst: AnchorSubst): string = + getFilename(s.filenames, subst.info.fileIndex) + proc currFilename(s: PRstSharedState): string = getFilename(s.filenames, s.currFileIdx) proc newRstSharedState*(options: RstParseOptions, filename: string, findFile: FindFileHandler, + findRefFile: FindRefFileHandler, msgHandler: MsgHandler, hasToc: bool): PRstSharedState = let r = defaultRole(options) @@ -507,6 +536,9 @@ proc newRstSharedState*(options: RstParseOptions, options: options, msgHandler: if not isNil(msgHandler): msgHandler else: defaultMsgHandler, findFile: if not isNil(findFile): findFile else: defaultFindFile, + findRefFile: + if not isNil(findRefFile): findRefFile + else: defaultFindRefFile, hasToc: hasToc ) setCurrFilename(result, filename) @@ -525,6 +557,14 @@ proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string) = proc rstMessage(s: PRstSharedState, msgKind: MsgKind, arg: string) = s.msgHandler(s.currFilename, LineRstInit, ColRstInit, msgKind, arg) +proc rstMessage(s: PRstSharedState, msgKind: MsgKind, arg: string; + line, col: int) = + s.msgHandler(s.currFilename, line, col, msgKind, arg) + +proc rstMessage(s: PRstSharedState, filename: string, msgKind: MsgKind, + arg: string) = + s.msgHandler(filename, LineRstInit, ColRstInit, msgKind, arg) + proc rstMessage*(filenames: RstFileTable, f: MsgHandler, info: TLineInfo, msgKind: MsgKind, arg: string) = ## Print warnings using `info`, i.e. in 2nd-pass warnings for @@ -756,6 +796,14 @@ proc internalRefPriority(k: RstAnchorKind): int = of footnoteAnchor: result = 4 of headlineAnchor: result = 3 +proc `$`(subst: AnchorSubst): string = # for debug + let s = + case subst.kind + of arInternalRst: "type=" & $subst.anchorType + of arExternalRst: "type=" & $subst.anchorTypeExt + of arNim: "langsym=" & $subst.langSym + result = "(kind=$1, priority=$2, $3)" % [$subst.kind, $subst.priority, s] + proc addAnchorRst(p: var RstParser, name: string, target: PRstNode, anchorType: RstAnchorKind) = ## Associates node `target` (which has field `anchor`) with an @@ -771,31 +819,49 @@ proc addAnchorRst(p: var RstParser, name: string, target: PRstNode, info: prevLineInfo(p), anchorType: anchorType)) p.curAnchors.setLen 0 -proc addAnchorNim*(s: var PRstSharedState, refn: string, tooltip: string, +proc addAnchorExtRst(s: var PRstSharedState, key: string, refn: string, + anchorType: RstAnchorKind, info: TLineInfo) = + let name = key.toLowerAscii + let prio = internalRefPriority(anchorType) + s.anchors.mgetOrPut(name, newSeq[AnchorSubst]()).add( + AnchorSubst(kind: arExternalRst, refnameExt: refn, priority: prio, + info: info, + anchorTypeExt: anchorType)) + +proc addAnchorNim*(s: var PRstSharedState, external: bool, refn: string, tooltip: string, langSym: LangSymbol, priority: int, info: TLineInfo) = ## Adds an anchor `refn`, which follows ## the rule `arNim` (i.e. a symbol in ``*.nim`` file) s.anchors.mgetOrPut(langSym.name, newSeq[AnchorSubst]()).add( - AnchorSubst(kind: arNim, refname: refn, langSym: langSym, + AnchorSubst(kind: arNim, external: external, refname: refn, langSym: langSym, tooltip: tooltip, priority: priority, info: info)) proc findMainAnchorNim(s: PRstSharedState, signature: PRstNode, info: TLineInfo): seq[AnchorSubst] = - let langSym = toLangSymbol(signature) + var langSym: LangSymbol + try: + langSym = toLangSymbol(signature) + except ValueError: # parsing failed, not a Nim symbol + return let substitutions = s.anchors.getOrDefault(langSym.name, newSeq[AnchorSubst]()) if substitutions.len == 0: return - # map symKind (like "proc") -> found symbols/groups: - var found: Table[string, seq[AnchorSubst]] - for s in substitutions: - if s.kind == arNim: - if match(s.langSym, langSym): - found.mgetOrPut(s.langSym.symKind, newSeq[AnchorSubst]()).add s - for symKind, sList in found: + # logic to select only groups instead of concrete symbols + # with overloads, note that the same symbol can be defined + # in multiple modules and `importdoc`ed: + type GroupKey = tuple[symKind: string, origModule: string] + # map (symKind, file) (like "proc", "os.nim") -> found symbols/groups: + var found: Table[GroupKey, seq[AnchorSubst]] + for subst in substitutions: + if subst.kind == arNim: + if match(subst.langSym, langSym): + let key: GroupKey = (subst.langSym.symKind, getFilename(s, subst)) + found.mgetOrPut(key, newSeq[AnchorSubst]()).add subst + for key, sList in found: if sList.len == 1: result.add sList[0] else: # > 1, there are overloads, potential ambiguity in this `symKind` @@ -812,14 +878,16 @@ proc findMainAnchorNim(s: PRstSharedState, signature: PRstNode, result.add s foundGroup = true break - doAssert foundGroup, "docgen has not generated the group" + doAssert(foundGroup, + "docgen has not generated the group for $1 (file $2)" % [ + langSym.name, getFilename(s, sList[0]) ]) proc findMainAnchorRst(s: PRstSharedState, linkText: string, info: TLineInfo): seq[AnchorSubst] = let name = linkText.toLowerAscii let substitutions = s.anchors.getOrDefault(name, newSeq[AnchorSubst]()) for s in substitutions: - if s.kind == arInternalRst: + if s.kind in {arInternalRst, arExternalRst}: result.add s proc addFootnoteNumManual(p: var RstParser, num: int) = @@ -3251,6 +3319,15 @@ proc dirRaw(p: var RstParser): PRstNode = else: dirRawAux(p, result, rnRaw, parseSectionWrapper) +proc dirImportdoc(p: var RstParser): PRstNode = + result = parseDirective(p, rnDirective, {}, parseLiteralBlock) + assert result.sons[2].kind == rnLiteralBlock + assert result.sons[2].sons[0].kind == rnLeaf + let filenames: seq[string] = split(result.sons[2].sons[0].text, seps = {','}) + proc rmSpaces(s: string): string = s.split.join("") + for origFilename in filenames: + p.s.idxImports[origFilename.rmSpaces] = ImportdocInfo(fromInfo: lineInfo(p)) + proc selectDir(p: var RstParser, d: string): PRstNode = result = nil let tok = p.tok[p.idx-2] # report on directive in ".. directive::" @@ -3271,6 +3348,7 @@ proc selectDir(p: var RstParser, d: string): PRstNode = of "hint": result = dirAdmonition(p, d) of "image": result = dirImage(p) of "important": result = dirAdmonition(p, d) + of "importdoc": result = dirImportdoc(p) of "include": result = dirInclude(p) of "index": result = dirIndex(p) of "note": result = dirAdmonition(p, d) @@ -3401,11 +3479,90 @@ proc rstParsePass1*(fragment: string, getTokens(fragment, p.tok) result = parseDoc(p) -proc preparePass2*(s: PRstSharedState, mainNode: PRstNode) = +proc extractLinkEnd(x: string): string = + ## From links like `path/to/file.html#/%` extract `file.html#/%`. + let i = find(x, '#') + let last = + if i >= 0: i + else: x.len - 1 + let j = rfind(x, '/', start=0, last=last) + if j >= 0: + result = x[j+1 .. ^1] + else: + result = x + +proc loadIdxFile(s: var PRstSharedState, origFilename: string) = + doAssert roSandboxDisabled in s.options + var info: TLineInfo + info.fileIndex = addFilename(s, origFilename) + var (dir, basename, ext) = origFilename.splitFile + if ext notin [".md", ".rst", ".nim", ""]: + rstMessage(s.filenames, s.msgHandler, s.idxImports[origFilename].fromInfo, + meCannotOpenFile, origFilename & ": unknown extension") + let idxFilename = dir / basename & ".idx" + let (idxPath, linkRelPath) = s.findRefFile(idxFilename) + s.idxImports[origFilename].linkRelPath = linkRelPath + var + fileEntries: seq[IndexEntry] + title: IndexEntry + try: + (fileEntries, title) = parseIdxFile(idxPath) + except IOError: + rstMessage(s.filenames, s.msgHandler, s.idxImports[origFilename].fromInfo, + meCannotOpenFile, idxPath) + except ValueError as e: + s.msgHandler(idxPath, LineRstInit, ColRstInit, meInvalidField, e.msg) + + var isMarkup = false # for sanity check to avoid mixing .md <-> .nim + for entry in fileEntries: + # Though target .idx already has inside it the path to HTML relative + # project's root, we won't rely on it and use `linkRelPath` instead. + let refn = extractLinkEnd(entry.link) + # select either markup (rst/md) or Nim cases: + if entry.kind in {ieMarkupTitle, ieNimTitle}: + s.idxImports[origFilename].title = entry.keyword + case entry.kind + of ieIdxRole, ieHeading, ieMarkupTitle: + if ext == ".nim" and entry.kind == ieMarkupTitle: + rstMessage(s, idxPath, meInvalidField, + $ieMarkupTitle & " in supposedly .nim-derived file") + if entry.kind == ieMarkupTitle: + isMarkup = true + info.line = entry.line.uint16 + addAnchorExtRst(s, key = entry.keyword, refn = refn, + anchorType = headlineAnchor, info=info) + of ieNim, ieNimGroup, ieNimTitle: + if ext in [".md", ".rst"] or isMarkup: + rstMessage(s, idxPath, meInvalidField, + $entry.kind & " in supposedly markup-derived file") + s.nimFileImported = true + var langSym: LangSymbol + if entry.kind in {ieNim, ieNimTitle}: + var q: RstParser + initParser(q, s) + info.line = entry.line.uint16 + setLen(q.tok, 0) + q.idx = 0 + getTokens(entry.linkTitle, q.tok) + var sons = newSeq[PRstNode](q.tok.len) + for i in 0 ..< q.tok.len: sons[i] = newLeaf(q.tok[i].symbol) + let linkTitle = newRstNode(rnInner, sons) + langSym = linkTitle.toLangSymbol + else: # entry.kind == ieNimGroup + langSym = langSymbolGroup(kind=entry.linkTitle, name=entry.keyword) + addAnchorNim(s, external = true, refn = refn, tooltip = entry.linkDesc, + langSym = langSym, priority = -4, # lowest + info=info) + doAssert s.idxImports[origFilename].title != "" + +proc preparePass2*(s: var PRstSharedState, mainNode: PRstNode, importdoc = true) = ## Records titles in node `mainNode` and orders footnotes. countTitles(s, mainNode) fixHeadlines(s) orderFootnotes(s) + if importdoc: + for origFilename in s.idxImports.keys: + loadIdxFile(s, origFilename) proc resolveLink(s: PRstSharedState, n: PRstNode) : PRstNode = # Associate this link alias with its target and change node kind to @@ -3423,6 +3580,9 @@ proc resolveLink(s: PRstSharedState, n: PRstNode) : PRstNode = tooltip: string target: PRstNode info: TLineInfo + externFilename: string + # when external anchor: origin filename where anchor was defined + isTitle: bool proc cmp(x, y: LinkDef): int = result = cmp(x.priority, y.priority) if result == 0: @@ -3435,26 +3595,67 @@ proc resolveLink(s: PRstSharedState, n: PRstNode) : PRstNode = target: y.value, info: y.info, tooltip: "(" & $y.kind & ")") let substRst = findMainAnchorRst(s, alias.addNodes, n.info) + template getExternFilename(subst: AnchorSubst): string = + if subst.kind == arExternalRst or + (subst.kind == arNim and subst.external): + getFilename(s, subst) + else: "" for subst in substRst: - foundLinks.add LinkDef(ar: arInternalRst, priority: subst.priority, - target: newLeaf(subst.target.anchor), + var refname, fullRefname: string + if subst.kind == arInternalRst: + refname = subst.target.anchor + fullRefname = refname + else: # arExternalRst + refname = subst.refnameExt + fullRefname = s.idxImports[getFilename(s, subst)].linkRelPath & + "/" & refname + let anchorType = + if subst.kind == arInternalRst: subst.anchorType + else: subst.anchorTypeExt # arExternalRst + foundLinks.add LinkDef(ar: subst.kind, priority: subst.priority, + target: newLeaf(fullRefname), info: subst.info, - tooltip: "(" & $subst.anchorType & ")") + externFilename: getExternFilename(subst), + isTitle: isDocumentationTitle(refname), + tooltip: "(" & $anchorType & ")") # find anchors automatically generated from Nim symbols - if roNimFile in s.options: + if roNimFile in s.options or s.nimFileImported: let substNim = findMainAnchorNim(s, signature=alias, n.info) for subst in substNim: - foundLinks.add LinkDef(ar: arNim, priority: subst.priority, - target: newLeaf(subst.refname), + let fullRefname = + if subst.external: + s.idxImports[getFilename(s, subst)].linkRelPath & + "/" & subst.refname + else: subst.refname + foundLinks.add LinkDef(ar: subst.kind, priority: subst.priority, + target: newLeaf(fullRefname), + externFilename: getExternFilename(subst), + isTitle: isDocumentationTitle(subst.refname), info: subst.info, tooltip: subst.tooltip) foundLinks.sort(cmp = cmp, order = Descending) let aliasStr = addNodes(alias) if foundLinks.len >= 1: - let kind = if foundLinks[0].ar == arHyperlink: rnHyperlink - elif foundLinks[0].ar == arNim: rnNimdocRef + if foundLinks[0].externFilename != "": + s.idxImports[foundLinks[0].externFilename].used = true + let kind = if foundLinks[0].ar in {arHyperlink, arExternalRst}: rnHyperlink + elif foundLinks[0].ar == arNim: + if foundLinks[0].externFilename == "": rnNimdocRef + else: rnHyperlink else: rnInternalRef result = newRstNode(kind) - result.sons = @[newRstNode(rnInner, desc.sons), foundLinks[0].target] + let documentName = # filename without ext for `.nim`, title for `.md` + if foundLinks[0].ar == arNim: + changeFileExt(foundLinks[0].externFilename.extractFilename, "") + elif foundLinks[0].externFilename != "": + s.idxImports[foundLinks[0].externFilename].title + else: foundLinks[0].externFilename.extractFilename + let linkText = + if foundLinks[0].externFilename != "": + if foundLinks[0].isTitle: newLeaf(addNodes(desc)) + else: newLeaf(documentName & ": " & addNodes(desc)) + else: + newRstNode(rnInner, desc.sons) + result.sons = @[linkText, foundLinks[0].target] if kind == rnNimdocRef: result.tooltip = foundLinks[0].tooltip if foundLinks.len > 1: # report ambiguous link var targets = newSeq[string]() @@ -3568,20 +3769,28 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode = inc i result.sons = newSons +proc completePass2*(s: PRstSharedState) = + for (filename, importdocInfo) in s.idxImports.pairs: + if not importdocInfo.used: + rstMessage(s.filenames, s.msgHandler, importdocInfo.fromInfo, + mwUnusedImportdoc, filename) + proc rstParse*(text, filename: string, line, column: int, options: RstParseOptions, findFile: FindFileHandler = nil, + findRefFile: FindRefFileHandler = nil, msgHandler: MsgHandler = nil): tuple[node: PRstNode, filenames: RstFileTable, hasToc: bool] = ## Parses the whole `text`. The result is ready for `rstgen.renderRstToOut`, ## note that 2nd tuple element should be fed to `initRstGenerator` ## argument `filenames` (it is being filled here at least with `filename` ## and possibly with other files from RST ``.. include::`` statement). - var sharedState = newRstSharedState(options, filename, findFile, + var sharedState = newRstSharedState(options, filename, findFile, findRefFile, msgHandler, hasToc=false) let unresolved = rstParsePass1(text, line, column, sharedState) preparePass2(sharedState, unresolved) result.node = resolveSubs(sharedState, unresolved) + completePass2(sharedState) result.filenames = sharedState.filenames result.hasToc = sharedState.hasToc diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index 597ee1553..57bc00fcb 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -39,7 +39,8 @@ ## No backreferences are generated since finding all references of a footnote ## can be done by simply searching for ``[footnoteName]``. -import strutils, os, hashes, strtabs, rstast, rst, highlite, tables, sequtils, +import strutils, os, hashes, strtabs, rstast, rst, rstidx, + highlite, tables, sequtils, algorithm, parseutils, std/strbasics @@ -59,7 +60,7 @@ type outLatex # output is Latex MetaEnum* = enum - metaNone, metaTitle, metaSubtitle, metaAuthor, metaVersion + metaNone, metaTitleRaw, metaTitle, metaSubtitle, metaAuthor, metaVersion EscapeMode* = enum # in Latex text inside options [] and URLs is # escaped slightly differently than in normal text @@ -321,31 +322,8 @@ proc renderAux(d: PDoc, n: PRstNode, html, tex: string, result: var string) = # ---------------- index handling -------------------------------------------- -proc quoteIndexColumn(text: string): string = - ## Returns a safe version of `text` for serialization to the ``.idx`` file. - ## - ## The returned version can be put without worries in a line based tab - ## separated column text file. The following character sequence replacements - ## will be performed for that goal: - ## - ## * ``"\\"`` => ``"\\\\"`` - ## * ``"\n"`` => ``"\\n"`` - ## * ``"\t"`` => ``"\\t"`` - result = newStringOfCap(text.len + 3) - for c in text: - case c - of '\\': result.add "\\" - of '\L': result.add "\\n" - of '\C': discard - of '\t': result.add "\\t" - else: result.add c - -proc unquoteIndexColumn(text: string): string = - ## Returns the unquoted version generated by ``quoteIndexColumn``. - result = text.multiReplace(("\\t", "\t"), ("\\n", "\n"), ("\\\\", "\\")) - -proc setIndexTerm*(d: var RstGenerator, htmlFile, id, term: string, - linkTitle, linkDesc = "") = +proc setIndexTerm*(d: var RstGenerator; k: IndexEntryKind, htmlFile, id, term: string, + linkTitle, linkDesc = "", line = 0) = ## Adds a `term` to the index using the specified hyperlink identifier. ## ## A new entry will be added to the index using the format @@ -368,21 +346,8 @@ proc setIndexTerm*(d: var RstGenerator, htmlFile, id, term: string, ## <#writeIndexFile,RstGenerator,string>`_. The purpose of the index is ## documented in the `docgen tools guide ## `_. - var - entry = term - isTitle = false - entry.add('\t') - entry.add(htmlFile) - if id.len > 0: - entry.add('#') - entry.add(id) - else: - isTitle = true - if linkTitle.len > 0 or linkDesc.len > 0: - entry.add('\t' & linkTitle.quoteIndexColumn) - entry.add('\t' & linkDesc.quoteIndexColumn) - entry.add("\n") - + let (entry, isTitle) = formatIndexEntry(k, htmlFile, id, term, + linkTitle, linkDesc, line) if isTitle: d.theIndex.insert(entry) else: d.theIndex.add(entry) @@ -395,6 +360,15 @@ proc hash(n: PRstNode): int = result = result !& hash(n.sons[i]) result = !$result +proc htmlFileRelPath(d: PDoc): string = + if d.outDir.len == 0: + # /foo/bar/zoo.nim -> zoo.html + changeFileExt(extractFilename(d.filename), HtmlExt) + else: # d is initialized in docgen.nim + # outDir = /foo -\ + # destFile = /foo/bar/zoo.html -|-> bar/zoo.html + d.destFile.relativePath(d.outDir, '/') + proc renderIndexTerm*(d: PDoc, n: PRstNode, result: var string) = ## Renders the string decorated within \`foobar\`\:idx\: markers. ## @@ -411,17 +385,12 @@ proc renderIndexTerm*(d: PDoc, n: PRstNode, result: var string) = var term = "" renderAux(d, n, term) - setIndexTerm(d, changeFileExt(extractFilename(d.filename), HtmlExt), id, term, d.currentSection) + setIndexTerm(d, ieIdxRole, + htmlFileRelPath(d), id, term, d.currentSection) dispA(d.target, result, "$2", "\\nimindexterm{$1}{$2}", [id, term]) type - IndexEntry* = object - keyword*: string - link*: string - linkTitle*: string ## contains a prettier text for the href - linkDesc*: string ## the title attribute of the final href - IndexedDocs* = Table[IndexEntry, seq[IndexEntry]] ## \ ## Contains the index sequences for doc types. ## @@ -432,21 +401,6 @@ type ## The value indexed by this IndexEntry is a sequence with the real index ## entries found in the ``.idx`` file. -proc cmp(a, b: IndexEntry): int = - ## Sorts two ``IndexEntry`` first by `keyword` field, then by `link`. - result = cmpIgnoreStyle(a.keyword, b.keyword) - if result == 0: - result = cmpIgnoreStyle(a.link, b.link) - -proc hash(x: IndexEntry): Hash = - ## Returns the hash for the combined fields of the type. - ## - ## The hash is computed as the chained hash of the individual string hashes. - result = x.keyword.hash !& x.link.hash - result = result !& x.linkTitle.hash - result = result !& x.linkDesc.hash - result = !$result - when defined(gcDestructors): template `<-`(a, b: var IndexEntry) = a = move(b) else: @@ -455,6 +409,7 @@ else: shallowCopy a.link, b.link shallowCopy a.linkTitle, b.linkTitle shallowCopy a.linkDesc, b.linkDesc + shallowCopy a.module, b.module proc sortIndex(a: var openArray[IndexEntry]) = # we use shellsort here; fast and simple @@ -494,16 +449,20 @@ proc generateSymbolIndex(symbols: seq[IndexEntry]): string = result = "
          " var i = 0 while i < symbols.len: - let keyword = symbols[i].keyword + let keyword = esc(outHtml, symbols[i].keyword) let cleanedKeyword = keyword.escapeLink result.addf("
          $1:
            \n", [keyword, cleanedKeyword]) var j = i - while j < symbols.len and keyword == symbols[j].keyword: + while j < symbols.len and symbols[i].keyword == symbols[j].keyword: let url = symbols[j].link.escapeLink - text = if symbols[j].linkTitle.len > 0: symbols[j].linkTitle else: url - desc = if symbols[j].linkDesc.len > 0: symbols[j].linkDesc else: "" + module = symbols[j].module + text = + if symbols[j].linkTitle.len > 0: + esc(outHtml, module & ": " & symbols[j].linkTitle) + else: url + desc = symbols[j].linkDesc if desc.len > 0: result.addf("""
          • $2
            • @@ -517,13 +476,6 @@ proc generateSymbolIndex(symbols: seq[IndexEntry]): string = i = j result.add("
          ") -proc isDocumentationTitle(hyperlink: string): bool = - ## Returns true if the hyperlink is actually a documentation title. - ## - ## Documentation titles lack the hash. See `mergeIndexes() - ## <#mergeIndexes,string>`_ for a more detailed explanation. - result = hyperlink.find('#') < 0 - proc stripTocLevel(s: string): tuple[level: int, text: string] = ## Returns the *level* of the toc along with the text without it. for c in 0 ..< s.len: @@ -557,17 +509,15 @@ proc generateDocumentationToc(entries: seq[IndexEntry]): string = level = 1 levels.newSeq(entries.len) for entry in entries: - let (rawLevel, rawText) = stripTocLevel(entry.linkTitle or entry.keyword) + let (rawLevel, rawText) = stripTocLevel(entry.linkTitle) if rawLevel < 1: # This is a normal symbol, push it *inside* one level from the last one. levels[L].level = level + 1 - # Also, ignore the linkTitle and use directly the keyword. - levels[L].text = entry.keyword else: # The level did change, update the level indicator. level = rawLevel levels[L].level = rawLevel - levels[L].text = rawText + levels[L].text = rawText inc L # Now generate hierarchical lists based on the precalculated levels. @@ -598,7 +548,7 @@ proc generateDocumentationIndex(docs: IndexedDocs): string = for title in titles: let tocList = generateDocumentationToc(docs.getOrDefault(title)) result.add("\n") + title.link & "\">" & title.linkTitle & "\n" & tocList & "
        \n") proc generateDocumentationJumps(docs: IndexedDocs): string = ## Returns a plain list of hyperlinks to documentation TOCs in HTML. @@ -610,7 +560,7 @@ proc generateDocumentationJumps(docs: IndexedDocs): string = var chunks: seq[string] = @[] for title in titles: - chunks.add("" & title.keyword & "") + chunks.add("" & title.linkTitle & "") result.add(chunks.join(", ") & ".
        ") @@ -639,39 +589,11 @@ proc readIndexDir*(dir: string): # Scan index files and build the list of symbols. for path in walkDirRec(dir): if path.endsWith(IndexExt): - var - fileEntries: seq[IndexEntry] - title: IndexEntry - f = 0 - newSeq(fileEntries, 500) - setLen(fileEntries, 0) - for line in lines(path): - let s = line.find('\t') - if s < 0: continue - setLen(fileEntries, f+1) - fileEntries[f].keyword = line.substr(0, s-1) - fileEntries[f].link = line.substr(s+1) - # See if we detect a title, a link without a `#foobar` trailing part. - if title.keyword.len == 0 and fileEntries[f].link.isDocumentationTitle: - title.keyword = fileEntries[f].keyword - title.link = fileEntries[f].link - - if fileEntries[f].link.find('\t') > 0: - let extraCols = fileEntries[f].link.split('\t') - fileEntries[f].link = extraCols[0] - assert extraCols.len == 3 - fileEntries[f].linkTitle = extraCols[1].unquoteIndexColumn - fileEntries[f].linkDesc = extraCols[2].unquoteIndexColumn - else: - fileEntries[f].linkTitle = "" - fileEntries[f].linkDesc = "" - inc f + var (fileEntries, title) = parseIdxFile(path) # Depending on type add this to the list of symbols or table of APIs. - if title.keyword.len == 0: - for i in 0 ..< f: - # Don't add to symbols TOC entries (they start with a whitespace). - let toc = fileEntries[i].linkTitle - if toc.len > 0 and toc[0] == ' ': + if title.kind == ieNimTitle: + for i in 0 ..< fileEntries.len: + if fileEntries[i].kind != ieNim: continue # Ok, non TOC entry, add it. setLen(result.symbols, L + 1) @@ -687,7 +609,7 @@ proc readIndexDir*(dir: string): result.modules.add(x.changeFileExt("")) else: # Generate the symbolic anchor for index quickjumps. - title.linkTitle = "doc_toc_" & $result.docs.len + title.aux = "doc_toc_" & $result.docs.len result.docs[title] = fileEntries proc mergeIndexes*(dir: string): string = @@ -747,24 +669,6 @@ proc mergeIndexes*(dir: string): string = # ---------------------------------------------------------------------------- -proc stripTocHtml(s: string): string = - ## Ugly quick hack to remove HTML tags from TOC titles. - ## - ## A TocEntry.header field already contains rendered HTML tags. Instead of - ## implementing a proper version of renderRstToOut() which recursively - ## renders an rst tree to plain text, we simply remove text found between - ## angled brackets. Given the limited possibilities of rst inside TOC titles - ## this should be enough. - result = s - var first = result.find('<') - while first >= 0: - let last = result.find('>', first) - if last < 0: - # Abort, since we didn't found a closing angled bracket. - return - result.delete(first..last) - first = result.find('<', first) - proc renderHeadline(d: PDoc, n: PRstNode, result: var string) = var tmp = "" for i in countup(0, len(n) - 1): renderRstToOut(d, n.sons[i], tmp) @@ -785,19 +689,12 @@ proc renderHeadline(d: PDoc, n: PRstNode, result: var string) = # Generate index entry using spaces to indicate TOC level for the output HTML. assert n.level >= 0 - let - htmlFileRelPath = if d.outDir.len == 0: - # /foo/bar/zoo.nim -> zoo.html - changeFileExt(extractFilename(d.filename), HtmlExt) - else: # d is initialized in docgen.nim - # outDir = /foo -\ - # destFile = /foo/bar/zoo.html -|-> bar/zoo.html - d.destFile.relativePath(d.outDir, '/') - setIndexTerm(d, htmlFileRelPath, n.anchor, tmp.stripTocHtml, - spaces(max(0, n.level)) & tmp) + setIndexTerm(d, ieHeading, htmlFile = d.htmlFileRelPath, id = n.anchor, + term = n.addNodes, linkTitle = spaces(max(0, n.level)) & tmp) proc renderOverline(d: PDoc, n: PRstNode, result: var string) = if n.level == 0 and d.meta[metaTitle].len == 0: + d.meta[metaTitleRaw] = n.addNodes for i in countup(0, len(n)-1): renderRstToOut(d, n.sons[i], d.meta[metaTitle]) d.currentSection = d.meta[metaTitle] @@ -813,6 +710,8 @@ proc renderOverline(d: PDoc, n: PRstNode, result: var string) = dispA(d.target, result, "
        $3
        ", "\\rstov$4[$5]{$3}$2\n", [$n.level, n.anchor.idS, tmp, $chr(n.level - 1 + ord('A')), tocName]) + setIndexTerm(d, ieHeading, htmlFile = d.htmlFileRelPath, id = n.anchor, + term = n.addNodes, linkTitle = spaces(max(0, n.level)) & tmp) proc renderTocEntry(d: PDoc, n: PRstNode, result: var string) = var header = "" @@ -1197,6 +1096,18 @@ proc renderHyperlink(d: PDoc, text, link: PRstNode, result: var string, "\\hyperlink{$2}{$1} (p.~\\pageref{$2})", [textStr, linkStr, nimDocStr, tooltipStr]) +proc traverseForIndex*(d: PDoc, n: PRstNode) = + ## A version of [renderRstToOut] that only fills entries for ``.idx`` files. + var discarded: string + if n == nil: return + case n.kind + of rnIdx: renderIndexTerm(d, n, discarded) + of rnHeadline, rnMarkdownHeadline: renderHeadline(d, n, discarded) + of rnOverline: renderOverline(d, n, discarded) + else: + for i in 0 ..< len(n): + traverseForIndex(d, n.sons[i]) + proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = if n == nil: return case n.kind @@ -1451,6 +1362,7 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = of rnTitle: d.meta[metaTitle] = "" renderRstToOut(d, n.sons[0], d.meta[metaTitle]) + d.meta[metaTitleRaw] = n.sons[0].addNodes # ----------------------------------------------------------------------------- @@ -1616,11 +1528,13 @@ proc rstToHtml*(s: string, options: RstParseOptions, proc myFindFile(filename: string): string = # we don't find any files in online mode: result = "" + proc myFindRefFile(filename: string): (string, string) = + result = ("", "") const filen = "input" let (rst, filenames, t) = rstParse(s, filen, line=LineRstInit, column=ColRstInit, - options, myFindFile, msgHandler) + options, myFindFile, myFindRefFile, msgHandler) var d: RstGenerator initRstGenerator(d, outHtml, config, filen, myFindFile, msgHandler, filenames, hasToc = t) diff --git a/lib/packages/docutils/rstidx.nim b/lib/packages/docutils/rstidx.nim new file mode 100644 index 000000000..c109636d7 --- /dev/null +++ b/lib/packages/docutils/rstidx.nim @@ -0,0 +1,138 @@ +# +# Nim's Runtime Library +# (c) Copyright 2022 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. + +## Nim `idx`:idx: file format related definitions. + +import strutils, std/syncio, hashes +from os import splitFile + +type + IndexEntryKind* = enum ## discriminator tag + ieMarkupTitle = "markupTitle" + ## RST/Markdown title, text in `keyword` + + ## HTML text in `linkTitle` + ieNimTitle = "nimTitle" + ## Nim title + ieHeading = "heading" ## RST/Markdown markup heading, escaped + ieIdxRole = "idx" ## RST :idx: definition, escaped + ieNim = "nim" ## Nim symbol, unescaped + ieNimGroup = "nimgrp" ## Nim overload group, unescaped + IndexEntry* = object + kind*: IndexEntryKind ## 0. + keyword*: string ## 1. + link*: string ## 2. + linkTitle*: string ## 3. contains a prettier text for the href + linkDesc*: string ## 4. the title attribute of the final href + line*: int ## 5. + module*: string ## origin file, NOT a field in ``.idx`` file + aux*: string ## auxuliary field, NOT a field in ``.idx`` file + +proc isDocumentationTitle*(hyperlink: string): bool = + ## Returns true if the hyperlink is actually a documentation title. + ## + ## Documentation titles lack the hash. See `mergeIndexes() + ## <#mergeIndexes,string>`_ for a more detailed explanation. + result = hyperlink.find('#') < 0 + +proc `$`*(e: IndexEntry): string = + """("$1", "$2", "$3", "$4", $5)""" % [ + e.keyword, e.link, e.linkTitle, e.linkDesc, $e.line] + +proc quoteIndexColumn(text: string): string = + ## Returns a safe version of `text` for serialization to the ``.idx`` file. + ## + ## The returned version can be put without worries in a line based tab + ## separated column text file. The following character sequence replacements + ## will be performed for that goal: + ## + ## * ``"\\"`` => ``"\\\\"`` + ## * ``"\n"`` => ``"\\n"`` + ## * ``"\t"`` => ``"\\t"`` + result = newStringOfCap(text.len + 3) + for c in text: + case c + of '\\': result.add "\\" + of '\L': result.add "\\n" + of '\C': discard + of '\t': result.add "\\t" + else: result.add c + +proc unquoteIndexColumn*(text: string): string = + ## Returns the unquoted version generated by ``quoteIndexColumn``. + result = text.multiReplace(("\\t", "\t"), ("\\n", "\n"), ("\\\\", "\\")) + +proc formatIndexEntry*(kind: IndexEntryKind; htmlFile, id, term, linkTitle, + linkDesc: string, line: int): + tuple[entry: string, isTitle: bool] = + result.entry = $kind + result.entry.add('\t') + result.entry.add term + result.entry.add('\t') + result.entry.add(htmlFile) + if id.len > 0: + result.entry.add('#') + result.entry.add(id) + result.isTitle = false + else: + result.isTitle = true + result.entry.add('\t' & linkTitle.quoteIndexColumn) + result.entry.add('\t' & linkDesc.quoteIndexColumn) + result.entry.add('\t' & $line) + result.entry.add("\n") + +proc parseIndexEntryKind(s: string): IndexEntryKind = + result = case s: + of "nim": ieNim + of "nimgrp": ieNimGroup + of "heading": ieHeading + of "idx": ieIdxRole + of "nimTitle": ieNimTitle + of "markupTitle": ieMarkupTitle + else: raise newException(ValueError, "unknown index entry value $1" % [s]) + +proc parseIdxFile*(path: string): + tuple[fileEntries: seq[IndexEntry], title: IndexEntry] = + var + f = 0 + newSeq(result.fileEntries, 500) + setLen(result.fileEntries, 0) + let (_, base, _) = path.splitFile + for line in lines(path): + let s = line.find('\t') + if s < 0: continue + setLen(result.fileEntries, f+1) + let cols = line.split('\t') + result.fileEntries[f].kind = parseIndexEntryKind(cols[0]) + result.fileEntries[f].keyword = cols[1] + result.fileEntries[f].link = cols[2] + if result.title.keyword.len == 0: + result.fileEntries[f].module = base + else: + result.fileEntries[f].module = result.title.keyword + + result.fileEntries[f].linkTitle = cols[3].unquoteIndexColumn + result.fileEntries[f].linkDesc = cols[4].unquoteIndexColumn + result.fileEntries[f].line = parseInt(cols[5]) + + if result.fileEntries[f].kind in {ieNimTitle, ieMarkupTitle}: + result.title = result.fileEntries[f] + inc f + +proc cmp*(a, b: IndexEntry): int = + ## Sorts two ``IndexEntry`` first by `keyword` field, then by `link`. + result = cmpIgnoreStyle(a.keyword, b.keyword) + if result == 0: + result = cmpIgnoreStyle(a.link, b.link) + +proc hash*(x: IndexEntry): Hash = + ## Returns the hash for the combined fields of the type. + ## + ## The hash is computed as the chained hash of the individual string hashes. + result = x.keyword.hash !& x.link.hash + result = result !& x.linkTitle.hash + result = result !& x.linkDesc.hash + result = !$result diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 5f8f116d3..ecceed671 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -11,6 +11,8 @@ ## retrieving environment variables, working with directories, ## running shell commands, etc. +## .. importdoc:: symlinks.nim, appdirs.nim, dirs.nim, ospaths2.nim + runnableExamples: let myFile = "/path/to/my/file.nim" assert splitPath(myFile) == (head: "/path/to/my", tail: "file.nim") diff --git a/lib/std/appdirs.nim b/lib/std/appdirs.nim index e648fe0c1..c945ba8ec 100644 --- a/lib/std/appdirs.nim +++ b/lib/std/appdirs.nim @@ -1,5 +1,7 @@ ## This module implements helpers for determining special directories used by apps. +## .. importdoc:: paths.nim + from std/private/osappdirs import nil import std/paths import std/envvars diff --git a/lib/std/private/osappdirs.nim b/lib/std/private/osappdirs.nim index 8e6ae00f8..581b511ff 100644 --- a/lib/std/private/osappdirs.nim +++ b/lib/std/private/osappdirs.nim @@ -1,3 +1,5 @@ +## .. importdoc:: paths.nim, dirs.nim + include system/inclrtl import std/envvars import std/private/ospaths2 diff --git a/lib/std/private/oscommon.nim b/lib/std/private/oscommon.nim index 5b4e1d7e3..3aac58636 100644 --- a/lib/std/private/oscommon.nim +++ b/lib/std/private/oscommon.nim @@ -5,6 +5,8 @@ import std/[oserrors] when defined(nimPreviewSlimSystem): import std/[syncio, assertions, widestrs] +## .. importdoc:: osdirs.nim, os.nim + const weirdTarget* = defined(nimscript) or defined(js) diff --git a/lib/std/private/osdirs.nim b/lib/std/private/osdirs.nim index add9ed424..4af418ead 100644 --- a/lib/std/private/osdirs.nim +++ b/lib/std/private/osdirs.nim @@ -1,3 +1,5 @@ +## .. importdoc:: osfiles.nim, appdirs.nim, paths.nim + include system/inclrtl import std/oserrors diff --git a/lib/std/private/osfiles.nim b/lib/std/private/osfiles.nim index 301f14600..a7a595d97 100644 --- a/lib/std/private/osfiles.nim +++ b/lib/std/private/osfiles.nim @@ -7,6 +7,7 @@ export fileExists import ospaths2, ossymlinks +## .. importdoc:: osdirs.nim, os.nim when defined(nimPreviewSlimSystem): import std/[syncio, assertions, widestrs] @@ -420,4 +421,4 @@ proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", removeFile(source) except: discard tryRemoveFile(dest) - raise \ No newline at end of file + raise diff --git a/lib/std/private/ospaths2.nim b/lib/std/private/ospaths2.nim index 75c34ecf5..5e3bece68 100644 --- a/lib/std/private/ospaths2.nim +++ b/lib/std/private/ospaths2.nim @@ -10,6 +10,8 @@ export ReadDirEffect, WriteDirEffect when defined(nimPreviewSlimSystem): import std/[syncio, assertions, widestrs] +## .. importdoc:: osappdirs.nim, osdirs.nim, osseps.nim, os.nim + const weirdTarget = defined(nimscript) or defined(js) when weirdTarget: diff --git a/lib/std/private/osseps.nim b/lib/std/private/osseps.nim index 1ea587e3c..f2d49d886 100644 --- a/lib/std/private/osseps.nim +++ b/lib/std/private/osseps.nim @@ -3,6 +3,8 @@ # Improved based on info in 'compiler/platform.nim' +## .. importdoc:: ospaths2.nim + const doslikeFileSystem* = defined(windows) or defined(OS2) or defined(DOS) diff --git a/lib/std/private/ossymlinks.nim b/lib/std/private/ossymlinks.nim index cb6287bde..53f59f939 100644 --- a/lib/std/private/ossymlinks.nim +++ b/lib/std/private/ossymlinks.nim @@ -31,6 +31,7 @@ elif defined(js): else: {.pragma: noNimJs.} +## .. importdoc:: os.nim proc createSymlink*(src, dest: string) {.noWeirdTarget.} = ## Create a symbolic link at `dest` which points to the item specified diff --git a/lib/std/symlinks.nim b/lib/std/symlinks.nim index 9e77bbe2a..54ab7b677 100644 --- a/lib/std/symlinks.nim +++ b/lib/std/symlinks.nim @@ -1,10 +1,11 @@ ## This module implements symlink (symbolic link) handling. +## .. importdoc:: os.nim + from paths import Path, ReadDirEffect from std/private/ossymlinks import symlinkExists, createSymlink, expandSymlink - proc symlinkExists*(link: Path): bool {.inline, tags: [ReadDirEffect].} = ## Returns true if the symlink `link` exists. Will return true ## regardless of whether the link points to a directory or file. diff --git a/nimdoc/extlinks/project/doc/manual.md b/nimdoc/extlinks/project/doc/manual.md new file mode 100644 index 000000000..d44b5ca39 --- /dev/null +++ b/nimdoc/extlinks/project/doc/manual.md @@ -0,0 +1,17 @@ +=================== +Nothing User Manual +=================== + +.. importdoc:: ../main.nim, .. / sub / submodule.nim, ../../util.nim + +First section +============= + +Second *section* & +================== + +Ref. [``] or [submoduleInt] from [module nimdoc/extlinks/project/sub/submodule]. + +Ref. [proc mainfunction*(): int]. + +Ref. [utilfunction(x: int)]. diff --git a/nimdoc/extlinks/project/expected/_._/util.html b/nimdoc/extlinks/project/expected/_._/util.html new file mode 100644 index 000000000..9b9b29a4f --- /dev/null +++ b/nimdoc/extlinks/project/expected/_._/util.html @@ -0,0 +1,104 @@ + + + + + + + +nimdoc/extlinks/util + + + + + + + + + + + + +
        +
        +

        nimdoc/extlinks/util

        +
        +
        +
        + + +
        + +
        + Search: +
        +
        + Group by: + +
        + + +
        +
        + +
        + +

        +
        +

        Procs

        +
        +
        +
        + 
        proc utilfunction(x: int): int {....raises: [], tags: [], forbids: [].}
        +
        + + + +
        +
        + +
        + +
        +
        + +
        +
        + +
        + +
        + Made with Nim. Generated: 1970-01-02 03:46:40 UTC +
        +
        +
        + + + + + + diff --git a/nimdoc/extlinks/project/expected/_._/util.idx b/nimdoc/extlinks/project/expected/_._/util.idx new file mode 100644 index 000000000..d83d8c97d --- /dev/null +++ b/nimdoc/extlinks/project/expected/_._/util.idx @@ -0,0 +1,2 @@ +nimTitle util _._/util.html module nimdoc/extlinks/util 0 +nim utilfunction _._/util.html#utilfunction,int proc utilfunction(x: int): int 1 diff --git a/nimdoc/extlinks/project/expected/doc/manual.html b/nimdoc/extlinks/project/expected/doc/manual.html new file mode 100644 index 000000000..7b964f4ac --- /dev/null +++ b/nimdoc/extlinks/project/expected/doc/manual.html @@ -0,0 +1,44 @@ + + + + + + + +Nothing User Manual + + + + + + + + + + + + +
        +
        +

        Nothing User Manual

        + +

        First section

        +

        Second section &

        Ref. submodule: `</a>` or submodule: submoduleInt from module nimdoc/extlinks/project/sub/submodule.

        +

        Ref. main: proc mainfunction*(): int.

        +

        Ref. util: utilfunction(x: int).

        + + + +
        + +
        + Made with Nim. Generated: 1970-01-02 03:46:40 UTC +
        +
        +
        + + + + + + diff --git a/nimdoc/extlinks/project/expected/doc/manual.idx b/nimdoc/extlinks/project/expected/doc/manual.idx new file mode 100644 index 000000000..158a758f0 --- /dev/null +++ b/nimdoc/extlinks/project/expected/doc/manual.idx @@ -0,0 +1,3 @@ +markupTitle Nothing User Manual doc/manual.html Nothing User Manual 0 +heading First section doc/manual.html#first-section First section 0 +heading Second section & doc/manual.html#second-section-amp Second section & 0 diff --git a/nimdoc/extlinks/project/expected/main.html b/nimdoc/extlinks/project/expected/main.html new file mode 100644 index 000000000..5facedb3b --- /dev/null +++ b/nimdoc/extlinks/project/expected/main.html @@ -0,0 +1,144 @@ + + + + + + + +nimdoc/extlinks/project/main + + + + + + + + + + + + +
        +
        +

        nimdoc/extlinks/project/main

        +
        +
        +
        + + +
        + +
        + Search: +
        +
        + Group by: + +
        + + +
        +
        + +
        + +

        +

        my heading

        See also module nimdoc/extlinks/util or nimdoc/extlinks/project/sub/submodule module.

        +

        Ref. submodule: `</a>` proc.

        +

        Ref. Nothing User Manual: First section or Nothing User Manual: Second section & from Nothing User Manual.

        +

        + +
        +

        Types

        +
        +
        + 
        A = object
+  x: int
+
        +
        + + + +
        +
        + +
        +
        +
        +

        Procs

        +
        +
        +
        + 
        proc mainfunction(): int {....raises: [], tags: [], forbids: [].}
        +
        + + + +
        +
        + +
        + +
        +
        + +
        +
        + +
        + +
        + Made with Nim. Generated: 1970-01-02 03:46:40 UTC +
        +
        +
        + + + + + + diff --git a/nimdoc/extlinks/project/expected/main.idx b/nimdoc/extlinks/project/expected/main.idx new file mode 100644 index 000000000..d01f2e4c5 --- /dev/null +++ b/nimdoc/extlinks/project/expected/main.idx @@ -0,0 +1,4 @@ +nimTitle main main.html module nimdoc/extlinks/project/main 0 +nim A main.html#A object A 17 +nim mainfunction main.html#mainfunction proc mainfunction(): int 20 +heading my heading main.html#my-heading my heading 0 diff --git a/nimdoc/extlinks/project/expected/sub/submodule.html b/nimdoc/extlinks/project/expected/sub/submodule.html new file mode 100644 index 000000000..913138d6e --- /dev/null +++ b/nimdoc/extlinks/project/expected/sub/submodule.html @@ -0,0 +1,130 @@ + + + + + + + +nimdoc/extlinks/project/sub/submodule + + + + + + + + + + + + +
        +
        +

        nimdoc/extlinks/project/sub/submodule

        +
        +
        +
        + + +
        + +
        + Search: +
        +
        + Group by: + +
        + + +
        +
        + +
        + +

        Ref. main: object A from module nimdoc/extlinks/project/main.

        +

        Ref. util: utilfunction(x: int).

        +

        Ref. Nothing User Manual: Second section & from Nothing User Manual.

        +

        +
        +

        Types

        +
        +
        + 
        submoduleInt = distinct int
        +
        + + + +
        +
        + +
        +
        +
        +

        Procs

        +
        +
        +
        + 
        proc `</a>`(x, y: int): bool {....raises: [], tags: [], forbids: [].}
        +
        + + Attempt to break HTML formatting. + +
        +
        + +
        + +
        +
        + +
        +
        + +
        + +
        + Made with Nim. Generated: 1970-01-02 03:46:40 UTC +
        +
        +
        + + + + + + diff --git a/nimdoc/extlinks/project/expected/sub/submodule.idx b/nimdoc/extlinks/project/expected/sub/submodule.idx new file mode 100644 index 000000000..2b02c889e --- /dev/null +++ b/nimdoc/extlinks/project/expected/sub/submodule.idx @@ -0,0 +1,3 @@ +nimTitle submodule sub/submodule.html module nimdoc/extlinks/project/sub/submodule 0 +nim `` sub/submodule.html#,int,int proc ``(x, y: int): bool 9 +nim submoduleInt sub/submodule.html#submoduleInt type submoduleInt 13 diff --git a/nimdoc/extlinks/project/expected/theindex.html b/nimdoc/extlinks/project/expected/theindex.html new file mode 100644 index 000000000..9d2f6723c --- /dev/null +++ b/nimdoc/extlinks/project/expected/theindex.html @@ -0,0 +1,58 @@ + + + + + + + +Index + + + + + + + + + + + + + + + + + + + diff --git a/nimdoc/extlinks/project/main.nim b/nimdoc/extlinks/project/main.nim new file mode 100644 index 000000000..36b778af6 --- /dev/null +++ b/nimdoc/extlinks/project/main.nim @@ -0,0 +1,23 @@ +## my heading +## ========== +## +## .. importdoc:: sub/submodule.nim, ../util.nim, doc/manual.md +## +## .. See also [Second&&&] and particularly [first section] and [Second section &]. +## +## See also [module nimdoc/extlinks/util] or [nimdoc/extlinks/project/sub/submodule module]. +## +## Ref. [`` proc]. +## +## Ref. [First section] or [Second section &] from [Nothing User Manual]. + + +import ../util, sub/submodule + +type A* = object + x: int + +proc mainfunction*(): int = + # just to suppress "not used" warnings: + if ``(1, 2): + result = utilfunction(0) diff --git a/nimdoc/extlinks/project/sub/submodule.nim b/nimdoc/extlinks/project/sub/submodule.nim new file mode 100644 index 000000000..876e00684 --- /dev/null +++ b/nimdoc/extlinks/project/sub/submodule.nim @@ -0,0 +1,13 @@ +## .. importdoc:: ../../util.nim, ../main.nim, ../doc/manual.md +## +## Ref. [object A] from [module nimdoc/extlinks/project/main]. +## +## Ref. [utilfunction(x: int)]. +## +## Ref. [Second section &] from [Nothing User Manual]. + +proc ``*(x, y: int): bool = + ## Attempt to break HTML formatting. + result = x < y + +type submoduleInt* = distinct int diff --git a/nimdoc/extlinks/util.nim b/nimdoc/extlinks/util.nim new file mode 100644 index 000000000..f208f98c1 --- /dev/null +++ b/nimdoc/extlinks/util.nim @@ -0,0 +1,2 @@ +proc utilfunction*(x: int): int = + x + 42 diff --git a/nimdoc/test_out_index_dot_html/expected/foo.idx b/nimdoc/test_out_index_dot_html/expected/foo.idx index a8dabb67e..ac76aa532 100644 --- a/nimdoc/test_out_index_dot_html/expected/foo.idx +++ b/nimdoc/test_out_index_dot_html/expected/foo.idx @@ -1 +1,2 @@ -foo index.html#foo foo: foo() +nimTitle foo index.html module nimdoc/test_out_index_dot_html/foo 0 +nim foo index.html#foo proc foo() 1 diff --git a/nimdoc/test_out_index_dot_html/expected/theindex.html b/nimdoc/test_out_index_dot_html/expected/theindex.html index 2a0ea7900..ea3486d4b 100644 --- a/nimdoc/test_out_index_dot_html/expected/theindex.html +++ b/nimdoc/test_out_index_dot_html/expected/theindex.html @@ -24,7 +24,7 @@ Modules: index.

        API symbols

        foo:
        • foo: foo()
          • + data-doc-search-tag="foo: proc foo()" href="index.html#foo">foo: proc foo()
        diff --git a/nimdoc/tester.nim b/nimdoc/tester.nim index ef82ae1b9..e94caae7c 100644 --- a/nimdoc/tester.nim +++ b/nimdoc/tester.nim @@ -12,11 +12,16 @@ var const baseDir = "nimdoc" +let + baseDirAbs = getCurrentDir() / baseDir type NimSwitches = object doc: seq[string] + docStage2: seq[string] buildIndex: seq[string] + md2html: seq[string] + md2htmlStage2: seq[string] proc exec(cmd: string) = if execShellCmd(cmd) != 0: @@ -25,6 +30,9 @@ proc exec(cmd: string) = proc testNimDoc(prjDir, docsDir: string; switches: NimSwitches; fixup = false) = let nimDocSwitches = switches.doc.join(" ") + nimDocStage2Switches = switches.docStage2.join(" ") + nimMd2HtmlSwitches = switches.md2html.join(" ") + nimMd2HtmlStage2Switches = switches.md2htmlStage2.join(" ") nimBuildIndexSwitches = switches.buildIndex.join(" ") putEnv("SOURCE_DATE_EPOCH", "100000") @@ -32,9 +40,23 @@ proc testNimDoc(prjDir, docsDir: string; switches: NimSwitches; fixup = false) = if nimDocSwitches != "": exec("$1 doc $2" % [nimExe, nimDocSwitches]) + echo("$1 doc $2" % [nimExe, nimDocSwitches]) + + if nimMd2HtmlSwitches != "": + exec("$1 md2html $2" % [nimExe, nimMd2HtmlSwitches]) + echo("$1 md2html $2" % [nimExe, nimMd2HtmlSwitches]) + + if nimDocStage2Switches != "": + exec("$1 doc $2" % [nimExe, nimDocStage2Switches]) + echo("$1 doc $2" % [nimExe, nimDocStage2Switches]) + + if nimMd2HtmlStage2Switches != "": + exec("$1 md2html $2" % [nimExe, nimMd2HtmlStage2Switches]) + echo("$1 md2html $2" % [nimExe, nimMd2HtmlStage2Switches]) if nimBuildIndexSwitches != "": exec("$1 buildIndex $2" % [nimExe, nimBuildIndexSwitches]) + echo("$1 buildIndex $2" % [nimExe, nimBuildIndexSwitches]) for expected in walkDirRec(prjDir / "expected/", checkDir=true): let produced = expected.replace('\\', '/').replace("/expected/", "/$1/" % [docsDir]) @@ -88,5 +110,46 @@ let test3Switches = NimSwitches(doc: @["$1/$2.nim" % [test3Dir, test3PrjName]]) testNimDoc(test3Dir, test3DocsDir, test3Switches, fixup) + +# Test concise external links (RFC#125) that work with `.idx` files. +# extlinks +# ├── project +# │   ├── main.nim +# │   ├── manual.md +# │   └── sub +# │   └── submodule.nim +# └── util.nim +# +# `main.nim` imports `submodule.nim` and `../utils.nim`. +# `main.nim`, `submodule.nim`, `manual.md` do importdoc and reference each other. +let + test4PrjName = "extlinks/project" + test4Dir = baseDir / test4PrjName + test4DirAbs = baseDirAbs / test4PrjName + test4MainModule = "main" + test4MarkupDoc = "doc" / "manual.md" + test4DocsDir = "htmldocs" + # 1st stage is with --index:only, 2nd is final + test4Switches = NimSwitches( + doc: @["--project", + "--outdir:$1/$2" % [test4Dir, test4DocsDir], + "--index:only", + "$1/$2.nim" % [test4Dir, test4MainModule]], + md2html: + @["--outdir:$1/$2" % [test4Dir, test4DocsDir], + "--docroot:$1" % [test4DirAbs], + "--index:only", + "$1/$2" % [test4Dir, test4MarkupDoc]], + docStage2: + @["--project", + "--outdir:$1/$2" % [test4Dir, test4DocsDir], + "$1/$2.nim" % [test4Dir, test4MainModule]], + md2htmlStage2: + @["--outdir:$1/$2" % [test4Dir, test4DocsDir], + "--docroot:$1" % [test4DirAbs], + "$1/$2" % [test4Dir, test4MarkupDoc]], + ) +testNimDoc(test4Dir, test4DocsDir, test4Switches, fixup) + if failures > 0: quit "$# failures occurred; see note in nimdoc/tester.nim regarding -d:nimTestsNimdocFixup" % $failures diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html index 7d59e9e86..f0f30536f 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html @@ -226,7 +226,7 @@ cmp: proc (x: T; y: K): int {.closure.}): int" href="#binarySearch,openArray[T],K,proc(T,K)">binary search.

        Note that proc can be used in postfix form: binarySearch proc.

        -

        Ref. type like G and type G and G[T] and type G*[T].

        +

        Ref. type like G and type G and G[T] and type G*[T].

        Group ref. with capital letters works: fN11 or fn11

        Ref. [] is the same as proc `[]`(G[T]) because there are no overloads. The full form: proc `[]`*[T](x: G[T]): TRef. []= aka `[]=`(G[T], int, T).Ref. $ aka proc $ or proc `$`.Ref. $(a: ref SomeType).Ref. foo_bar aka iterator foo_bar_.Ref. fn[T; U,V: SomeFloat]().Ref. 'big or func `'big` or `'big`(string).

        Pandoc Markdown

        Now repeat all the auto links of above in Pandoc Markdown Syntax.

        @@ -240,11 +240,11 @@ Ref. binary search.

        Note that proc can be used in postfix form: binarySearch proc.

        -

        Ref. type like G and type G and G[T] and type G*[T].

        +

        Ref. type like G and type G and G[T] and type G*[T].

        Group ref. with capital letters works: fN11 or fn11

        Ref. `[]` is the same as proc `[]`(G[T]) because there are no overloads. The full form: proc `[]`*[T](x: G[T]): T Ref. `[]=` aka `[]=`(G[T], int, T). Ref. $ aka proc $ or proc `$`. Ref. $(a: ref SomeType). Ref. foo_bar aka iterator foo_bar_. Ref. fn[T; U,V: SomeFloat](). Ref. 'big or func `'big` or `'big`(string).

        -

        Link name syntax

        Pandoc Markdown has synax for changing text of links: Ref. this proc or another symbol.

        +

        Link name syntax

        Pandoc Markdown has synax for changing text of links: Ref. this proc or another symbol.

        Symbols documentation

        Let us repeat auto links from symbols section below:

        There is also variant f(G[string]). See also f(G[int]).

        diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.idx b/nimdoc/testproject/expected/subdir/subdir_b/utils.idx index 007101b37..c2f148a92 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.idx +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.idx @@ -1,41 +1,46 @@ -funWithGenerics subdir/subdir_b/utils.html#funWithGenerics,T,U utils: funWithGenerics[T, U: SomeFloat](a: T; b: U) -enumValueA subdir/subdir_b/utils.html#enumValueA SomeType.enumValueA -enumValueB subdir/subdir_b/utils.html#enumValueB SomeType.enumValueB -enumValueC subdir/subdir_b/utils.html#enumValueC SomeType.enumValueC -SomeType subdir/subdir_b/utils.html#SomeType utils: SomeType -G subdir/subdir_b/utils.html#G utils: G -someType subdir/subdir_b/utils.html#someType_2 utils: someType(): SomeType -fn2 subdir/subdir_b/utils.html#fn2 utils: fn2() -fn2 subdir/subdir_b/utils.html#fn2,int utils: fn2(x: int) -fn2 subdir/subdir_b/utils.html#fn2,int,float utils: fn2(x: int; y: float) -binarySearch subdir/subdir_b/utils.html#binarySearch,openArray[T],K,proc(T,K) utils: binarySearch[T, K](a: openArray[T]; key: K;\n cmp: proc (x: T; y: K): int {.closure.}): int -fn3 subdir/subdir_b/utils.html#fn3 utils: fn3(): auto -fn4 subdir/subdir_b/utils.html#fn4 utils: fn4(): auto -fn5 subdir/subdir_b/utils.html#fn5 utils: fn5() -fn6 subdir/subdir_b/utils.html#fn6 utils: fn6() -fn7 subdir/subdir_b/utils.html#fn7 utils: fn7() -fn8 subdir/subdir_b/utils.html#fn8 utils: fn8(): auto -fn9 subdir/subdir_b/utils.html#fn9,int utils: fn9(a: int): int -fn10 subdir/subdir_b/utils.html#fn10,int utils: fn10(a: int): int -fN11 subdir/subdir_b/utils.html#fN11 utils: fN11() -fN11 subdir/subdir_b/utils.html#fN11,int utils: fN11(x: int) -aEnum subdir/subdir_b/utils.html#aEnum.t utils: aEnum(): untyped -bEnum subdir/subdir_b/utils.html#bEnum.t utils: bEnum(): untyped -fromUtilsGen subdir/subdir_b/utils.html#fromUtilsGen.t utils: fromUtilsGen(): untyped -f subdir/subdir_b/utils.html#f,G[int] utils: f(x: G[int]) -f subdir/subdir_b/utils.html#f,G[string] utils: f(x: G[string]) -`[]` subdir/subdir_b/utils.html#[],G[T] utils: `[]`[T](x: G[T]): T -`[]=` subdir/subdir_b/utils.html#[]=,G[T],int,T utils: `[]=`[T](a: var G[T]; index: int; value: T) -`$` subdir/subdir_b/utils.html#$,G[T] utils: `$`[T](a: G[T]): string -`$` subdir/subdir_b/utils.html#$,ref.SomeType utils: `$`[T](a: ref SomeType): string -fooBar subdir/subdir_b/utils.html#fooBar.i,seq[SomeType] utils: fooBar(a: seq[SomeType]): int -fn subdir/subdir_b/utils.html#fn utils: fn[T; U, V: SomeFloat]() -`'big` subdir/subdir_b/utils.html#'big,string utils: `'big`(a: string): SomeType -This is now a header subdir/subdir_b/utils.html#this-is-now-a-header This is now a header -Next header subdir/subdir_b/utils.html#this-is-now-a-header-next-header Next header -And so on subdir/subdir_b/utils.html#next-header-and-so-on And so on -More headers subdir/subdir_b/utils.html#more-headers More headers -Up to level 6 subdir/subdir_b/utils.html#more-headers-up-to-level-6 Up to level 6 -Pandoc Markdown subdir/subdir_b/utils.html#pandoc-markdown Pandoc Markdown -Link name syntax subdir/subdir_b/utils.html#pandoc-markdown-link-name-syntax Link name syntax -Symbols documentation subdir/subdir_b/utils.html#pandoc-markdown-symbols-documentation Symbols documentation +nimTitle utils subdir/subdir_b/utils.html module subdir/subdir_b/utils 0 +nim funWithGenerics subdir/subdir_b/utils.html#funWithGenerics,T,U proc funWithGenerics[T, U: SomeFloat](a: T; b: U) 1 +nim enumValueA subdir/subdir_b/utils.html#enumValueA SomeType.enumValueA 45 +nim enumValueB subdir/subdir_b/utils.html#enumValueB SomeType.enumValueB 45 +nim enumValueC subdir/subdir_b/utils.html#enumValueC SomeType.enumValueC 45 +nim SomeType subdir/subdir_b/utils.html#SomeType enum SomeType 45 +nim G subdir/subdir_b/utils.html#G object G 49 +nim someType subdir/subdir_b/utils.html#someType_2 proc someType(): SomeType 52 +nim fn2 subdir/subdir_b/utils.html#fn2 proc fn2() 57 +nim fn2 subdir/subdir_b/utils.html#fn2,int proc fn2(x: int) 58 +nim fn2 subdir/subdir_b/utils.html#fn2,int,float proc fn2(x: int; y: float) 61 +nim binarySearch subdir/subdir_b/utils.html#binarySearch,openArray[T],K,proc(T,K) proc binarySearch[T, K](a: openArray[T]; key: K;\n cmp: proc (x: T; y: K): int {.closure.}): int 63 +nim fn3 subdir/subdir_b/utils.html#fn3 proc fn3(): auto 66 +nim fn4 subdir/subdir_b/utils.html#fn4 proc fn4(): auto 67 +nim fn5 subdir/subdir_b/utils.html#fn5 proc fn5() 69 +nim fn6 subdir/subdir_b/utils.html#fn6 proc fn6() 70 +nim fn7 subdir/subdir_b/utils.html#fn7 proc fn7() 72 +nim fn8 subdir/subdir_b/utils.html#fn8 proc fn8(): auto 75 +nim fn9 subdir/subdir_b/utils.html#fn9,int proc fn9(a: int): int 78 +nim fn10 subdir/subdir_b/utils.html#fn10,int proc fn10(a: int): int 79 +nim fN11 subdir/subdir_b/utils.html#fN11 proc fN11() 85 +nim fN11 subdir/subdir_b/utils.html#fN11,int proc fN11(x: int) 86 +nim aEnum subdir/subdir_b/utils.html#aEnum.t template aEnum(): untyped 90 +nim bEnum subdir/subdir_b/utils.html#bEnum.t template bEnum(): untyped 95 +nim fromUtilsGen subdir/subdir_b/utils.html#fromUtilsGen.t template fromUtilsGen(): untyped 106 +nim f subdir/subdir_b/utils.html#f,G[int] proc f(x: G[int]) 130 +nim f subdir/subdir_b/utils.html#f,G[string] proc f(x: G[string]) 133 +nim `[]` subdir/subdir_b/utils.html#[],G[T] proc `[]`[T](x: G[T]): T 140 +nim `[]=` subdir/subdir_b/utils.html#[]=,G[T],int,T proc `[]=`[T](a: var G[T]; index: int; value: T) 144 +nim `$` subdir/subdir_b/utils.html#$,G[T] proc `$`[T](a: G[T]): string 148 +nim `$` subdir/subdir_b/utils.html#$,ref.SomeType proc `$`[T](a: ref SomeType): string 152 +nim fooBar subdir/subdir_b/utils.html#fooBar.i,seq[SomeType] iterator fooBar(a: seq[SomeType]): int 156 +nim fn subdir/subdir_b/utils.html#fn proc fn[T; U, V: SomeFloat]() 160 +nim `'big` subdir/subdir_b/utils.html#'big,string proc `'big`(a: string): SomeType 164 +nimgrp $ subdir/subdir_b/utils.html#$-procs-all proc 148 +nimgrp fn11 subdir/subdir_b/utils.html#fN11-procs-all proc 85 +nimgrp fn2 subdir/subdir_b/utils.html#fn2-procs-all proc 57 +nimgrp f subdir/subdir_b/utils.html#f-procs-all proc 130 +heading This is now a header subdir/subdir_b/utils.html#this-is-now-a-header This is now a header 0 +heading Next header subdir/subdir_b/utils.html#this-is-now-a-header-next-header Next header 0 +heading And so on subdir/subdir_b/utils.html#next-header-and-so-on And so on 0 +heading More headers subdir/subdir_b/utils.html#more-headers More headers 0 +heading Up to level 6 subdir/subdir_b/utils.html#more-headers-up-to-level-6 Up to level 6 0 +heading Pandoc Markdown subdir/subdir_b/utils.html#pandoc-markdown Pandoc Markdown 0 +heading Link name syntax subdir/subdir_b/utils.html#pandoc-markdown-link-name-syntax Link name syntax 0 +heading Symbols documentation subdir/subdir_b/utils.html#pandoc-markdown-symbols-documentation Symbols documentation 0 diff --git a/nimdoc/testproject/expected/testproject.idx b/nimdoc/testproject/expected/testproject.idx index 4d0172313..e69bedf33 100644 --- a/nimdoc/testproject/expected/testproject.idx +++ b/nimdoc/testproject/expected/testproject.idx @@ -1,64 +1,67 @@ -someVariable testproject.html#someVariable testproject: someVariable -C_A testproject.html#C_A testproject: C_A -C_B testproject.html#C_B testproject: C_B -C_C testproject.html#C_C testproject: C_C -C_D testproject.html#C_D testproject: C_D -bar testproject.html#bar,T,T testproject: bar[T](a, b: T): T -baz testproject.html#baz,T,T testproject: baz[T](a, b: T): T -buzz testproject.html#buzz,T,T testproject: buzz[T](a, b: T): T -FooBuzz testproject.html#FooBuzz testproject: FooBuzz -bar testproject.html#bar testproject: bar(f: FooBuzz) -aVariable testproject.html#aVariable testproject: aVariable -A testproject.html#A testproject: A -B testproject.html#B testproject: B -someFunc testproject.html#someFunc testproject: someFunc() -fromUtils1 testproject.html#fromUtils1.i testproject: fromUtils1(): int -fromUtils2 testproject.html#fromUtils2.t testproject: fromUtils2() -fromUtils3 testproject.html#fromUtils3 testproject: fromUtils3() -isValid testproject.html#isValid,T testproject: isValid[T](x: T): bool -enumValueA2 testproject.html#enumValueA2 Foo.enumValueA2 -Foo testproject.html#Foo testproject: Foo -z1 testproject.html#z1 testproject: z1(): Foo -z2 testproject.html#z2 testproject: z2() -z3 testproject.html#z3 testproject: z3() -z4 testproject.html#z4 testproject: z4() -z5 testproject.html#z5 testproject: z5(): int -z6 testproject.html#z6 testproject: z6(): int -z6t testproject.html#z6t.t testproject: z6t(): int -z7 testproject.html#z7 testproject: z7(): int -z8 testproject.html#z8 testproject: z8(): int -z9 testproject.html#z9 testproject: z9() -z10 testproject.html#z10 testproject: z10() -z11 testproject.html#z11 testproject: z11() -z12 testproject.html#z12 testproject: z12(): int -z13 testproject.html#z13 testproject: z13() -baz testproject.html#baz testproject: baz() -z17 testproject.html#z17 testproject: z17() -p1 testproject.html#p1 testproject: p1() -addfBug14485 testproject.html#addfBug14485 testproject: addfBug14485() -c_printf testproject.html#c_printf,cstring testproject: c_printf(frmt: cstring): cint -c_nonexistent testproject.html#c_nonexistent,cstring testproject: c_nonexistent(frmt: cstring): cint -low testproject.html#low,T testproject: low[T: Ordinal | enum | range](x: T): T -low2 testproject.html#low2,T testproject: low2[T: Ordinal | enum | range](x: T): T -tripleStrLitTest testproject.html#tripleStrLitTest testproject: tripleStrLitTest() -method1 testproject.html#method1.e,Moo testproject: method1(self: Moo) -method2 testproject.html#method2.e,Moo testproject: method2(self: Moo): int -method3 testproject.html#method3.e,Moo testproject: method3(self: Moo): int -iter1 testproject.html#iter1.i,int testproject: iter1(n: int): int -iter2 testproject.html#iter2.i,int testproject: iter2(n: int): int -bar testproject.html#bar.m testproject: bar(): untyped -z16 testproject.html#z16.m testproject: z16() -z18 testproject.html#z18.m testproject: z18(): int -foo testproject.html#foo.t,SomeType,SomeType testproject: foo(a, b: SomeType) -myfn testproject.html#myfn.t testproject: myfn() -z14 testproject.html#z14.t testproject: z14() -z15 testproject.html#z15.t testproject: z15() -asyncFun1 testproject.html#asyncFun1 testproject: asyncFun1(): Future[int] -asyncFun2 testproject.html#asyncFun2 testproject: asyncFun2(): owned(Future[void]) -asyncFun3 testproject.html#asyncFun3 testproject: asyncFun3(): owned(Future[void]) -testNimDocTrailingExample testproject.html#testNimDocTrailingExample.t testproject: testNimDocTrailingExample() -Circle testproject.html#Circle Shapes.Circle -Triangle testproject.html#Triangle Shapes.Triangle -Rectangle testproject.html#Rectangle Shapes.Rectangle -Shapes testproject.html#Shapes testproject: Shapes -anything testproject.html#anything testproject: anything() +nimTitle testproject testproject.html module testproject 0 +nim someVariable testproject.html#someVariable var someVariable 13 +nim C_A testproject.html#C_A const C_A 26 +nim C_B testproject.html#C_B const C_B 27 +nim C_C testproject.html#C_C const C_C 28 +nim C_D testproject.html#C_D const C_D 29 +nim bar testproject.html#bar,T,T proc bar[T](a, b: T): T 31 +nim baz testproject.html#baz,T,T proc baz[T](a, b: T): T 34 +nim buzz testproject.html#buzz,T,T proc buzz[T](a, b: T): T 38 +nim FooBuzz testproject.html#FooBuzz type FooBuzz 43 +nim bar testproject.html#bar proc bar(f: FooBuzz) 47 +nim aVariable testproject.html#aVariable var aVariable 52 +nim A testproject.html#A enum A 92 +nim B testproject.html#B enum B 97 +nim someFunc testproject.html#someFunc proc someFunc() 56 +nim fromUtils1 testproject.html#fromUtils1.i iterator fromUtils1(): int 112 +nim fromUtils2 testproject.html#fromUtils2.t template fromUtils2() 119 +nim fromUtils3 testproject.html#fromUtils3 proc fromUtils3() 57 +nim isValid testproject.html#isValid,T proc isValid[T](x: T): bool 59 +nim enumValueA2 testproject.html#enumValueA2 Foo.enumValueA2 66 +nim Foo testproject.html#Foo enum Foo 66 +nim z1 testproject.html#z1 proc z1(): Foo 69 +nim z2 testproject.html#z2 proc z2() 73 +nim z3 testproject.html#z3 proc z3() 78 +nim z4 testproject.html#z4 proc z4() 81 +nim z5 testproject.html#z5 proc z5(): int 87 +nim z6 testproject.html#z6 proc z6(): int 91 +nim z6t testproject.html#z6t.t template z6t(): int 95 +nim z7 testproject.html#z7 proc z7(): int 99 +nim z8 testproject.html#z8 proc z8(): int 103 +nim z9 testproject.html#z9 proc z9() 111 +nim z10 testproject.html#z10 proc z10() 114 +nim z11 testproject.html#z11 proc z11() 119 +nim z12 testproject.html#z12 proc z12(): int 124 +nim z13 testproject.html#z13 proc z13() 129 +nim baz testproject.html#baz proc baz() 134 +nim z17 testproject.html#z17 proc z17() 144 +nim p1 testproject.html#p1 proc p1() 156 +nim addfBug14485 testproject.html#addfBug14485 proc addfBug14485() 177 +nim c_printf testproject.html#c_printf,cstring proc c_printf(frmt: cstring): cint 193 +nim c_nonexistent testproject.html#c_nonexistent,cstring proc c_nonexistent(frmt: cstring): cint 197 +nim low testproject.html#low,T proc low[T: Ordinal | enum | range](x: T): T 200 +nim low2 testproject.html#low2,T proc low2[T: Ordinal | enum | range](x: T): T 210 +nim tripleStrLitTest testproject.html#tripleStrLitTest proc tripleStrLitTest() 223 +nim method1 testproject.html#method1.e,Moo method method1(self: Moo) 264 +nim method2 testproject.html#method2.e,Moo method method2(self: Moo): int 266 +nim method3 testproject.html#method3.e,Moo method method3(self: Moo): int 269 +nim iter1 testproject.html#iter1.i,int iterator iter1(n: int): int 274 +nim iter2 testproject.html#iter2.i,int iterator iter2(n: int): int 278 +nim bar testproject.html#bar.m macro bar(): untyped 285 +nim z16 testproject.html#z16.m macro z16() 288 +nim z18 testproject.html#z18.m macro z18(): int 297 +nim foo testproject.html#foo.t,SomeType,SomeType template foo(a, b: SomeType) 302 +nim myfn testproject.html#myfn.t template myfn() 307 +nim z14 testproject.html#z14.t template z14() 328 +nim z15 testproject.html#z15.t template z15() 333 +nim asyncFun1 testproject.html#asyncFun1 proc asyncFun1(): Future[int] 358 +nim asyncFun2 testproject.html#asyncFun2 proc asyncFun2(): owned(Future[void]) 361 +nim asyncFun3 testproject.html#asyncFun3 proc asyncFun3(): owned(Future[void]) 362 +nim testNimDocTrailingExample testproject.html#testNimDocTrailingExample.t template testNimDocTrailingExample() 371 +nim Circle testproject.html#Circle Shapes.Circle 380 +nim Triangle testproject.html#Triangle Shapes.Triangle 380 +nim Rectangle testproject.html#Rectangle Shapes.Rectangle 380 +nim Shapes testproject.html#Shapes enum Shapes 380 +nim anything testproject.html#anything proc anything() 387 +nimgrp bar testproject.html#bar-procs-all proc 31 +nimgrp baz testproject.html#baz-procs-all proc 34 diff --git a/nimdoc/testproject/expected/theindex.html b/nimdoc/testproject/expected/theindex.html index 9f73040f2..7b4a95561 100644 --- a/nimdoc/testproject/expected/theindex.html +++ b/nimdoc/testproject/expected/theindex.html @@ -24,377 +24,377 @@ Modules: subdir/subdir_b/utils, testproject.

        API symbols

        `$`:
        • utils: `$`[T](a: G[T]): string
          • + data-doc-search-tag="utils: proc `$`[T](a: G[T]): string" href="subdir/subdir_b/utils.html#%24%2CG%5BT%5D">utils: proc `$`[T](a: G[T]): string
        • utils: `$`[T](a: ref SomeType): string
          • + data-doc-search-tag="utils: proc `$`[T](a: ref SomeType): string" href="subdir/subdir_b/utils.html#%24%2Cref.SomeType">utils: proc `$`[T](a: ref SomeType): string
        `'big`:
        • utils: `'big`(a: string): SomeType
          • + data-doc-search-tag="utils: proc `'big`(a: string): SomeType" href="subdir/subdir_b/utils.html#%27big%2Cstring">utils: proc `'big`(a: string): SomeType
        `[]=`:
        • utils: `[]=`[T](a: var G[T]; index: int; value: T)
          • + data-doc-search-tag="utils: proc `[]=`[T](a: var G[T]; index: int; value: T)" href="subdir/subdir_b/utils.html#%5B%5D%3D%2CG%5BT%5D%2Cint%2CT">utils: proc `[]=`[T](a: var G[T]; index: int; value: T)
        `[]`:
        • utils: `[]`[T](x: G[T]): T
          • + data-doc-search-tag="utils: proc `[]`[T](x: G[T]): T" href="subdir/subdir_b/utils.html#%5B%5D%2CG%5BT%5D">utils: proc `[]`[T](x: G[T]): T
        A:
        • testproject: A
          • + data-doc-search-tag="testproject: enum A" href="testproject.html#A">testproject: enum A
        addfBug14485:
        • testproject: addfBug14485()
          • + data-doc-search-tag="testproject: proc addfBug14485()" href="testproject.html#addfBug14485">testproject: proc addfBug14485()
        aEnum:
        • utils: aEnum(): untyped
          • + data-doc-search-tag="utils: template aEnum(): untyped" href="subdir/subdir_b/utils.html#aEnum.t">utils: template aEnum(): untyped
        anything:
        • testproject: anything()
          • + data-doc-search-tag="testproject: proc anything()" href="testproject.html#anything">testproject: proc anything()
        asyncFun1:
        asyncFun2:
        asyncFun3:
        aVariable:
        • testproject: aVariable
          • + data-doc-search-tag="testproject: var aVariable" href="testproject.html#aVariable">testproject: var aVariable
        B:
        • testproject: B
          • + data-doc-search-tag="testproject: enum B" href="testproject.html#B">testproject: enum B
        bar:
        • testproject: bar(f: FooBuzz)
          • + data-doc-search-tag="testproject: proc bar(f: FooBuzz)" href="testproject.html#bar">testproject: proc bar(f: FooBuzz)
        • testproject: bar[T](a, b: T): T
          • + data-doc-search-tag="testproject: proc bar[T](a, b: T): T" href="testproject.html#bar%2CT%2CT">testproject: proc bar[T](a, b: T): T
        • testproject: bar(): untyped
          • + data-doc-search-tag="testproject: macro bar(): untyped" href="testproject.html#bar.m">testproject: macro bar(): untyped
        baz:
        • testproject: baz()
          • + data-doc-search-tag="testproject: proc baz()" href="testproject.html#baz">testproject: proc baz()
        • testproject: baz[T](a, b: T): T
          • + data-doc-search-tag="testproject: proc baz[T](a, b: T): T" href="testproject.html#baz%2CT%2CT">testproject: proc baz[T](a, b: T): T
        bEnum:
        • utils: bEnum(): untyped
          • + data-doc-search-tag="utils: template bEnum(): untyped" href="subdir/subdir_b/utils.html#bEnum.t">utils: template bEnum(): untyped
        binarySearch:
        buzz:
        • testproject: buzz[T](a, b: T): T
          • + data-doc-search-tag="testproject: proc buzz[T](a, b: T): T" href="testproject.html#buzz%2CT%2CT">testproject: proc buzz[T](a, b: T): T
        C_A:
        • testproject: C_A
          • + data-doc-search-tag="testproject: const C_A" href="testproject.html#C_A">testproject: const C_A
        C_B:
        • testproject: C_B
          • + data-doc-search-tag="testproject: const C_B" href="testproject.html#C_B">testproject: const C_B
        C_C:
        • testproject: C_C
          • + data-doc-search-tag="testproject: const C_C" href="testproject.html#C_C">testproject: const C_C
        C_D:
        • testproject: C_D
          • + data-doc-search-tag="testproject: const C_D" href="testproject.html#C_D">testproject: const C_D
        Circle:
        • Shapes.Circle
          • + data-doc-search-tag="testproject: Shapes.Circle" href="testproject.html#Circle">testproject: Shapes.Circle
        c_nonexistent:
        c_printf:
        enumValueA:
        • SomeType.enumValueA
          • + data-doc-search-tag="utils: SomeType.enumValueA" href="subdir/subdir_b/utils.html#enumValueA">utils: SomeType.enumValueA
        enumValueA2:
        • Foo.enumValueA2
          • + data-doc-search-tag="testproject: Foo.enumValueA2" href="testproject.html#enumValueA2">testproject: Foo.enumValueA2
        enumValueB:
        • SomeType.enumValueB
          • + data-doc-search-tag="utils: SomeType.enumValueB" href="subdir/subdir_b/utils.html#enumValueB">utils: SomeType.enumValueB
        enumValueC:
        • SomeType.enumValueC
          • + data-doc-search-tag="utils: SomeType.enumValueC" href="subdir/subdir_b/utils.html#enumValueC">utils: SomeType.enumValueC
        f:
        • utils: f(x: G[int])
          • + data-doc-search-tag="utils: proc f(x: G[int])" href="subdir/subdir_b/utils.html#f%2CG%5Bint%5D">utils: proc f(x: G[int])
        • utils: f(x: G[string])
          • + data-doc-search-tag="utils: proc f(x: G[string])" href="subdir/subdir_b/utils.html#f%2CG%5Bstring%5D">utils: proc f(x: G[string])
        fn:
        • utils: fn[T; U, V: SomeFloat]()
          • + data-doc-search-tag="utils: proc fn[T; U, V: SomeFloat]()" href="subdir/subdir_b/utils.html#fn">utils: proc fn[T; U, V: SomeFloat]()
        fn10:
        • utils: fn10(a: int): int
          • + data-doc-search-tag="utils: proc fn10(a: int): int" href="subdir/subdir_b/utils.html#fn10%2Cint">utils: proc fn10(a: int): int
        fN11:
        • utils: fN11()
          • + data-doc-search-tag="utils: proc fN11()" href="subdir/subdir_b/utils.html#fN11">utils: proc fN11()
        • utils: fN11(x: int)
          • + data-doc-search-tag="utils: proc fN11(x: int)" href="subdir/subdir_b/utils.html#fN11%2Cint">utils: proc fN11(x: int)
        fn2:
        • utils: fn2()
          • + data-doc-search-tag="utils: proc fn2()" href="subdir/subdir_b/utils.html#fn2">utils: proc fn2()
        • utils: fn2(x: int)
          • + data-doc-search-tag="utils: proc fn2(x: int)" href="subdir/subdir_b/utils.html#fn2%2Cint">utils: proc fn2(x: int)
        • utils: fn2(x: int; y: float)
          • + data-doc-search-tag="utils: proc fn2(x: int; y: float)" href="subdir/subdir_b/utils.html#fn2%2Cint%2Cfloat">utils: proc fn2(x: int; y: float)
        fn3:
        • utils: fn3(): auto
          • + data-doc-search-tag="utils: proc fn3(): auto" href="subdir/subdir_b/utils.html#fn3">utils: proc fn3(): auto
        fn4:
        • utils: fn4(): auto
          • + data-doc-search-tag="utils: proc fn4(): auto" href="subdir/subdir_b/utils.html#fn4">utils: proc fn4(): auto
        fn5:
        • utils: fn5()
          • + data-doc-search-tag="utils: proc fn5()" href="subdir/subdir_b/utils.html#fn5">utils: proc fn5()
        fn6:
        • utils: fn6()
          • + data-doc-search-tag="utils: proc fn6()" href="subdir/subdir_b/utils.html#fn6">utils: proc fn6()
        fn7:
        • utils: fn7()
          • + data-doc-search-tag="utils: proc fn7()" href="subdir/subdir_b/utils.html#fn7">utils: proc fn7()
        fn8:
        • utils: fn8(): auto
          • + data-doc-search-tag="utils: proc fn8(): auto" href="subdir/subdir_b/utils.html#fn8">utils: proc fn8(): auto
        fn9:
        • utils: fn9(a: int): int
          • + data-doc-search-tag="utils: proc fn9(a: int): int" href="subdir/subdir_b/utils.html#fn9%2Cint">utils: proc fn9(a: int): int
        Foo:
        • testproject: Foo
          • + data-doc-search-tag="testproject: enum Foo" href="testproject.html#Foo">testproject: enum Foo
        foo:
        • testproject: foo(a, b: SomeType)
          • + data-doc-search-tag="testproject: template foo(a, b: SomeType)" href="testproject.html#foo.t%2CSomeType%2CSomeType">testproject: template foo(a, b: SomeType)
        fooBar:
        • utils: fooBar(a: seq[SomeType]): int
          • + data-doc-search-tag="utils: iterator fooBar(a: seq[SomeType]): int" href="subdir/subdir_b/utils.html#fooBar.i%2Cseq%5BSomeType%5D">utils: iterator fooBar(a: seq[SomeType]): int
        FooBuzz:
        • testproject: FooBuzz
          • + data-doc-search-tag="testproject: type FooBuzz" href="testproject.html#FooBuzz">testproject: type FooBuzz
        fromUtils1:
        • testproject: fromUtils1(): int
          • + data-doc-search-tag="testproject: iterator fromUtils1(): int" href="testproject.html#fromUtils1.i">testproject: iterator fromUtils1(): int
        fromUtils2:
        • testproject: fromUtils2()
          • + data-doc-search-tag="testproject: template fromUtils2()" href="testproject.html#fromUtils2.t">testproject: template fromUtils2()
        fromUtils3:
        • testproject: fromUtils3()
          • + data-doc-search-tag="testproject: proc fromUtils3()" href="testproject.html#fromUtils3">testproject: proc fromUtils3()
        fromUtilsGen:
        • utils: fromUtilsGen(): untyped
          • + data-doc-search-tag="utils: template fromUtilsGen(): untyped" href="subdir/subdir_b/utils.html#fromUtilsGen.t">utils: template fromUtilsGen(): untyped
        funWithGenerics:
        G:
        • utils: G
          • + data-doc-search-tag="utils: object G" href="subdir/subdir_b/utils.html#G">utils: object G
        isValid:
        • testproject: isValid[T](x: T): bool
          • + data-doc-search-tag="testproject: proc isValid[T](x: T): bool" href="testproject.html#isValid%2CT">testproject: proc isValid[T](x: T): bool
        iter1:
        • testproject: iter1(n: int): int
          • + data-doc-search-tag="testproject: iterator iter1(n: int): int" href="testproject.html#iter1.i%2Cint">testproject: iterator iter1(n: int): int
        iter2:
        • testproject: iter2(n: int): int
          • + data-doc-search-tag="testproject: iterator iter2(n: int): int" href="testproject.html#iter2.i%2Cint">testproject: iterator iter2(n: int): int
        low:
        low2:
        method1:
        • testproject: method1(self: Moo)
          • + data-doc-search-tag="testproject: method method1(self: Moo)" href="testproject.html#method1.e%2CMoo">testproject: method method1(self: Moo)
        method2:
        • testproject: method2(self: Moo): int
          • + data-doc-search-tag="testproject: method method2(self: Moo): int" href="testproject.html#method2.e%2CMoo">testproject: method method2(self: Moo): int
        method3:
        • testproject: method3(self: Moo): int
          • + data-doc-search-tag="testproject: method method3(self: Moo): int" href="testproject.html#method3.e%2CMoo">testproject: method method3(self: Moo): int
        myfn:
        • testproject: myfn()
          • + data-doc-search-tag="testproject: template myfn()" href="testproject.html#myfn.t">testproject: template myfn()
        p1:
        • testproject: p1()
          • + data-doc-search-tag="testproject: proc p1()" href="testproject.html#p1">testproject: proc p1()
        Rectangle:
        • Shapes.Rectangle
          • + data-doc-search-tag="testproject: Shapes.Rectangle" href="testproject.html#Rectangle">testproject: Shapes.Rectangle
        Shapes:
        • testproject: Shapes
          • + data-doc-search-tag="testproject: enum Shapes" href="testproject.html#Shapes">testproject: enum Shapes
        someFunc:
        • testproject: someFunc()
          • + data-doc-search-tag="testproject: proc someFunc()" href="testproject.html#someFunc">testproject: proc someFunc()
        SomeType:
        • utils: SomeType
          • + data-doc-search-tag="utils: enum SomeType" href="subdir/subdir_b/utils.html#SomeType">utils: enum SomeType
        someType:
        • utils: someType(): SomeType
          • + data-doc-search-tag="utils: proc someType(): SomeType" href="subdir/subdir_b/utils.html#someType_2">utils: proc someType(): SomeType
        someVariable:
        • testproject: someVariable
          • + data-doc-search-tag="testproject: var someVariable" href="testproject.html#someVariable">testproject: var someVariable
        testNimDocTrailingExample:
        • testproject: testNimDocTrailingExample()
          • + data-doc-search-tag="testproject: template testNimDocTrailingExample()" href="testproject.html#testNimDocTrailingExample.t">testproject: template testNimDocTrailingExample()
        Triangle:
        • Shapes.Triangle
          • + data-doc-search-tag="testproject: Shapes.Triangle" href="testproject.html#Triangle">testproject: Shapes.Triangle
        tripleStrLitTest:
        • testproject: tripleStrLitTest()
          • + data-doc-search-tag="testproject: proc tripleStrLitTest()" href="testproject.html#tripleStrLitTest">testproject: proc tripleStrLitTest()
        z1:
        • testproject: z1(): Foo
          • + data-doc-search-tag="testproject: proc z1(): Foo" href="testproject.html#z1">testproject: proc z1(): Foo
        z10:
        • testproject: z10()
          • + data-doc-search-tag="testproject: proc z10()" href="testproject.html#z10">testproject: proc z10()
        z11:
        • testproject: z11()
          • + data-doc-search-tag="testproject: proc z11()" href="testproject.html#z11">testproject: proc z11()
        z12:
        • testproject: z12(): int
          • + data-doc-search-tag="testproject: proc z12(): int" href="testproject.html#z12">testproject: proc z12(): int
        z13:
        • testproject: z13()
          • + data-doc-search-tag="testproject: proc z13()" href="testproject.html#z13">testproject: proc z13()
        z14:
        • testproject: z14()
          • + data-doc-search-tag="testproject: template z14()" href="testproject.html#z14.t">testproject: template z14()
        z15:
        • testproject: z15()
          • + data-doc-search-tag="testproject: template z15()" href="testproject.html#z15.t">testproject: template z15()
        z16:
        • testproject: z16()
          • + data-doc-search-tag="testproject: macro z16()" href="testproject.html#z16.m">testproject: macro z16()
        z17:
        • testproject: z17()
          • + data-doc-search-tag="testproject: proc z17()" href="testproject.html#z17">testproject: proc z17()
        z18:
        • testproject: z18(): int
          • + data-doc-search-tag="testproject: macro z18(): int" href="testproject.html#z18.m">testproject: macro z18(): int
        z2:
        • testproject: z2()
          • + data-doc-search-tag="testproject: proc z2()" href="testproject.html#z2">testproject: proc z2()
        z3:
        • testproject: z3()
          • + data-doc-search-tag="testproject: proc z3()" href="testproject.html#z3">testproject: proc z3()
        z4:
        • testproject: z4()
          • + data-doc-search-tag="testproject: proc z4()" href="testproject.html#z4">testproject: proc z4()
        z5:
        • testproject: z5(): int
          • + data-doc-search-tag="testproject: proc z5(): int" href="testproject.html#z5">testproject: proc z5(): int
        z6:
        • testproject: z6(): int
          • + data-doc-search-tag="testproject: proc z6(): int" href="testproject.html#z6">testproject: proc z6(): int
        z6t:
        • testproject: z6t(): int
          • + data-doc-search-tag="testproject: template z6t(): int" href="testproject.html#z6t.t">testproject: template z6t(): int
        z7:
        • testproject: z7(): int
          • + data-doc-search-tag="testproject: proc z7(): int" href="testproject.html#z7">testproject: proc z7(): int
        z8:
        • testproject: z8(): int
          • + data-doc-search-tag="testproject: proc z8(): int" href="testproject.html#z8">testproject: proc z8(): int
        z9:
        • testproject: z9()
          • + data-doc-search-tag="testproject: proc z9()" href="testproject.html#z9">testproject: proc z9()
        diff --git a/tests/stdlib/tdochelpers.nim b/tests/stdlib/tdochelpers.nim index 15d538891..300ce76dc 100644 --- a/tests/stdlib/tdochelpers.nim +++ b/tests/stdlib/tdochelpers.nim @@ -202,3 +202,19 @@ suite "Integration with Nim": name: "Copyflag") check inputRst.fromRst == expected check inputMd.fromMd == expected + + test "prefixed module": + let inputRst = "`module std / paths`_" + let inputMd = "[module std / paths]" + let expected = LangSymbol(symKind: "module", + name: "std/paths") + check inputRst.fromRst == expected + check inputMd.fromMd == expected + + test "postfixed module": + let inputRst = "`std / paths module`_" + let inputMd = "[std / paths module]" + let expected = LangSymbol(symKind: "module", + name: "std/paths") + check inputRst.fromRst == expected + check inputMd.fromMd == expected diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index 3575d17aa..da787d274 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -60,7 +60,7 @@ proc toAst(input: string, result = "" var (rst, _, _) = rstParse(input, filen, line=LineRstInit, column=ColRstInit, - rstOptions, myFindFile, testMsgHandler) + rstOptions, myFindFile, nil, testMsgHandler) result = treeRepr(rst) except EParseError as e: if e.msg != "": diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 4fffc7533..f63aea030 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -240,27 +240,28 @@ proc buildDocSamples(nimArgs, destPath: string) = exec(findNim().quoteShell() & " doc $# -o:$# $#" % [nimArgs, destPath / "docgen_sample.html", "doc" / "docgen_sample.nim"]) -proc buildDocPackages(nimArgs, destPath: string) = +proc buildDocPackages(nimArgs, destPath: string, indexOnly: bool) = # compiler docs; later, other packages (perhaps tools, testament etc) let nim = findNim().quoteShell() # to avoid broken links to manual from compiler dir, but a multi-package # structure could be supported later proc docProject(outdir, options, mainproj: string) = - exec("$nim doc --project --outdir:$outdir $nimArgs --git.url:$gitUrl $options $mainproj" % [ + exec("$nim doc --project --outdir:$outdir $nimArgs --git.url:$gitUrl $index $options $mainproj" % [ "nim", nim, "outdir", outdir, "nimArgs", nimArgs, "gitUrl", gitUrl, "options", options, "mainproj", mainproj, + "index", if indexOnly: "--index:only" else: "" ]) let extra = "-u:boot" # xxx keep in sync with what's in $nim_prs_D/config/nimdoc.cfg, or, rather, # start using nims instead of nimdoc.cfg docProject(destPath/"compiler", extra, "compiler/index.nim") -proc buildDoc(nimArgs, destPath: string) = +proc buildDoc(nimArgs, destPath: string, indexOnly: bool) = # call nim for the documentation: let rst2html = getMd2html() var @@ -268,17 +269,19 @@ proc buildDoc(nimArgs, destPath: string) = officialPackagesList.len + officialPackagesListWithoutIndex.len) i = 0 let nim = findNim().quoteShell() + + let index = if indexOnly: "--index:only" else: "" for d in items(rst2html): - commands[i] = nim & " md2html $# --git.url:$# -o:$# --index:on $#" % + commands[i] = nim & " md2html $# --git.url:$# -o:$# $# $#" % [nimArgs, gitUrl, - destPath / changeFileExt(splitFile(d).name, "html"), d] + destPath / changeFileExt(splitFile(d).name, "html"), index, d] i.inc for d in items(doc): let extra = if isJsOnly(d): "--backend:js" else: "" var nimArgs2 = nimArgs if d.isRelativeTo("compiler"): doAssert false - commands[i] = nim & " doc $# $# --git.url:$# --outdir:$# --index:on $#" % - [extra, nimArgs2, gitUrl, destPath, d] + commands[i] = nim & " doc $# $# --git.url:$# --outdir:$# $# $#" % + [extra, nimArgs2, gitUrl, destPath, index, d] i.inc for d in items(withoutIndex): commands[i] = nim & " doc $# --git.url:$# -o:$# $#" % @@ -300,12 +303,6 @@ proc buildDoc(nimArgs, destPath: string) = i.inc mexec(commands) - exec(nim & " buildIndex -o:$1/theindex.html $1" % [destPath]) - # caveat: this works so long it's called before `buildDocPackages` which - # populates `compiler/` with unrelated idx files that shouldn't be in index, - # so should work in CI but you may need to remove your generated html files - # locally after calling `./koch docs`. The clean fix would be for `idx` files - # to be transient with `--project` (eg all in memory). proc nim2pdf(src: string, dst: string, nimArgs: string) = # xxx expose as a `nim` command or in some other reusable way. @@ -351,9 +348,23 @@ proc buildDocsDir*(args: string, dir: string) = gitClonePackages(@["asyncftpclient", "punycode", "smtp", "db_connector"]) createDir(dir) buildDocSamples(args, dir) - buildDoc(args, dir) # bottleneck + + # generate `.idx` files and top-level `theindex.html`: + buildDoc(args, dir, indexOnly=true) # bottleneck + let nim = findNim().quoteShell() + exec(nim & " buildIndex -o:$1/theindex.html $1" % [dir]) + # caveat: this works so long it's called before `buildDocPackages` which + # populates `compiler/` with unrelated idx files that shouldn't be in index, + # so should work in CI but you may need to remove your generated html files + # locally after calling `./koch docs`. The clean fix would be for `idx` files + # to be transient with `--project` (eg all in memory). + buildDocPackages(args, dir, indexOnly=true) + + # generate HTML and package-level `theindex.html`: + buildDoc(args, dir, indexOnly=false) # bottleneck + buildDocPackages(args, dir, indexOnly=false) + copyFile(dir / "overview.html", dir / "index.html") - buildDocPackages(args, dir) copyFile(docHackJsSource, dir / docHackJsSource.lastPathPart) proc buildDocs*(args: string, localOnly = false, localOutDir = "") = From b34412ff0fb450278d8c93f6e5d7bf0e8d3f0a0b Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 6 Jan 2023 13:24:58 +0800 Subject: [PATCH 2346/2472] adds changelog for more architectures support and fixes a link (#21229) --- changelogs/changelog_2_0_0.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/changelogs/changelog_2_0_0.md b/changelogs/changelog_2_0_0.md index 29d906d62..9ff493b54 100644 --- a/changelogs/changelog_2_0_0.md +++ b/changelogs/changelog_2_0_0.md @@ -26,7 +26,7 @@ - Enabling `-d:nimPreviewSlimSystem` also removes the following deprecated symbols in the `system` module: - Aliases with `Error` suffix to exception types that have a `Defect` suffix - (see [exceptions](https://nim-lang.org/docs/exceptions.html)): + (see [exceptions](https://nim-lang.github.io/Nim/exceptions.html)): `ArithmeticError`, `DivByZeroError`, `OverflowError`, `AccessViolationError`, `AssertionError`, `OutOfMemError`, `IndexError`, `FieldError`, `RangeError`, `StackOverflowError`, `ReraiseError`, @@ -317,6 +317,7 @@ - Nim now supports `out` parameters and ["strict definitions"](https://nim-lang.github.io/Nim/manual_experimental.html#strict-definitions-and-nimout-parameters). - Nim now offers a [strict mode](https://nim-lang.github.io/Nim/manual_experimental.html#strict-case-objects) for `case objects`. +- IBM Z architecture and macOS m1 arm64 architecture are supported. ## Compiler changes From 05c6db6500de0551882a2d6cc612c9afb6ca674c Mon Sep 17 00:00:00 2001 From: metagn Date: Sun, 8 Jan 2023 08:49:49 +0300 Subject: [PATCH 2347/2472] underscores for routine parameters (#21192) * underscores for routine parameters fixes #13443, fixes #13804, refs #21121 * add changelog + more tests * support generics and ensure inferred lambdas work --- changelogs/changelog_2_0_0.md | 15 +++++ compiler/lookups.nim | 1 + compiler/semtempl.nim | 5 +- compiler/semtypes.nim | 4 +- tests/proc/tunderscoreparam.nim | 109 ++++++++++++++++++++++++++++++++ 5 files changed, 131 insertions(+), 3 deletions(-) create mode 100644 tests/proc/tunderscoreparam.nim diff --git a/changelogs/changelog_2_0_0.md b/changelogs/changelog_2_0_0.md index 9ff493b54..c8427cc2a 100644 --- a/changelogs/changelog_2_0_0.md +++ b/changelogs/changelog_2_0_0.md @@ -135,6 +135,21 @@ - The experimental strictFuncs feature now disallows a store to the heap via a `ref` or `ptr` indirection. +- Underscores (`_`) as routine parameters are now ignored and cannot be used in the routine body. + The following code now does not compile: + + ```nim + proc foo(_: int): int = _ + 1 + echo foo(1) + ``` + + Instead, the following code now compiles: + + ```nim + proc foo(_, _: int): int = 123 + echo foo(1, 2) + ``` + - - Added the `--legacy:verboseTypeMismatch` switch to get legacy type mismatch error messages. ## Standard library additions and changes diff --git a/compiler/lookups.nim b/compiler/lookups.nim index d50fc6e27..3f4fcb4d0 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -323,6 +323,7 @@ proc wrongRedefinition*(c: PContext; info: TLineInfo, s: string; # xxx pending bootstrap >= 1.4, replace all those overloads with a single one: # proc addDecl*(c: PContext, sym: PSym, info = sym.info, scope = c.currentScope) {.inline.} = proc addDeclAt*(c: PContext; scope: PScope, sym: PSym, info: TLineInfo) = + if sym.name.s == "_": return let conflict = scope.addUniqueSym(sym) if conflict != nil: if sym.kind == skModule and conflict.kind == skModule and sym.owner == conflict.owner: diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim index 0016e9edf..3b2f2dd9e 100644 --- a/compiler/semtempl.nim +++ b/compiler/semtempl.nim @@ -642,8 +642,9 @@ proc semTemplateDef(c: PContext, n: PNode): PNode = # body by the absence of the sfGenSym flag: for i in 1.. Date: Sun, 8 Jan 2023 16:37:40 +0800 Subject: [PATCH 2348/2472] disable bump (#21241) --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index c134be6f9..77d89a420 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -46,7 +46,7 @@ pkg "BipBuffer" pkg "blscurve", allowFailure = true # pending https://github.com/status-im/nim-blscurve/issues/39 pkg "bncurve" pkg "brainfuck", "nim c -d:release -r tests/compile.nim" -pkg "bump", "nim c --gc:arc --path:. -r tests/tbump.nim", "https://github.com/disruptek/bump" +pkg "bump", "nim c --gc:arc --path:. -r tests/tbump.nim", "https://github.com/disruptek/bump", allowFailure = true pkg "c2nim", "nim c testsuite/tester.nim" pkg "cascade" pkg "cello", url = "https://github.com/nim-lang/cello", useHead = true From 76e4f8afb1423eb04f26059f1ed01ca0375bee63 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sun, 8 Jan 2023 22:39:25 +0800 Subject: [PATCH 2349/2472] fixes #21209; exclude threads options for JS (#21233) --- compiler/commands.nim | 2 ++ tests/js/t21209.nim | 6 ++++++ tests/js/t21209.nims | 1 + 3 files changed, 9 insertions(+) create mode 100644 tests/js/t21209.nim create mode 100644 tests/js/t21209.nims diff --git a/compiler/commands.nim b/compiler/commands.nim index 73140036a..59b221647 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -649,6 +649,8 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; of "backend", "b": let backend = parseEnum(arg.normalize, TBackend.default) if backend == TBackend.default: localError(conf, info, "invalid backend: '$1'" % arg) + if backend == backendJs: # bug #21209 + conf.globalOptions.excl {optThreadAnalysis, optThreads} conf.backend = backend of "doccmd": conf.docCmd = arg of "define", "d": diff --git a/tests/js/t21209.nim b/tests/js/t21209.nim new file mode 100644 index 000000000..4de34f035 --- /dev/null +++ b/tests/js/t21209.nim @@ -0,0 +1,6 @@ +discard """ + action: "compile" + cmd: "nim check --warning[UnusedImport]:off $file" +""" + +import std/times diff --git a/tests/js/t21209.nims b/tests/js/t21209.nims new file mode 100644 index 000000000..318e28f97 --- /dev/null +++ b/tests/js/t21209.nims @@ -0,0 +1 @@ +--b:js \ No newline at end of file From a55604a3d8605e398f27df364cc215b7114395eb Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sun, 8 Jan 2023 22:40:27 +0800 Subject: [PATCH 2350/2472] fixes #21232; highlite.getNextToken() returns gtOperator for '/' in C (#21234) --- lib/packages/docutils/highlite.nim | 3 +++ tests/stdlib/thighlite.nim | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/lib/packages/docutils/highlite.nim b/lib/packages/docutils/highlite.nim index 3f65c8cc4..c3caf31b4 100644 --- a/lib/packages/docutils/highlite.nim +++ b/lib/packages/docutils/highlite.nim @@ -498,6 +498,9 @@ proc clikeNextToken(g: var GeneralTokenizer, keywords: openArray[string], of '\0': break else: inc(pos) + else: + g.kind = gtOperator + while g.buf[pos] in OpChars: inc(pos) of '#': inc(pos) if hasPreprocessor in flags: diff --git a/tests/stdlib/thighlite.nim b/tests/stdlib/thighlite.nim index 5134215c1..d88bc7fec 100644 --- a/tests/stdlib/thighlite.nim +++ b/tests/stdlib/thighlite.nim @@ -1,6 +1,7 @@ import unittest, strutils import ../../lib/packages/docutils/highlite +import std/objectdollar block: # Nim tokenizing test "string literals and escape seq": @@ -30,3 +31,12 @@ block: # Cmd (shell) tokenizing ("file.nim", gtIdentifier), ("\n", gtWhitespace), ("out: file [SuccessX]", gtProgramOutput) ]) + +block: # bug #21232 + let code = "/" + var toknizr: GeneralTokenizer + + initGeneralTokenizer(toknizr, code) + + getNextToken(toknizr, langC) + check $toknizr == """(kind: gtOperator, start: 0, length: 1, buf: "/", pos: 1, state: gtEof, lang: langC)""" From b68b28fd2448b15e989b9e8b07e33eee4f9b8822 Mon Sep 17 00:00:00 2001 From: Bung Date: Sun, 8 Jan 2023 22:34:51 -0800 Subject: [PATCH 2351/2472] fix typeRel of typeDesc (#21227) --- compiler/sigmatch.nim | 8 ++++++-- tests/typerel/tint.nim | 4 ++++ 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 tests/typerel/tint.nim diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 75e3e5428..0dbd5e5f1 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -1840,7 +1840,9 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, elif f.base.kind == tyNone: result = isGeneric else: - result = typeRel(c, f.base, a.base, flags) + let r = typeRel(c, f.base, a.base, flags) + if r >= isIntConv: + result = r if result != isNone: put(c, f, a) @@ -1848,7 +1850,9 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, if tfUnresolved in f.flags: result = typeRel(c, prev.base, a, flags) elif a.kind == tyTypeDesc: - result = typeRel(c, prev.base, a.base, flags) + let r = typeRel(c, prev.base, a.base, flags) + if r >= isIntConv: + result = r else: result = isNone diff --git a/tests/typerel/tint.nim b/tests/typerel/tint.nim new file mode 100644 index 000000000..de0e78a81 --- /dev/null +++ b/tests/typerel/tint.nim @@ -0,0 +1,4 @@ + +template a(T: type int32) = discard +template a(T: type int64) = discard +a(int) From 8be46f3d16fc4c3b05d927aa1e9738ac865a266e Mon Sep 17 00:00:00 2001 From: Russell Brinson <57331546+rbrins@users.noreply.github.com> Date: Wed, 11 Jan 2023 12:20:39 -0500 Subject: [PATCH 2352/2472] parseopt.nim documentation clarity - default values & cmdEnd in getopt (#21047) parseopt.nim documentation clarity Added example for default values to cmd line parameters. Additionally, added lines in getopt documentation about case switching still requiring the `cmdEnd` kind. Hopefully this clears up any vagueness for those following along in the example but omitting the `cmdEnd` in the case because the documentation said it wasn't needed. --- lib/pure/parseopt.nim | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/lib/pure/parseopt.nim b/lib/pure/parseopt.nim index f8d03e092..9052067db 100644 --- a/lib/pure/parseopt.nim +++ b/lib/pure/parseopt.nim @@ -74,7 +74,28 @@ ## ## The `getopt iterator<#getopt.i,OptParser>`_, which is provided for ## convenience, can be used to iterate through all command line options as well. +## +## To set a default value for a variable assigned through `getopt` and accept arguments from the cmd line. +## Assign the default value to a variable before parsing. +## Then set the variable to the new value while parsing. ## +## Here is an example: +## .. code-block:: +## import std/parseopt +## +## var varName: string = "defaultValue" +## +## for kind, key, val in getopt(): +## case kind +## of cmdArgument: +## discard +## of cmdLongOption, cmdShortOption: +## case key: +## of "varName": # --varName: in the console when executing +## varName = val # do input sanitization in production systems +## of cmdEnd: +## discard +## ## `shortNoVal` and `longNoVal` ## ============================ ## @@ -210,6 +231,8 @@ proc initOptParser*(cmdline = "", shortNoVal: set[char] = {}, ## parameters<#nimshortnoval-and-nimlongnoval>`_ for more information on ## how this affects parsing. ## + ## This does not provide a way of passing default values to arguments. + ## ## See also: ## * `getopt iterator<#getopt.i,OptParser>`_ runnableExamples: @@ -403,7 +426,8 @@ iterator getopt*(p: var OptParser): tuple[kind: CmdLineKind, key, ## Convenience iterator for iterating over the given ## `OptParser<#OptParser>`_. ## - ## There is no need to check for `cmdEnd` while iterating. + ## There is no need to check for `cmdEnd` while iterating. If using `getopt` + ## with case switching, checking for `cmdEnd` is required. ## ## See also: ## * `initOptParser proc<#initOptParser,string,set[char],seq[string]>`_ @@ -451,7 +475,8 @@ iterator getopt*(cmdline: seq[string] = @[], ## parameters<#nimshortnoval-and-nimlongnoval>`_ for more information on ## how this affects parsing. ## - ## There is no need to check for `cmdEnd` while iterating. + ## There is no need to check for `cmdEnd` while iterating. If using `getopt` + ## with case switching, checking for `cmdEnd` is required. ## ## See also: ## * `initOptParser proc<#initOptParser,seq[string],set[char],seq[string]>`_ From e4e947232b9e054cd55c9fefe3ea97f47619a5ca Mon Sep 17 00:00:00 2001 From: Anna Date: Wed, 11 Jan 2023 22:23:43 +0500 Subject: [PATCH 2353/2472] testament: make valgrind optional (#21238) Add a simple switch to enable/disable valgrind support. On by default --- doc/testament.md | 1 + testament/testament.nim | 12 +++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/doc/testament.md b/doc/testament.md index ee57be4f1..08c3cd835 100644 --- a/doc/testament.md +++ b/doc/testament.md @@ -50,6 +50,7 @@ Options --colors:on|off Turn messages coloring on|off. --backendLogging:on|off Disable or enable backend logging. By default turned on. --megatest:on|off Enable or disable megatest. Default is on. +--valgrind:on|off Enable or disable valgrind support. Default is on. --skipFrom:file Read tests to skip from `file` - one test per line, # comments ignored diff --git a/testament/testament.nim b/testament/testament.nim index ac99a2373..296c313aa 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -29,6 +29,7 @@ var backendLogging = true var simulate = false var optVerbose = false var useMegatest = true +var valgrindEnabled = true proc verboseCmd(cmd: string) = if optVerbose: @@ -60,6 +61,7 @@ Options: --colors:on|off Turn messages coloring on|off. --backendLogging:on|off Disable or enable backend logging. By default turned on. --megatest:on|off Enable or disable megatest. Default is on. + --valgrind:on|off Enable or disable valgrind support. Default is on. --skipFrom:file Read tests to skip from `file` - one test per line, # comments ignored On Azure Pipelines, testament will also publish test results via Azure Pipelines' Test Management API @@ -487,7 +489,7 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, args = @["--unhandled-rejections=strict", exeFile] & args else: exeCmd = exeFile.dup(normalizeExe) - if expected.useValgrind != disabled: + if valgrindEnabled and expected.useValgrind != disabled: var valgrindOptions = @["--error-exitcode=1"] if expected.useValgrind != leaking: valgrindOptions.add "--leak-check=yes" @@ -664,6 +666,14 @@ proc main() = useMegatest = false else: quit Usage + of "valgrind": + case p.val: + of "on": + valgrindEnabled = true + of "off": + valgrindEnabled = false + else: + quit Usage of "backendlogging": case p.val: of "on": From 1e52423774e81f71d10a712575fd65231705362e Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Thu, 12 Jan 2023 12:44:33 +1100 Subject: [PATCH 2354/2472] Fix getting custom pragma from generic object (#20481) * Merge devel Add another test case * Fix test Use getCustomPragmaVal instead of hasCustomPragma Co-authored-by: Clay Sweetser --- lib/core/macros.nim | 2 +- tests/stdlib/tmacros.nim | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index beb3aa695..9cb694bd8 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -1591,7 +1591,7 @@ proc customPragmaNode(n: NimNode): NimNode = elif impl.kind in {nnkIdentDefs, nnkConstDef} and impl[0].kind == nnkPragmaExpr: return impl[0][1] else: - let timpl = typ.getImpl() + let timpl = getImpl(if typ.kind == nnkBracketExpr: typ[0] else: typ) if timpl.len>0 and timpl[0].len>1: return timpl[0][1] else: diff --git a/tests/stdlib/tmacros.nim b/tests/stdlib/tmacros.nim index 7ec2fed9f..c054b66c7 100644 --- a/tests/stdlib/tmacros.nim +++ b/tests/stdlib/tmacros.nim @@ -157,3 +157,21 @@ block: # bug #19020 static: doAssert $bar.getCustomPragmaVal(typ) == "foo" doAssert $bar.getCustomPragmaVal(typ) == "foo" + +block hasCustomPragmaGeneric: + template examplePragma() {.pragma.} + type + Foo[T] {.examplePragma.} = object + x {.examplePragma.}: T + var f: Foo[string] + doAssert f.hasCustomPragma(examplePragma) + doAssert f.x.hasCustomPragma(examplePragma) + +block getCustomPragmaValGeneric: + template examplePragma(x: int) {.pragma.} + type + Foo[T] {.examplePragma(42).} = object + x {.examplePragma(25).}: T + var f: Foo[string] + doAssert f.getCustomPragmaVal(examplePragma) == 42 + doAssert f.x.getCustomPragmaVal(examplePragma) == 25 From 5f41be3b4503a76fb68018c3ff5b05b35d6d68c1 Mon Sep 17 00:00:00 2001 From: Bung Date: Fri, 13 Jan 2023 06:36:13 -0800 Subject: [PATCH 2355/2472] fix #20253 (#21174) * fix #20253 * change NimbleStableCommit * Update koch.nim Co-authored-by: Andreas Rumpf --- compiler/sempass2.nim | 5 ++++- tests/misc/t20253.nim | 10 ++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 tests/misc/t20253.nim diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index f13d8d8d4..747540c78 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -1488,7 +1488,10 @@ proc trackProc*(c: PContext; s: PSym, body: PNode) = s.kind in {skProc, skFunc, skConverter, skMethod} and s.magic == mNone: var res = s.ast[resultPos].sym # get result symbol if res.id notin t.init: - message(g.config, body.info, warnProveInit, "result") + if tfRequiresInit in s.typ[0].flags: + localError(g.config, body.info, "'$1' requires explicit initialization" % "result") + else: + message(g.config, body.info, warnProveInit, "result") let p = s.ast[pragmasPos] let raisesSpec = effectSpec(p, wRaises) if not isNil(raisesSpec): diff --git a/tests/misc/t20253.nim b/tests/misc/t20253.nim new file mode 100644 index 000000000..d47c36c55 --- /dev/null +++ b/tests/misc/t20253.nim @@ -0,0 +1,10 @@ +discard """ + errormsg: "'result' requires explicit initialization" + line: 10 +""" + +type Meow {.requiresInit.} = object + init: bool + +proc initMeow(): Meow = + discard From 71c60bf6f760d3a1dfc12135866cd46f817ee439 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sat, 14 Jan 2023 19:30:02 +0800 Subject: [PATCH 2356/2472] bump nimble (#21259) --- koch.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/koch.nim b/koch.nim index 6b3d327f5..27953e05a 100644 --- a/koch.nim +++ b/koch.nim @@ -10,7 +10,7 @@ # const - NimbleStableCommit = "afd03bc000f4013c48f52381728327dabfccfc4c" # master + NimbleStableCommit = "7efb226ef908297e8791cade20d991784b4e8bfc" # master # examples of possible values: #head, #ea82b54, 1.2.3 FusionStableHash = "#372ee4313827ef9f2ea388840f7d6b46c2b1b014" HeadHash = "#head" From 30da566d9dc81feab9ba3015d293d93b5c6785af Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Tue, 17 Jan 2023 12:09:26 +0800 Subject: [PATCH 2357/2472] switch to official DelaunayNim URL (#21269) follow up https://github.com/Nycto/DelaunayNim/pull/4#event-8240077930 --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 77d89a420..d9835ef23 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -60,7 +60,7 @@ pkg "comprehension", "nimble test", "https://github.com/alehander92/comprehensio pkg "criterion", allowFailure = true # pending https://github.com/disruptek/criterion/issues/3 (wrongly closed) pkg "datamancer" pkg "dashing", "nim c tests/functional.nim" -pkg "delaunay", url = "https://github.com/nim-lang/DelaunayNim", useHead = true +pkg "delaunay" pkg "docopt" pkg "easygl", "nim c -o:egl -r src/easygl.nim", "https://github.com/jackmott/easygl" pkg "elvis" From cc08a9015e446471be8b734c2895d99c91c59cb3 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Wed, 18 Jan 2023 06:15:23 +0800 Subject: [PATCH 2358/2472] fixes #21263; consider all candidates for concept matches (#21265) --- compiler/concepts.nim | 2 +- compiler/lookups.nim | 17 +++++++++++++++++ tests/concepts/tconcepts_issues.nim | 25 +++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/compiler/concepts.nim b/compiler/concepts.nim index a810b3642..bcf2d4d0e 100644 --- a/compiler/concepts.nim +++ b/compiler/concepts.nim @@ -274,7 +274,7 @@ proc matchSym(c: PContext; candidate: PSym, n: PNode; m: var MatchCon): bool = proc matchSyms(c: PContext, n: PNode; kinds: set[TSymKind]; m: var MatchCon): bool = ## Walk the current scope, extract candidates which the same name as 'n[namePos]', ## 'n' is the nkProcDef or similar from the concept that we try to match. - let candidates = searchInScopesFilterBy(c, n[namePos].sym.name, kinds) + let candidates = searchInScopesAllCandidatesFilterBy(c, n[namePos].sym.name, kinds) for candidate in candidates: #echo "considering ", typeToString(candidate.typ), " ", candidate.magic m.magic = candidate.magic diff --git a/compiler/lookups.nim b/compiler/lookups.nim index 3f4fcb4d0..e7bca08bc 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -228,6 +228,23 @@ proc debugScopes*(c: PContext; limit=0, max = int.high) {.deprecated.} = if i == limit: return inc i +proc searchInScopesAllCandidatesFilterBy*(c: PContext, s: PIdent, filter: TSymKinds): seq[PSym] = + result = @[] + for scope in allScopes(c.currentScope): + var ti: TIdentIter + var candidate = initIdentIter(ti, scope.symbols, s) + while candidate != nil: + if candidate.kind in filter: + result.add candidate + candidate = nextIdentIter(ti, scope.symbols) + + if result.len == 0: + var marked = initIntSet() + for im in c.imports.mitems: + for s in symbols(im, marked, s, c.graph): + if s.kind in filter: + result.add s + proc searchInScopesFilterBy*(c: PContext, s: PIdent, filter: TSymKinds): seq[PSym] = result = @[] block outer: diff --git a/tests/concepts/tconcepts_issues.nim b/tests/concepts/tconcepts_issues.nim index 802582f57..3efb9964a 100644 --- a/tests/concepts/tconcepts_issues.nim +++ b/tests/concepts/tconcepts_issues.nim @@ -27,6 +27,8 @@ false true -1 Meow +10 0.0 +1 2.0 ''' joinable: false """ @@ -500,3 +502,26 @@ var r, b: Fp2[6, uint64] prod(r, b) + +block: # bug #21263 + type + DateDayFraction = concept # no T, an atom + proc date(a: Self): int + proc fraction(b: Self): float + Date = distinct int + DateDayFractionImpl = object + date : int + fraction : float + + proc date(a: Date): int = a.int + proc fraction(a:Date): float = 0.0 + + proc date(a: DateDayFractionImpl): int = a.date + proc fraction(b: DateDayFractionImpl): float = b.fraction + + + proc print(a: DateDayFraction) = + echo a.date, " ", a.fraction + + print(10.Date) # ok + print(DateDayFractionImpl(date: 1, fraction: 2)) # error From c4035d7f7c424caa81ee449d2032ecd8e8967ebd Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Wed, 18 Jan 2023 16:03:03 +0800 Subject: [PATCH 2359/2472] fixes unittest warnings (#21271) --- lib/pure/unittest.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/pure/unittest.nim b/lib/pure/unittest.nim index c87fe38f1..9dafa8f03 100644 --- a/lib/pure/unittest.nim +++ b/lib/pure/unittest.nim @@ -544,11 +544,14 @@ template test*(name, body) {.dirty.} = for formatter in formatters: formatter.testStarted(name) + {.warning[BareExcept]:off.} try: when declared(testSetupIMPLFlag): testSetupIMPL() when declared(testTeardownIMPLFlag): defer: testTeardownIMPL() + {.warning[BareExcept]:on.} body + {.warning[BareExcept]:off.} except: let e = getCurrentException() @@ -570,6 +573,7 @@ template test*(name, body) {.dirty.} = ) testEnded(testResult) checkpoints = @[] + {.warning[BareExcept]:on.} proc checkpoint*(msg: string) = ## Set a checkpoint identified by `msg`. Upon test failure all From fc35f83eee55610af3931f95771b6d1bce1fc845 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Wed, 18 Jan 2023 18:52:18 +0800 Subject: [PATCH 2360/2472] fixes #21260; add check for illegal recursion for defaults (#21270) * fixes #21260; add check for illegal recursion for defaults * fixes differently --- compiler/sem.nim | 36 ++++++++++++++---------------------- compiler/semdata.nim | 1 + compiler/semmagic.nim | 3 +-- compiler/semobjconstr.nim | 15 +++++++++------ tests/types/t21260.nim | 13 +++++++++++++ 5 files changed, 38 insertions(+), 30 deletions(-) create mode 100644 tests/types/t21260.nim diff --git a/compiler/sem.nim b/compiler/sem.nim index f5283fa3d..c1bb89fc8 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -550,18 +550,17 @@ proc pickCaseBranchIndex(caseExpr, matched: PNode): int = if endsWithElse: return caseExpr.len - 1 -proc defaultFieldsForTheUninitialized(c: PContext, recNode: PNode, id: var IntSet): seq[PNode] -proc defaultNodeField(c: PContext, a: PNode, aTyp: PType, id: var IntSet): PNode -proc defaultNodeField(c: PContext, a: PNode): PNode +proc defaultFieldsForTheUninitialized(c: PContext, recNode: PNode): seq[PNode] proc defaultNodeField(c: PContext, a: PNode, aTyp: PType): PNode +proc defaultNodeField(c: PContext, a: PNode): PNode const defaultFieldsSkipTypes = {tyGenericInst, tyAlias, tySink} -proc defaultFieldsForTuple(c: PContext, recNode: PNode, id: var IntSet, hasDefault: var bool): seq[PNode] = +proc defaultFieldsForTuple(c: PContext, recNode: PNode, hasDefault: var bool): seq[PNode] = case recNode.kind of nkRecList: for field in recNode: - result.add defaultFieldsForTuple(c, field, id, hasDefault) + result.add defaultFieldsForTuple(c, field, hasDefault) of nkSym: let field = recNode.sym let recType = recNode.typ.skipTypes(defaultFieldsSkipTypes) @@ -570,7 +569,7 @@ proc defaultFieldsForTuple(c: PContext, recNode: PNode, id: var IntSet, hasDefau result.add newTree(nkExprColonExpr, recNode, field.ast) else: if recType.kind in {tyObject, tyArray, tyTuple}: - let asgnExpr = defaultNodeField(c, recNode, recNode.typ, id) + let asgnExpr = defaultNodeField(c, recNode, recNode.typ) if asgnExpr != nil: hasDefault = true asgnExpr.flags.incl nfUseDefaultField @@ -589,11 +588,11 @@ proc defaultFieldsForTuple(c: PContext, recNode: PNode, id: var IntSet, hasDefau else: doAssert false -proc defaultFieldsForTheUninitialized(c: PContext, recNode: PNode, id: var IntSet): seq[PNode] = +proc defaultFieldsForTheUninitialized(c: PContext, recNode: PNode): seq[PNode] = case recNode.kind of nkRecList: for field in recNode: - result.add defaultFieldsForTheUninitialized(c, field, id) + result.add defaultFieldsForTheUninitialized(c, field) of nkRecCase: let discriminator = recNode[0] var selectedBranch: int @@ -607,14 +606,14 @@ proc defaultFieldsForTheUninitialized(c: PContext, recNode: PNode, id: var IntSe selectedBranch = recNode.pickCaseBranchIndex defaultValue defaultValue.flags.incl nfUseDefaultField result.add newTree(nkExprColonExpr, discriminator, defaultValue) - result.add defaultFieldsForTheUninitialized(c, recNode[selectedBranch][^1], id) + result.add defaultFieldsForTheUninitialized(c, recNode[selectedBranch][^1]) of nkSym: let field = recNode.sym let recType = recNode.typ.skipTypes(defaultFieldsSkipTypes) if field.ast != nil: #Try to use default value result.add newTree(nkExprColonExpr, recNode, field.ast) elif recType.kind in {tyObject, tyArray, tyTuple}: - let asgnExpr = defaultNodeField(c, recNode, recType, id) + let asgnExpr = defaultNodeField(c, recNode, recType) if asgnExpr != nil: asgnExpr.typ = recType asgnExpr.flags.incl nfUseDefaultField @@ -622,19 +621,17 @@ proc defaultFieldsForTheUninitialized(c: PContext, recNode: PNode, id: var IntSe else: doAssert false -proc defaultNodeField(c: PContext, a: PNode, aTyp: PType, id: var IntSet): PNode = +proc defaultNodeField(c: PContext, a: PNode, aTyp: PType): PNode = let aTypSkip = aTyp.skipTypes(defaultFieldsSkipTypes) if aTypSkip.kind == tyObject: - if id.containsOrIncl(aTypSkip.id): - return - let child = defaultFieldsForTheUninitialized(c, aTypSkip.n, id) + let child = defaultFieldsForTheUninitialized(c, aTypSkip.n) if child.len > 0: var asgnExpr = newTree(nkObjConstr, newNodeIT(nkType, a.info, aTypSkip)) asgnExpr.typ = aTypSkip asgnExpr.sons.add child result = semExpr(c, asgnExpr) elif aTypSkip.kind == tyArray: - let child = defaultNodeField(c, a, aTypSkip[1], id) + let child = defaultNodeField(c, a, aTypSkip[1]) if child != nil: let node = newNode(nkIntLit) @@ -647,20 +644,15 @@ proc defaultNodeField(c: PContext, a: PNode, aTyp: PType, id: var IntSet): PNode elif aTypSkip.kind == tyTuple: var hasDefault = false if aTypSkip.n != nil: - let children = defaultFieldsForTuple(c, aTypSkip.n, id, hasDefault) + let children = defaultFieldsForTuple(c, aTypSkip.n, hasDefault) if hasDefault and children.len > 0: result = newNodeI(nkTupleConstr, a.info) result.typ = aTyp result.sons.add children result = semExpr(c, result) -proc defaultNodeField(c: PContext, a: PNode, aTyp: PType): PNode = - var s = initIntSet() - defaultNodeField(c, a, aTyp, s) - proc defaultNodeField(c: PContext, a: PNode): PNode = - var s = initIntSet() - result = defaultNodeField(c, a, a.typ, s) + result = defaultNodeField(c, a, a.typ) include semtempl, semgnrc, semstmts, semexprs diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 29c2c139f..5b94cc770 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -75,6 +75,7 @@ type # overload resolution. efNoDiagnostics, efTypeAllowed # typeAllowed will be called after + efWantNoDefaults TExprFlags* = set[TExprFlag] diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim index 7025dc867..751ca8fe7 100644 --- a/compiler/semmagic.nim +++ b/compiler/semmagic.nim @@ -20,9 +20,8 @@ proc addDefaultFieldForNew(c: PContext, n: PNode): PNode = var asgnExpr = newTree(nkObjConstr, newNodeIT(nkType, result[1].info, typ)) asgnExpr.typ = typ var t = typ.skipTypes({tyGenericInst, tyAlias, tySink})[0] - var id = initIntSet() while true: - asgnExpr.sons.add defaultFieldsForTheUninitialized(c, t.n, id) + asgnExpr.sons.add defaultFieldsForTheUninitialized(c, t.n) let base = t[0] if base == nil: break diff --git a/compiler/semobjconstr.nim b/compiler/semobjconstr.nim index 206602677..15e53a639 100644 --- a/compiler/semobjconstr.nim +++ b/compiler/semobjconstr.nim @@ -326,10 +326,13 @@ proc semConstructFields(c: PContext, n: PNode, constrCtx: var ObjConstrContext, result.status = initUnknown result.defaults.add newTree(nkExprColonExpr, n, field.ast) else: - let defaultExpr = defaultNodeField(c, n) - if defaultExpr != nil: - result.status = initUnknown - result.defaults.add newTree(nkExprColonExpr, n, defaultExpr) + if efWantNoDefaults notin flags: # cannot compute defaults at the typeRightPass + let defaultExpr = defaultNodeField(c, n) + if defaultExpr != nil: + result.status = initUnknown + result.defaults.add newTree(nkExprColonExpr, n, defaultExpr) + else: + result.status = initNone else: result.status = initNone else: @@ -364,7 +367,7 @@ proc initConstrContext(t: PType, initExpr: PNode): ObjConstrContext = proc computeRequiresInit(c: PContext, t: PType): bool = assert t.kind == tyObject var constrCtx = initConstrContext(t, newNode(nkObjConstr)) - let initResult = semConstructTypeAux(c, constrCtx, {}) + let initResult = semConstructTypeAux(c, constrCtx, {efWantNoDefaults}) constrCtx.missingFields.len > 0 proc defaultConstructionError(c: PContext, t: PType, info: TLineInfo) = @@ -374,7 +377,7 @@ proc defaultConstructionError(c: PContext, t: PType, info: TLineInfo) = assert objType != nil if objType.kind == tyObject: var constrCtx = initConstrContext(objType, newNodeI(nkObjConstr, info)) - let initResult = semConstructTypeAux(c, constrCtx, {}) + let initResult = semConstructTypeAux(c, constrCtx, {efWantNoDefaults}) if constrCtx.missingFields.len > 0: localError(c.config, info, "The $1 type doesn't have a default value. The following fields must be initialized: $2." % [typeToString(t), listSymbolNames(constrCtx.missingFields)]) diff --git a/tests/types/t21260.nim b/tests/types/t21260.nim new file mode 100644 index 000000000..90d6613c1 --- /dev/null +++ b/tests/types/t21260.nim @@ -0,0 +1,13 @@ +discard """ + errormsg: "illegal recursion in type 'Foo'" + line: 8 +""" + +type + Kind = enum kA, kB + Foo = object + case k: Kind: + of kA: + foo: Foo + of kB: + discard From fd207827c5929ba5eb244dd2442457944a55b7d0 Mon Sep 17 00:00:00 2001 From: PhilippMDoerner Date: Wed, 18 Jan 2023 19:10:05 +0100 Subject: [PATCH 2361/2472] Add docs informing about gotcha of clients (#21262) * Add docs informing about gotcha of clients It is a hidden problem that AsyncHttpClient-instances *can not* deal with multiple requests at once. Similar to normal HttpClients they can only deal with sending one request at a time. This is not told anywhere in the documentation, but critical information that should be available to the user. * Update lib/pure/httpclient.nim fixes typo Co-authored-by: ringabout <43030857+ringabout@users.noreply.github.com> Co-authored-by: ringabout <43030857+ringabout@users.noreply.github.com> --- lib/pure/httpclient.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index c832d4285..d2cf64149 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -46,6 +46,10 @@ ## **Note:** You need to run asynchronous examples in an async proc ## otherwise you will get an `Undeclared identifier: 'await'` error. ## +## **Note:** An asynchronous client instance can only deal with one +## request at a time. To send multiple requests in parallel, use +## multiple client instances. +## ## Using HTTP POST ## =============== ## From fc5b7a01eccfbfeba6ded8765ca956eab0952db5 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 19 Jan 2023 02:10:34 +0800 Subject: [PATCH 2362/2472] fixes #21243; ships `build_all.sh` on Unix [backport] (#21264) fixes #21243; ships `build_all.sh` on Unix --- compiler/installer.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler/installer.ini b/compiler/installer.ini index 1e57455c6..f67ca5c25 100644 --- a/compiler/installer.ini +++ b/compiler/installer.ini @@ -121,6 +121,7 @@ Files: "bin/nim" InstallScript: "yes" UninstallScript: "yes" Files: "bin/nim-gdb" +Files: "build_all.sh" [InnoSetup] From f04f3e8bf78ab4187a395a024c94cd545c275720 Mon Sep 17 00:00:00 2001 From: Phil Krylov Date: Wed, 18 Jan 2023 19:11:15 +0100 Subject: [PATCH 2363/2472] std/parsesql: Add OFFSET keyword support. (#21255) Co-authored-by: Clay Sweetser --- lib/pure/parsesql.nim | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/pure/parsesql.nim b/lib/pure/parsesql.nim index 3af840e29..074faf64c 100644 --- a/lib/pure/parsesql.nim +++ b/lib/pure/parsesql.nim @@ -60,7 +60,7 @@ const reservedKeywords = @[ # statements - "select", "from", "where", "group", "limit", "having", + "select", "from", "where", "group", "limit", "offset", "having", # functions "count", ] @@ -509,6 +509,7 @@ type nkFromItemPair, nkGroup, nkLimit, + nkOffset, nkHaving, nkOrder, nkJoin, @@ -1126,6 +1127,11 @@ proc parseSelect(p: var SqlParser): SqlNode = var l = newNode(nkLimit) l.add(parseExpr(p)) result.add(l) + if isKeyw(p, "offset"): + getTok(p) + var o = newNode(nkOffset) + o.add(parseExpr(p)) + result.add(o) proc parseStmt(p: var SqlParser; parent: SqlNode) = if isKeyw(p, "create"): @@ -1388,6 +1394,9 @@ proc ra(n: SqlNode, s: var SqlWriter) = of nkLimit: s.addKeyw("limit") s.addMulti(n) + of nkOffset: + s.addKeyw("offset") + s.addMulti(n) of nkHaving: s.addKeyw("having") s.addMulti(n) From 00ef27f4d18bcd5f56132d9ac5dfab840ef4e6fe Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 18 Jan 2023 21:37:26 +0100 Subject: [PATCH 2364/2472] minor parseopt.nim improvements (#21256) * minor parseopt.nim improvements * attempt to make CI happy Co-authored-by: Clay Sweetser --- lib/pure/json.nim | 1 + lib/pure/parseopt.nim | 72 ++++++++++++++++++++----------------------- 2 files changed, 34 insertions(+), 39 deletions(-) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 78d6bd5bd..e881fbff7 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -1360,6 +1360,7 @@ proc to*[T](node: JsonNode, t: typedesc[T]): T = doAssert data.list == @[1, 2, 3, 4] var jsonPath = "" + result = default(T) initFromJson(result, node, jsonPath) when false: diff --git a/lib/pure/parseopt.nim b/lib/pure/parseopt.nim index 9052067db..059f8e0f5 100644 --- a/lib/pure/parseopt.nim +++ b/lib/pure/parseopt.nim @@ -74,9 +74,9 @@ ## ## The `getopt iterator<#getopt.i,OptParser>`_, which is provided for ## convenience, can be used to iterate through all command line options as well. -## +## ## To set a default value for a variable assigned through `getopt` and accept arguments from the cmd line. -## Assign the default value to a variable before parsing. +## Assign the default value to a variable before parsing. ## Then set the variable to the new value while parsing. ## ## Here is an example: @@ -84,7 +84,7 @@ ## import std/parseopt ## ## var varName: string = "defaultValue" -## +## ## for kind, key, val in getopt(): ## case kind ## of cmdArgument: @@ -95,7 +95,7 @@ ## varName = val # do input sanitization in production systems ## of cmdEnd: ## discard -## +## ## `shortNoVal` and `longNoVal` ## ============================ ## @@ -213,36 +213,6 @@ proc parseWord(s: string, i: int, w: var string, add(w, s[result]) inc(result) -proc initOptParser*(cmdline: seq[string], shortNoVal: set[char] = {}, - longNoVal: seq[string] = @[]; - allowWhitespaceAfterColon = true): OptParser - -proc initOptParser*(cmdline = "", shortNoVal: set[char] = {}, - longNoVal: seq[string] = @[]; - allowWhitespaceAfterColon = true): OptParser = - ## Initializes the command line parser. - ## - ## If `cmdline == ""`, the real command line as provided by the - ## `os` module is retrieved instead if it is available. If the - ## command line is not available, a `ValueError` will be raised. - ## - ## `shortNoVal` and `longNoVal` are used to specify which options - ## do not take values. See the `documentation about these - ## parameters<#nimshortnoval-and-nimlongnoval>`_ for more information on - ## how this affects parsing. - ## - ## This does not provide a way of passing default values to arguments. - ## - ## See also: - ## * `getopt iterator<#getopt.i,OptParser>`_ - runnableExamples: - var p = initOptParser() - p = initOptParser("--left --debug:3 -l -r:2") - p = initOptParser("--left --debug:3 -l -r:2", - shortNoVal = {'l'}, longNoVal = @["left"]) - - initOptParser(parseCmdLine(cmdline), shortNoVal, longNoVal, allowWhitespaceAfterColon) - proc initOptParser*(cmdline: seq[string], shortNoVal: set[char] = {}, longNoVal: seq[string] = @[]; allowWhitespaceAfterColon = true): OptParser = @@ -287,6 +257,32 @@ proc initOptParser*(cmdline: seq[string], shortNoVal: set[char] = {}, result.key = "" result.val = "" +proc initOptParser*(cmdline = "", shortNoVal: set[char] = {}, + longNoVal: seq[string] = @[]; + allowWhitespaceAfterColon = true): OptParser = + ## Initializes the command line parser. + ## + ## If `cmdline == ""`, the real command line as provided by the + ## `os` module is retrieved instead if it is available. If the + ## command line is not available, a `ValueError` will be raised. + ## + ## `shortNoVal` and `longNoVal` are used to specify which options + ## do not take values. See the `documentation about these + ## parameters<#nimshortnoval-and-nimlongnoval>`_ for more information on + ## how this affects parsing. + ## + ## This does not provide a way of passing default values to arguments. + ## + ## See also: + ## * `getopt iterator<#getopt.i,OptParser>`_ + runnableExamples: + var p = initOptParser() + p = initOptParser("--left --debug:3 -l -r:2") + p = initOptParser("--left --debug:3 -l -r:2", + shortNoVal = {'l'}, longNoVal = @["left"]) + + initOptParser(parseCmdLine(cmdline), shortNoVal, longNoVal, allowWhitespaceAfterColon) + proc handleShortOption(p: var OptParser; cmd: string) = var i = p.pos p.kind = cmdShortOption @@ -379,7 +375,7 @@ proc next*(p: var OptParser) {.rtl, extern: "npo$1".} = handleShortOption(p, p.cmds[p.idx]) else: p.kind = cmdArgument - p.key = p.cmds[p.idx] + p.key = p.cmds[p.idx] inc p.idx p.pos = 0 @@ -398,7 +394,6 @@ when declared(quoteShellCommand): ## p.next() ## if p.kind == cmdLongOption and p.key == "": # Look for "--" ## break - ## else: continue ## doAssert p.cmdLineRest == "foo.txt bar.txt" result = p.cmds[p.idx .. ^1].quoteShellCommand @@ -416,7 +411,6 @@ proc remainingArgs*(p: OptParser): seq[string] {.rtl, extern: "npo$1".} = ## p.next() ## if p.kind == cmdLongOption and p.key == "": # Look for "--" ## break - ## else: continue ## doAssert p.remainingArgs == @["foo.txt", "bar.txt"] result = @[] for i in p.idx..`_. ## - ## There is no need to check for `cmdEnd` while iterating. If using `getopt` + ## There is no need to check for `cmdEnd` while iterating. If using `getopt` ## with case switching, checking for `cmdEnd` is required. ## ## See also: @@ -475,7 +469,7 @@ iterator getopt*(cmdline: seq[string] = @[], ## parameters<#nimshortnoval-and-nimlongnoval>`_ for more information on ## how this affects parsing. ## - ## There is no need to check for `cmdEnd` while iterating. If using `getopt` + ## There is no need to check for `cmdEnd` while iterating. If using `getopt` ## with case switching, checking for `cmdEnd` is required. ## ## See also: From 7c6dcfd968867ec9b2fee5ca3ef8f251a6faa350 Mon Sep 17 00:00:00 2001 From: Ivan Yonchovski Date: Wed, 18 Jan 2023 22:38:48 +0200 Subject: [PATCH 2365/2472] Implicitly set noNimblePath when nimble.lock is present (#21266) Fixes https://github.com/nim-lang/nimble/issues/1004 --- compiler/nim.nim | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/compiler/nim.nim b/compiler/nim.nim index c2b70dda2..420579b97 100644 --- a/compiler/nim.nim +++ b/compiler/nim.nim @@ -41,6 +41,15 @@ when defined(profiler) or defined(memProfiler): {.hint: "Profiling support is turned on!".} import nimprof +proc nimbleLockExists(config: ConfigRef): bool = + const nimbleLock = "nimble.lock" + let pd = if not config.projectPath.isEmpty: config.projectPath else: AbsoluteDir(getCurrentDir()) + if optSkipParentConfigFiles notin config.globalOptions: + for dir in parentDirs(pd.string, fromRoot=true, inclusive=false): + if fileExists(dir / nimbleLock): + return true + return fileExists(pd.string / nimbleLock) + proc processCmdLine(pass: TCmdLinePass, cmd: string; config: ConfigRef) = var p = parseopt.initOptParser(cmd) var argsCount = 0 @@ -74,6 +83,11 @@ proc processCmdLine(pass: TCmdLinePass, cmd: string; config: ConfigRef) = config.arguments.len > 0 and config.cmd notin {cmdTcc, cmdNimscript, cmdCrun}: rawMessage(config, errGenerated, errArgsNeedRunOption) + if config.nimbleLockExists: + # disable nimble path if nimble.lock is present. + # see https://github.com/nim-lang/nimble/issues/1004 + disableNimblePath(config) + proc getNimRunExe(conf: ConfigRef): string = # xxx consider defining `conf.getConfigVar("nimrun.exe")` to allow users to # customize the binary to run the command with, e.g. for custom `nodejs` or `wine`. From ac982d8a96d4602838d46abc61a8f5385f8c766d Mon Sep 17 00:00:00 2001 From: metagn Date: Wed, 18 Jan 2023 23:40:00 +0300 Subject: [PATCH 2366/2472] close #21257 (#21275) * close #21257 * fix generics --- compiler/semtypes.nim | 4 +++- tests/errmsgs/t21257.nim | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 tests/errmsgs/t21257.nim diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 844bc2070..aaffa1ea7 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -1310,13 +1310,15 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode, if hasDefault: def = a[^1] block determineType: + var defTyp = typ if genericParams != nil and genericParams.len > 0: + defTyp = nil def = semGenericStmt(c, def) if hasUnresolvedArgs(c, def): def.typ = makeTypeFromExpr(c, def.copyTree) break determineType - def = semExprWithType(c, def, {efDetermineType}) + def = semExprWithType(c, def, {efDetermineType}, defTyp) if def.referencesAnotherParam(getCurrOwner(c)): def.flags.incl nfDefaultRefsParam diff --git a/tests/errmsgs/t21257.nim b/tests/errmsgs/t21257.nim new file mode 100644 index 000000000..eecdb1dfe --- /dev/null +++ b/tests/errmsgs/t21257.nim @@ -0,0 +1,20 @@ +discard """ + action: compile + cmd: "nim check $file" +""" + +type AC_WINCTRL_Fields* = distinct uint8 + +type AC_STATUSA_WSTATE0* {.pure.} = enum + ABOVE = 0x0, + INSIDE = 0x1, + BELOW = 0x2, + +type AC_WINCTRL_WINTSEL0* {.pure.} = enum + ABOVE = 0x0, + INSIDE = 0x1, + BELOW = 0x2, + OUTSIDE = 0x3, + +proc write*(WINTSEL0: AC_WINCTRL_WINTSEL0 = ABOVE) = + discard From 9afb466d733bc16f5ac0fef7477c2ac919046eb2 Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Thu, 19 Jan 2023 11:46:41 +0100 Subject: [PATCH 2367/2472] Force lambda lifting for getImplTransformed. Hacky. Fixes #19818 (#21031) --- compiler/lambdalifting.nim | 4 ++-- compiler/transf.nim | 6 +++--- compiler/vm.nim | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 9790eefd8..f6bc21c01 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -859,7 +859,7 @@ proc liftIterToProc*(g: ModuleGraph; fn: PSym; body: PNode; ptrType: PType; fn.typ.callConv = oldCC proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool; - idgen: IdGenerator): PNode = + idgen: IdGenerator, force: bool): PNode = # XXX backend == backendJs does not suffice! The compiletime stuff needs # the transformation even when compiling to JS ... @@ -868,7 +868,7 @@ proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool; if body.kind == nkEmpty or ( g.config.backend == backendJs and not isCompileTime) or - fn.skipGenericOwner.kind != skModule: + (fn.skipGenericOwner.kind != skModule and not force): # ignore forward declaration: result = body diff --git a/compiler/transf.nim b/compiler/transf.nim index 30d64282f..7277e6898 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -31,7 +31,7 @@ type TransformBodyFlag* = enum dontUseCache, useCache -proc transformBody*(g: ModuleGraph; idgen: IdGenerator, prc: PSym, flag: TransformBodyFlag): PNode +proc transformBody*(g: ModuleGraph; idgen: IdGenerator, prc: PSym, flag: TransformBodyFlag, force = false): PNode import closureiters, lambdalifting @@ -1145,7 +1145,7 @@ template liftDefer(c, root) = if c.deferDetected: liftDeferAux(root) -proc transformBody*(g: ModuleGraph; idgen: IdGenerator; prc: PSym; flag: TransformBodyFlag): PNode = +proc transformBody*(g: ModuleGraph; idgen: IdGenerator; prc: PSym; flag: TransformBodyFlag, force = false): PNode = assert prc.kind in routineKinds if prc.transformedBody != nil: @@ -1155,7 +1155,7 @@ proc transformBody*(g: ModuleGraph; idgen: IdGenerator; prc: PSym; flag: Transfo else: prc.transformedBody = newNode(nkEmpty) # protects from recursion var c = openTransf(g, prc.getModule, "", idgen) - result = liftLambdas(g, prc, getBody(g, prc), c.tooEarly, c.idgen) + result = liftLambdas(g, prc, getBody(g, prc), c.tooEarly, c.idgen, force) result = processTransf(c, result, prc) liftDefer(c, result) result = liftLocalsIfRequested(prc, result, g.cache, g.config, c.idgen) diff --git a/compiler/vm.nim b/compiler/vm.nim index d87fa33ba..b8c23290e 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -1271,7 +1271,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = let ast = a.sym.ast.shallowCopy for i in 0.. Date: Fri, 20 Jan 2023 12:48:12 +0800 Subject: [PATCH 2368/2472] switch to official chronos URL (#21283) ref https://github.com/status-im/nim-chronos/pull/348 --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index d9835ef23..90d12df4e 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -52,7 +52,7 @@ pkg "cascade" pkg "cello", url = "https://github.com/nim-lang/cello", useHead = true pkg "chroma" pkg "chronicles", "nim c -o:chr -r chronicles.nim", url = "https://github.com/nim-lang/nim-chronicles" -pkg "chronos", "nim c -r -d:release tests/testall", url = "https://github.com/nim-lang/nim-chronos" +pkg "chronos", "nim c -r -d:release tests/testall" pkg "cligen", "nim c --path:. -r cligen.nim" pkg "combparser", "nimble test --gc:orc" pkg "compactdict" From b82b5d44afbd860075f806f71a62d595f2376d6e Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 20 Jan 2023 18:43:45 +0800 Subject: [PATCH 2369/2472] fixes #21278; `deques.shrink` off by one bug (#21284) fixes #21278; deques.shrink off ny one bug --- lib/pure/collections/deques.nim | 2 +- tests/stdlib/tdeques.nim | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/pure/collections/deques.nim b/lib/pure/collections/deques.nim index 7e9ed32f3..60e0fd351 100644 --- a/lib/pure/collections/deques.nim +++ b/lib/pure/collections/deques.nim @@ -440,7 +440,7 @@ proc shrink*[T](deq: var Deque[T], fromFirst = 0, fromLast = 0) = deq.head = (deq.head + 1) and deq.mask for i in 0 ..< fromLast: - destroy(deq.data[deq.tail]) + destroy(deq.data[(deq.tail - 1) and deq.mask]) deq.tail = (deq.tail - 1) and deq.mask dec deq.count, fromFirst + fromLast diff --git a/tests/stdlib/tdeques.nim b/tests/stdlib/tdeques.nim index 8a788d337..fcfc278a9 100644 --- a/tests/stdlib/tdeques.nim +++ b/tests/stdlib/tdeques.nim @@ -183,6 +183,12 @@ proc main() = clear(a) doAssert len(a) == 0 + block: # bug #21278 + var a = [10, 20, 30, 40].toDeque + + a.shrink(fromFirst = 0, fromLast = 1) + doAssert $a == "[10, 20, 30]" + static: main() main() From f5ec4fbde3d8fb5250e2944d397448219e9053bc Mon Sep 17 00:00:00 2001 From: ghais Date: Sat, 21 Jan 2023 05:07:40 -0500 Subject: [PATCH 2370/2472] Update the reference to parseSpec (#21289) The current reference pointed to the wrong location --- tests/readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/readme.md b/tests/readme.md index fe59a7757..f638ddc10 100644 --- a/tests/readme.md +++ b/tests/readme.md @@ -9,7 +9,7 @@ top-level tests like `tests/tbar.nim`. Each test can contain a spec in a ``discard """ ... """`` block. -**Check out the [``parseSpec`` procedure](https://github.com/nim-lang/Nim/blob/devel/testament/specs.nim#L158) in the ``specs`` module for a full and reliable reference** +**Check out the [``parseSpec`` procedure](https://github.com/nim-lang/Nim/blob/devel/testament/specs.nim#L315) in the ``specs`` module for a full and reliable reference** ## action From 36e489e69b08ac6a34dfd5c22abc3873c7d86838 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Tue, 24 Jan 2023 20:33:13 +0800 Subject: [PATCH 2371/2472] add `md2html`, `md2tex` to cmd helpers (#21295) --- doc/advopt.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/advopt.txt b/doc/advopt.txt index efff428e8..122a7c6ea 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -4,8 +4,11 @@ Advanced commands: //compileToOC, objc compile project to Objective C code //js compile project to Javascript //e run a Nimscript file + //md2html convert a Markdown file to HTML + use `--docCmd:skip` to skip compiling snippets //rst2html convert a reStructuredText file to HTML use `--docCmd:skip` to skip compiling snippets + //md2tex convert a Markdown file to LaTeX //rst2tex convert a reStructuredText file to LaTeX //doc2tex extract the documentation to a LaTeX file //jsondoc extract the documentation to a json file From 5e7f7109e19b33bc142aa7b06e9044420232a578 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Tue, 24 Jan 2023 20:35:19 +0800 Subject: [PATCH 2372/2472] fixes #21290; deindent if the last same level is a text node (#21293) * fixes #21290; deindent if the last same level is a text node * add one more test --- lib/pure/xmltree.nim | 37 +++++++++++++++++++++---------------- tests/stdlib/txmltree.nim | 18 +++++++++++++++++- 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/lib/pure/xmltree.nim b/lib/pure/xmltree.nim index 7a2cebe87..c3923ca50 100644 --- a/lib/pure/xmltree.nim +++ b/lib/pure/xmltree.nim @@ -723,20 +723,8 @@ proc addIndent(result: var string, indent: int, addNewLines: bool) = for i in 1 .. indent: result.add(' ') -proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, - addNewLines = true) = - ## Adds the textual representation of `n` to string `result`. - runnableExamples: - var - a = newElement("firstTag") - b = newText("my text") - c = newComment("my comment") - s = "" - s.add(c) - s.add(a) - s.add(b) - assert s == "my text" - +proc addImpl(result: var string, n: XmlNode, indent = 0, indWidth = 2, + addNewLines = true, lastNodeIsText = false) = proc noWhitespace(n: XmlNode): bool = for i in 0 ..< n.len: if n[i].kind in {xnText, xnEntity}: return true @@ -756,7 +744,7 @@ proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, case n.k of xnElement: - if indent > 0: + if indent > 0 and not lastNodeIsText: result.addIndent(indent, addNewLines) let @@ -785,8 +773,10 @@ proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, else: indent+indWidth result.add('>') + var lastNodeIsText = false for i in 0 ..< n.len: - result.add(n[i], indentNext, indWidth, addNewLines) + result.addImpl(n[i], indentNext, indWidth, addNewLines, lastNodeIsText) + lastNodeIsText = n[i].kind == xnText if not n.noWhitespace(): result.addIndent(indent, addNewLines) @@ -811,6 +801,21 @@ proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, result.add(n.fText) result.add(';') +proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, + addNewLines = true) {.inline.} = + ## Adds the textual representation of `n` to string `result`. + runnableExamples: + var + a = newElement("firstTag") + b = newText("my text") + c = newComment("my comment") + s = "" + s.add(c) + s.add(a) + s.add(b) + assert s == "my text" + result.addImpl(n, indent, indWidth, addNewLines) + proc `$`*(n: XmlNode): string = ## Converts `n` into its string representation. ## diff --git a/tests/stdlib/txmltree.nim b/tests/stdlib/txmltree.nim index f6b7c62e8..4362f5ec3 100644 --- a/tests/stdlib/txmltree.nim +++ b/tests/stdlib/txmltree.nim @@ -1,4 +1,4 @@ -import std/[xmltree, assertions] +import std/[xmltree, assertions, xmlparser] block: @@ -83,3 +83,19 @@ block: x.add newElement("sonTag") x.add newEntity("my entity") doAssert $x == "my text&my entity;" + +block: # bug #21290 + let x = newXmlTree("foo",[ + newXmlTree("bar",[ + newText("Hola"), + newXmlTree("qux",[ + newXmlTree("plugh",[]) + ]) + ]) + ]) + + let s = $x + doAssert $parseXml(s) == s + doAssert s == """ + Hola +""" From d54a7f078d29445bc79ec5f117a2ceb73761b4c4 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Tue, 24 Jan 2023 22:37:22 +0800 Subject: [PATCH 2373/2472] fixes #21261; always checking `nimTestErrorFlag` in the main module (#21288) * fixes #21261; always checking nimTestErrorFlag in the main module * add a test --- compiler/cgen.nim | 3 +++ tests/exception/m21261.nim | 1 + tests/exception/t21261.nim | 9 +++++++++ 3 files changed, 13 insertions(+) create mode 100644 tests/exception/m21261.nim create mode 100644 tests/exception/t21261.nim diff --git a/compiler/cgen.nim b/compiler/cgen.nim index f6457f1e0..5bf9400c5 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1801,6 +1801,9 @@ proc genInitCode(m: BModule) = if optStackTrace in m.initProc.options and preventStackTrace notin m.flags: prc.add(deinitFrame(m.initProc)) + elif sfMainModule in m.module.flags and m.config.exc == excGoto: + if getCompilerProc(m.g.graph, "nimTestErrorFlag") != nil: + m.appcg(prc, "\t#nimTestErrorFlag();$n", []) prc.addf("}$N", []) diff --git a/tests/exception/m21261.nim b/tests/exception/m21261.nim new file mode 100644 index 000000000..11b12fb5b --- /dev/null +++ b/tests/exception/m21261.nim @@ -0,0 +1 @@ +raise (ref Exception)(msg: "something") \ No newline at end of file diff --git a/tests/exception/t21261.nim b/tests/exception/t21261.nim new file mode 100644 index 000000000..84817d854 --- /dev/null +++ b/tests/exception/t21261.nim @@ -0,0 +1,9 @@ +discard """ + exitcode: 1 + outputsub: ''' +m21261.nim(1) m21261 +Error: unhandled exception: something [Exception] +''' +""" + +import m21261 \ No newline at end of file From c4d3d650baa88820dadeafa81ca75b6e495084f2 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 26 Jan 2023 03:56:19 +0800 Subject: [PATCH 2374/2472] fixes #21273; fixes an io.readLine off by one bug [backport 1.0] (#21276) fixes #21273; io.readLine off by one --- lib/std/syncio.nim | 7 ++++--- tests/stdlib/tio.nim | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/lib/std/syncio.nim b/lib/std/syncio.nim index 46f113450..83b2d6d13 100644 --- a/lib/std/syncio.nim +++ b/lib/std/syncio.nim @@ -484,11 +484,12 @@ proc readLine*(f: File, line: var string): bool {.tags: [ReadIOEffect], if last > 0 and line[last-1] == '\c': line.setLen(last-1) return last > 1 or fgetsSuccess - # We have to distinguish between two possible cases: + elif last > 0 and line[last-1] == '\0': + # We have to distinguish among three possible cases: # \0\l\0 => line ending in a null character. # \0\l\l => last line without newline, null was put there by fgets. - elif last > 0 and line[last-1] == '\0': - if last < pos + sp - 1 and line[last+1] != '\0': + # \0\l => last line without newline, null was put there by fgets. + if last >= pos + sp - 1 or line[last+1] != '\0': # bug #21273 dec last line.setLen(last) return last > 0 or fgetsSuccess diff --git a/tests/stdlib/tio.nim b/tests/stdlib/tio.nim index 0e20d6495..ca411379c 100644 --- a/tests/stdlib/tio.nim +++ b/tests/stdlib/tio.nim @@ -36,3 +36,21 @@ block: # readChars break doAssert n2s == @[2,2,2,1,0] doAssert s2 == s + + +import std/strutils + +block: # bug #21273 + let FILE = buildDir / "D20220119T134305.txt" + + let hex = "313632313920313632343720313632353920313632363020313632393020323035363520323037323120323131353020323239393820323331303520323332313020323332343820323332363820" + + + writeFile FILE, parseHexStr(hex) + + doAssert readFile(FILE).toHex == hex + + let f = open(FILE) + var s = newString(80) + while f.readLine(s): + doAssert s.toHex == hex From af0f3639ec6fabc00bf8357615b62641ee0b4508 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 26 Jan 2023 03:58:52 +0800 Subject: [PATCH 2375/2472] remove dead code in VM (#21296) remove deadcode in VM --- compiler/ast.nim | 2 +- compiler/vm.nim | 6 ------ compiler/vmdef.nim | 2 +- compiler/vmgen.nim | 3 --- 4 files changed, 2 insertions(+), 11 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index c85a94860..aa0ca98c9 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -706,7 +706,7 @@ type mNctPut, mNctLen, mNctGet, mNctHasNext, mNctNext, mNIntVal, mNFloatVal, mNSymbol, mNIdent, mNGetType, mNStrVal, mNSetIntVal, - mNSetFloatVal, mNSetSymbol, mNSetIdent, mNSetType, mNSetStrVal, mNLineInfo, + mNSetFloatVal, mNSetSymbol, mNSetIdent, mNSetStrVal, mNLineInfo, mNNewNimNode, mNCopyNimNode, mNCopyNimTree, mStrToIdent, mNSigHash, mNSizeOf, mNBindSym, mNCallSite, mEqIdent, mEqNimrodNode, mSameNodeType, mGetImpl, mNGenSym, diff --git a/compiler/vm.nim b/compiler/vm.nim index b8c23290e..c6ca228ff 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -2071,12 +2071,6 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = dest.ident = regs[rb].node.ident else: stackTrace(c, tos, pc, errFieldXNotFound & "ident") - of opcNSetType: - decodeB(rkNode) - let b = regs[rb].node - internalAssert c.config, b.kind == nkSym and b.sym.kind == skType - internalAssert c.config, regs[ra].node != nil - regs[ra].node.typ = b.sym.typ of opcNSetStrVal: decodeB(rkNode) var dest = regs[ra].node diff --git a/compiler/vmdef.nim b/compiler/vmdef.nim index 114a49557..0af15e5c9 100644 --- a/compiler/vmdef.nim +++ b/compiler/vmdef.nim @@ -127,7 +127,7 @@ type opcNGetSize, opcNSetIntVal, - opcNSetFloatVal, opcNSetSymbol, opcNSetIdent, opcNSetType, opcNSetStrVal, + opcNSetFloatVal, opcNSetSymbol, opcNSetIdent, opcNSetStrVal, opcNNewNimNode, opcNCopyNimNode, opcNCopyNimTree, opcNDel, opcGenSym, opcNccValue, opcNccInc, opcNcsAdd, opcNcsIncl, opcNcsLen, opcNcsAt, diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 6a2ab1d11..ba3a05481 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1313,9 +1313,6 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = of mNSetIdent: unused(c, n, dest) genBinaryStmt(c, n, opcNSetIdent) - of mNSetType: - unused(c, n, dest) - genBinaryStmt(c, n, opcNSetType) of mNSetStrVal: unused(c, n, dest) genBinaryStmt(c, n, opcNSetStrVal) From 23bd812b0c531dbb6a9854a81030468bbae29cd8 Mon Sep 17 00:00:00 2001 From: Heechul Ryu Date: Thu, 26 Jan 2023 03:43:55 -0800 Subject: [PATCH 2376/2472] Update nims.md to use `nim e` instead of `nim` in advanced shebang (#21301) Update nims.md - suggest a better shebang that works with any filename - related to https://github.com/nim-lang/Nim/issues/17190 --- doc/nims.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/nims.md b/doc/nims.md index a69ea1789..2409bb577 100644 --- a/doc/nims.md +++ b/doc/nims.md @@ -211,7 +211,7 @@ ends with ``.nims``: echo "hello world" ``` -Use `#!/usr/bin/env -S nim --hints:off` to disable hints. +Use `#!/usr/bin/env -S nim e --hints:off` to disable hints and relax the file extension constraint. Benefits From fc068ee06d058b35d2d5890cf1fa3c356e549d1f Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 27 Jan 2023 06:03:59 +0800 Subject: [PATCH 2377/2472] fixes #18134; registers `formatBiggestFloat` in VM (#21299) fixes #18134; registers formatBiggestFloat in vmops strformat supports float format in VM --- compiler/vmops.nim | 8 ++- lib/pure/strutils.nim | 107 ++++++++++++++++++------------------ tests/stdlib/tstrformat.nim | 19 ++++--- 3 files changed, 73 insertions(+), 61 deletions(-) diff --git a/compiler/vmops.nim b/compiler/vmops.nim index bb943b8a0..2b5b2c938 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -36,7 +36,9 @@ from std/osproc import nil when defined(nimPreviewSlimSystem): import std/syncio else: - from std/formatfloat import addFloatRoundtrip, addFloatSprintf + from std/formatfloat import addFloatRoundtrip, addFloatSprintf + +from std/strutils import formatBiggestFloat, FloatFormatMode # There are some useful procs in vmconv. import vmconv, vmmarshal @@ -374,6 +376,10 @@ proc registerAdditionalOps*(c: PCtx) = let x = a.getFloat(1) addFloatSprintf(p.strVal, x) + registerCallback c, "stdlib.strutils.formatBiggestFloat", proc(a: VmArgs) = + setResult(a, formatBiggestFloat(a.getFloat(0), FloatFormatMode(a.getInt(1)), + a.getInt(2), chr(a.getInt(3)))) + wrapIterator("stdlib.envvars.envPairsImplSeq"): envPairs() registerCallback c, "stdlib.marshal.toVM", proc(a: VmArgs) = diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index dbdd0d6a1..c458d605e 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -2413,60 +2413,63 @@ func formatBiggestFloat*(f: BiggestFloat, format: FloatFormatMode = ffDefault, doAssert x.formatBiggestFloat() == "123.4560000000000" doAssert x.formatBiggestFloat(ffDecimal, 4) == "123.4560" doAssert x.formatBiggestFloat(ffScientific, 2) == "1.23e+02" - when defined(js): - var precision = precision - if precision == -1: - # use the same default precision as c_sprintf - precision = 6 - var res: cstring - case format - of ffDefault: - {.emit: "`res` = `f`.toString();".} - of ffDecimal: - {.emit: "`res` = `f`.toFixed(`precision`);".} - of ffScientific: - {.emit: "`res` = `f`.toExponential(`precision`);".} - result = $res - if 1.0 / f == -Inf: - # JavaScript removes the "-" from negative Zero, add it back here - result = "-" & $res - for i in 0 ..< result.len: - # Depending on the locale either dot or comma is produced, - # but nothing else is possible: - if result[i] in {'.', ','}: result[i] = decimalSep + when nimvm: + discard "implemented in the vmops" else: - const floatFormatToChar: array[FloatFormatMode, char] = ['g', 'f', 'e'] - var - frmtstr {.noinit.}: array[0..5, char] - buf {.noinit.}: array[0..2500, char] - L: cint - frmtstr[0] = '%' - if precision >= 0: - frmtstr[1] = '#' - frmtstr[2] = '.' - frmtstr[3] = '*' - frmtstr[4] = floatFormatToChar[format] - frmtstr[5] = '\0' - L = c_sprintf(cast[cstring](addr buf), cast[cstring](addr frmtstr), precision, f) + when defined(js): + var precision = precision + if precision == -1: + # use the same default precision as c_sprintf + precision = 6 + var res: cstring + case format + of ffDefault: + {.emit: "`res` = `f`.toString();".} + of ffDecimal: + {.emit: "`res` = `f`.toFixed(`precision`);".} + of ffScientific: + {.emit: "`res` = `f`.toExponential(`precision`);".} + result = $res + if 1.0 / f == -Inf: + # JavaScript removes the "-" from negative Zero, add it back here + result = "-" & $res + for i in 0 ..< result.len: + # Depending on the locale either dot or comma is produced, + # but nothing else is possible: + if result[i] in {'.', ','}: result[i] = decimalSep else: - frmtstr[1] = floatFormatToChar[format] - frmtstr[2] = '\0' - L = c_sprintf(cast[cstring](addr buf), cast[cstring](addr frmtstr), f) - result = newString(L) - for i in 0 ..< L: - # Depending on the locale either dot or comma is produced, - # but nothing else is possible: - if buf[i] in {'.', ','}: result[i] = decimalSep - else: result[i] = buf[i] - when defined(windows): - # VS pre 2015 violates the C standard: "The exponent always contains at - # least two digits, and only as many more digits as necessary to - # represent the exponent." [C11 §7.21.6.1] - # The following post-processing fixes this behavior. - if result.len > 4 and result[^4] == '+' and result[^3] == '0': - result[^3] = result[^2] - result[^2] = result[^1] - result.setLen(result.len - 1) + const floatFormatToChar: array[FloatFormatMode, char] = ['g', 'f', 'e'] + var + frmtstr {.noinit.}: array[0..5, char] + buf {.noinit.}: array[0..2500, char] + L: cint + frmtstr[0] = '%' + if precision >= 0: + frmtstr[1] = '#' + frmtstr[2] = '.' + frmtstr[3] = '*' + frmtstr[4] = floatFormatToChar[format] + frmtstr[5] = '\0' + L = c_sprintf(cast[cstring](addr buf), cast[cstring](addr frmtstr), precision, f) + else: + frmtstr[1] = floatFormatToChar[format] + frmtstr[2] = '\0' + L = c_sprintf(cast[cstring](addr buf), cast[cstring](addr frmtstr), f) + result = newString(L) + for i in 0 ..< L: + # Depending on the locale either dot or comma is produced, + # but nothing else is possible: + if buf[i] in {'.', ','}: result[i] = decimalSep + else: result[i] = buf[i] + when defined(windows): + # VS pre 2015 violates the C standard: "The exponent always contains at + # least two digits, and only as many more digits as necessary to + # represent the exponent." [C11 §7.21.6.1] + # The following post-processing fixes this behavior. + if result.len > 4 and result[^4] == '+' and result[^3] == '0': + result[^3] = result[^2] + result[^2] = result[^1] + result.setLen(result.len - 1) func formatFloat*(f: float, format: FloatFormatMode = ffDefault, precision: range[-1..32] = 16; decimalSep = '.'): string {. diff --git a/tests/stdlib/tstrformat.nim b/tests/stdlib/tstrformat.nim index b44a11e68..d332e8375 100644 --- a/tests/stdlib/tstrformat.nim +++ b/tests/stdlib/tstrformat.nim @@ -475,15 +475,17 @@ proc main() = # Note: times.format adheres to the format protocol. Test that this # works: + when nimvm: + discard + else: + var dt = dateTime(2000, mJan, 01, 00, 00, 00) + check &"{dt:yyyy-MM-dd}", "2000-01-01" - var dt = initDateTime(01, mJan, 2000, 00, 00, 00) - check &"{dt:yyyy-MM-dd}", "2000-01-01" + var tm = fromUnix(0) + discard &"{tm}" - var tm = fromUnix(0) - discard &"{tm}" - - var noww = now() - check &"{noww}", $noww + var noww = now() + check &"{noww}", $noww # Unicode string tests check &"""{"αβγ"}""", "αβγ" @@ -558,5 +560,6 @@ proc main() = doAssert &"""{(if true: "'" & "'" & ')' else: "")}""" == "'')" doAssert &"{(if true: \"\'\" & \"'\" & ')' else: \"\")}" == "'')" doAssert fmt"""{(if true: "'" & ')' else: "")}""" == "')" -# xxx static: main() + +static: main() main() From 4647c7b59634e0d8bd24d815337e7a7a1f0830bd Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 27 Jan 2023 06:49:04 +0800 Subject: [PATCH 2378/2472] fixes #14409; fixes #10674 VM callbacks switch to table-index seqs (#21297) * fixes #14409; fixes#10674 VM callbacks switch to table-index seqs * fixes package name * reduce runtime cost --- compiler/vm.nim | 4 ++-- compiler/vmdef.nim | 18 ++++++++++++---- compiler/vmgen.nim | 34 +++++++++++++++++------------- compiler/vmops.nim | 2 +- tests/compilerapi/tcompilerapi.nim | 2 +- 5 files changed, 37 insertions(+), 23 deletions(-) diff --git a/compiler/vm.nim b/compiler/vm.nim index c6ca228ff..4f5f8485b 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -1351,7 +1351,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = let prc = if not isClosure: bb.sym else: bb[0].sym if prc.offset < -1: # it's a callback: - c.callbacks[-prc.offset-2].value( + c.callbacks[-prc.offset-2]( VmArgs(ra: ra, rb: rb, rc: rc, slots: cast[ptr UncheckedArray[TFullReg]](addr regs[0]), currentException: c.currentExceptionA, currentLineInfo: c.debug[pc]) @@ -1674,7 +1674,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = rb = instr.regB rc = instr.regC idx = int(regs[rb+rc-1].intVal) - callback = c.callbacks[idx].value + callback = c.callbacks[idx] args = VmArgs(ra: ra, rb: rb, rc: rc, slots: cast[ptr UncheckedArray[TFullReg]](addr regs[0]), currentException: c.currentExceptionA, currentLineInfo: c.debug[pc]) diff --git a/compiler/vmdef.nim b/compiler/vmdef.nim index 0af15e5c9..f369908ba 100644 --- a/compiler/vmdef.nim +++ b/compiler/vmdef.nim @@ -10,7 +10,7 @@ ## This module contains the type definitions for the new evaluation engine. ## An instruction is 1-3 int32s in memory, it is a register based VM. -import tables +import std/[tables, strutils] import ast, idents, options, modulegraphs, lineinfos @@ -259,7 +259,8 @@ type traceActive*: bool loopIterations*: int comesFromHeuristic*: TLineInfo # Heuristic for better macro stack traces - callbacks*: seq[tuple[key: string, value: VmCallback]] + callbacks*: seq[VmCallback] + callbackIndex*: Table[string, int] errorFlag*: string cache*: IdentCache config*: ConfigRef @@ -292,7 +293,7 @@ proc newCtx*(module: PSym; cache: IdentCache; g: ModuleGraph; idgen: IdGenerator PCtx(code: @[], debug: @[], globals: newNode(nkStmtListExpr), constants: newNode(nkStmtList), types: @[], prc: PProc(blocks: @[]), module: module, loopIterations: g.config.maxLoopIterationsVM, - comesFromHeuristic: unknownLineInfo, callbacks: @[], errorFlag: "", + comesFromHeuristic: unknownLineInfo, callbacks: @[], callbackIndex: initTable[string, int](), errorFlag: "", cache: cache, config: g.config, graph: g, idgen: idgen) proc refresh*(c: PCtx, module: PSym; idgen: IdGenerator) = @@ -301,9 +302,18 @@ proc refresh*(c: PCtx, module: PSym; idgen: IdGenerator) = c.loopIterations = c.config.maxLoopIterationsVM c.idgen = idgen +proc reverseName(s: string): string = + result = newStringOfCap(s.len) + let y = s.split('.') + for i in 1..y.len: + result.add y[^i] + if i != y.len: + result.add '.' + proc registerCallback*(c: PCtx; name: string; callback: VmCallback): int {.discardable.} = result = c.callbacks.len - c.callbacks.add((name, callback)) + c.callbacks.add(callback) + c.callbackIndex[reverseName(name)] = result const firstABxInstr* = opcTJmp diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index ba3a05481..bca961a63 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -2011,25 +2011,29 @@ proc genTupleConstr(c: PCtx, n: PNode, dest: var TDest) = proc genProc*(c: PCtx; s: PSym): int -proc matches(s: PSym; x: string): bool = - let y = x.split('.') +proc toKey(s: PSym): string = var s = s - for i in 1..y.len: - if s == nil or (y[^i].cmpIgnoreStyle(s.name.s) != 0 and y[^i] != "*"): - return false - s = if sfFromGeneric in s.flags: s.owner.owner else: s.owner - while s != nil and s.kind == skPackage and s.owner != nil: s = s.owner - result = true + while s != nil: + result.add s.name.s + if s.owner != nil: + if sfFromGeneric in s.flags: + s = s.owner.owner + else: + s = s.owner + result.add "." + else: + break proc procIsCallback(c: PCtx; s: PSym): bool = if s.offset < -1: return true - var i = -2 - for key, value in items(c.callbacks): - if s.matches(key): - doAssert s.offset == -1 - s.offset = i - return true - dec i + let key = toKey(s) + if c.callbackIndex.contains(key): + let index = c.callbackIndex[key] + doAssert s.offset == -1 + s.offset = -2 - index + result = true + else: + result = false proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) = when defined(nimCompilerStacktraceHints): diff --git a/compiler/vmops.nim b/compiler/vmops.nim index 2b5b2c938..f1a2e1e69 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -256,7 +256,7 @@ proc registerAdditionalOps*(c: PCtx) = wrap2si(readLines, ioop) systemop getCurrentExceptionMsg systemop getCurrentException - registerCallback c, "stdlib.*.staticWalkDir", proc (a: VmArgs) {.nimcall.} = + registerCallback c, "stdlib.osdirs.staticWalkDir", proc (a: VmArgs) {.nimcall.} = setResult(a, staticWalkDirImpl(getString(a, 0), getBool(a, 1))) registerCallback c, "stdlib.compilesettings.querySetting", proc (a: VmArgs) = setResult(a, querySettingImpl(c.config, getInt(a, 0))) diff --git a/tests/compilerapi/tcompilerapi.nim b/tests/compilerapi/tcompilerapi.nim index ab995cb6d..828dcd0d9 100644 --- a/tests/compilerapi/tcompilerapi.nim +++ b/tests/compilerapi/tcompilerapi.nim @@ -23,7 +23,7 @@ proc initInterpreter(script: string): Interpreter = proc main() = let i = initInterpreter("myscript.nim") - i.implementRoutine("*", "exposed", "addFloats", proc (a: VmArgs) = + i.implementRoutine("nim", "exposed", "addFloats", proc (a: VmArgs) = setResult(a, getFloat(a, 0) + getFloat(a, 1) + getFloat(a, 2)) ) i.evalScript() From 7031ea65cd220360b8e9f566fd28f01bc0bf53c4 Mon Sep 17 00:00:00 2001 From: Ivan Yonchovski Date: Fri, 27 Jan 2023 08:11:30 +0200 Subject: [PATCH 2379/2472] Implemented basic macro expand functionality (#20579) * Implemented level based macro expand functionality - it can handle single macro call or expand whole function/proc/etc and it - In addition, I have altered the parser to provide the endInfo for the node. The usefulness of the `endInfo` is not limited to the `expandMacro` functionality but also it is useful for `ideOutline` functionality and I have altered the ideOutline functionality to use `endInfo`. Note `endInfo` most of the time is lost during the AST transformation thus in `nimsuggest.nim` I am using freshly parsed tree to get the location information. * Make sure we stop expanding correctly * Test CI * Fix tv3_outline.nim --- compiler/ast.nim | 8 ++ compiler/lexer.nim | 6 + compiler/options.nim | 14 ++- compiler/parser.nim | 70 ++++++++++- compiler/semexprs.nim | 30 ++++- compiler/sempass2.nim | 3 + compiler/semstmts.nim | 2 +- compiler/suggest.nim | 12 +- compiler/vm.nim | 3 + nimsuggest/nimsuggest.nim | 193 ++++++++++++++++++++++++------- nimsuggest/tester.nim | 15 ++- nimsuggest/tests/tv3.nim | 4 - nimsuggest/tests/tv3_outline.nim | 45 +++++++ 13 files changed, 345 insertions(+), 60 deletions(-) create mode 100644 nimsuggest/tests/tv3_outline.nim diff --git a/compiler/ast.nim b/compiler/ast.nim index aa0ca98c9..aa70560ed 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -796,6 +796,8 @@ type ident*: PIdent else: sons*: TNodeSeq + when defined(nimsuggest): + endInfo*: TLineInfo TStrTable* = object # a table[PIdent] of PSym counter*: int @@ -892,6 +894,8 @@ type typ*: PType name*: PIdent info*: TLineInfo + when defined(nimsuggest): + endInfo*: TLineInfo owner*: PSym flags*: TSymFlags ast*: PNode # syntax tree of proc, iterator, etc.: @@ -1690,6 +1694,8 @@ proc copyNode*(src: PNode): PNode = of nkIdent: result.ident = src.ident of nkStrLit..nkTripleStrLit: result.strVal = src.strVal else: discard + when defined(nimsuggest): + result.endInfo = src.endInfo template transitionNodeKindCommon(k: TNodeKind) = let obj {.inject.} = n[] @@ -1742,6 +1748,8 @@ template copyNodeImpl(dst, src, processSonsStmt) = if src == nil: return dst = newNode(src.kind) dst.info = src.info + when defined(nimsuggest): + result.endInfo = src.endInfo dst.typ = src.typ dst.flags = src.flags * PersistentNodeFlags dst.comment = src.comment diff --git a/compiler/lexer.nim b/compiler/lexer.nim index ec27add8a..a62d40e54 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -127,6 +127,8 @@ type cache*: IdentCache when defined(nimsuggest): previousToken: TLineInfo + tokenEnd*: TLineInfo + previousTokenEnd*: TLineInfo config*: ConfigRef proc getLineInfo*(L: Lexer, tok: Token): TLineInfo {.inline.} = @@ -1224,6 +1226,10 @@ proc skip(L: var Lexer, tok: var Token) = proc rawGetTok*(L: var Lexer, tok: var Token) = template atTokenEnd() {.dirty.} = when defined(nimsuggest): + L.previousTokenEnd.line = L.tokenEnd.line + L.previousTokenEnd.col = L.tokenEnd.col + L.tokenEnd.line = tok.line.uint16 + L.tokenEnd.col = getColNumber(L, L.bufpos).int16 # we attach the cursor to the last *strong* token if tok.tokType notin weakTokens: L.previousToken.line = tok.line.uint16 diff --git a/compiler/options.nim b/compiler/options.nim index fc970e420..8c477f0a5 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -195,7 +195,7 @@ type IdeCmd* = enum ideNone, ideSug, ideCon, ideDef, ideUse, ideDus, ideChk, ideChkFile, ideMod, ideHighlight, ideOutline, ideKnown, ideMsg, ideProject, ideGlobalSymbols, - ideRecompile, ideChanged, ideType, ideDeclaration + ideRecompile, ideChanged, ideType, ideDeclaration, ideExpand Feature* = enum ## experimental features; DO NOT RENAME THESE! dotOperators, @@ -278,6 +278,9 @@ type scope*, localUsages*, globalUsages*: int # more usages is better tokenLen*: int version*: int + endLine*: uint16 + endCol*: int + Suggestions* = seq[Suggest] ProfileInfo* = object @@ -408,6 +411,11 @@ type nimMainPrefix*: string vmProfileData*: ProfileData + expandProgress*: bool + expandLevels*: int + expandNodeResult*: string + expandPosition*: TLineInfo + proc parseNimVersion*(a: string): NimVer = # could be moved somewhere reusable if a.len > 0: @@ -996,6 +1004,9 @@ proc isDynlibOverride*(conf: ConfigRef; lib: string): bool = result = optDynlibOverrideAll in conf.globalOptions or conf.dllOverrides.hasKey(lib.canonDynlibName) +proc expandDone*(conf: ConfigRef): bool = + result = conf.ideCmd == ideExpand and conf.expandLevels == 0 and conf.expandProgress + proc parseIdeCmd*(s: string): IdeCmd = case s: of "sug": ideSug @@ -1035,6 +1046,7 @@ proc `$`*(c: IdeCmd): string = of ideProject: "project" of ideGlobalSymbols: "globalSymbols" of ideDeclaration: "declaration" + of ideExpand: "expand" of ideRecompile: "recompile" of ideChanged: "changed" of ideType: "type" diff --git a/compiler/parser.nim b/compiler/parser.nim index 0c8edd2a8..14882d415 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -354,6 +354,12 @@ proc colcom(p: var Parser, n: PNode) = const tkBuiltInMagics = {tkType, tkStatic, tkAddr} +template setEndInfo() = + when defined(nimsuggest): + result.endInfo = TLineInfo(fileIndex: p.lex.fileIdx, + line: p.lex.previousTokenEnd.line, + col: p.lex.previousTokenEnd.col) + proc parseSymbol(p: var Parser, mode = smNormal): PNode = #| symbol = '`' (KEYW|IDENT|literal|(operator|'('|')'|'['|']'|'{'|'}'|'=')+)+ '`' #| | IDENT | KEYW @@ -406,6 +412,7 @@ proc parseSymbol(p: var Parser, mode = smNormal): PNode = # if it is a keyword: #if not isKeyword(p.tok.tokType): getTok(p) result = p.emptyNode + setEndInfo() proc equals(p: var Parser, a: PNode): PNode = if p.tok.tokType == tkEquals: @@ -577,6 +584,7 @@ proc parseCast(p: var Parser): PNode = result.add(exprColonEqExpr(p)) optPar(p) eat(p, tkParRi) + setEndInfo() proc setBaseFlags(n: PNode, base: NumericalBase) = case base @@ -599,6 +607,7 @@ proc parseGStrLit(p: var Parser, a: PNode): PNode = getTok(p) else: result = a + setEndInfo() proc complexOrSimpleStmt(p: var Parser): PNode proc simpleExpr(p: var Parser, mode = pmNormal): PNode @@ -703,6 +712,7 @@ proc parsePar(p: var Parser): PNode = skipComment(p, a) optPar(p) eat(p, tkParRi) + setEndInfo() proc identOrLiteral(p: var Parser, mode: PrimaryMode): PNode = #| literal = | INT_LIT | INT8_LIT | INT16_LIT | INT32_LIT | INT64_LIT @@ -941,6 +951,7 @@ proc parseOperators(p: var Parser, headNode: PNode, a.add(b) result = a opPrec = getPrecedence(p.tok) + setEndInfo() proc simpleExprAux(p: var Parser, limit: int, mode: PrimaryMode): PNode = var mode = mode @@ -990,6 +1001,7 @@ proc parsePragma(p: var Parser): PNode = when defined(nimpretty): dec p.em.doIndentMore dec p.em.keepIndents + setEndInfo() proc identVis(p: var Parser; allowDot=false): PNode = #| identVis = symbol OPR? # postfix position @@ -1058,6 +1070,7 @@ proc parseIdentColonEquals(p: var Parser, flags: DeclaredIdentFlags): PNode = result.add(parseExpr(p)) else: result.add(newNodeP(nkEmpty, p)) + setEndInfo() proc parseTuple(p: var Parser, indentAllowed = false): PNode = #| tupleTypeBracket = '[' optInd (identColonEquals (comma/semicolon)?)* optPar ']' @@ -1102,6 +1115,7 @@ proc parseTuple(p: var Parser, indentAllowed = false): PNode = parMessage(p, errGenerated, "the syntax for tuple types is 'tuple[...]', not 'tuple(...)'") else: result = newNodeP(nkTupleClassTy, p) + setEndInfo() proc parseParamList(p: var Parser, retColon = true): PNode = #| paramList = '(' declColonEquals ^* (comma/semicolon) ')' @@ -1150,6 +1164,7 @@ proc parseParamList(p: var Parser, retColon = true): PNode = when defined(nimpretty): dec p.em.doIndentMore dec p.em.keepIndents + setEndInfo() proc optPragmas(p: var Parser): PNode = if p.tok.tokType == tkCurlyDotLe and (p.tok.indent < 0 or realInd(p)): @@ -1170,6 +1185,7 @@ proc parseDoBlock(p: var Parser; info: TLineInfo): PNode = result = newProcNode(nkDo, info, body = result, params = params, name = p.emptyNode, pattern = p.emptyNode, genericParams = p.emptyNode, pragmas = pragmas, exceptions = p.emptyNode) + setEndInfo() proc parseProcExpr(p: var Parser; isExpr: bool; kind: TNodeKind): PNode = #| routineExpr = ('proc' | 'func' | 'iterator') paramListColon pragma? ('=' COMMENT? stmt)? @@ -1192,6 +1208,7 @@ proc parseProcExpr(p: var Parser; isExpr: bool; kind: TNodeKind): PNode = if kind == nkFuncDef: parMessage(p, "func keyword is not allowed in type descriptions, use proc with {.noSideEffect.} pragma instead") result.add(pragmas) + setEndInfo() proc isExprStart(p: Parser): bool = case p.tok.tokType @@ -1211,6 +1228,7 @@ proc parseSymbolList(p: var Parser, result: PNode) = if p.tok.tokType != tkComma: break getTok(p) optInd(p, s) + setEndInfo() proc parseTypeDescKAux(p: var Parser, kind: TNodeKind, mode: PrimaryMode): PNode = @@ -1239,6 +1257,7 @@ proc parseTypeDescKAux(p: var Parser, kind: TNodeKind, parseSymbolList(p, list) if mode == pmTypeDef and not isTypedef: result = parseOperators(p, result, -1, mode) + setEndInfo() proc parseVarTuple(p: var Parser): PNode @@ -1264,6 +1283,7 @@ proc parseFor(p: var Parser): PNode = result.add(parseExpr(p)) colcom(p, result) result.add(parseStmt(p)) + setEndInfo() template nimprettyDontTouch(body) = when defined(nimpretty): @@ -1302,6 +1322,7 @@ proc parseExpr(p: var Parser): PNode = nimprettyDontTouch: result = parseTry(p, isExpr=true) else: result = simpleExpr(p) + setEndInfo() proc parseEnum(p: var Parser): PNode proc parseObject(p: var Parser): PNode @@ -1411,6 +1432,7 @@ proc parseTypeDesc(p: var Parser, fullExpr = false): PNode = else: result = simpleExpr(p, pmTypeDesc) result = binaryNot(p, result) + setEndInfo() proc parseTypeDefValue(p: var Parser): PNode = #| typeDefValue = ((tupleDecl | enumDecl | objectDecl | conceptDecl | @@ -1441,6 +1463,7 @@ proc parseTypeDefValue(p: var Parser): PNode = result.add(commandParam(p, isFirstParam, pmTypeDef)) result = postExprBlocks(p, result) result = binaryNot(p, result) + setEndInfo() proc makeCall(n: PNode): PNode = ## Creates a call if the given node isn't already a call. @@ -1561,6 +1584,7 @@ proc parseExprStmt(p: var Parser): PNode = else: result = a result = postExprBlocks(p, result) + setEndInfo() proc parseModuleName(p: var Parser, kind: TNodeKind): PNode = result = parseExpr(p) @@ -1572,6 +1596,7 @@ proc parseModuleName(p: var Parser, kind: TNodeKind): PNode = getTok(p) result.add(a) result.add(parseExpr(p)) + setEndInfo() proc parseImport(p: var Parser, kind: TNodeKind): PNode = #| importStmt = 'import' optInd expr @@ -1600,6 +1625,7 @@ proc parseImport(p: var Parser, kind: TNodeKind): PNode = getTok(p) optInd(p, a) #expectNl(p) + setEndInfo() proc parseIncludeStmt(p: var Parser): PNode = #| includeStmt = 'include' optInd expr ^+ comma @@ -1616,6 +1642,7 @@ proc parseIncludeStmt(p: var Parser): PNode = getTok(p) optInd(p, a) #expectNl(p) + setEndInfo() proc parseFromStmt(p: var Parser): PNode = #| fromStmt = 'from' expr 'import' optInd expr (comma expr)* @@ -1636,6 +1663,7 @@ proc parseFromStmt(p: var Parser): PNode = getTok(p) optInd(p, a) #expectNl(p) + setEndInfo() proc parseReturnOrRaise(p: var Parser, kind: TNodeKind): PNode = #| returnStmt = 'return' optInd expr? @@ -1657,6 +1685,7 @@ proc parseReturnOrRaise(p: var Parser, kind: TNodeKind): PNode = var e = parseExpr(p) e = postExprBlocks(p, e) result.add(e) + setEndInfo() proc parseIfOrWhen(p: var Parser, kind: TNodeKind): PNode = #| condStmt = expr colcom stmt COMMENT? @@ -1681,6 +1710,7 @@ proc parseIfOrWhen(p: var Parser, kind: TNodeKind): PNode = colcom(p, branch) branch.add(parseStmt(p)) result.add(branch) + setEndInfo() proc parseIfOrWhenExpr(p: var Parser, kind: TNodeKind): PNode = #| condExpr = expr colcom expr optInd @@ -1705,6 +1735,7 @@ proc parseIfOrWhenExpr(p: var Parser, kind: TNodeKind): PNode = colcom(p, branch) branch.add(parseStmt(p)) result.add(branch) + setEndInfo() proc parseWhile(p: var Parser): PNode = #| whileStmt = 'while' expr colcom stmt @@ -1714,6 +1745,7 @@ proc parseWhile(p: var Parser): PNode = result.add(parseExpr(p)) colcom(p, result) result.add(parseStmt(p)) + setEndInfo() proc parseCase(p: var Parser): PNode = #| ofBranch = 'of' exprList colcom stmt @@ -1761,6 +1793,7 @@ proc parseCase(p: var Parser): PNode = if wasIndented: p.currInd = oldInd + setEndInfo() proc parseTry(p: var Parser; isExpr: bool): PNode = #| tryStmt = 'try' colcom stmt &(IND{=}? 'except'|'finally') @@ -1789,12 +1822,14 @@ proc parseTry(p: var Parser; isExpr: bool): PNode = b.add(parseStmt(p)) result.add(b) if b == nil: parMessage(p, "expected 'except'") + setEndInfo() proc parseExceptBlock(p: var Parser, kind: TNodeKind): PNode = result = newNodeP(kind, p) getTok(p) colcom(p, result) result.add(parseStmt(p)) + setEndInfo() proc parseBlock(p: var Parser): PNode = #| blockStmt = 'block' symbol? colcom stmt @@ -1805,6 +1840,7 @@ proc parseBlock(p: var Parser): PNode = else: result.add(parseSymbol(p)) colcom(p, result) result.add(parseStmt(p)) + setEndInfo() proc parseStaticOrDefer(p: var Parser; k: TNodeKind): PNode = #| staticStmt = 'static' colcom stmt @@ -1813,6 +1849,7 @@ proc parseStaticOrDefer(p: var Parser; k: TNodeKind): PNode = getTok(p) colcom(p, result) result.add(parseStmt(p)) + setEndInfo() proc parseAsm(p: var Parser): PNode = #| asmStmt = 'asm' pragma? (STR_LIT | RSTR_LIT | TRIPLESTR_LIT) @@ -1829,6 +1866,7 @@ proc parseAsm(p: var Parser): PNode = result.add(p.emptyNode) return getTok(p) + setEndInfo() proc parseGenericParam(p: var Parser): PNode = #| genericParam = symbol (comma symbol)* (colon expr)? ('=' optInd expr)? @@ -1864,6 +1902,7 @@ proc parseGenericParam(p: var Parser): PNode = result.add(parseExpr(p)) else: result.add(p.emptyNode) + setEndInfo() proc parseGenericParamList(p: var Parser): PNode = #| genericParamList = '[' optInd @@ -1882,12 +1921,14 @@ proc parseGenericParamList(p: var Parser): PNode = skipComment(p, a) optPar(p) eat(p, tkBracketRi) + setEndInfo() proc parsePattern(p: var Parser): PNode = #| pattern = '{' stmt '}' eat(p, tkCurlyLe) result = parseStmt(p) eat(p, tkCurlyRi) + setEndInfo() proc parseRoutine(p: var Parser, kind: TNodeKind): PNode = #| indAndComment = (IND{>} COMMENT)? | COMMENT? @@ -1932,6 +1973,7 @@ proc parseRoutine(p: var Parser, kind: TNodeKind): PNode = #else: # assert false, p.lex.config$body.info # avoids hard to track bugs, fail early. # Yeah, that worked so well. There IS a bug in this logic, now what? + setEndInfo() proc newCommentStmt(p: var Parser): PNode = #| commentStmt = COMMENT @@ -1967,6 +2009,7 @@ proc parseSection(p: var Parser, kind: TNodeKind, result.add(defparser(p)) else: parMessage(p, errIdentifierExpected, p.tok) + setEndInfo() proc parseEnum(p: var Parser): PNode = #| enumDecl = 'enum' optInd (symbol pragma? optInd ('=' optInd expr COMMENT?)? comma?)+ @@ -2013,6 +2056,7 @@ proc parseEnum(p: var Parser): PNode = break if result.len <= 1: parMessage(p, errIdentifierExpected, p.tok) + setEndInfo() proc parseObjectPart(p: var Parser): PNode proc parseObjectWhen(p: var Parser): PNode = @@ -2038,6 +2082,7 @@ proc parseObjectWhen(p: var Parser): PNode = branch.add(parseObjectPart(p)) flexComment(p, branch) result.add(branch) + setEndInfo() proc parseObjectCase(p: var Parser): PNode = #| objectBranch = 'of' exprList colcom objectPart @@ -2079,6 +2124,7 @@ proc parseObjectCase(p: var Parser): PNode = if b.kind == nkElse: break if wasIndented: p.currInd = oldInd + setEndInfo() proc parseObjectPart(p: var Parser): PNode = #| objectPart = IND{>} objectPart^+IND{=} DED @@ -2111,6 +2157,7 @@ proc parseObjectPart(p: var Parser): PNode = result = p.emptyNode else: result = p.emptyNode + setEndInfo() proc parseObject(p: var Parser): PNode = #| objectDecl = 'object' ('of' typeDesc)? COMMENT? objectPart @@ -2131,6 +2178,7 @@ proc parseObject(p: var Parser): PNode = result.add(p.emptyNode) else: result.add(parseObjectPart(p)) + setEndInfo() proc parseTypeClassParam(p: var Parser): PNode = let modifier = @@ -2148,6 +2196,7 @@ proc parseTypeClassParam(p: var Parser): PNode = result.add(p.parseSymbol) else: result = p.parseSymbol + setEndInfo() proc parseTypeClass(p: var Parser): PNode = #| conceptParam = ('var' | 'out')? symbol @@ -2191,6 +2240,7 @@ proc parseTypeClass(p: var Parser): PNode = result.add(p.emptyNode) else: result.add(parseStmt(p)) + setEndInfo() proc parseTypeDef(p: var Parser): PNode = #| @@ -2224,6 +2274,7 @@ proc parseTypeDef(p: var Parser): PNode = else: result.add(p.emptyNode) indAndComment(p, result) # special extension! + setEndInfo() proc parseVarTuple(p: var Parser): PNode = #| varTuple = '(' optInd identWithPragma ^+ comma optPar ')' '=' optInd expr @@ -2240,6 +2291,7 @@ proc parseVarTuple(p: var Parser): PNode = result.add(p.emptyNode) # no type desc optPar(p) eat(p, tkParRi) + setEndInfo() proc parseVariable(p: var Parser): PNode = #| colonBody = colcom stmt postExprBlocks? @@ -2252,6 +2304,7 @@ proc parseVariable(p: var Parser): PNode = else: result = parseIdentColonEquals(p, {withPragma, withDot}) result[^1] = postExprBlocks(p, result[^1]) indAndComment(p, result) + setEndInfo() proc parseConstant(p: var Parser): PNode = #| constant = (varTuple / identWithPragma) (colon typeDesc)? '=' optInd expr indAndComment @@ -2271,6 +2324,7 @@ proc parseConstant(p: var Parser): PNode = result.add(parseExpr(p)) result[^1] = postExprBlocks(p, result[^1]) indAndComment(p, result) + setEndInfo() proc parseBind(p: var Parser, k: TNodeKind): PNode = #| bindStmt = 'bind' optInd qualifiedIdent ^+ comma @@ -2286,6 +2340,7 @@ proc parseBind(p: var Parser, k: TNodeKind): PNode = getTok(p) optInd(p, a) #expectNl(p) + setEndInfo() proc parseStmtPragma(p: var Parser): PNode = #| pragmaStmt = pragma (':' COMMENT? stmt)? @@ -2297,6 +2352,7 @@ proc parseStmtPragma(p: var Parser): PNode = skipComment(p, result) result.add a result.add parseStmt(p) + setEndInfo() proc simpleStmt(p: var Parser): PNode = #| simpleStmt = ((returnStmt | raiseStmt | yieldStmt | discardStmt | breakStmt @@ -2439,6 +2495,7 @@ proc parseStmt(p: var Parser): PNode = if p.tok.tokType != tkSemiColon: break getTok(p) if err and p.tok.tokType == tkEof: break + setEndInfo() proc parseAll(p: var Parser): PNode = ## Parses the rest of the input stream held by the parser into a PNode. @@ -2454,6 +2511,7 @@ proc parseAll(p: var Parser): PNode = getTok(p) if p.tok.indent != 0: parMessage(p, errInvalidIndentation) + setEndInfo() proc checkFirstLineIndentation*(p: var Parser) = if p.tok.indent != 0 and p.tok.strongSpaceA: @@ -2487,6 +2545,7 @@ proc parseTopLevelStmt(p: var Parser): PNode = result = complexOrSimpleStmt(p) if result.kind == nkEmpty: parMessage(p, errExprExpected, p.tok) break + setEndInfo() proc parseString*(s: string; cache: IdentCache; config: ConfigRef; filename: string = ""; line: int = 0; @@ -2498,9 +2557,10 @@ proc parseString*(s: string; cache: IdentCache; config: ConfigRef; var stream = llStreamOpen(s) stream.lineOffset = line - var parser: Parser - parser.lex.errorHandler = errorHandler - openParser(parser, AbsoluteFile filename, stream, cache, config) + var p: Parser + p.lex.errorHandler = errorHandler + openParser(p, AbsoluteFile filename, stream, cache, config) - result = parser.parseAll - closeParser(parser) + result = p.parseAll + closeParser(p) + setEndInfo() diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 612af482a..7454c7f8e 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -980,6 +980,14 @@ proc afterCallActions(c: PContext; n, orig: PNode, flags: TExprFlags; expectedTy return errorNode(c, n) result = n + + when defined(nimsuggest): + if c.config.expandProgress: + if c.config.expandLevels == 0: + return n + else: + c.config.expandLevels -= 1 + let callee = result[0].sym case callee.kind of skMacro: result = semMacroExpr(c, result, orig, callee, flags, expectedType) @@ -1890,6 +1898,9 @@ proc semReturn(c: PContext, n: PNode): PNode = localError(c.config, n.info, "'return' not allowed here") proc semProcBody(c: PContext, n: PNode; expectedType: PType = nil): PNode = + when defined(nimsuggest): + if c.graph.config.expandDone(): + return n openScope(c) result = semExpr(c, n, expectedType = expectedType) if c.p.resultSym != nil and not isEmptyType(result.typ): @@ -2895,7 +2906,6 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}, expectedType: PType defer: if isCompilerDebug(): echo ("<", c.config$n.info, n, ?.result.typ) - template directLiteral(typeKind: TTypeKind) = if result.typ == nil: if expectedType != nil and ( @@ -2907,6 +2917,19 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}, expectedType: PType result.typ = getSysType(c.graph, n.info, typeKind) result = n + when defined(nimsuggest): + var expandStarted = false + if c.config.ideCmd == ideExpand and not c.config.expandProgress and + ((n.kind in {nkFuncDef, nkProcDef, nkIteratorDef, nkTemplateDef, nkMethodDef, nkConverterDef} and + n.info.exactEquals(c.config.expandPosition)) or + (n.kind in {nkCall, nkCommand} and + n[0].info.exactEquals(c.config.expandPosition))): + expandStarted = true + c.config.expandProgress = true + if c.config.expandLevels == 0: + c.config.expandNodeResult = $n + suggestQuit() + if c.config.cmd == cmdIdeTools: suggestExpr(c, n) if nfSem in n.flags: return case n.kind @@ -3234,3 +3257,8 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}, expectedType: PType localError(c.config, n.info, "invalid expression: " & renderTree(n, {renderNoComments})) if result != nil: incl(result.flags, nfSem) + + when defined(nimsuggest): + if expandStarted: + c.config.expandNodeResult = $result + suggestQuit() diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index 747540c78..e989eb4a6 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -1450,6 +1450,9 @@ proc hasRealBody(s: PSym): bool = proc trackProc*(c: PContext; s: PSym, body: PNode) = let g = c.graph + when defined(nimsuggest): + if g.config.expandDone(): + return var effects = s.typ.n[0] if effects.kind != nkEffectList: return # effects already computed? diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 96883255c..c7e48db4c 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -2156,7 +2156,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, if s.kind notin {skMacro, skTemplate} and s.magic == mNone: paramsTypeCheck(c, s.typ) maybeAddResult(c, s, n) - let resultType = + let resultType = if s.kind == skMacro: sysTypeFromName(c.graph, n.info, "NimNode") elif not isInlineIterator(s.typ): diff --git a/compiler/suggest.nim b/compiler/suggest.nim index e0b0fb516..637010ad5 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -120,7 +120,9 @@ proc getTokenLenFromSource(conf: ConfigRef; ident: string; info: TLineInfo): int proc symToSuggest*(g: ModuleGraph; s: PSym, isLocal: bool, section: IdeCmd, info: TLineInfo; quality: range[0..100]; prefix: PrefixMatch; inTypeContext: bool; scope: int; - useSuppliedInfo = false): Suggest = + useSuppliedInfo = false, + endLine: uint16 = 0, + endCol = 0): Suggest = new(result) result.section = section result.quality = quality @@ -176,6 +178,8 @@ proc symToSuggest*(g: ModuleGraph; s: PSym, isLocal: bool, section: IdeCmd, info else: getTokenLenFromSource(g.config, s.name.s, infox) result.version = g.config.suggestVersion + result.endLine = endLine + result.endCol = endCol proc `$`*(suggest: Suggest): string = result = $suggest.section @@ -216,6 +220,12 @@ proc `$`*(suggest: Suggest): string = result.add(sep) result.add($suggest.prefix) + if (suggest.version == 3 and suggest.section in {ideOutline, ideExpand}): + result.add(sep) + result.add($suggest.endLine) + result.add(sep) + result.add($suggest.endCol) + proc suggestResult*(conf: ConfigRef; s: Suggest) = if not isNil(conf.suggestionResultHook): conf.suggestionResultHook(s) diff --git a/compiler/vm.nim b/compiler/vm.nim index 4f5f8485b..424685e40 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -2333,6 +2333,9 @@ const evalPass* = makePass(myOpen, myProcess, myClose) proc evalConstExprAux(module: PSym; idgen: IdGenerator; g: ModuleGraph; prc: PSym, n: PNode, mode: TEvalMode): PNode = + when defined(nimsuggest): + if g.config.expandDone(): + return n #if g.config.errorCounter > 0: return n let n = transformExpr(g, idgen, module, n) setupGlobalCtx(module, g, idgen) diff --git a/nimsuggest/nimsuggest.nim b/nimsuggest/nimsuggest.nim index c5e015ce9..a0a3b8e82 100644 --- a/nimsuggest/nimsuggest.nim +++ b/nimsuggest/nimsuggest.nim @@ -27,7 +27,7 @@ import compiler / [options, commands, modules, sem, passes, passaux, msgs, sigmatch, ast, idents, modulegraphs, prefixmatches, lineinfos, cmdlinehelper, - pathutils, condsyms] + pathutils, condsyms, syntaxes] when defined(nimPreviewSlimSystem): import std/typedthreads @@ -88,7 +88,7 @@ var requests: Channel[string] results: Channel[Suggest] -proc executeNoHooksV3(cmd: IdeCmd, file: AbsoluteFile, dirtyfile: AbsoluteFile, line, col: int; +proc executeNoHooksV3(cmd: IdeCmd, file: AbsoluteFile, dirtyfile: AbsoluteFile, line, col: int; tag: string, graph: ModuleGraph); proc writelnToChannel(line: string) = @@ -140,6 +140,9 @@ proc sexp(s: Suggest): SexpNode = ]) if s.section == ideSug: result.add convertSexp(s.prefix) + if s.section in {ideOutline, ideExpand} and s.version == 3: + result.add convertSexp(s.endLine.int) + result.add convertSexp(s.endCol) proc sexp(s: seq[Suggest]): SexpNode = result = newSList() @@ -175,12 +178,23 @@ proc symFromInfo(graph: ModuleGraph; trackPos: TLineInfo): PSym = if m != nil and m.ast != nil: result = findNode(m.ast, trackPos) -proc executeNoHooks(cmd: IdeCmd, file, dirtyfile: AbsoluteFile, line, col: int; +template benchmark(benchmarkName: untyped, code: untyped) = + block: + myLog "Started [" & benchmarkName & "]..." + let t0 = epochTime() + code + let elapsed = epochTime() - t0 + let elapsedStr = elapsed.formatFloat(format = ffDecimal, precision = 3) + myLog "CPU Time [" & benchmarkName & "] " & elapsedStr & "s" + +proc executeNoHooks(cmd: IdeCmd, file, dirtyfile: AbsoluteFile, line, col: int, tag: string, graph: ModuleGraph) = let conf = graph.config if conf.suggestVersion == 3: - executeNoHooksV3(cmd, file, dirtyfile, line, col, graph) + let command = fmt "cmd = {cmd} {file}:{line}:{col}" + benchmark command: + executeNoHooksV3(cmd, file, dirtyfile, line, col, tag, graph) return myLog("cmd: " & $cmd & ", file: " & file.string & @@ -219,7 +233,10 @@ proc executeNoHooks(cmd: IdeCmd, file, dirtyfile: AbsoluteFile, line, col: int; else: localError(conf, conf.m.trackPos, "found no symbol at this position " & (conf $ conf.m.trackPos)) -proc execute(cmd: IdeCmd, file, dirtyfile: AbsoluteFile, line, col: int; +proc executeNoHooks(cmd: IdeCmd, file, dirtyfile: AbsoluteFile, line, col: int, graph: ModuleGraph) = + executeNoHooks(cmd, file, dirtyfile, line, col, graph) + +proc execute(cmd: IdeCmd, file, dirtyfile: AbsoluteFile, line, col: int; tag: string, graph: ModuleGraph) = if cmd == ideChk: graph.config.structuredErrorHook = errorHook @@ -227,7 +244,7 @@ proc execute(cmd: IdeCmd, file, dirtyfile: AbsoluteFile, line, col: int; else: graph.config.structuredErrorHook = nil graph.config.writelnHook = myLog - executeNoHooks(cmd, file, dirtyfile, line, col, graph) + executeNoHooks(cmd, file, dirtyfile, line, col, tag, graph) proc executeEpc(cmd: IdeCmd, args: SexpNode; graph: ModuleGraph) = @@ -238,7 +255,7 @@ proc executeEpc(cmd: IdeCmd, args: SexpNode; var dirtyfile = AbsoluteFile"" if len(args) > 3: dirtyfile = AbsoluteFile args[3].getStr("") - execute(cmd, file, dirtyfile, int(line), int(column), graph) + execute(cmd, file, dirtyfile, int(line), int(column), args[3].getStr, graph) proc returnEpc(socket: Socket, uid: BiggestInt, s: SexpNode|string, returnSymbol = "return") = @@ -459,6 +476,7 @@ proc execCmd(cmd: string; graph: ModuleGraph; cachedMsgs: CachedMsgs) = of "changed": conf.ideCmd = ideChanged of "globalsymbols": conf.ideCmd = ideGlobalSymbols of "declaration": conf.ideCmd = ideDeclaration + of "expand": conf.ideCmd = ideExpand of "chkfile": conf.ideCmd = ideChkFile of "recompile": conf.ideCmd = ideRecompile of "type": conf.ideCmd = ideType @@ -478,6 +496,7 @@ proc execCmd(cmd: string; graph: ModuleGraph; cachedMsgs: CachedMsgs) = i += parseInt(cmd, line, i) i += skipWhile(cmd, seps, i) i += parseInt(cmd, col, i) + let tag = substr(cmd, i) if conf.ideCmd == ideKnown: results.send(Suggest(section: ideKnown, quality: ord(fileInfoKnown(conf, AbsoluteFile orig)))) @@ -486,18 +505,9 @@ proc execCmd(cmd: string; graph: ModuleGraph; cachedMsgs: CachedMsgs) = else: if conf.ideCmd == ideChk: for cm in cachedMsgs: errorHook(conf, cm.info, cm.msg, cm.sev) - execute(conf.ideCmd, AbsoluteFile orig, AbsoluteFile dirtyfile, line, col, graph) + execute(conf.ideCmd, AbsoluteFile orig, AbsoluteFile dirtyfile, line, col, tag, graph) sentinel() -template benchmark(benchmarkName: string, code: untyped) = - block: - myLog "Started [" & benchmarkName & "]..." - let t0 = epochTime() - code - let elapsed = epochTime() - t0 - let elapsedStr = elapsed.formatFloat(format = ffDecimal, precision = 3) - myLog "CPU Time [" & benchmarkName & "] " & elapsedStr & "s" - proc recompileFullProject(graph: ModuleGraph) = benchmark "Recompilation(clean)": graph.resetForBackend() @@ -509,9 +519,9 @@ proc recompileFullProject(graph: ModuleGraph) = proc mainThread(graph: ModuleGraph) = let conf = graph.config - if gLogging: - for it in conf.searchPaths: - log(it.string) + myLog "searchPaths: " + for it in conf.searchPaths: + myLog(" " & it.string) proc wrHook(line: string) {.closure.} = if gMode == mepc: @@ -732,16 +742,20 @@ func deduplicateSymInfoPair[SymInfoPair](xs: seq[SymInfoPair]): seq[SymInfoPair] result.add(itm) result.reverse() +proc findSymData(graph: ModuleGraph, trackPos: TLineInfo): + ref SymInfoPair = + for s in graph.fileSymbols(trackPos.fileIndex).deduplicateSymInfoPair: + if isTracked(s.info, trackPos, s.sym.name.s.len): + new(result) + result[] = s + break + proc findSymData(graph: ModuleGraph, file: AbsoluteFile; line, col: int): ref SymInfoPair = let fileIdx = fileInfoIdx(graph.config, file) trackPos = newLineInfo(fileIdx, line, col) - for s in graph.fileSymbols(fileIdx).deduplicateSymInfoPair: - if isTracked(s.info, trackPos, s.sym.name.s.len): - new(result) - result[] = s - break + result = findSymData(graph, trackPos) proc markDirtyIfNeeded(graph: ModuleGraph, file: string, originalFileIdx: FileIndex) = let sha = $sha1.secureHashFile(file) @@ -752,7 +766,8 @@ proc markDirtyIfNeeded(graph: ModuleGraph, file: string, originalFileIdx: FileIn else: myLog fmt "No changes in file {file} compared to last compilation" -proc suggestResult(graph: ModuleGraph, sym: PSym, info: TLineInfo, defaultSection = ideNone) = +proc suggestResult(graph: ModuleGraph, sym: PSym, info: TLineInfo, + defaultSection = ideNone, endLine: uint16 = 0, endCol = 0) = let section = if defaultSection != ideNone: defaultSection elif sym.info.exactEquals(info): @@ -760,7 +775,8 @@ proc suggestResult(graph: ModuleGraph, sym: PSym, info: TLineInfo, defaultSectio else: ideUse let suggest = symToSuggest(graph, sym, isLocal=false, section, - info, 100, PrefixMatch.None, false, 0) + info, 100, PrefixMatch.None, false, 0, + endLine = endLine, endCol = endCol) suggestResult(graph.config, suggest) const @@ -771,7 +787,75 @@ proc symbolEqual(left, right: PSym): bool = # More relaxed symbol comparison return left.info.exactEquals(right.info) and left.name == right.name -proc executeNoHooksV3(cmd: IdeCmd, file: AbsoluteFile, dirtyfile: AbsoluteFile, line, col: int; +proc findDef(n: PNode, line: uint16, col: int16): PNode = + if n.kind in {nkProcDef, nkIteratorDef, nkTemplateDef, nkMethodDef, nkMacroDef}: + if n.info.line == line: + return n + else: + for i in 0 ..< safeLen(n): + let res = findDef(n[i], line, col) + if res != nil: return res + +proc findByTLineInfo(trackPos: TLineInfo, infoPairs: seq[SymInfoPair]): + ref SymInfoPair = + for s in infoPairs: + if s.info.exactEquals trackPos: + new(result) + result[] = s + break + +proc outlineNode(graph: ModuleGraph, n: PNode, endInfo: TLineInfo, infoPairs: seq[SymInfoPair]): bool = + proc checkSymbol(sym: PSym, info: TLineInfo): bool = + result = (sym.owner.kind in {skModule, skType} or sym.kind in {skProc, skMethod, skIterator, skTemplate, skType}) + + if n.kind == nkSym and n.sym.checkSymbol(n.info): + graph.suggestResult(n.sym, n.sym.info, ideOutline, endInfo.line, endInfo.col) + return true + elif n.kind == nkIdent: + let symData = findByTLineInfo(n.info, infoPairs) + if symData != nil and symData.sym.checkSymbol(symData.info): + let sym = symData.sym + graph.suggestResult(sym, sym.info, ideOutline, endInfo.line, endInfo.col) + return true + +proc handleIdentOrSym(graph: ModuleGraph, n: PNode, endInfo: TLineInfo, infoPairs: seq[SymInfoPair]): bool = + for child in n: + if child.kind in {nkIdent, nkSym}: + if graph.outlineNode(child, endInfo, infoPairs): + return true + elif child.kind == nkPostfix: + if graph.handleIdentOrSym(child, endInfo, infoPairs): + return true + +proc iterateOutlineNodes(graph: ModuleGraph, n: PNode, infoPairs: seq[SymInfoPair]) = + var matched = true + if n.kind == nkIdent: + let symData = findByTLineInfo(n.info, infoPairs) + if symData != nil and symData.sym.kind == skEnumField and symData.info.exactEquals(symData.sym.info): + let sym = symData.sym + graph.suggestResult(sym, sym.info, ideOutline, n.endInfo.line, n.endInfo.col) + elif (n.kind in {nkFuncDef, nkProcDef, nkTypeDef, nkMacroDef, nkTemplateDef, nkConverterDef, nkEnumFieldDef, nkConstDef}): + matched = handleIdentOrSym(graph, n, n.endInfo, infoPairs) + else: + matched = false + + if n.kind != nkFormalParams: + for child in n: + graph.iterateOutlineNodes(child, infoPairs) + +proc calculateExpandRange(n: PNode, info: TLineInfo): TLineInfo = + if ((n.kind in {nkFuncDef, nkProcDef, nkIteratorDef, nkTemplateDef, nkMethodDef, nkConverterDef} and + n.info.exactEquals(info)) or + (n.kind in {nkCall, nkCommand} and n[0].info.exactEquals(info))): + result = n.endInfo + else: + for child in n: + result = child.calculateExpandRange(info) + if result != unknownLineInfo: + return result + result = unknownLineInfo + +proc executeNoHooksV3(cmd: IdeCmd, file: AbsoluteFile, dirtyfile: AbsoluteFile, line, col: int; tag: string, graph: ModuleGraph) = let conf = graph.config conf.writelnHook = proc (s: string) = discard @@ -783,7 +867,7 @@ proc executeNoHooksV3(cmd: IdeCmd, file: AbsoluteFile, dirtyfile: AbsoluteFile, conf.ideCmd = cmd - myLog fmt "cmd: {cmd}, file: {file}[{line}:{col}], dirtyFile: {dirtyfile}" + myLog fmt "cmd: {cmd}, file: {file}[{line}:{col}], dirtyFile: {dirtyfile}, tag: {tag}" var fileIndex: FileIndex @@ -811,7 +895,7 @@ proc executeNoHooksV3(cmd: IdeCmd, file: AbsoluteFile, dirtyfile: AbsoluteFile, graph.unmarkAllDirty() # these commands require partially compiled project - elif cmd in {ideSug, ideOutline, ideHighlight, ideDef, ideChkFile, ideType, ideDeclaration} and + elif cmd in {ideSug, ideOutline, ideHighlight, ideDef, ideChkFile, ideType, ideDeclaration, ideExpand} and (graph.needsCompilation(fileIndex) or cmd == ideSug): # for ideSug use v2 implementation if cmd == ideSug: @@ -861,16 +945,8 @@ proc executeNoHooksV3(cmd: IdeCmd, file: AbsoluteFile, dirtyfile: AbsoluteFile, # future calls. graph.markDirtyIfNeeded(file.string, fileIndex) of ideOutline: - let - module = graph.getModule fileIndex - symbols = graph.fileSymbols(fileIndex) - .deduplicateSymInfoPair - .filterIt(it.sym.info.exactEquals(it.info) and - (it.sym.owner == module or - it.sym.kind in searchableSymKinds)) - - for s in symbols: - graph.suggestResult(s.sym, s.info, ideOutline) + let n = parseFile(fileIndex, graph.cache, graph.config) + graph.iterateOutlineNodes(n, graph.fileSymbols(fileIndex).deduplicateSymInfoPair) of ideChk: myLog fmt "Reporting errors for {graph.suggestErrors.len} file(s)" for sug in graph.suggestErrorsIter: @@ -933,6 +1009,39 @@ proc executeNoHooksV3(cmd: IdeCmd, file: AbsoluteFile, dirtyfile: AbsoluteFile, else: # we are on definition or usage, look for declaration graph.suggestResult(first.sym, first.info, ideDeclaration) + of ideExpand: + var level: int = high(int) + let index = skipWhitespace(tag, 0); + let trimmed = substr(tag, index) + if not (trimmed == "" or trimmed == "all"): + discard parseInt(trimmed, level, 0) + + conf.expandPosition = newLineInfo(fileIndex, line, col) + conf.expandLevels = level + conf.expandProgress = false + conf.expandNodeResult = "" + + graph.markDirty fileIndex + graph.markClientsDirty fileIndex + graph.recompilePartially() + var suggest = Suggest() + suggest.section = ideExpand + suggest.version = 3 + suggest.line = line + suggest.column = col + suggest.doc = graph.config.expandNodeResult + if suggest.doc != "": + let + n = parseFile(fileIndex, graph.cache, graph.config) + endInfo = n.calculateExpandRange(conf.expandPosition) + + suggest.endLine = endInfo.line + suggest.endCol = endInfo.col + + suggestResult(graph.config, suggest) + + graph.markDirty fileIndex + graph.markClientsDirty fileIndex else: myLog fmt "Discarding {cmd}" @@ -1018,9 +1127,9 @@ else: if self.loadConfigsAndProcessCmdLine(cache, conf, graph): mockCommand(graph) if gLogging: - log("Search paths:") + myLog("Search paths:") for it in conf.searchPaths: - log(" " & it.string) + myLog(" " & it.string) retval.doStopCompile = proc (): bool = false return NimSuggest(graph: retval, idle: 0, cachedMsgs: @[]) diff --git a/nimsuggest/tester.nim b/nimsuggest/tester.nim index 6e068e067..060335959 100644 --- a/nimsuggest/tester.nim +++ b/nimsuggest/tester.nim @@ -66,7 +66,7 @@ proc parseTest(filename: string; epcMode=false): Test = elif x.startsWith(">"): # since 'markers' here are not complete yet, we do the $substitutions # afterwards - result.script.add((x.substr(1).replaceWord("$path", tpath), "")) + result.script.add((x.substr(1).replaceWord("$path", tpath).replaceWord("$file", filename), "")) elif x.len > 0: # expected output line: let x = x % ["file", filename, "lib", libpath] @@ -218,7 +218,12 @@ proc sexpToAnswer(s: SexpNode): string = result.add doc result.add '\t' result.addInt a[8].getNum - if a.len >= 10: + if a.len >= 11: + result.add '\t' + result.addInt a[9].getNum + result.add '\t' + result.addInt a[10].getNum + elif a.len >= 10: result.add '\t' result.add a[9].getStr result.add '\L' @@ -229,8 +234,8 @@ proc doReport(filename, answer, resp: string; report: var string) = var hasDiff = false for i in 0..min(resp.len-1, answer.len-1): if resp[i] != answer[i]: - report.add "\n Expected: " & resp.substr(i, i+200) - report.add "\n But got: " & answer.substr(i, i+200) + report.add "\n Expected:\n" & resp + report.add "\n But got:\n" & answer hasDiff = true break if not hasDiff: @@ -342,8 +347,8 @@ proc main() = if os.paramCount() > 0: let x = os.paramStr(1) let xx = expandFilename x + # run only stdio when running single test failures += runTest(xx) - failures += runEpcTest(xx) else: let files = toSeq(walkFiles(tpath / "t*.nim")) for i, x in files: diff --git a/nimsuggest/tests/tv3.nim b/nimsuggest/tests/tv3.nim index 9d8b1ef2d..fd736a1d8 100644 --- a/nimsuggest/tests/tv3.nim +++ b/nimsuggest/tests/tv3.nim @@ -16,10 +16,6 @@ def skField tv3.Foo.bar string $file 5 4 "" 100 use skField tv3.Foo.bar string $file 8 9 "" 100 >def $1 def skField tv3.Foo.bar string $file 5 4 "" 100 ->outline $1 -outline skType tv3.Foo Foo $file 4 2 "" 100 -outline skField tv3.Foo.bar string $file 5 4 "" 100 -outline skProc tv3.test proc (f: Foo){.gcsafe.} $file 7 5 "" 100 >sug $1 sug skField bar string $file 5 4 "" 100 Prefix >globalSymbols test diff --git a/nimsuggest/tests/tv3_outline.nim b/nimsuggest/tests/tv3_outline.nim new file mode 100644 index 000000000..6370948d9 --- /dev/null +++ b/nimsuggest/tests/tv3_outline.nim @@ -0,0 +1,45 @@ +# tests v3 outline + +type + Foo* = ref object of RootObj + bar*: string + FooEnum = enum value1, value2 + FooPrivate = ref object of RootObj + barPrivate: string + +macro m(arg: untyped): untyped = discard +template t(arg: untyped): untyped = discard +proc p(): void = discard +iterator i(): int = discard +converter c(s: string): int = discard +method m(f: Foo): void = discard +func f(): void = discard + +let a = 1 +var b = 2 +const con = 2 + +proc outer(): void = + proc inner() = discard + +proc procWithLocal(): void = + let local = 10 + +discard """ +$nimsuggest --v3 --tester $file +>outline $file +outline skType tv3_outline.Foo Foo $file 4 2 "" 100 5 16 +outline skType tv3_outline.FooEnum FooEnum $file 6 2 "" 100 6 31 +outline skEnumField tv3_outline.FooEnum.value1 FooEnum $file 6 17 "" 100 6 23 +outline skEnumField tv3_outline.FooEnum.value2 FooEnum $file 6 25 "" 100 6 31 +outline skType tv3_outline.FooPrivate FooPrivate $file 7 2 "" 100 8 22 +outline skMacro tv3_outline.m macro (arg: untyped): untyped{.noSideEffect, gcsafe.} $file 10 6 "" 100 10 40 +outline skTemplate tv3_outline.t template (arg: untyped): untyped $file 11 9 "" 100 11 43 +outline skProc tv3_outline.p proc (){.noSideEffect, gcsafe.} $file 12 5 "" 100 12 24 +outline skConverter tv3_outline.c converter (s: string): int{.noSideEffect, gcsafe.} $file 14 10 "" 100 14 37 +outline skFunc tv3_outline.f proc (){.noSideEffect, gcsafe.} $file 16 5 "" 100 16 24 +outline skConst tv3_outline.con int literal(2) $file 20 6 "" 100 20 13 +outline skProc tv3_outline.outer proc (){.noSideEffect, gcsafe.} $file 22 5 "" 100 23 24 +outline skProc tv3_outline.outer.inner proc (){.noSideEffect, gcsafe.} $file 23 7 "" 100 23 24 +outline skProc tv3_outline.procWithLocal proc (){.noSideEffect, gcsafe.} $file 25 5 "" 100 26 16 +""" From a7bae919adb952362cb53206140872d2b7424b47 Mon Sep 17 00:00:00 2001 From: Bung Date: Fri, 27 Jan 2023 00:50:44 -0800 Subject: [PATCH 2380/2472] Fix #13093 C++ Atomics: operator= is implicitly deleted because the default definition would be ill-formed (#21169) * add test * fix #17982 Invalid C++ code generation when returning discardable var T * fix #13093 * cpp atomic good example * clearify the condition --- compiler/cgen.nim | 4 ++++ compiler/semstmts.nim | 6 +++++- lib/pure/concurrency/atomics.nim | 2 +- tests/cpp/t13093.nim | 24 ++++++++++++++++++++++++ tests/cpp/t17982.nim | 20 ++++++++++++++++++++ 5 files changed, 54 insertions(+), 2 deletions(-) create mode 100644 tests/cpp/t13093.nim create mode 100644 tests/cpp/t17982.nim diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 5bf9400c5..869c1bfdd 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -491,6 +491,10 @@ proc constructLoc(p: BProc, loc: var TLoc, isTemp = false) = nilLoc.r = rope("NIM_NIL") genRefAssign(p, loc, nilLoc) else: + if typ.kind == tyVar: + let tt = typ.skipTypes({tyVar}) + if isImportedType(tt) and tfRequiresInit in tt.flags: + return linefmt(p, cpsStmts, "$1 = ($2)0;$n", [rdLoc(loc), getTypeDesc(p.module, typ, mapTypeChooser(loc))]) else: diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index c7e48db4c..12a346f63 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -717,7 +717,11 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = tyUserTypeClassInst}) if actualType.kind in {tyObject, tyDistinct} and actualType.requiresInit: - defaultConstructionError(c, v.typ, v.info) + # imported type use requiresInit pragma prevent implicit initialization + if (tfRequiresInit in actualType.flags and sfImportc in actualType.sym.flags): + discard + else: + defaultConstructionError(c, v.typ, v.info) else: checkNilable(c, v) # allow let to not be initialised if imported from C: diff --git a/lib/pure/concurrency/atomics.nim b/lib/pure/concurrency/atomics.nim index c7b881bc5..49204bd4d 100644 --- a/lib/pure/concurrency/atomics.nim +++ b/lib/pure/concurrency/atomics.nim @@ -89,7 +89,7 @@ when defined(cpp) or defined(nimdoc): ## with other moSequentiallyConsistent operations. type - Atomic*[T] {.importcpp: "std::atomic", completeStruct.} = object + Atomic*[T] {.importcpp: "std::atomic", completeStruct, requiresInit.} = object ## An atomic object with underlying type `T`. raw: T diff --git a/tests/cpp/t13093.nim b/tests/cpp/t13093.nim new file mode 100644 index 000000000..17c730d16 --- /dev/null +++ b/tests/cpp/t13093.nim @@ -0,0 +1,24 @@ +discard """ + targets: "cpp" + action: reject + errormsg: "The PledgeObj type requires the following fields to be initialized: refCount" +""" + +import atomics + +type + Pledge* = object + p: PledgePtr + + PledgePtr = ptr PledgeObj + PledgeObj = object + refCount: Atomic[int32] + +proc main() = + var pledge: Pledge + pledge.p = createShared(PledgeObj) + let tmp = PledgeObj() # <---- not allowed: atomics are not copyable + + pledge.p[] = tmp + +main() diff --git a/tests/cpp/t17982.nim b/tests/cpp/t17982.nim new file mode 100644 index 000000000..5413f06ff --- /dev/null +++ b/tests/cpp/t17982.nim @@ -0,0 +1,20 @@ +discard """ + targets: "cpp" + action: "compile" +""" + +type + String* {.importcpp: "std::string", header: "string".} = object + +proc initString*(): String + {.importcpp: "std::string()", header: "string".} + +proc append*(this: var String, str: String): String + # bug seems to trigger when `#`, `@`, or `$1` is used inside `importcpp` + {.importcpp: "#.append(@)", header: "string", discardable.} # <- changed from `importcpp: "append"` + +var + s1 = initString() + s2 = initString() + +s1.append s2 From 1431f90d8c3d3ed0afcaee62073d5808cea227ff Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sat, 28 Jan 2023 04:41:59 +0800 Subject: [PATCH 2381/2472] Revert "Fix #13093 C++ Atomics: operator= is implicitly deleted because the default definition would be ill-formed " (#21307) Revert "Fix #13093 C++ Atomics: operator= is implicitly deleted because the default definition would be ill-formed (#21169)" This reverts commit a7bae919adb952362cb53206140872d2b7424b47. --- compiler/cgen.nim | 4 ---- compiler/semstmts.nim | 6 +----- lib/pure/concurrency/atomics.nim | 2 +- tests/cpp/t13093.nim | 24 ------------------------ tests/cpp/t17982.nim | 20 -------------------- 5 files changed, 2 insertions(+), 54 deletions(-) delete mode 100644 tests/cpp/t13093.nim delete mode 100644 tests/cpp/t17982.nim diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 869c1bfdd..5bf9400c5 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -491,10 +491,6 @@ proc constructLoc(p: BProc, loc: var TLoc, isTemp = false) = nilLoc.r = rope("NIM_NIL") genRefAssign(p, loc, nilLoc) else: - if typ.kind == tyVar: - let tt = typ.skipTypes({tyVar}) - if isImportedType(tt) and tfRequiresInit in tt.flags: - return linefmt(p, cpsStmts, "$1 = ($2)0;$n", [rdLoc(loc), getTypeDesc(p.module, typ, mapTypeChooser(loc))]) else: diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 12a346f63..c7e48db4c 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -717,11 +717,7 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = tyUserTypeClassInst}) if actualType.kind in {tyObject, tyDistinct} and actualType.requiresInit: - # imported type use requiresInit pragma prevent implicit initialization - if (tfRequiresInit in actualType.flags and sfImportc in actualType.sym.flags): - discard - else: - defaultConstructionError(c, v.typ, v.info) + defaultConstructionError(c, v.typ, v.info) else: checkNilable(c, v) # allow let to not be initialised if imported from C: diff --git a/lib/pure/concurrency/atomics.nim b/lib/pure/concurrency/atomics.nim index 49204bd4d..c7b881bc5 100644 --- a/lib/pure/concurrency/atomics.nim +++ b/lib/pure/concurrency/atomics.nim @@ -89,7 +89,7 @@ when defined(cpp) or defined(nimdoc): ## with other moSequentiallyConsistent operations. type - Atomic*[T] {.importcpp: "std::atomic", completeStruct, requiresInit.} = object + Atomic*[T] {.importcpp: "std::atomic", completeStruct.} = object ## An atomic object with underlying type `T`. raw: T diff --git a/tests/cpp/t13093.nim b/tests/cpp/t13093.nim deleted file mode 100644 index 17c730d16..000000000 --- a/tests/cpp/t13093.nim +++ /dev/null @@ -1,24 +0,0 @@ -discard """ - targets: "cpp" - action: reject - errormsg: "The PledgeObj type requires the following fields to be initialized: refCount" -""" - -import atomics - -type - Pledge* = object - p: PledgePtr - - PledgePtr = ptr PledgeObj - PledgeObj = object - refCount: Atomic[int32] - -proc main() = - var pledge: Pledge - pledge.p = createShared(PledgeObj) - let tmp = PledgeObj() # <---- not allowed: atomics are not copyable - - pledge.p[] = tmp - -main() diff --git a/tests/cpp/t17982.nim b/tests/cpp/t17982.nim deleted file mode 100644 index 5413f06ff..000000000 --- a/tests/cpp/t17982.nim +++ /dev/null @@ -1,20 +0,0 @@ -discard """ - targets: "cpp" - action: "compile" -""" - -type - String* {.importcpp: "std::string", header: "string".} = object - -proc initString*(): String - {.importcpp: "std::string()", header: "string".} - -proc append*(this: var String, str: String): String - # bug seems to trigger when `#`, `@`, or `$1` is used inside `importcpp` - {.importcpp: "#.append(@)", header: "string", discardable.} # <- changed from `importcpp: "append"` - -var - s1 = initString() - s2 = initString() - -s1.append s2 From 4459c272ecb8ddb964fa2d34a4dcef3368d807b5 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sun, 29 Jan 2023 22:11:24 +0800 Subject: [PATCH 2382/2472] remove `enumUtilsExist` defines (#21309) --- compiler/wordrecg.nim | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/compiler/wordrecg.nim b/compiler/wordrecg.nim index a9ef31dc2..29a50ac52 100644 --- a/compiler/wordrecg.nim +++ b/compiler/wordrecg.nim @@ -130,20 +130,7 @@ const wFor, wIf, wReturn, wStatic, wTemplate, wTry, wWhile, wUsing} -const enumUtilsExist = compiles: - import std/enumutils - -when enumUtilsExist: - from std/enumutils import genEnumCaseStmt - from strutils import normalize - proc findStr*[T: enum](a, b: static[T], s: string, default: T): T = - genEnumCaseStmt(T, s, default, ord(a), ord(b), normalize) - -else: - from strutils import cmpIgnoreStyle - proc findStr*[T: enum](a, b: static[T], s: string, default: T): T {.deprecated.} = - # used for compiler bootstrapping only - for i in a..b: - if cmpIgnoreStyle($i, s) == 0: - return i - result = default +from std/enumutils import genEnumCaseStmt +from strutils import normalize +proc findStr*[T: enum](a, b: static[T], s: string, default: T): T = + genEnumCaseStmt(T, s, default, ord(a), ord(b), normalize) From ebaa07b955182dd98a1bfe06de4195fe1d306c5e Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Tue, 31 Jan 2023 23:11:44 +0800 Subject: [PATCH 2383/2472] remove md5_old (#21312) --- compiler/md5_old.nim | 297 -------------------------------------- compiler/modulegraphs.nim | 2 +- compiler/sighashes.nim | 2 +- 3 files changed, 2 insertions(+), 299 deletions(-) delete mode 100644 compiler/md5_old.nim diff --git a/compiler/md5_old.nim b/compiler/md5_old.nim deleted file mode 100644 index da4a51198..000000000 --- a/compiler/md5_old.nim +++ /dev/null @@ -1,297 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2010 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -# `std/md5` without VM and JavaScript support, to circumvent a bug with -# openarrays on Nim < 1.4. - -when defined(nimHasStyleChecks): - {.push styleChecks: off.} - -type - MD5State = array[0..3, uint32] - MD5Block = array[0..15, uint32] - MD5CBits = array[0..7, uint8] - MD5Digest* = array[0..15, uint8] - ## MD5 checksum of a string, obtained with the `toMD5 proc <#toMD5,string>`_. - MD5Buffer = array[0..63, uint8] - MD5Context* {.final.} = object - state: MD5State - count: array[0..1, uint32] - buffer: MD5Buffer - -const - padding: array[0..63, uint8] = [ - 0x80'u8, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0 - ] - -proc F(x, y, z: uint32): uint32 {.inline.} = - result = (x and y) or ((not x) and z) - -proc G(x, y, z: uint32): uint32 {.inline.} = - result = (x and z) or (y and (not z)) - -proc H(x, y, z: uint32): uint32 {.inline.} = - result = x xor y xor z - -proc I(x, y, z: uint32): uint32 {.inline.} = - result = y xor (x or (not z)) - -proc rot(x: var uint32, n: uint8) {.inline.} = - x = (x shl n) or (x shr (32'u32 - n)) - -proc FF(a: var uint32, b, c, d, x: uint32, s: uint8, ac: uint32) = - a = a + F(b, c, d) + x + ac - rot(a, s) - a = a + b - -proc GG(a: var uint32, b, c, d, x: uint32, s: uint8, ac: uint32) = - a = a + G(b, c, d) + x + ac - rot(a, s) - a = a + b - -proc HH(a: var uint32, b, c, d, x: uint32, s: uint8, ac: uint32) = - a = a + H(b, c, d) + x + ac - rot(a, s) - a = a + b - -proc II(a: var uint32, b, c, d, x: uint32, s: uint8, ac: uint32) = - a = a + I(b, c, d) + x + ac - rot(a, s) - a = a + b - -proc encode(dest: var MD5Block, src: openArray[uint8]) = - var j = 0 - for i in 0..high(dest): - dest[i] = uint32(ord(src[j])) or - uint32(ord(src[j+1])) shl 8 or - uint32(ord(src[j+2])) shl 16 or - uint32(ord(src[j+3])) shl 24 - inc(j, 4) - -proc decode(dest: var openArray[uint8], src: openArray[uint32]) = - var i = 0 - for j in 0..high(src): - dest[i] = uint8(src[j] and 0xff'u32) - dest[i+1] = uint8(src[j] shr 8 and 0xff'u32) - dest[i+2] = uint8(src[j] shr 16 and 0xff'u32) - dest[i+3] = uint8(src[j] shr 24 and 0xff'u32) - inc(i, 4) - -template slice(s: string | cstring, a, b): openArray[uint8] = - s.toOpenArrayByte(a, b) - -template slice(s: openArray[uint8], a, b): openArray[uint8] = - s.toOpenArray(a, b) - -proc transform(buffer: openArray[uint8], state: var MD5State) = - var - myBlock: MD5Block - encode(myBlock, buffer) - var a = state[0] - var b = state[1] - var c = state[2] - var d = state[3] - FF(a, b, c, d, myBlock[0], 7'u8, 0xD76AA478'u32) - FF(d, a, b, c, myBlock[1], 12'u8, 0xE8C7B756'u32) - FF(c, d, a, b, myBlock[2], 17'u8, 0x242070DB'u32) - FF(b, c, d, a, myBlock[3], 22'u8, 0xC1BDCEEE'u32) - FF(a, b, c, d, myBlock[4], 7'u8, 0xF57C0FAF'u32) - FF(d, a, b, c, myBlock[5], 12'u8, 0x4787C62A'u32) - FF(c, d, a, b, myBlock[6], 17'u8, 0xA8304613'u32) - FF(b, c, d, a, myBlock[7], 22'u8, 0xFD469501'u32) - FF(a, b, c, d, myBlock[8], 7'u8, 0x698098D8'u32) - FF(d, a, b, c, myBlock[9], 12'u8, 0x8B44F7AF'u32) - FF(c, d, a, b, myBlock[10], 17'u8, 0xFFFF5BB1'u32) - FF(b, c, d, a, myBlock[11], 22'u8, 0x895CD7BE'u32) - FF(a, b, c, d, myBlock[12], 7'u8, 0x6B901122'u32) - FF(d, a, b, c, myBlock[13], 12'u8, 0xFD987193'u32) - FF(c, d, a, b, myBlock[14], 17'u8, 0xA679438E'u32) - FF(b, c, d, a, myBlock[15], 22'u8, 0x49B40821'u32) - GG(a, b, c, d, myBlock[1], 5'u8, 0xF61E2562'u32) - GG(d, a, b, c, myBlock[6], 9'u8, 0xC040B340'u32) - GG(c, d, a, b, myBlock[11], 14'u8, 0x265E5A51'u32) - GG(b, c, d, a, myBlock[0], 20'u8, 0xE9B6C7AA'u32) - GG(a, b, c, d, myBlock[5], 5'u8, 0xD62F105D'u32) - GG(d, a, b, c, myBlock[10], 9'u8, 0x02441453'u32) - GG(c, d, a, b, myBlock[15], 14'u8, 0xD8A1E681'u32) - GG(b, c, d, a, myBlock[4], 20'u8, 0xE7D3FBC8'u32) - GG(a, b, c, d, myBlock[9], 5'u8, 0x21E1CDE6'u32) - GG(d, a, b, c, myBlock[14], 9'u8, 0xC33707D6'u32) - GG(c, d, a, b, myBlock[3], 14'u8, 0xF4D50D87'u32) - GG(b, c, d, a, myBlock[8], 20'u8, 0x455A14ED'u32) - GG(a, b, c, d, myBlock[13], 5'u8, 0xA9E3E905'u32) - GG(d, a, b, c, myBlock[2], 9'u8, 0xFCEFA3F8'u32) - GG(c, d, a, b, myBlock[7], 14'u8, 0x676F02D9'u32) - GG(b, c, d, a, myBlock[12], 20'u8, 0x8D2A4C8A'u32) - HH(a, b, c, d, myBlock[5], 4'u8, 0xFFFA3942'u32) - HH(d, a, b, c, myBlock[8], 11'u8, 0x8771F681'u32) - HH(c, d, a, b, myBlock[11], 16'u8, 0x6D9D6122'u32) - HH(b, c, d, a, myBlock[14], 23'u8, 0xFDE5380C'u32) - HH(a, b, c, d, myBlock[1], 4'u8, 0xA4BEEA44'u32) - HH(d, a, b, c, myBlock[4], 11'u8, 0x4BDECFA9'u32) - HH(c, d, a, b, myBlock[7], 16'u8, 0xF6BB4B60'u32) - HH(b, c, d, a, myBlock[10], 23'u8, 0xBEBFBC70'u32) - HH(a, b, c, d, myBlock[13], 4'u8, 0x289B7EC6'u32) - HH(d, a, b, c, myBlock[0], 11'u8, 0xEAA127FA'u32) - HH(c, d, a, b, myBlock[3], 16'u8, 0xD4EF3085'u32) - HH(b, c, d, a, myBlock[6], 23'u8, 0x04881D05'u32) - HH(a, b, c, d, myBlock[9], 4'u8, 0xD9D4D039'u32) - HH(d, a, b, c, myBlock[12], 11'u8, 0xE6DB99E5'u32) - HH(c, d, a, b, myBlock[15], 16'u8, 0x1FA27CF8'u32) - HH(b, c, d, a, myBlock[2], 23'u8, 0xC4AC5665'u32) - II(a, b, c, d, myBlock[0], 6'u8, 0xF4292244'u32) - II(d, a, b, c, myBlock[7], 10'u8, 0x432AFF97'u32) - II(c, d, a, b, myBlock[14], 15'u8, 0xAB9423A7'u32) - II(b, c, d, a, myBlock[5], 21'u8, 0xFC93A039'u32) - II(a, b, c, d, myBlock[12], 6'u8, 0x655B59C3'u32) - II(d, a, b, c, myBlock[3], 10'u8, 0x8F0CCC92'u32) - II(c, d, a, b, myBlock[10], 15'u8, 0xFFEFF47D'u32) - II(b, c, d, a, myBlock[1], 21'u8, 0x85845DD1'u32) - II(a, b, c, d, myBlock[8], 6'u8, 0x6FA87E4F'u32) - II(d, a, b, c, myBlock[15], 10'u8, 0xFE2CE6E0'u32) - II(c, d, a, b, myBlock[6], 15'u8, 0xA3014314'u32) - II(b, c, d, a, myBlock[13], 21'u8, 0x4E0811A1'u32) - II(a, b, c, d, myBlock[4], 6'u8, 0xF7537E82'u32) - II(d, a, b, c, myBlock[11], 10'u8, 0xBD3AF235'u32) - II(c, d, a, b, myBlock[2], 15'u8, 0x2AD7D2BB'u32) - II(b, c, d, a, myBlock[9], 21'u8, 0xEB86D391'u32) - state[0] = state[0] + a - state[1] = state[1] + b - state[2] = state[2] + c - state[3] = state[3] + d - -proc md5Init*(c: var MD5Context) {.raises: [], tags: [], gcsafe.} -proc md5Update*(c: var MD5Context, input: openArray[uint8]) {.raises: [], - tags: [], gcsafe.} -proc md5Final*(c: var MD5Context, digest: var MD5Digest) {.raises: [], tags: [], gcsafe.} - -proc md5Update*(c: var MD5Context, input: cstring, len: int) {.raises: [], - tags: [], gcsafe.} = - ## Updates the `MD5Context` with the `input` data of length `len`. - ## - ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this - ## function explicitly. - md5Update(c, input.slice(0, len - 1)) - - -proc toMD5*(s: string): MD5Digest = - ## Computes the `MD5Digest` value for a string `s`. - ## - ## **See also:** - ## * `getMD5 proc <#getMD5,string>`_ which returns a string representation - ## of the `MD5Digest` - ## * `$ proc <#$,MD5Digest>`_ for converting MD5Digest to string - runnableExamples: - assert $toMD5("abc") == "900150983cd24fb0d6963f7d28e17f72" - - var c: MD5Context - md5Init(c) - md5Update(c, s.slice(0, s.len - 1)) - md5Final(c, result) - -proc `$`*(d: MD5Digest): string = - ## Converts a `MD5Digest` value into its string representation. - const digits = "0123456789abcdef" - result = "" - for i in 0..15: - add(result, digits[(d[i].int shr 4) and 0xF]) - add(result, digits[d[i].int and 0xF]) - -proc getMD5*(s: string): string = - ## Computes an MD5 value of `s` and returns its string representation. - ## - ## **See also:** - ## * `toMD5 proc <#toMD5,string>`_ which returns the `MD5Digest` of a string - runnableExamples: - assert getMD5("abc") == "900150983cd24fb0d6963f7d28e17f72" - - var - c: MD5Context - d: MD5Digest - md5Init(c) - md5Update(c, s.slice(0, s.len - 1)) - md5Final(c, d) - result = $d - -proc `==`*(D1, D2: MD5Digest): bool = - ## Checks if two `MD5Digest` values are identical. - for i in 0..15: - if D1[i] != D2[i]: return false - return true - - -proc clearBuffer(c: var MD5Context) {.inline.} = - zeroMem(addr(c.buffer), sizeof(MD5Buffer)) - -proc md5Init*(c: var MD5Context) = - ## Initializes an `MD5Context`. - ## - ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this - ## function explicitly. - c.state[0] = 0x67452301'u32 - c.state[1] = 0xEFCDAB89'u32 - c.state[2] = 0x98BADCFE'u32 - c.state[3] = 0x10325476'u32 - c.count[0] = 0'u32 - c.count[1] = 0'u32 - clearBuffer(c) - -proc writeBuffer(c: var MD5Context, index: int, - input: openArray[uint8], inputIndex, len: int) {.inline.} = - copyMem(addr(c.buffer[index]), unsafeAddr(input[inputIndex]), len) - -proc md5Update*(c: var MD5Context, input: openArray[uint8]) = - ## Updates the `MD5Context` with the `input` data. - ## - ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this - ## function explicitly. - var Index = int((c.count[0] shr 3) and 0x3F) - c.count[0] = c.count[0] + (uint32(input.len) shl 3) - if c.count[0] < (uint32(input.len) shl 3): c.count[1] = c.count[1] + 1'u32 - c.count[1] = c.count[1] + (uint32(input.len) shr 29) - var PartLen = 64 - Index - if input.len >= PartLen: - writeBuffer(c, Index, input, 0, PartLen) - transform(c.buffer, c.state) - var i = PartLen - while i + 63 < input.len: - transform(input.slice(i, i + 63), c.state) - inc(i, 64) - if i < input.len: - writeBuffer(c, 0, input, i, input.len - i) - elif input.len > 0: - writeBuffer(c, Index, input, 0, input.len) - -proc md5Final*(c: var MD5Context, digest: var MD5Digest) = - ## Finishes the `MD5Context` and stores the result in `digest`. - ## - ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this - ## function explicitly. - var - Bits: MD5CBits - PadLen: int - decode(Bits, c.count) - var Index = int((c.count[0] shr 3) and 0x3F) - if Index < 56: PadLen = 56 - Index - else: PadLen = 120 - Index - md5Update(c, padding.slice(0, PadLen - 1)) - md5Update(c, Bits) - decode(digest, c.state) - clearBuffer(c) - - -when defined(nimHasStyleChecks): - {.pop.} #{.push styleChecks: off.} diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index e8f77a491..598959a37 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -11,7 +11,7 @@ ## represents a complete Nim project. Single modules can either be kept in RAM ## or stored in a rod-file. -import intsets, tables, hashes, md5_old +import intsets, tables, hashes, md5 import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils, packages import ic / [packed_ast, ic] diff --git a/compiler/sighashes.nim b/compiler/sighashes.nim index a642d5cb3..3508a146a 100644 --- a/compiler/sighashes.nim +++ b/compiler/sighashes.nim @@ -9,7 +9,7 @@ ## Computes hash values for routine (proc, method etc) signatures. -import ast, tables, ropes, md5_old, modulegraphs +import ast, tables, ropes, md5, modulegraphs from hashes import Hash import types From b5f64f55d02a8ba980244596dcf310dd76b48fd8 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Wed, 1 Feb 2023 02:22:10 +0800 Subject: [PATCH 2384/2472] fixes #16790; fixes #19075; put big arrays on the constant seqs; don't inline them in the VM; big performance boost (#21318) * don't inline arrays in VM * add a test for #19075 --- compiler/vmgen.nim | 7 +++++-- tests/vm/t19075.nim | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 tests/vm/t19075.nim diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index bca961a63..bdf7b679e 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -32,7 +32,7 @@ when defined(nimPreviewSlimSystem): import std/assertions import - strutils, ast, types, msgs, renderer, vmdef, + strutils, ast, types, msgs, renderer, vmdef, trees, intsets, magicsys, options, lowerings, lineinfos, transf, astmsgs from modulegraphs import getBody @@ -2054,7 +2054,10 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) = genLit(c, n, dest) of skConst: let constVal = if s.astdef != nil: s.astdef else: s.typ.n - gen(c, constVal, dest) + if dontInlineConstant(n, constVal): + genLit(c, constVal, dest) + else: + gen(c, constVal, dest) of skEnumField: # we never reach this case - as of the time of this comment, # skEnumField is folded to an int in semfold.nim, but this code diff --git a/tests/vm/t19075.nim b/tests/vm/t19075.nim new file mode 100644 index 000000000..89ca9cb19 --- /dev/null +++ b/tests/vm/t19075.nim @@ -0,0 +1,19 @@ +discard """ + timeout: 10 + joinable: false +""" + +# bug #19075 +const size = 50_000 + +const stuff = block: + var a: array[size, int] + a + +const zeugs = block: + var zeugs: array[size, int] + for i in 0.. Date: Wed, 1 Feb 2023 14:58:12 +0800 Subject: [PATCH 2385/2472] fixes #21317; 1.6.4 regression; etyBaseIndex should return fat pointers [backport 1.6] (#21320) fixes #21317; regression; etyBaseIndex should return fat pointers --- compiler/jsgen.nim | 7 ++++--- tests/system/trefs.nim | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 tests/system/trefs.nim diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index d8fb6d57d..60399d3cd 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -2484,17 +2484,18 @@ proc genProc(oldProc: PProc, prc: PSym): Rope = if prc.typ[0] != nil and sfPure notin prc.flags: resultSym = prc.ast[resultPos].sym let mname = mangleName(p.module, resultSym) - let returnAddress = not isIndirect(resultSym) and + # otherwise uses "fat pointers" + let useRawPointer = not isIndirect(resultSym) and resultSym.typ.kind in {tyVar, tyPtr, tyLent, tyRef, tyOwned} and mapType(p, resultSym.typ) == etyBaseIndex - if returnAddress: + if useRawPointer: resultAsgn = p.indentLine(("var $# = null;$n") % [mname]) resultAsgn.add p.indentLine("var $#_Idx = 0;$n" % [mname]) else: let resVar = createVar(p, resultSym.typ, isIndirect(resultSym)) resultAsgn = p.indentLine(("var $# = $#;$n") % [mname, resVar]) gen(p, prc.ast[resultPos], a) - if returnAddress: + if mapType(p, resultSym.typ) == etyBaseIndex: returnStmt = "return [$#, $#];$n" % [a.address, a.res] else: returnStmt = "return $#;$n" % [a.res] diff --git a/tests/system/trefs.nim b/tests/system/trefs.nim new file mode 100644 index 000000000..8c36aec52 --- /dev/null +++ b/tests/system/trefs.nim @@ -0,0 +1,15 @@ +discard """ + targets: "c js" +""" + +# bug #21317 +proc parseHook*(v: var ref int) = + var a: ref int + new(a) + a[] = 123 + v = a + +proc fromJson2*(): ref int = + parseHook(result) + +doAssert fromJson2()[] == 123 From ff8ab06720976f18d75257a22bf8e23873514c67 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Wed, 1 Feb 2023 15:13:40 +0800 Subject: [PATCH 2386/2472] fixes #19396; Nimdoc hide nonexported fields (#21305) * suppresses non-exported fields of types and adds command-line option to re-enable this if desired * corrected the doctest that produced a CI error * an embarrassingly bad error in reasoning * modified a nimdoc test to reflect updated behavior * needed another change to bring utils.html doctest in sync with update * add info * fix nimdoc * lint * render postfix * fixes a problem * fixes nimdoc * fix nimdoc --------- Co-authored-by: johnperry-math Co-authored-by: johnperry-math --- compiler/commands.nim | 3 +++ compiler/docgen.nim | 22 +++++++++++-------- compiler/docgen2.nim | 2 +- compiler/options.nim | 4 ++++ compiler/renderer.nim | 13 ++++++----- compiler/vm.nim | 2 +- doc/docgen.md | 1 + nimdoc/extlinks/project/expected/main.html | 6 ++--- .../expected/subdir/subdir_b/utils.html | 6 ++--- nimdoc/testproject/expected/testproject.html | 12 ++++++++++ nimdoc/testproject/expected/testproject.idx | 1 + nimdoc/testproject/expected/theindex.html | 4 ++++ nimdoc/testproject/testproject.nim | 4 ++++ 13 files changed, 56 insertions(+), 24 deletions(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index 59b221647..7c0f7ab0a 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -1008,6 +1008,9 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; expectNoArg(conf, switch, arg, pass, info) conf.exc = low(ExceptionSystem) defineSymbol(conf.symbols, "noCppExceptions") + of "shownonexports": + expectNoArg(conf, switch, arg, pass, info) + showNonExportedFields(conf) of "exceptions": case arg.normalize of "cpp": conf.exc = excCpp diff --git a/compiler/docgen.nim b/compiler/docgen.nim index cab334e7a..8105cc807 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -1008,7 +1008,7 @@ proc toLangSymbol(k: TSymKind, n: PNode, baseName: string): LangSymbol = if k == skType: result.symTypeKind = getTypeKind(n) -proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) = +proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags, nonExports: bool = false) = if (docFlags != kForceExport) and not isVisible(d, nameNode): return let name = getName(nameNode) @@ -1062,8 +1062,11 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) = tooltip = detailedName, langSym = rstLangSymbol, priority = symbolPriority(k), info = lineinfo) - nodeToHighlightedHtml(d, n, result, {renderNoBody, renderNoComments, - renderDocComments, renderSyms, renderExpandUsing}, symbolOrIdEnc) + let renderFlags = + if nonExports: {renderNoBody, renderNoComments, renderDocComments, renderSyms, + renderExpandUsing, renderNonExportedFields} + else: {renderNoBody, renderNoComments, renderDocComments, renderSyms, renderExpandUsing} + nodeToHighlightedHtml(d, n, result, renderFlags, symbolOrIdEnc) let seeSrc = genSeeSrc(d, toFullPath(d.conf, n.info), n.info.line.int) @@ -1311,12 +1314,13 @@ proc documentRaises*(cache: IdentCache; n: PNode) = if p5 != nil: n[pragmasPos].add p5 if p6 != nil: n[pragmasPos].add p6 -proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) = +proc generateDoc*(d: PDoc, n, orig: PNode, config: ConfigRef, docFlags: DocFlags = kDefault) = ## Goes through nim nodes recursively and collects doc comments. ## Main function for `doc`:option: command, ## which is implemented in ``docgen2.nim``. template genItemAux(skind) = genItem(d, n, n[namePos], skind, docFlags) + let showNonExports = optShowNonExportedFields in config.globalOptions case n.kind of nkPragma: let pragmaNode = findPragma(n, wDeprecated) @@ -1343,20 +1347,20 @@ proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) = if n[i].kind != nkCommentStmt: # order is always 'type var let const': genItem(d, n[i], n[i][0], - succ(skType, ord(n.kind)-ord(nkTypeSection)), docFlags) + succ(skType, ord(n.kind)-ord(nkTypeSection)), docFlags, showNonExports) of nkStmtList: - for i in 0.. Types
        @@ -97,8 +96,7 @@ 
        A = object
-  x: int
-
        +
        diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html index f0f30536f..ee2618ab3 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html @@ -59,8 +59,7 @@
        Types
          -
        • G
          • +
        • G
        • SomeType
          • @@ -254,8 +253,7 @@ Ref. 
          G[T] = object
-  val: T
-
          +
          diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html index 7d10d7900..45a53bb93 100644 --- a/nimdoc/testproject/expected/testproject.html +++ b/nimdoc/testproject/expected/testproject.html @@ -65,6 +65,8 @@ Circle, ## A circle Triangle, ## A three-sided shape Rectangle ## A four-sided shape">Shapes +
        • T19396
        @@ -399,6 +401,16 @@ Some shapes. +
        +
        +
        + 
        T19396 = object
+  a*: int
+
        +
        + + +
        diff --git a/nimdoc/testproject/expected/testproject.idx b/nimdoc/testproject/expected/testproject.idx index e69bedf33..46ffaee72 100644 --- a/nimdoc/testproject/expected/testproject.idx +++ b/nimdoc/testproject/expected/testproject.idx @@ -63,5 +63,6 @@ nim Triangle testproject.html#Triangle Shapes.Triangle 380 nim Rectangle testproject.html#Rectangle Shapes.Rectangle 380 nim Shapes testproject.html#Shapes enum Shapes 380 nim anything testproject.html#anything proc anything() 387 +nim T19396 testproject.html#T19396 object T19396 392 nimgrp bar testproject.html#bar-procs-all proc 31 nimgrp baz testproject.html#baz-procs-all proc 34 diff --git a/nimdoc/testproject/expected/theindex.html b/nimdoc/testproject/expected/theindex.html index 7b4a95561..f9488eb2d 100644 --- a/nimdoc/testproject/expected/theindex.html +++ b/nimdoc/testproject/expected/theindex.html @@ -308,6 +308,10 @@
      • testproject: var someVariable
        • +
        T19396:
        testNimDocTrailingExample:
        • testproject: template testNimDocTrailingExample()
          • diff --git a/nimdoc/testproject/testproject.nim b/nimdoc/testproject/testproject.nim index d236552ac..b5fa2ac37 100644 --- a/nimdoc/testproject/testproject.nim +++ b/nimdoc/testproject/testproject.nim @@ -388,3 +388,7 @@ when true: # issue #15184 ## ## There is no block quote after blank lines at the beginning. discard + +type T19396* = object # bug #19396 + a*: int + b: float From 900fe8f501838077a5a1ef3d6793a1e901083bf8 Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Wed, 1 Feb 2023 20:00:10 +1100 Subject: [PATCH 2387/2472] Add `contains` to `std/macrocache` (#21304) * Add test cases * Implement contains for CacheSeq * Implement contains for CacheTable * Fix implementation of hasKey * Remove contains for CacheSeq Fix runnable examples I was accidently using --doccmd:skip so I didn't spot the failure locally * Implement hasKey as a VM callback instead of magic * Implement suggestions from PR Co-Authored-By: ringabout * Update lib/core/macrocache.nim --------- Co-authored-by: ringabout Co-authored-by: ringabout <43030857+ringabout@users.noreply.github.com> --- compiler/vmops.nim | 6 ++++++ lib/core/macrocache.nim | 26 ++++++++++++++++++++++++++ tests/macros/tmacros_various.nim | 7 +++++++ 3 files changed, 39 insertions(+) diff --git a/compiler/vmops.nim b/compiler/vmops.nim index f1a2e1e69..f51280ec5 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -284,6 +284,12 @@ proc registerAdditionalOps*(c: PCtx) = stackTrace2(c, "isExported() requires a symbol. '$#' is of kind '$#'" % [$n, $n.kind], n) setResult(a, sfExported in n.sym.flags) + registerCallback c, "stdlib.macrocache.hasKey", proc (a: VmArgs) = + let + table = getString(a, 0) + key = getString(a, 1) + setResult(a, table in c.graph.cacheTables and key in c.graph.cacheTables[table]) + registerCallback c, "stdlib.vmutils.vmTrace", proc (a: VmArgs) = c.config.isVmTrace = getBool(a, 0) diff --git a/lib/core/macrocache.nim b/lib/core/macrocache.nim index d4b103793..ffd07abb1 100644 --- a/lib/core/macrocache.nim +++ b/lib/core/macrocache.nim @@ -181,6 +181,32 @@ proc `[]`*(t: CacheTable; key: string): NimNode {.magic: "NctGet".} = # get the NimNode back assert mcTable["toAdd"].kind == nnkStmtList +proc hasKey*(t: CacheTable; key: string): bool = + ## Returns true if `key` is in the table `t`. + ## + ## See also: + ## * [contains proc][contains(CacheTable, string)] for use with the `in` operator + runnableExamples: + import std/macros + const mcTable = CacheTable"hasKeyEx" + static: + assert not mcTable.hasKey("foo") + mcTable["foo"] = newEmptyNode() + # Will now be true since we inserted a value + assert mcTable.hasKey("foo") + discard "Implemented in vmops" + +proc contains*(t: CacheTable; key: string): bool {.inline.} = + ## Alias of [hasKey][hasKey(CacheTable, string)] for use with the `in` operator. + runnableExamples: + import std/macros + const mcTable = CacheTable"containsEx" + static: + mcTable["foo"] = newEmptyNode() + # Will be true since we gave it a value before + assert "foo" in mcTable + t.hasKey(key) + proc hasNext(t: CacheTable; iter: int): bool {.magic: "NctHasNext".} proc next(t: CacheTable; iter: int): (string, NimNode, int) {.magic: "NctNext".} diff --git a/tests/macros/tmacros_various.nim b/tests/macros/tmacros_various.nim index 08be4602c..2446912ea 100644 --- a/tests/macros/tmacros_various.nim +++ b/tests/macros/tmacros_various.nim @@ -351,3 +351,10 @@ block: # bug #15118 block: flop("b") + +static: + block: + const containsTable = CacheTable"containsTable" + doAssert "foo" notin containsTable + containsTable["foo"] = newLit 42 + doAssert "foo" in containsTable From 43b1b9d077c6a96580f202101e643fdcd34e30fb Mon Sep 17 00:00:00 2001 From: Ivan Yonchovski Date: Wed, 1 Feb 2023 11:38:22 +0200 Subject: [PATCH 2388/2472] Fix the nimble build on Windows (#21314) Fix the build on Windows - `nimble install` fails on Windows, the `./` is not needed. --- nim.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nim.nimble b/nim.nimble index 5d1be49b1..713fc9256 100644 --- a/nim.nimble +++ b/nim.nimble @@ -9,6 +9,6 @@ skipDirs = @["build" , "changelogs" , "ci" , "csources_v2" , "drnim" , "nimdoc", before install: when defined(windows): - exec "./build_all.bat" + exec "build_all.bat" else: exec "./build_all.sh" From 17115cbc7375cdd47affd06dded62591d887441e Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 2 Feb 2023 23:44:14 +0800 Subject: [PATCH 2389/2472] fixes SSL version check logic [backport] (#21324) * fixed version check logic [backport] * add ciphersuites * debug nimble * fixes returns omission * finally * remove debug message * add ciphersuites --------- Co-authored-by: Araq --- lib/pure/asynchttpserver.nim | 4 ++-- lib/pure/net.nim | 6 +++--- lib/pure/unicode.nim | 2 +- lib/wrappers/openssl.nim | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index 6694c4bc2..845875a1e 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -158,7 +158,7 @@ proc parseProtocol(protocol: string): tuple[orig: string, major, minor: int] = proc sendStatus(client: AsyncSocket, status: string): Future[void] = client.send("HTTP/1.1 " & status & "\c\L\c\L") -func hasChunkedEncoding(request: Request): bool = +func hasChunkedEncoding(request: Request): bool = ## Searches for a chunked transfer encoding const transferEncoding = "Transfer-Encoding" @@ -300,7 +300,7 @@ proc processRequest( while true: lineFut.mget.setLen(0) lineFut.clean() - + # The encoding format alternates between specifying a number of bytes to read # and the data to be read, of the previously specified size if sizeOrData mod 2 == 0: diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 76a39226f..1dfef3b30 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -621,7 +621,7 @@ when defineSsl: proc newContext*(protVersion = protSSLv23, verifyMode = CVerifyPeer, certFile = "", keyFile = "", cipherList = CiphersIntermediate, - caDir = "", caFile = ""): SslContext = + caDir = "", caFile = "", ciphersuites = CiphersModern): SslContext = ## Creates an SSL context. ## ## Protocol version is currently ignored by default and TLS is used. @@ -675,10 +675,10 @@ when defineSsl: raiseSSLError() when not defined(openssl10) and not defined(libressl): let sslVersion = getOpenSSLVersion() - if sslVersion >= 0x010101000 and not sslVersion == 0x020000000: + if sslVersion >= 0x010101000 and sslVersion != 0x020000000: # In OpenSSL >= 1.1.1, TLSv1.3 cipher suites can only be configured via # this API. - if newCTX.SSL_CTX_set_ciphersuites(cipherList) != 1: + if newCTX.SSL_CTX_set_ciphersuites(ciphersuites) != 1: raiseSSLError() # Automatically the best ECDH curve for client exchange. Without this, ECDH # ciphers will be ignored by the server. diff --git a/lib/pure/unicode.nim b/lib/pure/unicode.nim index 09233035f..c8d18831a 100644 --- a/lib/pure/unicode.nim +++ b/lib/pure/unicode.nim @@ -887,7 +887,7 @@ proc reversed*(s: openArray[char]): string = proc graphemeLen*(s: openArray[char]; i: Natural): Natural = ## The number of bytes belonging to byte index ``s[i]``, - ## including following combining code unit. + ## including following combining code units. runnableExamples: let a = "añyóng" doAssert a.graphemeLen(1) == 2 ## ñ diff --git a/lib/wrappers/openssl.nim b/lib/wrappers/openssl.nim index b3fe8a608..fcf52a8d9 100644 --- a/lib/wrappers/openssl.nim +++ b/lib/wrappers/openssl.nim @@ -467,10 +467,10 @@ else: raiseInvalidLibrary MainProc proc SSL_CTX_set_ciphersuites*(ctx: SslCtx, str: cstring): cint = - var theProc {.global.}: proc(ctx: SslCtx, str: cstring) {.cdecl, gcsafe.} + var theProc {.global.}: proc(ctx: SslCtx, str: cstring): cint {.cdecl, gcsafe.} if theProc.isNil: theProc = cast[typeof(theProc)](sslSymThrows("SSL_CTX_set_ciphersuites")) - theProc(ctx, str) + result = theProc(ctx, str) proc SSL_new*(context: SslCtx): SslPtr{.cdecl, dynlib: DLLSSLName, importc.} proc SSL_free*(ssl: SslPtr){.cdecl, dynlib: DLLSSLName, importc.} From 8e53fba083a7450b8c2e9771cba8d477468a520e Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 3 Feb 2023 02:11:20 +0800 Subject: [PATCH 2390/2472] replaces `pairs` with `keys` and `items`; saves 8 bytes (#21319) replace `pairs` with `keys` and `items` --- compiler/condsyms.nim | 2 +- compiler/semfold.nim | 6 +++--- compiler/seminst.nim | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index c05aaf10b..a499b7142 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -25,7 +25,7 @@ proc undefSymbol*(symbols: StringTableRef; symbol: string) = # result = if isDefined(symbol): gSymbols[symbol] else: nil iterator definedSymbolNames*(symbols: StringTableRef): string = - for key, val in pairs(symbols): + for key in keys(symbols): yield key proc countDefinedSymbols*(symbols: StringTableRef): int = diff --git a/compiler/semfold.nim b/compiler/semfold.nim index 7f966c6fd..07c3b3ae4 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -685,7 +685,7 @@ proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode n[0] = a of nkBracket, nkCurly: result = copyNode(n) - for i, son in n.pairs: + for son in n.items: var a = getConstExpr(m, son, idgen, g) if a == nil: return nil result.add a @@ -709,7 +709,7 @@ proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode # tuple constructor result = copyNode(n) if (n.len > 0) and (n[0].kind == nkExprColonExpr): - for i, expr in n.pairs: + for expr in n.items: let exprNew = copyNode(expr) # nkExprColonExpr exprNew.add expr[0] let a = getConstExpr(m, expr[1], idgen, g) @@ -717,7 +717,7 @@ proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode exprNew.add a result.add exprNew else: - for i, expr in n.pairs: + for expr in n.items: let a = getConstExpr(m, expr, idgen, g) if a == nil: return nil result.add a diff --git a/compiler/seminst.nim b/compiler/seminst.nim index bd5eb1ec3..80d455d1e 100644 --- a/compiler/seminst.nim +++ b/compiler/seminst.nim @@ -40,7 +40,7 @@ const iterator instantiateGenericParamList(c: PContext, n: PNode, pt: TIdTable): PSym = internalAssert c.config, n.kind == nkGenericParams - for i, a in n.pairs: + for a in n.items: internalAssert c.config, a.kind == nkSym var q = a.sym if q.typ.kind in {tyTypeDesc, tyGenericParam, tyStatic, tyConcept}+tyTypeClasses: From 9f651f05d56407f63b9334917fecad975bdd49b4 Mon Sep 17 00:00:00 2001 From: Ikko Eltociear Ashimine Date: Tue, 7 Feb 2023 17:15:44 +0900 Subject: [PATCH 2391/2472] nimgrep: fix typo (#21337) occurences -> occurrences --- doc/nimgrep.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/nimgrep.md b/doc/nimgrep.md index 8fb86a9d3..63f760051 100644 --- a/doc/nimgrep.md +++ b/doc/nimgrep.md @@ -77,7 +77,7 @@ That means you can always use only 1 such an option with logical OR, e.g. .. Note:: If you want logical AND on patterns you should compose 1 appropriate pattern, possibly combined with multi-line mode `(?s)`:literal:. - E.g. to require that multi-line context of matches has occurences of + E.g. to require that multi-line context of matches has occurrences of **both** PAT1 and PAT2 use positive lookaheads (`(?=PAT)`:literal:): ```cmd nimgrep --inContext:'(?s)(?=.*PAT1)(?=.*PAT2)' From cbb6001d1e2e484986387e815a61339cd3dc916f Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Wed, 8 Feb 2023 18:27:44 +0800 Subject: [PATCH 2392/2472] fixes backticks in the documentation (#21340) --- lib/pure/collections/tables.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index cd08dcb9a..57a0107fd 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -2393,7 +2393,7 @@ proc contains*[A](t: CountTable[A], key: A): bool = return hasKey[A](t, key) proc getOrDefault*[A](t: CountTable[A], key: A; default: int = 0): int = - ## Retrieves the value at `t[key]` if`key` is in `t`. Otherwise, the + ## Retrieves the value at `t[key]` if `key` is in `t`. Otherwise, the ## integer value of `default` is returned. ## ## See also: @@ -2713,7 +2713,7 @@ proc contains*[A](t: CountTableRef[A], key: A): bool = return hasKey[A](t, key) proc getOrDefault*[A](t: CountTableRef[A], key: A, default: int): int = - ## Retrieves the value at `t[key]` if`key` is in `t`. Otherwise, the + ## Retrieves the value at `t[key]` if `key` is in `t`. Otherwise, the ## integer value of `default` is returned. ## ## See also: From 9fb4c2b3c765f993ba0055b281de19786a484c61 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 8 Feb 2023 20:55:01 +0100 Subject: [PATCH 2393/2472] fixes #21333; bad codegen for the at operator; [backport:1.6] (#21344) --- compiler/ccgexprs.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 844a37997..fe7cb252e 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1647,6 +1647,7 @@ proc genArrToSeq(p: BProc, n: PNode, d: var TLoc) = return if d.k == locNone: getTemp(p, n.typ, d) + initLocExpr(p, n[1], a) # generate call to newSeq before adding the elements per hand: let L = toInt(lengthOrd(p.config, n[1].typ)) if optSeqDestructors in p.config.globalOptions: @@ -1658,7 +1659,6 @@ proc genArrToSeq(p: BProc, n: PNode, d: var TLoc) = var lit = newRopeAppender() intLiteral(L, lit) genNewSeqAux(p, d, lit, L == 0) - initLocExpr(p, n[1], a) # bug #5007; do not produce excessive C source code: if L < 10: for i in 0.. Date: Thu, 9 Feb 2023 20:00:56 +0800 Subject: [PATCH 2394/2472] add `nimib` to important packages (#20697) r Co-authored-by: Clay Sweetser --- testament/important_packages.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 90d12df4e..b3a4bfb82 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -105,6 +105,7 @@ pkg "nimfp", "nim c -o:nfp -r src/fp.nim" pkg "nimgame2", "nim c --mm:refc nimgame2/nimgame.nim" # XXX Doesn't work with deprecated 'randomize', will create a PR. pkg "nimgen", "nim c -o:nimgenn -r src/nimgen/runcfg.nim" +pkg "nimib" pkg "nimlsp" pkg "nimly", "nim c -r tests/test_readme_example.nim" pkg "nimongo", "nimble test_ci", allowFailure = true From 51f410e1d5c19e42e97057f0b88e87056b7c43e2 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 10 Feb 2023 05:14:39 +0800 Subject: [PATCH 2395/2472] megatest now checks refc too (#21341) * megatest now checks refc too * fixes refc --- testament/categories.nim | 2 ++ tests/enum/tenum.nim | 3 ++- tests/misc/t18079.nim | 4 ++++ tests/misc/tunsignedconv.nim | 3 ++- tests/typerel/ttypedesc_as_genericparam1_orc.nim | 6 +++++- 5 files changed, 15 insertions(+), 3 deletions(-) diff --git a/testament/categories.nim b/testament/categories.nim index e8b13746a..cae699327 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -718,6 +718,8 @@ proc processCategory(r: var TResults, cat: Category, case cat2 of "megatest": runJoinedTest(r, cat, testsDir, options) + if isNimRepoTests(): + runJoinedTest(r, cat, testsDir, options & " --mm:refc") else: var testsRun = 0 var files: seq[string] diff --git a/tests/enum/tenum.nim b/tests/enum/tenum.nim index b11b02ec6..88d85ddcc 100644 --- a/tests/enum/tenum.nim +++ b/tests/enum/tenum.nim @@ -174,4 +174,5 @@ block: # bug #12589 proc typ(): OGRwkbGeometryType = return wkbPoint25D - doAssert $typ() == "wkbPoint25D" + when not defined(gcRefc): + doAssert $typ() == "wkbPoint25D" diff --git a/tests/misc/t18079.nim b/tests/misc/t18079.nim index 352348daf..ae64bbff9 100644 --- a/tests/misc/t18079.nim +++ b/tests/misc/t18079.nim @@ -1,3 +1,7 @@ +discard """ + matrix: "--mm:orc" +""" + type Foo = object y: int diff --git a/tests/misc/tunsignedconv.nim b/tests/misc/tunsignedconv.nim index 989f39277..c32f85b4d 100644 --- a/tests/misc/tunsignedconv.nim +++ b/tests/misc/tunsignedconv.nim @@ -66,7 +66,8 @@ let limit = 1'u64 let rangeVar = 0'u64 ..< limit -doAssert repr(rangeVar) == """0 .. 0""", repr(rangeVar) +when not defined(gcRefc): + doAssert repr(rangeVar) == """0 .. 0""", repr(rangeVar) # bug #15210 diff --git a/tests/typerel/ttypedesc_as_genericparam1_orc.nim b/tests/typerel/ttypedesc_as_genericparam1_orc.nim index 0ee4d8f92..d528a7421 100644 --- a/tests/typerel/ttypedesc_as_genericparam1_orc.nim +++ b/tests/typerel/ttypedesc_as_genericparam1_orc.nim @@ -1 +1,5 @@ -doAssert repr(int) == "int" \ No newline at end of file +discard """ + matrix: "--mm:orc" +""" + +doAssert repr(int) == "int" From 642136ec4f2cd93cdd753bf16fa4aec89b8fee82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20M=20G=C3=B3mez?= Date: Thu, 9 Feb 2023 21:19:05 +0000 Subject: [PATCH 2396/2472] Adds an extra optional argument to vcc: `vctoolset` Fixes #21335 (#21336) --- tools/vccexe/vccexe.nim | 15 +++++++++++---- tools/vccexe/vcvarsall.nim | 7 ++++++- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/tools/vccexe/vccexe.nim b/tools/vccexe/vccexe.nim index abe68c0a0..d548666b9 100644 --- a/tools/vccexe/vccexe.nim +++ b/tools/vccexe/vccexe.nim @@ -41,6 +41,7 @@ const platformPrefix = "--platform" sdktypePrefix = "--sdktype" sdkversionPrefix = "--sdkversion" + vctoolsetPrefix = "--vctoolset" verbosePrefix = "--verbose" vccversionSepIdx = vccversionPrefix.len @@ -49,6 +50,7 @@ const platformSepIdx = platformPrefix.len sdktypeSepIdx = sdktypePrefix.len sdkversionSepIdx = sdkversionPrefix.len + vctoolsetSepIdx = vctoolsetPrefix.len vcvarsallDefaultPath = "vcvarsall.bat" @@ -97,6 +99,8 @@ Options: "8.1" to use the windows 8.1 SDK --verbose Echoes the command line for loading the Developer Command Prompt and the command line passed on to the secondary command. + --vctoolset Optionally specifies the Visual Studio compiler toolset to use. + By default, the environment is set to use the current Visual Studio compiler toolset. Other command line arguments are passed on to the secondary command specified by --command or to the @@ -108,7 +112,7 @@ proc parseVccexeCmdLine(argseq: seq[string], vccversionArg: var seq[string], printPathArg: var bool, vcvarsallArg: var string, commandArg: var string, noCommandArg: var bool, platformArg: var VccArch, sdkTypeArg: var VccPlatformType, - sdkVersionArg: var string, verboseArg: var bool, + sdkVersionArg: var string, vctoolsetArg: var string, verboseArg: var bool, clArgs: var seq[string]) = ## Cannot use usual command-line argument parser here ## Since vccexe command-line arguments are intermingled @@ -125,7 +129,7 @@ proc parseVccexeCmdLine(argseq: seq[string], responseargs = parseCmdLine(responsecontent) parseVccexeCmdLine(responseargs, vccversionArg, printPathArg, vcvarsallArg, commandArg, noCommandArg, platformArg, sdkTypeArg, - sdkVersionArg, verboseArg, clArgs) + sdkVersionArg, vctoolsetArg, verboseArg, clArgs) elif wargv.startsWith(vccversionPrefix): # Check for vccversion vccversionArg.add(wargv.substr(vccversionSepIdx + 1)) elif wargv.cmpIgnoreCase(printPathPrefix) == 0: # Check for printPath @@ -142,6 +146,8 @@ proc parseVccexeCmdLine(argseq: seq[string], sdkTypeArg = parseEnum[VccPlatformType](wargv.substr(sdktypeSepIdx + 1)) elif wargv.startsWith(sdkversionPrefix): # Check for sdkversion sdkVersionArg = wargv.substr(sdkversionSepIdx + 1) + elif wargv.startsWith(vctoolsetPrefix): # Check for vctoolset + vctoolsetArg = wargv.substr(vctoolsetSepIdx + 1) elif wargv.startsWith(verbosePrefix): verboseArg = true else: # Regular cl.exe argument -> store for final cl.exe invocation @@ -158,13 +164,14 @@ when isMainModule: var platformArg: VccArch var sdkTypeArg: VccPlatformType var sdkVersionArg: string + var vctoolsetArg: string var verboseArg: bool = false var clArgs: seq[string] = @[] let wrapperArgs = commandLineParams() parseVccexeCmdLine(wrapperArgs, vccversionArg, printPathArg, vcvarsallArg, - commandArg, noCommandArg, platformArg, sdkTypeArg, sdkVersionArg, + commandArg, noCommandArg, platformArg, sdkTypeArg, sdkVersionArg, vctoolsetArg, verboseArg, clArgs) @@ -195,7 +202,7 @@ when isMainModule: echo "$1: $2" % [head, vcvarsallArg] # Call vcvarsall to get the appropriate VCC process environment - var vcvars = vccVarsAll(vcvarsallArg, platformArg, sdkTypeArg, sdkVersionArg, verboseArg) + var vcvars = vccVarsAll(vcvarsallArg, platformArg, sdkTypeArg, sdkVersionArg, vctoolsetArg, verboseArg) if vcvars != nil: for vccEnvKey, vccEnvVal in vcvars: putEnv(vccEnvKey, vccEnvVal) diff --git a/tools/vccexe/vcvarsall.nim b/tools/vccexe/vcvarsall.nim index 29d13cc7e..73b103e3c 100644 --- a/tools/vccexe/vcvarsall.nim +++ b/tools/vccexe/vcvarsall.nim @@ -33,7 +33,7 @@ type vccplatUWP = "uwp", ## Universal Windows Platform (UWP) Application vccplatOneCore = "onecore" # Undocumented platform type in the Windows SDK, probably XBox One SDK platform type. -proc vccVarsAll*(path: string, arch: VccArch = vccarchUnspecified, platform_type: VccPlatformType = vccplatEmpty, sdk_version: string = "", verbose: bool = false): StringTableRef = +proc vccVarsAll*(path: string, arch: VccArch = vccarchUnspecified, platform_type: VccPlatformType = vccplatEmpty, sdk_version, vctoolset: string = "", verbose: bool = false): StringTableRef = ## Returns a string table containing the proper process environment to successfully execute VCC compile commands for the specified SDK version, CPU architecture and platform type. ## ## path @@ -44,6 +44,8 @@ proc vccVarsAll*(path: string, arch: VccArch = vccarchUnspecified, platform_type ## The compile target Platform Type. Defaults to the Windows Desktop platform, i.e. a regular Windows executable binary. ## sdk_version ## The Windows SDK version to use. + ## vctoolset + ## Visual Studio compiler toolset to use. ## verbose ## Echo the command-line passed on to the system to load the VCC environment. Defaults to `false`. @@ -63,6 +65,9 @@ proc vccVarsAll*(path: string, arch: VccArch = vccarchUnspecified, platform_type if sdk_version.len > 0: args.add(sdk_version) + + if vctoolset.len > 0: + args.add("-vcvars_ver="&vctoolset) let argStr = args.join " " From 6c15958a835b645f3acdc3d1d012b10d0df43454 Mon Sep 17 00:00:00 2001 From: Gabriel Huber Date: Sun, 12 Feb 2023 16:41:27 +0100 Subject: [PATCH 2397/2472] Change the default stdlib location for Linux packaging (#21328) * Correctly evaluate the Nim prefix on Posix * Document new packaging layout --- compiler/options.nim | 22 ++++++++++++---------- doc/packaging.md | 7 +++++-- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/compiler/options.nim b/compiler/options.nim index 833d86a16..862b42446 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -717,22 +717,24 @@ proc getPrefixDir*(conf: ConfigRef): AbsoluteDir = ## clone or using installed nim, so that these exist: `result/doc/advopt.txt` ## and `result/lib/system.nim` if not conf.prefixDir.isEmpty: result = conf.prefixDir - else: result = AbsoluteDir splitPath(getAppDir()).head + else: + let binParent = AbsoluteDir splitPath(getAppDir()).head + when defined(posix): + if binParent == AbsoluteDir"/usr": + result = AbsoluteDir"/usr/lib/nim" + elif binParent == AbsoluteDir"/usr/local": + result = AbsoluteDir"/usr/local/lib/nim" + else: + result = binParent + else: + result = binParent proc setDefaultLibpath*(conf: ConfigRef) = # set default value (can be overwritten): if conf.libpath.isEmpty: # choose default libpath: var prefix = getPrefixDir(conf) - when defined(posix): - if prefix == AbsoluteDir"/usr": - conf.libpath = AbsoluteDir"/usr/lib/nim" - elif prefix == AbsoluteDir"/usr/local": - conf.libpath = AbsoluteDir"/usr/local/lib/nim" - else: - conf.libpath = prefix / RelativeDir"lib" - else: - conf.libpath = prefix / RelativeDir"lib" + conf.libpath = prefix / RelativeDir"lib" # Special rule to support other tools (nimble) which import the compiler # modules and make use of them. diff --git a/doc/packaging.md b/doc/packaging.md index 9a1cc0f6e..522ca6d44 100644 --- a/doc/packaging.md +++ b/doc/packaging.md @@ -69,8 +69,11 @@ Hints on the build process: What to install: -- The expected stdlib location is /usr/lib/nim -- Global configuration files under /etc/nim +- The expected stdlib location is `/usr/lib/nim/lib`, previously it was just `/usr/lib/nim` +- `nimdoc.css` and `nimdoc.cls` from the `doc` folder should go into `/usr/lib/nim/doc/` +- `tools/nim-gdb.py` should go into `/usr/lib/nim/tools/` +- `tools/dochack/dochack.js` should be installed to `/usr/lib/nim/tools/dochack/` +- Global configuration files under `/etc/nim` - Optionally: manpages, documentation, shell completion - When installing documentation, .idx files are not required - The "compiler" directory contains compiler sources and should not be part of the compiler binary package From 684c3b3aeb90ae549382ed0f9c3286464c72809f Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Tue, 14 Feb 2023 14:00:29 +0800 Subject: [PATCH 2398/2472] fixes comments about type bound operations (#21365) --- compiler/liftdestructors.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim index 2b27a58fc..738f659b7 100644 --- a/compiler/liftdestructors.nim +++ b/compiler/liftdestructors.nim @@ -8,7 +8,7 @@ # ## This module implements lifting for type-bound operations -## (``=sink``, ``=``, ``=destroy``, ``=deepCopy``). +## (``=sink``, ``=copy``, ``=destroy``, ``=deepCopy``). import modulegraphs, lineinfos, idents, ast, renderer, semdata, sighashes, lowerings, options, types, msgs, magicsys, tables, ccgutils From 406d3021314c66fdf4c715efeeaea3c8f478fcaa Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Tue, 14 Feb 2023 14:29:30 +0800 Subject: [PATCH 2399/2472] allow omitting stmts using `finally` as post expr blocks; make it consistent with `else`, `except` etc. (#21361) allow omitting stmts using `finally` as post expr blocks --- compiler/parser.nim | 2 +- tests/parser/tpostexprblocks.nim | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/compiler/parser.nim b/compiler/parser.nim index 14882d415..abfc7b323 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -1496,7 +1496,7 @@ proc postExprBlocks(p: var Parser, x: PNode): PNode = result = makeCall(result) getTok(p) skipComment(p, result) - if p.tok.tokType notin {tkOf, tkElif, tkElse, tkExcept}: + if p.tok.tokType notin {tkOf, tkElif, tkElse, tkExcept, tkFinally}: var stmtList = newNodeP(nkStmtList, p) stmtList.add parseStmt(p) # to keep backwards compatibility (see tests/vm/tstringnil) diff --git a/tests/parser/tpostexprblocks.nim b/tests/parser/tpostexprblocks.nim index d272c712f..6cd4a8350 100644 --- a/tests/parser/tpostexprblocks.nim +++ b/tests/parser/tpostexprblocks.nim @@ -498,6 +498,13 @@ StmtList StmtList DiscardStmt Empty + + Call + Ident "foo" + Finally + StmtList + DiscardStmt + Empty ''' """ @@ -655,3 +662,7 @@ dumpTree: discard finally: discard + + foo: + finally: + discard From 1d06c2b6cf84bcf65b67e88667dba4fd5e8ba519 Mon Sep 17 00:00:00 2001 From: c-blake Date: Tue, 14 Feb 2023 02:00:30 -0500 Subject: [PATCH 2400/2472] This adds `parseutils.parseSize`, an inverse to `strutils.formatSize` (#21349) * This adds `parseutils.parseSize`, an inverse to `strutils.formatSize` which has existed since 2017. It is useful for parsing the compiler's own output logs (like SuccessX) or many other scenarios where "human readable" units have been chosen. The doc comment and tests explain accepted syntax in detail. Big units lead to small numbers, often with a fractional part, but we parse into an `int64` since that is what `formatSize` stringifies and this is an inverse over partial function slots. Although metric prefixes z & y for zettabyte & yottabyte are accepted, these will saturate the result at `int64.high` unless the qualified number is a small fraction. This should not be much of a problem until such sizes are common (at which point another overload with the parse result either `float64` or `int128` could be added). Tests avoids `test()` because of a weakly related static: test() failure as mentioned in https://github.com/nim-lang/Nim/pull/21325. This is a more elemental VM failure. As such, it needs its own failure exhibition issue that is a smaller test case. (I am working on that, but unless there is a burning need to `parseSize` at compile-time before run-time it need not hold up this PR.) * This worked with `int` but fails with `int64`. Try for green tests. * Lift 2-result matching into a `checkParseSize` template and format as a table of input & 2 expected outputs which seems nicer and to address https://github.com/nim-lang/Nim/pull/21349#pullrequestreview-1294407679 * Fix (probably) the i386 trouble by using `int64` consistently. * Improve documentation by mentioning saturation. * Improve documentation with `runnableExamples` and a little more detail in the main doc comment based on excellent code review by @juancarlospaco: https://github.com/nim-lang/Nim/pull/21349#pullrequestreview-1294564155 * Address some more @juancarlospaco code review concerns. * Remove a stray space. * Mention milli-bytes in docs to maybe help clarify why wild conventions are so prone to going case-insensitive-metric. * Add some parens. --- changelog.md | 2 +- lib/pure/parseutils.nim | 65 ++++++++++++++++++++++++++++++++++++ tests/stdlib/tparseutils.nim | 41 +++++++++++++++++++++++ 3 files changed, 107 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 0389faad9..2db577d0b 100644 --- a/changelog.md +++ b/changelog.md @@ -10,7 +10,7 @@ [//]: # "Additions:" - +- Added `parseutils.parseSize` - inverse to `strutils.formatSize` - to parse human readable sizes. [//]: # "Deprecations:" diff --git a/lib/pure/parseutils.nim b/lib/pure/parseutils.nim index 2bb23c626..c4d59d892 100644 --- a/lib/pure/parseutils.nim +++ b/lib/pure/parseutils.nim @@ -592,6 +592,71 @@ proc parseFloat*(s: openArray[char], number: var float): int {. if result != 0: number = bf +func toLowerAscii(c: char): char = + if c in {'A'..'Z'}: char(uint8(c) xor 0b0010_0000'u8) else: c + +func parseSize*(s: openArray[char], size: var int64, alwaysBin=false): int = + ## Parse a size qualified by binary or metric units into `size`. This format + ## is often called "human readable". Result is the number of processed chars + ## or 0 on parse errors and size is rounded to the nearest integer. Trailing + ## garbage like "/s" in "1k/s" is allowed and detected by `result < s.len`. + ## + ## To simplify use, following non-rare wild conventions, and since fractional + ## data like milli-bytes is so rare, unit matching is case-insensitive but for + ## the 'i' distinguishing binary-metric from metric (which cannot be 'I'). + ## + ## An optional trailing 'B|b' is ignored but processed. I.e., you must still + ## know if units are bytes | bits or infer this fact via the case of s[^1] (if + ## users can even be relied upon to use 'B' for byte and 'b' for bit or have + ## that be s[^1]). + ## + ## If `alwaysBin==true` then scales are always binary-metric, but e.g. "KiB" + ## is still accepted for clarity. If the value would exceed the range of + ## `int64`, `size` saturates to `int64.high`. Supported metric prefix chars + ## include k, m, g, t, p, e, z, y (but z & y saturate unless the number is a + ## small fraction). + ## + ## **See also:** + ## * https://en.wikipedia.org/wiki/Binary_prefix + ## * `formatSize module`_ for formatting + runnableExamples: + var res: int64 # caller must still know if 'b' refers to bytes|bits + doAssert parseSize("10.5 MB", res) == 7 + doAssert res == 10_500_000 # decimal metric Mega prefix + doAssert parseSize("64 mib", res) == 6 + doAssert res == 67108864 # 64 shl 20 + doAssert parseSize("1G/h", res, true) == 2 # '/' stops parse + doAssert res == 1073741824 # 1 shl 30, forced binary metric + const prefix = "b" & "kmgtpezy" # byte|bit & lowCase metric-ish prefixes + const scaleM = [1.0, 1e3, 1e6, 1e9, 1e12, 1e15, 1e18, 1e21, 1e24] # 10^(3*idx) + const scaleB = [1.0, 1024, 1048576, 1073741824, 1099511627776.0, # 2^(10*idx) + 1125899906842624.0, 1152921504606846976.0, # ldexp? + 1.180591620717411303424e21, 1.208925819614629174706176e24] + var number: float + var scale = 1.0 + result = parseFloat(s, number) + if number < 0: # While parseFloat accepts negatives .. + result = 0 #.. we do not since sizes cannot be < 0 + if result > 0: + let start = result # Save spot to maybe unwind white to EOS + while result < s.len and s[result] in Whitespace: + inc result + if result < s.len: # Illegal starting char => unity + if (let si = prefix.find(s[result].toLowerAscii); si >= 0): + inc result # Now parse the scale + scale = if alwaysBin: scaleB[si] else: scaleM[si] + if result < s.len and s[result] == 'i': + scale = scaleB[si] # Switch from default to binary-metric + inc result + if result < s.len and s[result].toLowerAscii == 'b': + inc result # Skip optional '[bB]' + else: # Unwind result advancement when there.. + result = start #..is no unit to the end of `s`. + var sizeF = number * scale + 0.5 # Saturate to int64.high when too big + size = if sizeF > 9223372036854774784.0: int64.high else: sizeF.int64 +# Above constant=2^63-1024 avoids C UB; github.com/nim-lang/Nim/issues/20102 or +# stackoverflow.com/questions/20923556/math-pow2-63-1-math-pow2-63-512-is-true + type InterpolatedKind* = enum ## Describes for `interpolatedFragments` ## which part of the interpolated string is diff --git a/tests/stdlib/tparseutils.nim b/tests/stdlib/tparseutils.nim index 102e74067..218dd08f6 100644 --- a/tests/stdlib/tparseutils.nim +++ b/tests/stdlib/tparseutils.nim @@ -55,5 +55,46 @@ proc test() = doAssert res == @[(17, "9.123456789012344"), (18, "11.123456789012344"), (17, "9.123456789012344"), (17, "8.123456789012344"), (16, "9.12345678901234"), (17, "9.123456789012344")] + test() static: test() + +block: # With this included, static: test() crashes the compiler (from a + # VM problem with parseSize calling parseFloat). + var sz: int64 + template checkParseSize(s, expectLen, expectVal) = + if (let got = parseSize(s, sz); got != expectLen): + raise newException(IOError, "got len " & $got & " != " & $expectLen) + if sz != expectVal: + raise newException(IOError, "got sz " & $sz & " != " & $expectVal) + # STRING LEN SZ + # Good, complete parses + checkParseSize "1 b" , 4, 1 + checkParseSize "1 B" , 4, 1 + checkParseSize "1k" , 2, 1000 + checkParseSize "1 kib" , 5, 1024 + checkParseSize "1 ki" , 4, 1024 + checkParseSize "1mi" , 3, 1048576 + checkParseSize "1 mi" , 4, 1048576 + checkParseSize "1 mib" , 5, 1048576 + checkParseSize "1 Mib" , 5, 1048576 + checkParseSize "1 MiB" , 5, 1048576 + checkParseSize "1.23GiB", 7, 1320702444 # 1320702443.52 rounded + checkParseSize "0.001k" , 6, 1 + checkParseSize "0.0004k", 7, 0 + checkParseSize "0.0006k", 7, 1 + # Incomplete parses + checkParseSize "1 " , 1, 1 # Trailing white IGNORED + checkParseSize "1 B " , 4, 1 # Trailing white IGNORED + checkParseSize "1 B/s" , 4, 1 # Trailing junk IGNORED + checkParseSize "1 kX" , 3, 1000 + checkParseSize "1 kiX" , 4, 1024 + checkParseSize "1j" , 1, 1 # Unknown prefix IGNORED + checkParseSize "1 jib" , 2, 1 # Unknown prefix post space + checkParseSize "1 ji" , 3, 1 + # Bad parses; `sz` should stay last good|incomplete value + checkParseSize "-1b" , 0, 1 # Negative numbers + checkParseSize "abc" , 0, 1 # Non-numeric + checkParseSize " 12" , 0, 1 # Leading white + # Value Edge cases + checkParseSize "9223372036854775807", 19, int64.high From cac49694c011e851707cb3a86a3263c741559b3d Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Tue, 14 Feb 2023 18:14:19 +1100 Subject: [PATCH 2401/2472] `std/asyncjs` allow transforming proc types (#21356) * Add test case * Implement JS async transform for nnkProcTy --- lib/js/asyncjs.nim | 10 +++++++++- tests/js/tasyncjs.nim | 5 +++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/js/asyncjs.nim b/lib/js/asyncjs.nim index 364f7a21a..8e2f85156 100644 --- a/lib/js/asyncjs.nim +++ b/lib/js/asyncjs.nim @@ -100,10 +100,18 @@ proc isFutureVoid(node: NimNode): bool = node[1].kind == nnkIdent and $node[1] == "void" proc generateJsasync(arg: NimNode): NimNode = - if arg.kind notin {nnkProcDef, nnkLambda, nnkMethodDef, nnkDo}: + if arg.kind notin {nnkProcDef, nnkLambda, nnkMethodDef, nnkDo, nnkProcTy}: error("Cannot transform this node kind into an async proc." & " proc/method definition or lambda node expected.") + # Transform type X = proc (): something {.async.} + # into type X = proc (): Future[something] + if arg.kind == nnkProcTy: + result = arg + if arg[0][0].kind == nnkEmpty: + result[0][0] = quote do: Future[void] + return result + result = arg var isVoid = false let jsResolve = ident("jsResolve") diff --git a/tests/js/tasyncjs.nim b/tests/js/tasyncjs.nim index 00753a16c..3de143643 100644 --- a/tests/js/tasyncjs.nim +++ b/tests/js/tasyncjs.nim @@ -94,4 +94,9 @@ proc main() {.async.} = doAssert "foobar: 7" in $reason.message echo "done" # justified here to make sure we're running this, since it's inside `async` +block asyncPragmaInType: + type Handler = proc () {.async.} + proc foo() {.async.} = discard + var x: Handler = foo + discard main() From 56a4d246a4288a7005de0b20e312fa4e4af89534 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Tue, 14 Feb 2023 20:11:14 +0800 Subject: [PATCH 2402/2472] Refines raises list in osproc (#21323) * Remove Exception from raises in closeImpl * Update osproc.nim * refine errors * add ValueError * cast raises * refactor raises lists * Update lib/pure/osproc.nim * Update lib/pure/osproc.nim --------- Co-authored-by: Antonis Geralis <43617260+planetis-m@users.noreply.github.com> Co-authored-by: Andreas Rumpf --- lib/pure/osproc.nim | 41 ++++++++++++++++++++------------------- lib/pure/streams.nim | 2 +- lib/std/socketstreams.nim | 2 +- 3 files changed, 23 insertions(+), 22 deletions(-) diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 5e130b47a..9a0e5895d 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -76,7 +76,7 @@ type proc execProcess*(command: string, workingDir: string = "", args: openArray[string] = [], env: StringTableRef = nil, options: set[ProcessOption] = {poStdErrToStdOut, poUsePath, poEvalCommand}): - string {.rtl, extern: "nosp$1", + string {.rtl, extern: "nosp$1", raises: [OSError, IOError], tags: [ExecIOEffect, ReadIOEffect, RootEffect].} ## A convenience procedure that executes ``command`` with ``startProcess`` ## and returns its output as a string. @@ -120,7 +120,7 @@ proc execCmd*(command: string): int {.rtl, extern: "nosp$1", proc startProcess*(command: string, workingDir: string = "", args: openArray[string] = [], env: StringTableRef = nil, options: set[ProcessOption] = {poStdErrToStdOut}): - owned(Process) {.rtl, extern: "nosp$1", + owned(Process) {.rtl, extern: "nosp$1", raises: [OSError, IOError], tags: [ExecIOEffect, ReadEnvEffect, RootEffect].} ## Starts a process. `Command` is the executable file, `workingDir` is the ## process's working directory. If ``workingDir == ""`` the current directory @@ -152,7 +152,7 @@ proc startProcess*(command: string, workingDir: string = "", ## <#execProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_ ## * `execCmd proc <#execCmd,string>`_ -proc close*(p: Process) {.rtl, extern: "nosp$1", tags: [WriteIOEffect].} +proc close*(p: Process) {.rtl, extern: "nosp$1", raises: [IOError, OSError], tags: [WriteIOEffect].} ## When the process has finished executing, cleanup related handles. ## ## .. warning:: If the process has not finished executing, this will forcibly @@ -201,7 +201,7 @@ proc kill*(p: Process) {.rtl, extern: "nosp$1", tags: [].} ## * `terminate proc <#terminate,Process>`_ ## * `posix_utils.sendSignal(pid: Pid, signal: int) `_ -proc running*(p: Process): bool {.rtl, extern: "nosp$1", tags: [].} +proc running*(p: Process): bool {.rtl, extern: "nosp$1", raises: [OSError], tags: [].} ## Returns true if the process `p` is still running. Returns immediately. proc processID*(p: Process): int {.rtl, extern: "nosp$1".} = @@ -212,7 +212,7 @@ proc processID*(p: Process): int {.rtl, extern: "nosp$1".} = return p.id proc waitForExit*(p: Process, timeout: int = -1): int {.rtl, - extern: "nosp$1", tags: [].} + extern: "nosp$1", raises: [OSError, ValueError], tags: [].} ## Waits for the process to finish and returns `p`'s error code. ## ## .. warning:: Be careful when using `waitForExit` for processes created without @@ -221,7 +221,7 @@ proc waitForExit*(p: Process, timeout: int = -1): int {.rtl, ## On posix, if the process has exited because of a signal, 128 + signal ## number will be returned. -proc peekExitCode*(p: Process): int {.rtl, extern: "nosp$1", tags: [].} +proc peekExitCode*(p: Process): int {.rtl, extern: "nosp$1", raises: [OSError], tags: [].} ## Return `-1` if the process is still running. Otherwise the process' exit code. ## ## On posix, if the process has exited because of a signal, 128 + signal @@ -237,7 +237,7 @@ proc inputStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [].} ## * `outputStream proc <#outputStream,Process>`_ ## * `errorStream proc <#errorStream,Process>`_ -proc outputStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [].} +proc outputStream*(p: Process): Stream {.rtl, extern: "nosp$1", raises: [IOError, OSError], tags: [].} ## Returns ``p``'s output stream for reading from. ## ## You cannot perform peek/write/setOption operations to this stream. @@ -289,7 +289,7 @@ proc peekableErrorStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [], ## * `errorStream proc <#errorStream,Process>`_ ## * `peekableOutputStream proc <#peekableOutputStream,Process>`_ -proc inputHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", +proc inputHandle*(p: Process): FileHandle {.rtl, raises: [], extern: "nosp$1", tags: [].} = ## Returns ``p``'s input file handle for writing to. ## @@ -302,7 +302,7 @@ proc inputHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", result = p.inHandle proc outputHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", - tags: [].} = + raises: [], tags: [].} = ## Returns ``p``'s output file handle for reading from. ## ## .. warning:: The returned `FileHandle` should not be closed manually as @@ -314,7 +314,7 @@ proc outputHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", result = p.outHandle proc errorHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", - tags: [].} = + raises: [], tags: [].} = ## Returns ``p``'s error file handle for reading from. ## ## .. warning:: The returned `FileHandle` should not be closed manually as @@ -325,7 +325,7 @@ proc errorHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", ## * `outputHandle proc <#outputHandle,Process>`_ result = p.errHandle -proc countProcessors*(): int {.rtl, extern: "nosp$1".} = +proc countProcessors*(): int {.rtl, extern: "nosp$1", raises: [].} = ## Returns the number of the processors/cores the machine has. ## Returns 0 if it cannot be detected. ## It is implemented just calling `cpuinfo.countProcessors`. @@ -339,6 +339,7 @@ proc execProcesses*(cmds: openArray[string], beforeRunEvent: proc(idx: int) = nil, afterRunEvent: proc(idx: int, p: Process) = nil): int {.rtl, extern: "nosp$1", + raises: [ValueError, OSError, IOError], tags: [ExecIOEffect, TimeEffect, ReadEnvEffect, RootEffect], effectsOf: [beforeRunEvent, afterRunEvent].} = ## Executes the commands `cmds` in parallel. @@ -452,7 +453,7 @@ proc execProcesses*(cmds: openArray[string], if afterRunEvent != nil: afterRunEvent(i, p) close(p) -iterator lines*(p: Process, keepNewLines = false): string {.since: (1, 3), tags: [ReadIOEffect].} = +iterator lines*(p: Process, keepNewLines = false): string {.since: (1, 3), raises: [OSError, IOError, ValueError], tags: [ReadIOEffect].} = ## Convenience iterator for working with `startProcess` to read data from a ## background process. ## @@ -481,7 +482,8 @@ iterator lines*(p: Process, keepNewLines = false): string {.since: (1, 3), tags: yield line discard waitForExit(p) -proc readLines*(p: Process): (seq[string], int) {.since: (1, 3).} = +proc readLines*(p: Process): (seq[string], int) {.since: (1, 3), + raises: [OSError, IOError, ValueError], tags: [ReadIOEffect].} = ## Convenience function for working with `startProcess` to read data from a ## background process. ## @@ -747,7 +749,7 @@ when defined(windows) and not defined(useNimRtl): const errFileNotFound = 2.int if lastError.int in {errInvalidParameter, errFileNotFound}: raiseOSError(lastError, - "Requested command not found: '$1'. OS error:" % command) + "Requested command not found: '" & command & "'. OS error:") else: raiseOSError(lastError, command) result.fProcessHandle = procInfo.hProcess @@ -955,13 +957,13 @@ elif not defined(useNimRtl): not defined(useClone) and not defined(linux) when useProcessAuxSpawn: proc startProcessAuxSpawn(data: StartProcessData): Pid {. - tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], gcsafe.} + raises: [OSError], tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], gcsafe.} else: proc startProcessAuxFork(data: StartProcessData): Pid {. - tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], gcsafe.} + raises: [OSError], tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], gcsafe.} {.push stacktrace: off, profiler: off.} proc startProcessAfterFork(data: ptr StartProcessData) {. - tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], cdecl, gcsafe.} + raises: [OSError], tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], cdecl, gcsafe.} {.pop.} proc startProcess(command: string, workingDir: string = "", @@ -1131,8 +1133,7 @@ elif not defined(useNimRtl): let sizeRead = read(data.pErrorPipe[readIdx], addr error, sizeof(error)) if sizeRead == sizeof(error): raiseOSError(osLastError(), - "Could not find command: '$1'. OS error: $2" % - [$data.sysCommand, $strerror(error)]) + "Could not find command: '" & $data.sysCommand & "'. OS error: " & $strerror(error)) return pid @@ -1578,7 +1579,7 @@ proc execCmdEx*(command: string, options: set[ProcessOption] = { poStdErrToStdOut, poUsePath}, env: StringTableRef = nil, workingDir = "", input = ""): tuple[ output: string, - exitCode: int] {.tags: + exitCode: int] {.raises: [OSError, IOError], tags: [ExecIOEffect, ReadIOEffect, RootEffect], gcsafe.} = ## A convenience proc that runs the `command`, and returns its `output` and ## `exitCode`. `env` and `workingDir` params behave as for `startProcess`. diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim index 945e85934..003f8ca4c 100644 --- a/lib/pure/streams.nim +++ b/lib/pure/streams.nim @@ -115,7 +115,7 @@ type ## * That these fields here shouldn't be used directly. ## They are accessible so that a stream implementation can override them. closeImpl*: proc (s: Stream) - {.nimcall, raises: [Exception, IOError, OSError], tags: [WriteIOEffect], gcsafe.} + {.nimcall, raises: [IOError, OSError], tags: [WriteIOEffect], gcsafe.} atEndImpl*: proc (s: Stream): bool {.nimcall, raises: [Defect, IOError, OSError], tags: [], gcsafe.} setPositionImpl*: proc (s: Stream, pos: int) diff --git a/lib/std/socketstreams.nim b/lib/std/socketstreams.nim index 5c882858d..b53d6a5d3 100644 --- a/lib/std/socketstreams.nim +++ b/lib/std/socketstreams.nim @@ -146,7 +146,7 @@ proc wsFlush(s: Stream) = s.lastFlush = s.buf.len proc rsClose(s: Stream) = - {.cast(tags: []).}: + {.cast(raises: [IOError, OSError]), cast(tags: []).}: # todo fixme maybe do something? var s = ReadSocketStream(s) s.data.close() From e24d0e5faf1f6f2906262457007261e2358f1e5d Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Wed, 15 Feb 2023 20:03:11 +0800 Subject: [PATCH 2403/2472] remove legacy code; the first iteration now can build Nim with cpp backend (#21373) --- koch.nim | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/koch.nim b/koch.nim index 27953e05a..fa498e017 100644 --- a/koch.nim +++ b/koch.nim @@ -295,11 +295,7 @@ proc boot(args: string) = let nimStart = findStartNim().quoteShell() for i in 0..2: - # Nim versions < (1, 1) expect Nim's exception type to have a 'raiseId' field for - # C++ interop. Later Nim versions do this differently and removed the 'raiseId' field. - # Thus we always bootstrap the first iteration with "c" and not with "cpp" as - # a workaround. - let defaultCommand = if useCpp and i > 0: "cpp" else: "c" + let defaultCommand = if useCpp: "cpp" else: "c" let bootOptions = if args.len == 0 or args.startsWith("-"): defaultCommand else: "" echo "iteration: ", i+1 var extraOption = "" From c91ef1a09f3284315e9e66c32532f5a8e3ba9297 Mon Sep 17 00:00:00 2001 From: c-blake Date: Wed, 15 Feb 2023 11:41:28 -0500 Subject: [PATCH 2404/2472] Fix `closeHandle` bug, add `setFileSize`, make `resize` work on Windows (#21375) * Add general purpose `setFileSize` (unexported for now). Use to simplify `memfiles.open` as well as make robust (via hard allocation, not merely `ftruncate` address space allocation) on systems with `posix_fallocate`. As part of this, fix a bad `closeHandle` return check bug on Windows and add `MemFile.resize` for Windows now that setFileSize makes that easier. * Adapt existing test to exercise newly portable `MemFile.resize`. * Since Apple has never provided `posix_fallocate`, provide a fallback. This is presently written in terms of `ftruncate`, but it can be improved to use `F_PREALLOCATE` instead, as mentioned in a comment. --- lib/posix/posix.nim | 11 +++- lib/pure/memfiles.nim | 123 ++++++++++++++++++++++++------------ tests/stdlib/tmemfiles2.nim | 5 +- 3 files changed, 92 insertions(+), 47 deletions(-) diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 43346608d..aee9308a6 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -195,8 +195,14 @@ proc open*(a1: cstring, a2: cint, mode: Mode | cint = 0.Mode): cint {.inline.} = proc posix_fadvise*(a1: cint, a2, a3: Off, a4: cint): cint {. importc, header: "".} -proc posix_fallocate*(a1: cint, a2, a3: Off): cint {. - importc, header: "".} + +proc ftruncate*(a1: cint, a2: Off): cint {.importc, header: "".} +when defined(osx): # 2001 POSIX evidently does not concern Apple + proc posix_fallocate*(a1: cint, a2, a3: Off): cint = + ftruncate(a1, a2 + a3) # Set size to off + len, max offset +else: # TODO: Use fcntl(fd, F_PREALLOCATE, ..) above + proc posix_fallocate*(a1: cint, a2, a3: Off): cint {. + importc, header: "".} when not defined(haiku) and not defined(openbsd): proc fmtmsg*(a1: int, a2: cstring, a3: cint, @@ -511,7 +517,6 @@ proc fpathconf*(a1, a2: cint): int {.importc, header: "".} proc fsync*(a1: cint): cint {.importc, header: "".} ## synchronize a file's buffer cache to the storage device -proc ftruncate*(a1: cint, a2: Off): cint {.importc, header: "".} proc getcwd*(a1: cstring, a2: int): cstring {.importc, header: "", sideEffect.} proc getuid*(): Uid {.importc, header: "", sideEffect.} ## returns the real user ID of the calling process diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim index 9ff8bcd74..58c4888f1 100644 --- a/lib/pure/memfiles.nim +++ b/lib/pure/memfiles.nim @@ -35,6 +35,31 @@ proc newEIO(msg: string): ref IOError = new(result) result.msg = msg +proc setFileSize(fh: FileHandle, newFileSize = -1): OSErrorCode = + ## Set the size of open file pointed to by `fh` to `newFileSize` if != -1. + ## Space is only allocated if that is cheaper than writing to the file. This + ## routine returns the last OSErrorCode found rather than raising to support + ## old rollback/clean-up code style. [ Should maybe move to std/osfiles. ] + if newFileSize == -1: + return + when defined(windows): + var sizeHigh = int32(newFileSize shr 32) + let sizeLow = int32(newFileSize and 0xffffffff) + let status = setFilePointer(fh, sizeLow, addr(sizeHigh), FILE_BEGIN) + let lastErr = osLastError() + if (status == INVALID_SET_FILE_POINTER and lastErr.int32 != NO_ERROR) or + setEndOfFile(fh) == 0: + result = lastErr + else: + var e: cint # posix_fallocate truncates up when needed. + when declared(posix_fallocate): + while (e = posix_fallocate(fh, 0, newFileSize); e == EINTR): + discard + if e in [EINVAL, EOPNOTSUPP] and ftruncate(fh, newFileSize) == -1: + result = osLastError() # fallback arguable; Most portable, but allows SEGV + elif e != 0: + result = osLastError() + type MemFile* = object ## represents a memory mapped file mem*: pointer ## a pointer to the memory mapped file. The pointer @@ -182,17 +207,8 @@ proc open*(filename: string, mode: FileMode = fmRead, if result.fHandle == INVALID_HANDLE_VALUE: fail(osLastError(), "error opening file") - if newFileSize != -1: - var - sizeHigh = int32(newFileSize shr 32) - sizeLow = int32(newFileSize and 0xffffffff) - - var status = setFilePointer(result.fHandle, sizeLow, addr(sizeHigh), - FILE_BEGIN) - let lastErr = osLastError() - if (status == INVALID_SET_FILE_POINTER and lastErr.int32 != NO_ERROR) or - (setEndOfFile(result.fHandle) == 0): - fail(lastErr, "error setting file size") + if (let e = setFileSize(result.fHandle.FileHandle, newFileSize); + e != 0.OSErrorCode): fail(e, "error setting file size") # since the strings are always 'nil', we simply always call # CreateFileMappingW which should be slightly faster anyway: @@ -226,7 +242,7 @@ proc open*(filename: string, mode: FileMode = fmRead, result.wasOpened = true if not allowRemap and result.fHandle != INVALID_HANDLE_VALUE: - if closeHandle(result.fHandle) == 0: + if closeHandle(result.fHandle) != 0: result.fHandle = INVALID_HANDLE_VALUE else: @@ -249,9 +265,8 @@ proc open*(filename: string, mode: FileMode = fmRead, # Is there an exception that wraps it? fail(osLastError(), "error opening file") - if newFileSize != -1: - if ftruncate(result.handle, newFileSize) == -1: - fail(osLastError(), "error setting file size") + if (let e = setFileSize(result.handle.FileHandle, newFileSize); + e != 0.OSErrorCode): fail(e, "error setting file size") if mappedSize != -1: result.size = mappedSize @@ -306,34 +321,58 @@ proc flush*(f: var MemFile; attempts: Natural = 3) = if lastErr != EBUSY.OSErrorCode: raiseOSError(lastErr, "error flushing mapping") -when defined(posix) or defined(nimdoc): - proc resize*(f: var MemFile, newFileSize: int) {.raises: [IOError, OSError].} = - ## resize and re-map the file underlying an `allowRemap MemFile`. - ## **Note**: this assumes the entire file is mapped read-write at offset zero. - ## Also, the value of `.mem` will probably change. - ## **Note**: This is not (yet) available on Windows. - when defined(posix): - if f.handle == -1: - raise newException(IOError, - "Cannot resize MemFile opened with allowRemap=false") - if ftruncate(f.handle, newFileSize) == -1: - raiseOSError(osLastError()) - when defined(linux): #Maybe NetBSD, too? - #On Linux this can be over 100 times faster than a munmap,mmap cycle. - proc mremap(old: pointer; oldSize, newSize: csize_t; flags: cint): - pointer {.importc: "mremap", header: "".} - let newAddr = mremap(f.mem, csize_t(f.size), csize_t(newFileSize), cint(1)) - if newAddr == cast[pointer](MAP_FAILED): - raiseOSError(osLastError()) - else: - if munmap(f.mem, f.size) != 0: - raiseOSError(osLastError()) - let newAddr = mmap(nil, newFileSize, PROT_READ or PROT_WRITE, - f.flags, f.handle, 0) - if newAddr == cast[pointer](MAP_FAILED): - raiseOSError(osLastError()) - f.mem = newAddr +proc resize*(f: var MemFile, newFileSize: int) {.raises: [IOError, OSError].} = + ## Resize & re-map the file underlying an `allowRemap MemFile`. If the OS/FS + ## supports it, file space is reserved to ensure room for new virtual pages. + ## Caller should wait often enough for `flush` to finish to limit use of + ## system RAM for write buffering, perhaps just prior to this call. + ## **Note**: this assumes the entire file is mapped read-write at offset 0. + ## Also, the value of `.mem` will probably change. + if newFileSize < 1: # Q: include system/bitmasks & use PageSize ? + raise newException(IOError, "Cannot resize MemFile to < 1 byte") + when defined(windows): + if not f.wasOpened: + raise newException(IOError, "Cannot resize unopened MemFile") + if f.fHandle == INVALID_HANDLE_VALUE: + raise newException(IOError, + "Cannot resize MemFile opened with allowRemap=false") + if unmapViewOfFile(f.mem) == 0 or closeHandle(f.mapHandle) == 0: # Un-do map + raiseOSError(osLastError()) + if newFileSize != f.size: # Seek to size & `setEndOfFile` => allocated. + if (let e = setFileSize(f.fHandle.FileHandle, newFileSize); + e != 0.OSErrorCode): raiseOSError(e) + f.mapHandle = createFileMappingW(f.fHandle, nil, PAGE_READWRITE, 0,0,nil) + if f.mapHandle == 0: # Re-do map + raiseOSError(osLastError()) + if (let m = mapViewOfFileEx(f.mapHandle, FILE_MAP_READ or FILE_MAP_WRITE, + 0, 0, WinSizeT(newFileSize), nil); m != nil): + f.mem = m f.size = newFileSize + else: + raiseOSError(osLastError()) + elif defined(posix): + if f.handle == -1: + raise newException(IOError, + "Cannot resize MemFile opened with allowRemap=false") + if newFileSize != f.size: + if (let e = setFileSize(f.handle.FileHandle, newFileSize); + e != 0.OSErrorCode): raiseOSError(e) + when defined(linux): #Maybe NetBSD, too? + # On Linux this can be over 100 times faster than a munmap,mmap cycle. + proc mremap(old: pointer; oldSize, newSize: csize_t; flags: cint): + pointer {.importc: "mremap", header: "".} + let newAddr = mremap(f.mem, csize_t(f.size), csize_t(newFileSize), 1.cint) + if newAddr == cast[pointer](MAP_FAILED): + raiseOSError(osLastError()) + else: + if munmap(f.mem, f.size) != 0: + raiseOSError(osLastError()) + let newAddr = mmap(nil, newFileSize, PROT_READ or PROT_WRITE, + f.flags, f.handle, 0) + if newAddr == cast[pointer](MAP_FAILED): + raiseOSError(osLastError()) + f.mem = newAddr + f.size = newFileSize proc close*(f: var MemFile) = ## closes the memory mapped file `f`. All changes are written back to the diff --git a/tests/stdlib/tmemfiles2.nim b/tests/stdlib/tmemfiles2.nim index 6fee3c1ae..c79f85ebf 100644 --- a/tests/stdlib/tmemfiles2.nim +++ b/tests/stdlib/tmemfiles2.nim @@ -15,8 +15,9 @@ var if fileExists(fn): removeFile(fn) -# Create a new file, data all zeros -mm = memfiles.open(fn, mode = fmReadWrite, newFileSize = 20) +# Create a new file, data all zeros, starting at size 10 +mm = memfiles.open(fn, mode = fmReadWrite, newFileSize = 10, allowRemap=true) +mm.resize 20 # resize up to 20 mm.close() # read, change From fc7385bfda7b3af93aa57c5cf446a90b470de348 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 16 Feb 2023 10:23:35 +0800 Subject: [PATCH 2405/2472] fixes #21360; discarding empty seqs/arrays now raises errors (#21374) * discarding empty seqs now raises errors * the same goes for sets --- compiler/semstmts.nim | 18 ++++++++++-------- tests/discard/tillegaldiscardtypes.nim | 14 ++++++++++++++ tests/typerel/ttypelessemptyset.nim | 2 +- 3 files changed, 25 insertions(+), 9 deletions(-) create mode 100644 tests/discard/tillegaldiscardtypes.nim diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index c7e48db4c..7dc976c19 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -40,6 +40,13 @@ const proc implicitlyDiscardable(n: PNode): bool +proc hasEmpty(typ: PType): bool = + if typ.kind in {tySequence, tyArray, tySet}: + result = typ.lastSon.kind == tyEmpty + elif typ.kind == tyTuple: + for s in typ.sons: + result = result or hasEmpty(s) + proc semDiscard(c: PContext, n: PNode): PNode = result = n checkSonsLen(n, 1, c.config) @@ -47,7 +54,9 @@ proc semDiscard(c: PContext, n: PNode): PNode = n[0] = semExprWithType(c, n[0]) let sonType = n[0].typ let sonKind = n[0].kind - if isEmptyType(sonType) or sonType.kind in {tyNone, tyTypeDesc} or sonKind == nkTypeOfExpr: + if isEmptyType(sonType) or hasEmpty(sonType) or + sonType.kind in {tyNone, tyTypeDesc} or + sonKind == nkTypeOfExpr: localError(c.config, n.info, errInvalidDiscard) if sonType.kind == tyProc and sonKind notin nkCallKinds: # tyProc is disallowed to prevent ``discard foo`` to be valid, when ``discard foo()`` is meant. @@ -410,13 +419,6 @@ proc semUsing(c: PContext; n: PNode): PNode = if a[^1].kind != nkEmpty: localError(c.config, a.info, "'using' sections cannot contain assignments") -proc hasEmpty(typ: PType): bool = - if typ.kind in {tySequence, tyArray, tySet}: - result = typ.lastSon.kind == tyEmpty - elif typ.kind == tyTuple: - for s in typ.sons: - result = result or hasEmpty(s) - proc hasUnresolvedParams(n: PNode; flags: TExprFlags): bool = result = tfUnresolved in n.typ.flags when false: diff --git a/tests/discard/tillegaldiscardtypes.nim b/tests/discard/tillegaldiscardtypes.nim new file mode 100644 index 000000000..b7877bcd2 --- /dev/null +++ b/tests/discard/tillegaldiscardtypes.nim @@ -0,0 +1,14 @@ +discard """ + cmd: "nim check $file" + errormsg: "statement returns no value that can be discarded" + nimout: ''' +tillegaldiscardtypes.nim(11, 3) Error: statement returns no value that can be discarded +tillegaldiscardtypes.nim(12, 3) Error: statement returns no value that can be discarded +''' +""" + +proc b(v: int) = # bug #21360 + discard @[] + discard [] + +b(0) \ No newline at end of file diff --git a/tests/typerel/ttypelessemptyset.nim b/tests/typerel/ttypelessemptyset.nim index 5f49c33fd..a4b6f003a 100644 --- a/tests/typerel/ttypelessemptyset.nim +++ b/tests/typerel/ttypelessemptyset.nim @@ -1,5 +1,5 @@ discard """ - errormsg: "internal error: invalid kind for lastOrd(tyEmpty)" + errormsg: "statement returns no value that can be discarded" """ var q = false discard (if q: {} else: {}) From 93e4f278a1b68c83944842175ab78b61fe5fb7ca Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 16 Feb 2023 13:34:32 +0800 Subject: [PATCH 2406/2472] add deprecated warnings for `{.deadcodeelim: on.}` (#21277) --- compiler/pragmas.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 92e3d1922..6d953dee7 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -929,7 +929,8 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, of wThreadVar: noVal(c, it) incl(sym.flags, {sfThread, sfGlobal}) - of wDeadCodeElimUnused: discard # deprecated, dead code elim always on + of wDeadCodeElimUnused: + warningDeprecated(c.config, n.info, "'{.deadcodeelim: on.}' is deprecated, now a noop") # deprecated, dead code elim always on of wNoForward: pragmaNoForward(c, it) of wReorder: pragmaNoForward(c, it, flag = sfReorder) of wMagic: processMagic(c, it, sym) From 3b9e9fd7b29de97b394811f350876732c66cff19 Mon Sep 17 00:00:00 2001 From: Century Systems Date: Thu, 16 Feb 2023 21:47:52 +0900 Subject: [PATCH 2407/2472] Add support for NuttX RTOS. (#21372) * Add support for NuttX RTOS. Signed-off-by: Takeyoshi Kikuchi * lib: pure: asyncdispatch: assign to result. Signed-off-by: Takeyoshi Kikuchi * lib: std: typedthreads: add support for parameters to adjust Thread Stack Size. Like FreeRTOS/Zephyr, add support for following configurations. -d:nimThreadStackSize=xxxxx -d:nimThreadStackGuard=yyyy Signed-off-by: Takeyoshi Kikuchi --------- Signed-off-by: Takeyoshi Kikuchi --- compiler/options.nim | 4 +++- compiler/platform.nim | 7 ++++++- lib/posix/posix_other.nim | 5 ++++- lib/posix/posix_other_consts.nim | 13 ++++++++++++- lib/pure/asyncdispatch.nim | 5 ++++- lib/pure/asynchttpserver.nim | 5 +++-- lib/pure/asyncnet.nim | 3 ++- lib/pure/ioselects/ioselectors_select.nim | 2 +- lib/pure/nativesockets.nim | 7 ++++--- lib/pure/net.nim | 3 ++- lib/pure/selectors.nim | 9 +++++++-- lib/std/typedthreads.nim | 3 ++- lib/system.nim | 1 + lib/system/dyncalls.nim | 2 +- 14 files changed, 52 insertions(+), 17 deletions(-) diff --git a/compiler/options.nim b/compiler/options.nim index 862b42446..51a8ff6f0 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -621,7 +621,7 @@ proc isDefined*(conf: ConfigRef; symbol: string): bool = osQnx, osAtari, osAix, osHaiku, osVxWorks, osSolaris, osNetbsd, osFreebsd, osOpenbsd, osDragonfly, osMacosx, osIos, - osAndroid, osNintendoSwitch, osFreeRTOS, osCrossos, osZephyr} + osAndroid, osNintendoSwitch, osFreeRTOS, osCrossos, osZephyr, osNuttX} of "linux": result = conf.target.targetOS in {osLinux, osAndroid} of "bsd": @@ -643,6 +643,8 @@ proc isDefined*(conf: ConfigRef; symbol: string): bool = result = conf.target.targetOS == osFreeRTOS of "zephyr": result = conf.target.targetOS == osZephyr + of "nuttx": + result = conf.target.targetOS == osNuttX of "littleendian": result = CPU[conf.target.targetCPU].endian == littleEndian of "bigendian": result = CPU[conf.target.targetCPU].endian == bigEndian of "cpu8": result = CPU[conf.target.targetCPU].bit == 8 diff --git a/compiler/platform.nim b/compiler/platform.nim index 4b4316bc2..b6d6a119e 100644 --- a/compiler/platform.nim +++ b/compiler/platform.nim @@ -26,7 +26,8 @@ type osNone, osDos, osWindows, osOs2, osLinux, osMorphos, osSkyos, osSolaris, osIrix, osNetbsd, osFreebsd, osOpenbsd, osDragonfly, osCrossos, osAix, osPalmos, osQnx, osAmiga, osAtari, osNetware, osMacos, osMacosx, osIos, osHaiku, osAndroid, osVxWorks - osGenode, osJS, osNimVM, osStandalone, osNintendoSwitch, osFreeRTOS, osZephyr, osAny + osGenode, osJS, osNimVM, osStandalone, osNintendoSwitch, osFreeRTOS, osZephyr, + osNuttX, osAny type TInfoOSProp* = enum @@ -193,6 +194,10 @@ const objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/", scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".", props: {ospPosix}), + (name: "NuttX", parDir: "..", dllFrmt: "lib$1.so", altDirSep: "/", + objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/", + scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".", + props: {ospPosix}), (name: "Any", parDir: "..", dllFrmt: "lib$1.so", altDirSep: "/", objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/", scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".", diff --git a/lib/posix/posix_other.nim b/lib/posix/posix_other.nim index 941e13192..1b6734b51 100644 --- a/lib/posix/posix_other.nim +++ b/lib/posix/posix_other.nim @@ -10,7 +10,7 @@ when defined(nimHasStyleChecks): {.push styleChecks: off.} -when defined(freertos) or defined(zephyr): +when defined(freertos) or defined(zephyr) or defined(nuttx): const hasSpawnH = false # should exist for every Posix system nowadays hasAioH = false @@ -640,6 +640,9 @@ when defined(linux) or defined(nimdoc): ## or UDP packets. (Requires Linux kernel > 3.9) else: const SO_REUSEPORT* = cint(15) +elif defined(nuttx): + # Not supported, use SO_REUSEADDR to avoid compilation errors. + var SO_REUSEPORT* {.importc: "SO_REUSEADDR", header: "".}: cint else: var SO_REUSEPORT* {.importc, header: "".}: cint diff --git a/lib/posix/posix_other_consts.nim b/lib/posix/posix_other_consts.nim index 59ac6f9ed..4b1644f20 100644 --- a/lib/posix/posix_other_consts.nim +++ b/lib/posix/posix_other_consts.nim @@ -99,7 +99,10 @@ var EXDEV* {.importc: "EXDEV", header: "".}: cint # var F_DUPFD* {.importc: "F_DUPFD", header: "".}: cint -var F_DUPFD_CLOEXEC* {.importc: "F_DUPFD", header: "".}: cint +when defined(nuttx): + var F_DUPFD_CLOEXEC* {.importc: "F_DUPFD_CLOEXEC", header: "".}: cint +else: + var F_DUPFD_CLOEXEC* {.importc: "F_DUPFD", header: "".}: cint var F_GETFD* {.importc: "F_GETFD", header: "".}: cint var F_SETFD* {.importc: "F_SETFD", header: "".}: cint var F_GETFL* {.importc: "F_GETFL", header: "".}: cint @@ -127,6 +130,11 @@ var O_RDONLY* {.importc: "O_RDONLY", header: "".}: cint var O_RDWR* {.importc: "O_RDWR", header: "".}: cint var O_WRONLY* {.importc: "O_WRONLY", header: "".}: cint var O_CLOEXEC* {.importc: "O_CLOEXEC", header: "".}: cint +when defined(nuttx): + var O_DIRECT* {.importc: "O_DIRECT", header: "".}: cint + var O_PATH* {.importc: "O_PATH", header: "".}: cint + var O_NOATIME* {.importc: "O_NOATIME", header: "".}: cint + var O_TMPFILE* {.importc: "O_TMPFILE", header: "".}: cint var POSIX_FADV_NORMAL* {.importc: "POSIX_FADV_NORMAL", header: "".}: cint var POSIX_FADV_SEQUENTIAL* {.importc: "POSIX_FADV_SEQUENTIAL", header: "".}: cint var POSIX_FADV_RANDOM* {.importc: "POSIX_FADV_RANDOM", header: "".}: cint @@ -465,6 +473,9 @@ var FD_SETSIZE* {.importc: "FD_SETSIZE", header: "".}: cint when defined(zephyr): # Zephyr specific hardcoded value var FD_MAX* {.importc: "CONFIG_POSIX_MAX_FDS ", header: "".}: cint +elif defined(nuttx): + # NuttX specific user configuration value + var NACTIVESOCKETS* {.importc: "CONFIG_NET_NACTIVESOCKETS", header: "".}: cint # var MSG_CTRUNC* {.importc: "MSG_CTRUNC", header: "".}: cint diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 214b7d12c..64100afa7 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -2024,7 +2024,7 @@ when defined(posix): import posix when defined(linux) or defined(windows) or defined(macosx) or defined(bsd) or - defined(solaris) or defined(zephyr) or defined(freertos): + defined(solaris) or defined(zephyr) or defined(freertos) or defined(nuttx): proc maxDescriptors*(): int {.raises: OSError.} = ## Returns the maximum number of active file descriptors for the current ## process. This involves a system call. For now `maxDescriptors` is @@ -2033,6 +2033,9 @@ when defined(linux) or defined(windows) or defined(macosx) or defined(bsd) or result = 16_700_000 elif defined(zephyr) or defined(freertos): result = FD_MAX + elif defined(nuttx): + # The maximum number of concurrently active UDP and TCP ports. + result = NACTIVESOCKETS else: var fdLim: RLimit if getrlimit(RLIMIT_NOFILE, fdLim) < 0: diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index 845875a1e..c09fe35ca 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -387,8 +387,9 @@ proc listen*(server: AsyncHttpServer; port: Port; address = ""; domain = AF_INET server.socket = newAsyncSocket(domain) if server.reuseAddr: server.socket.setSockOpt(OptReuseAddr, true) - if server.reusePort: - server.socket.setSockOpt(OptReusePort, true) + when not defined(nuttx): + if server.reusePort: + server.socket.setSockOpt(OptReusePort, true) server.socket.bindAddr(port, address) server.socket.listen() diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim index 982459af7..d35dbdfaf 100644 --- a/lib/pure/asyncnet.nim +++ b/lib/pure/asyncnet.nim @@ -106,7 +106,8 @@ export SOBool # TODO: Remove duplication introduced by PR #4683. const defineSsl = defined(ssl) or defined(nimdoc) -const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) +const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) or + defined(nuttx) when defineSsl: import openssl diff --git a/lib/pure/ioselects/ioselectors_select.nim b/lib/pure/ioselects/ioselectors_select.nim index 34c88d85e..a6f54e791 100644 --- a/lib/pure/ioselects/ioselectors_select.nim +++ b/lib/pure/ioselects/ioselectors_select.nim @@ -313,7 +313,7 @@ proc selectInto*[T](s: Selector[T], timeout: int, verifySelectParams(timeout) if timeout != -1: - when defined(genode) or defined(freertos) or defined(zephyr): + when defined(genode) or defined(freertos) or defined(zephyr) or defined(nuttx): tv.tv_sec = Time(timeout div 1_000) else: tv.tv_sec = timeout.int32 div 1_000 diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index 14a318412..b3465ef14 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -23,7 +23,8 @@ when hostOS == "solaris": {.passl: "-lsocket -lnsl".} const useWinVersion = defined(windows) or defined(nimdoc) -const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) +const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) or + defined(nuttx) when useWinVersion: import winlean @@ -304,7 +305,7 @@ proc getAddrInfo*(address: string, port: Port, domain: Domain = AF_INET, let socketPort = if sockType == SOCK_RAW: "" else: $port var gaiResult = getaddrinfo(address, socketPort.cstring, addr(hints), result) if gaiResult != 0'i32: - when useWinVersion or defined(freertos): + when useWinVersion or defined(freertos) or defined(nuttx): raiseOSError(osLastError()) else: raiseOSError(osLastError(), $gai_strerror(gaiResult)) @@ -351,7 +352,7 @@ proc getSockDomain*(socket: SocketHandle): Domain = else: raise newException(IOError, "Unknown socket family in getSockDomain") -when not useNimNetLite: +when not useNimNetLite: proc getServByName*(name, proto: string): Servent {.tags: [ReadIOEffect].} = ## Searches the database from the beginning and finds the first entry for ## which the service name specified by `name` matches the s_name member diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 1dfef3b30..d3a767eed 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -100,7 +100,8 @@ export nativesockets.Port, nativesockets.`$`, nativesockets.`==` export Domain, SockType, Protocol const useWinVersion = defined(windows) or defined(nimdoc) -const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) +const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) or + defined(nuttx) const defineSsl = defined(ssl) or defined(nimdoc) when useWinVersion: diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim index be5037fe2..ef8736cdf 100644 --- a/lib/pure/selectors.nim +++ b/lib/pure/selectors.nim @@ -37,7 +37,7 @@ const hasThreadSupport = compileOption("threads") and defined(threadsafe) const ioselSupportedPlatform* = defined(macosx) or defined(freebsd) or defined(netbsd) or defined(openbsd) or - defined(dragonfly) or + defined(dragonfly) or defined(nuttx) or (defined(linux) and not defined(android) and not defined(emscripten)) ## This constant is used to determine whether the destination platform is ## fully supported by `ioselectors` module. @@ -328,7 +328,7 @@ else: doAssert(timeout >= -1, "Cannot select with a negative value, got: " & $timeout) when defined(linux) or defined(windows) or defined(macosx) or defined(bsd) or - defined(solaris) or defined(zephyr) or defined(freertos): + defined(solaris) or defined(zephyr) or defined(freertos) or defined(nuttx): template maxDescriptors*(): int = ## Returns the maximum number of active file descriptors for the current ## process. This involves a system call. For now `maxDescriptors` is @@ -337,6 +337,9 @@ else: 16_700_000 elif defined(zephyr) or defined(freertos): FD_MAX + elif defined(nuttx): + # The maximum number of concurrently active UDP and TCP ports. + NACTIVESOCKETS else: var fdLim: RLimit var res = int(getrlimit(RLIMIT_NOFILE, fdLim)) @@ -360,5 +363,7 @@ else: include ioselects/ioselectors_select elif defined(zephyr): include ioselects/ioselectors_poll + elif defined(nuttx): + include ioselects/ioselectors_epoll else: include ioselects/ioselectors_poll diff --git a/lib/std/typedthreads.nim b/lib/std/typedthreads.nim index 970d381c3..b45ce8a61 100644 --- a/lib/std/typedthreads.nim +++ b/lib/std/typedthreads.nim @@ -46,7 +46,8 @@ when defined(genode): import genode/env -when hasAllocStack or defined(zephyr) or defined(freertos) or defined(cpu16) or defined(cpu8): +when hasAllocStack or defined(zephyr) or defined(freertos) or defined(nuttx) or + defined(cpu16) or defined(cpu8): const nimThreadStackSize {.intdefine.} = 8192 nimThreadStackGuard {.intdefine.} = 128 diff --git a/lib/system.nim b/lib/system.nim index d8fafc0ef..af7e0e4c8 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2688,6 +2688,7 @@ when notJSnotNims: not defined(nintendoswitch) and not defined(freertos) and not defined(zephyr) and + not defined(nuttx) and hostOS != "any" proc raiseEIO(msg: string) {.noinline, noreturn.} = diff --git a/lib/system/dyncalls.nim b/lib/system/dyncalls.nim index de22f7cbb..2162b234f 100644 --- a/lib/system/dyncalls.nim +++ b/lib/system/dyncalls.nim @@ -176,7 +176,7 @@ elif defined(genode): proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr = raiseAssert("nimGetProcAddr not implemented") -elif defined(nintendoswitch) or defined(freertos) or defined(zephyr): +elif defined(nintendoswitch) or defined(freertos) or defined(zephyr) or defined(nuttx): proc nimUnloadLibrary(lib: LibHandle) = cstderr.rawWrite("nimUnLoadLibrary not implemented") cstderr.rawWrite("\n") From 03dd853815173bcf63f80ef03b5a4db7177453ec Mon Sep 17 00:00:00 2001 From: "Qinsi (James) ZHU" Date: Fri, 17 Feb 2023 01:12:48 +0800 Subject: [PATCH 2408/2472] add .replace() with callback to jsre (#21371) --- lib/js/jsre.nim | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/js/jsre.nim b/lib/js/jsre.nim index 19888aaa9..69bd75c3b 100644 --- a/lib/js/jsre.nim +++ b/lib/js/jsre.nim @@ -34,6 +34,9 @@ func compile*(self: RegExp; pattern: cstring; flags: cstring) {.importjs: "#.com func replace*(pattern: cstring; self: RegExp; replacement: cstring): cstring {.importjs: "#.replace(#, #)".} ## Returns a new string with some or all matches of a pattern replaced by given replacement +func replace*(pattern: cstring, self: RegExp, cb: proc (args: varargs[cstring]): cstring): cstring {.importcpp.} + ## Returns a new string with some or all matches of a pattern replaced by given callback function + func split*(pattern: cstring; self: RegExp): seq[cstring] {.importjs: "(#.split(#) || [])".} ## Divides a string into an ordered list of substrings and returns the array @@ -88,5 +91,7 @@ runnableExamples: assert "do1ne".split(jsregex) == @["do".cstring, "ne".cstring] jsregex.compile(r"[lw]", r"i") assert "hello world".replace(jsregex,"X") == "heXlo world" + jsregex.compile(r"([a-z])\1*", r"g") + assert "abbcccdddd".replace(jsregex, proc (m: varargs[cstring]): cstring = ($m[0] & $(m.len)).cstring) == "a1b2c3d4" let digitsRegex: RegExp = newRegExp(r"\d") assert "foo".match(digitsRegex) == @[] From 13711b101d5ec101b3a2e18832d80e2b71eb3c80 Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Fri, 17 Feb 2023 21:24:25 +1100 Subject: [PATCH 2409/2472] `std/xmltree` Add the type of the node when the assertion fails (#21383) Print the type of the node when the assertion fails This way the user actually knows what the type was instead of just knowing it failed --- lib/pure/xmltree.nim | 52 +++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/lib/pure/xmltree.nim b/lib/pure/xmltree.nim index c3923ca50..82513bc98 100644 --- a/lib/pure/xmltree.nim +++ b/lib/pure/xmltree.nim @@ -70,6 +70,14 @@ const xmlHeader* = "\n" ## Header to use for complete XML output. +template expect(node: XmlNode, kind: set[XmlNodeKind]) = + ## Check the node's kind is within a set of values + assert node.k in kind, "Got " & $node.k + +template expect(node: XmlNode, kind: XmlNodeKind) = + ## Check the node's kind equals a value + assert node.k == kind, "Got " & $node.k + proc newXmlNode(kind: XmlNodeKind): XmlNode = ## Creates a new ``XmlNode``. result = XmlNode(k: kind) @@ -182,7 +190,7 @@ proc text*(n: XmlNode): lent string {.inline.} = assert $c == "" assert c.text == "my comment" - assert n.k in {xnText, xnComment, xnCData, xnEntity} + n.expect {xnText, xnComment, xnCData, xnEntity} result = n.fText proc `text=`*(n: XmlNode, text: sink string) {.inline.} = @@ -200,7 +208,7 @@ proc `text=`*(n: XmlNode, text: sink string) {.inline.} = e.text = "a new entity text" assert $e == "&a new entity text;" - assert n.k in {xnText, xnComment, xnCData, xnEntity} + n.expect {xnText, xnComment, xnCData, xnEntity} n.fText = text proc tag*(n: XmlNode): lent string {.inline.} = @@ -221,7 +229,7 @@ proc tag*(n: XmlNode): lent string {.inline.} = """ assert a.tag == "firstTag" - assert n.k == xnElement + n.expect xnElement result = n.fTag proc `tag=`*(n: XmlNode, tag: sink string) {.inline.} = @@ -244,7 +252,7 @@ proc `tag=`*(n: XmlNode, tag: sink string) {.inline.} = """ - assert n.k == xnElement + n.expect xnElement n.fTag = tag proc rawText*(n: XmlNode): string {.inline.} = @@ -315,7 +323,7 @@ proc add*(father, son: XmlNode) {.inline.} = f.add newEntity("my entity") assert $f == "my text&my entity;" - assert father.k == xnElement + father.expect xnElement add(father.s, son) proc add*(father: XmlNode, sons: openArray[XmlNode]) {.inline.} = @@ -335,7 +343,7 @@ proc add*(father: XmlNode, sons: openArray[XmlNode]) {.inline.} = f.add(@[newText("my text"), newElement("sonTag"), newEntity("my entity")]) assert $f == "my text&my entity;" - assert father.k == xnElement + father.expect xnElement add(father.s, sons) @@ -361,7 +369,7 @@ proc insert*(father, son: XmlNode, index: int) {.inline.} = """ - assert father.k == xnElement + father.expect xnElement if len(father.s) > index: insert(father.s, son, index) else: @@ -390,7 +398,7 @@ proc insert*(father: XmlNode, sons: openArray[XmlNode], index: int) {.inline.} = """ - assert father.k == xnElement + father.expect xnElement if len(father.s) > index: insert(father.s, sons, index) else: @@ -416,7 +424,7 @@ proc delete*(n: XmlNode, i: Natural) = """ - assert n.k == xnElement + n.expect xnElement n.s.delete(i) proc delete*(n: XmlNode, slice: Slice[int]) = @@ -439,7 +447,7 @@ proc delete*(n: XmlNode, slice: Slice[int]) = """ - assert n.k == xnElement + n.expect xnElement n.s.delete(slice) proc replace*(n: XmlNode, i: Natural, replacement: openArray[XmlNode]) = @@ -466,7 +474,7 @@ proc replace*(n: XmlNode, i: Natural, replacement: openArray[XmlNode]) = """ - assert n.k == xnElement + n.expect xnElement n.s.delete(i) n.s.insert(replacement, i) @@ -494,7 +502,7 @@ proc replace*(n: XmlNode, slice: Slice[int], replacement: openArray[XmlNode]) = """ - assert n.k == xnElement + n.expect xnElement n.s.delete(slice) n.s.insert(replacement, slice.a) @@ -525,12 +533,12 @@ proc `[]`*(n: XmlNode, i: int): XmlNode {.inline.} = assert $f[1] == "" assert $f[0] == "" - assert n.k == xnElement + n.expect xnElement result = n.s[i] proc `[]`*(n: var XmlNode, i: int): var XmlNode {.inline.} = ## Returns the `i`'th child of `n` so that it can be modified. - assert n.k == xnElement + n.expect xnElement result = n.s[i] proc clear*(n: var XmlNode) = @@ -582,12 +590,12 @@ iterator items*(n: XmlNode): XmlNode {.inline.} = # # &some entity; - assert n.k == xnElement + n.expect xnElement for i in 0 .. n.len-1: yield n[i] iterator mitems*(n: var XmlNode): var XmlNode {.inline.} = ## Iterates over all direct children of `n` so that they can be modified. - assert n.k == xnElement + n.expect xnElement for i in 0 .. n.len-1: yield n[i] proc toXmlAttributes*(keyValuePairs: varargs[tuple[key, @@ -619,7 +627,7 @@ proc attrs*(n: XmlNode): XmlAttributes {.inline.} = j.attrs = att assert j.attrs == att - assert n.k == xnElement + n.expect xnElement result = n.fAttr proc `attrs=`*(n: XmlNode, attr: XmlAttributes) {.inline.} = @@ -636,7 +644,7 @@ proc `attrs=`*(n: XmlNode, attr: XmlAttributes) {.inline.} = j.attrs = att assert j.attrs == att - assert n.k == xnElement + n.expect xnElement n.fAttr = attr proc attrsLen*(n: XmlNode): int {.inline.} = @@ -653,7 +661,7 @@ proc attrsLen*(n: XmlNode): int {.inline.} = j.attrs = att assert j.attrsLen == 2 - assert n.k == xnElement + n.expect xnElement if not isNil(n.fAttr): result = len(n.fAttr) proc attr*(n: XmlNode, name: string): string = @@ -671,7 +679,7 @@ proc attr*(n: XmlNode, name: string): string = assert j.attr("key1") == "first value" assert j.attr("key2") == "second value" - assert n.kind == xnElement + n.expect xnElement if n.attrs == nil: return "" return n.attrs.getOrDefault(name) @@ -834,7 +842,7 @@ proc child*(n: XmlNode, name: string): XmlNode = f.add newElement("thirdSon") assert $(f.child("secondSon")) == "" - assert n.kind == xnElement + n.expect xnElement for i in items(n): if i.kind == xnElement: if i.tag == name: @@ -870,7 +878,7 @@ proc findAll*(n: XmlNode, tag: string, result: var seq[XmlNode], a.findAll("BAD", s, caseInsensitive = true) assert $s == "@[c text, d text]" - assert n.k == xnElement + n.expect xnElement for child in n.items(): if child.k != xnElement: continue From 6b93db7070c984121e18b502d938d5e455d4ba70 Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Fri, 17 Feb 2023 05:25:15 -0500 Subject: [PATCH 2410/2472] Fix long link commands on macOS (v2) (#21382) Handle long link commands on macOS with a script, since ar does not support response files --- compiler/extccomp.nim | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 6515f6968..db97de65c 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -834,6 +834,15 @@ proc linkViaResponseFile(conf: ConfigRef; cmd: string) = finally: removeFile(linkerArgs) +proc linkViaShellScript(conf: ConfigRef; cmd: string) = + let linkerScript = conf.projectName & "_" & "linkerScript.sh" + writeFile(linkerScript, cmd) + let shell = getEnv("SHELL") + try: + execLinkCmd(conf, shell & " " & linkerScript) + finally: + removeFile(linkerScript) + proc getObjFilePath(conf: ConfigRef, f: Cfile): string = if noAbsolutePaths(conf): f.obj.extractFilename else: f.obj.string @@ -856,9 +865,12 @@ proc preventLinkCmdMaxCmdLen(conf: ConfigRef, linkCmd: string) = # Windows's command line limit is about 8K (8191 characters) so C compilers on # Windows support a feature where the command line can be passed via ``@linkcmd`` # to them. - const MaxCmdLen = when defined(windows): 8_000 else: 32_000 + const MaxCmdLen = when defined(windows): 8_000 elif defined(macosx): 260_000 else: 32_000 if linkCmd.len > MaxCmdLen: - linkViaResponseFile(conf, linkCmd) + when defined(macosx): + linkViaShellScript(conf, linkCmd) + else: + linkViaResponseFile(conf, linkCmd) else: execLinkCmd(conf, linkCmd) From e896977bd16b2471e4aaaa363690622420e99acc Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sun, 19 Feb 2023 14:49:19 +0800 Subject: [PATCH 2411/2472] closes #1072; add a test case (#21396) --- tests/specialops/tcallops.nim | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/specialops/tcallops.nim b/tests/specialops/tcallops.nim index 0508a37a1..c541a0c1d 100644 --- a/tests/specialops/tcallops.nim +++ b/tests/specialops/tcallops.nim @@ -37,3 +37,13 @@ doAssert a(b) == "(12)" doAssert a.b(c) == `()`(b, a, c) doAssert (a.b)(c) == `()`(a.b, c) doAssert `()`(a.b, c) == `()`(`()`(b, a), c) + +block: # bug #1072 + var x: int + + proc foo(some:int):int = some + proc `()`(l,r:string): string = discard + + block: + var foo = 42 + doAssert x.foo == 0 From c66dc913cefb08722c4bf9885948e26be1285c3f Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Tue, 21 Feb 2023 07:19:46 +1100 Subject: [PATCH 2412/2472] Support new runtime with nim-gdb (#21400) * Add support for orc strings * Cleaned up testing script Enums now printing properly Merged both old and new strings into the one printer Moving onto sets which seem kinda difficult * Sets working Instead of trying to imitate how Nim represents enums, I just call the dollar proc for each enum value While this runs into problems if the user doesn't call the dollar proc anywhere, I believe its a decent tradeoff I've cleaned up the error message for when it cannot find dollar proc (Might add in proper message on how to fix) * Support sequences V2 runtime seems to have sequences that don't have a len (Guessing its some kind of short seq optimisation?) but I've rolled the implementation into normal sequences since the implementation is practically the same * Clean up test program so it isn't using diff Also don't redirect the first nim compile to /dev/null so that we can check for any compilation errors I found the diff to be annoying to read (Seeing as the test script already performs diffing) * Tables are now supported * Add colours to test output It was getting difficult to tell apart test output from GDB output so I added colour to better tell them apart * Both old and new runtime are working Set exit code in python test script so that this could possibly be added to the CI. Only issue is that it can be flaky (GDB crashes randomly for some reason) * Remove old code that was commented out If I need it later I'll just use git * Remove branch that never runs * Remove the old test output [skip ci] --- .../untestable/gdb/gdb_pretty_printer_test.py | 30 +- .../gdb/gdb_pretty_printer_test_output.txt | 3 - .../gdb/gdb_pretty_printer_test_program.nim | 28 +- .../gdb/gdb_pretty_printer_test_run.sh | 20 +- tools/nim-gdb.py | 386 ++++++------------ 5 files changed, 160 insertions(+), 307 deletions(-) delete mode 100644 tests/untestable/gdb/gdb_pretty_printer_test_output.txt mode change 100644 => 100755 tests/untestable/gdb/gdb_pretty_printer_test_run.sh diff --git a/tests/untestable/gdb/gdb_pretty_printer_test.py b/tests/untestable/gdb/gdb_pretty_printer_test.py index 8f0f88e85..d28d01a60 100644 --- a/tests/untestable/gdb/gdb_pretty_printer_test.py +++ b/tests/untestable/gdb/gdb_pretty_printer_test.py @@ -1,10 +1,13 @@ import gdb +import re +import sys # this test should test the gdb pretty printers of the nim # library. But be aware this test is not complete. It only tests the # command line version of gdb. It does not test anything for the # machine interface of gdb. This means if if this test passes gdb # frontends might still be broken. +gdb.execute("set python print-stack full") gdb.execute("source ../../../tools/nim-gdb.py") # debug all instances of the generic function `myDebug`, should be 14 gdb.execute("rbreak myDebug") @@ -16,7 +19,7 @@ outputs = [ '"meTwo"', '{meOne, meThree}', 'MyOtherEnum(1)', - '5', + '{MyOtherEnum(0), MyOtherEnum(2)}', 'array = {1, 2, 3, 4, 5}', 'seq(0, 0)', 'seq(0, 10)', @@ -28,13 +31,17 @@ outputs = [ '{a = 1, b = "some string"}' ] +argRegex = re.compile("^.* = (?:No suitable Nim \$ operator found for type: \w+\s*)*(.*)$") +# Remove this error message which can pop up +noSuitableRegex = re.compile("(No suitable Nim \$ operator found for type: \w+\s*)") + for i, expected in enumerate(outputs): - gdb.write(f"{i+1}) expecting: {expected}: ", gdb.STDLOG) + gdb.write(f"\x1b[38;5;105m{i+1}) expecting: {expected}: \x1b[0m", gdb.STDLOG) gdb.flush() - - functionSymbol = gdb.selected_frame().block().function - assert functionSymbol.line == 41, str(functionSymbol.line) - + currFrame = gdb.selected_frame() + functionSymbol = currFrame.block().function + assert functionSymbol.line == 24, str(functionSymbol.line) + raw = "" if i == 6: # myArray is passed as pointer to int to myDebug. I look up myArray up in the stack gdb.execute("up") @@ -44,10 +51,13 @@ for i, expected in enumerate(outputs): gdb.execute("up") raw = gdb.parse_and_eval("myOtherArray") else: - raw = gdb.parse_and_eval("arg") - + rawArg = re.sub(noSuitableRegex, "", gdb.execute("info args", to_string = True)) + raw = rawArg.split("=", 1)[-1].strip() output = str(raw) - assert output == expected, "{0} : output: ({1}) != expected: ({2})".format(i, output, expected) - gdb.write(f"passed\n", gdb.STDLOG) + if output != expected: + gdb.write(f"\x1b[38;5;196m ({output}) != expected: ({expected})\x1b[0m\n", gdb.STDERR) + gdb.execute("quit 1") + else: + gdb.write("\x1b[38;5;34mpassed\x1b[0m\n", gdb.STDLOG) gdb.execute("continue") diff --git a/tests/untestable/gdb/gdb_pretty_printer_test_output.txt b/tests/untestable/gdb/gdb_pretty_printer_test_output.txt deleted file mode 100644 index 73d26016f..000000000 --- a/tests/untestable/gdb/gdb_pretty_printer_test_output.txt +++ /dev/null @@ -1,3 +0,0 @@ -Loading Nim Runtime support. -NimEnumPrinter: lookup global symbol 'NTI__z9cu80OJCfNgw9bUdzn5ZEzw_ failed for tyEnum_MyOtherEnum__z9cu80OJCfNgw9bUdzn5ZEzw. -8 diff --git a/tests/untestable/gdb/gdb_pretty_printer_test_program.nim b/tests/untestable/gdb/gdb_pretty_printer_test_program.nim index d2acdd282..b54fc1a7f 100644 --- a/tests/untestable/gdb/gdb_pretty_printer_test_program.nim +++ b/tests/untestable/gdb/gdb_pretty_printer_test_program.nim @@ -18,23 +18,6 @@ type MyObj = object a*: int b*: string - - # MyVariant = ref object - # id*: int - # case kind*: MyEnum - # of meOne: mInt*: int - # of meTwo, meThree: discard - # of meFour: - # moInt*: int - # babies*: seq[MyVariant] - # after: float - - # MyIntVariant = ref object - # stuff*: int - # case myKind*: range[0..32766] - # of 0: mFloat*: float - # of 2: mString*: string - # else: mBabies*: seq[MyIntVariant] var counter = 0 @@ -97,16 +80,7 @@ proc testProc(): void = var obj = MyObj(a: 1, b: "some string") myDebug(obj) #15 - # var varObj = MyVariant(id: 13, kind: meFour, moInt: 94, - # babies: @[MyVariant(id: 18, kind: meOne, mInt: 7, after: 1.0), - # MyVariant(id: 21, kind: meThree, after: 2.0)], - # after: 3.0) - # myDebug(varObj) #16 - - # var varObjInt = MyIntVariant(stuff: 5, myKind: 2, mString: "this is my sweet string") - # myDebug(varObjInt) #17 - - echo(counter) + assert counter == 15 testProc() diff --git a/tests/untestable/gdb/gdb_pretty_printer_test_run.sh b/tests/untestable/gdb/gdb_pretty_printer_test_run.sh old mode 100644 new mode 100755 index 525f54705..411c68435 --- a/tests/untestable/gdb/gdb_pretty_printer_test_run.sh +++ b/tests/untestable/gdb/gdb_pretty_printer_test_run.sh @@ -1,15 +1,13 @@ #!/usr/bin/env bash -# Exit if anything fails set -e -#!/usr/bin/env bash # Compile the test project with fresh debug information. -nim c --debugger:native gdb_pretty_printer_test_program.nim &> /dev/null +nim c --debugger:native --mm:orc --out:gdbNew gdb_pretty_printer_test_program.nim +echo "Running new runtime tests..." # 2>&1 redirects stderr to stdout (all output in stdout) -# <(...) is a bash feature that makes the output of a command into a -# file handle. -# diff compares the two files, the expected output, and the file -# handle that is created by the execution of gdb. -diff ./gdb_pretty_printer_test_output.txt <(gdb -x gdb_pretty_printer_test.py --batch-silent --args gdb_pretty_printer_test_program 2>&1) -# The exit code of diff is forwarded as the exit code of this -# script. So when the comparison fails, the exit code of this script -# won't be 0. So this script should be embeddable in a test suite. +gdb -x gdb_pretty_printer_test.py --batch-silent --args gdbNew 2>&1 + + +# Do it all again, but with old runtime +nim c --debugger:native --mm:refc --out:gdbOld gdb_pretty_printer_test_program.nim &> /dev/null +echo "Running old runtime tests" +gdb -x gdb_pretty_printer_test.py --batch-silent --args gdbOld 2>&1 diff --git a/tools/nim-gdb.py b/tools/nim-gdb.py index f35b9a033..e3af0dde6 100644 --- a/tools/nim-gdb.py +++ b/tools/nim-gdb.py @@ -16,6 +16,10 @@ def printErrorOnce(id, message): errorSet.add(id) gdb.write("printErrorOnce: " + message, gdb.STDERR) +def debugPrint(x): + gdb.write(str(x) + "\n", gdb.STDERR) + +NIM_STRING_TYPES = ["NimStringDesc", "NimStringV2"] ################################################################################ ##### Type pretty printers @@ -23,23 +27,28 @@ def printErrorOnce(id, message): type_hash_regex = re.compile("^([A-Za-z0-9]*)_([A-Za-z0-9]*)_+([A-Za-z0-9]*)$") +def getNimName(typ): + if m := type_hash_regex.match(typ): + return m.group(2) + return f"unknown <{typ}>" + def getNimRti(type_name): """ Return a ``gdb.Value`` object for the Nim Runtime Information of ``type_name``. """ # Get static const TNimType variable. This should be available for # every non trivial Nim type. m = type_hash_regex.match(type_name) - lookups = [ - "NTI" + m.group(2).lower() + "__" + m.group(3) + "_", - "NTI" + "__" + m.group(3) + "_", - "NTI" + m.group(2).replace("colon", "58").lower() + "__" + m.group(3) + "_" - ] if m: - for l in lookups: - try: - return gdb.parse_and_eval(l) - except: - pass + lookups = [ + "NTI" + m.group(2).lower() + "__" + m.group(3) + "_", + "NTI" + "__" + m.group(3) + "_", + "NTI" + m.group(2).replace("colon", "58").lower() + "__" + m.group(3) + "_" + ] + for l in lookups: + try: + return gdb.parse_and_eval(l) + except: + pass None def getNameFromNimRti(rti): @@ -68,7 +77,7 @@ class NimTypeRecognizer: 'NIM_BOOL': 'bool', - 'NIM_CHAR': 'char', 'NCSTRING': 'cstring', 'NimStringDesc': 'string' + 'NIM_CHAR': 'char', 'NCSTRING': 'cstring', 'NimStringDesc': 'string', 'NimStringV2': 'string' } # object_type_pattern = re.compile("^(\w*):ObjectType$") @@ -136,7 +145,7 @@ class DollarPrintFunction (gdb.Function): "Nim's equivalent of $ operator as a gdb function, available in expressions `print $dollar(myvalue)" dollar_functions = re.findall( - 'NimStringDesc \*(dollar__[A-z0-9_]+?)\(([^,)]*)\);', + '(?:NimStringDesc \*|NimStringV2)\s?(dollar__[A-z0-9_]+?)\(([^,)]*)\);', gdb.execute("info functions dollar__", True, True) ) @@ -146,12 +155,9 @@ class DollarPrintFunction (gdb.Function): @staticmethod def invoke_static(arg): - - if arg.type.code == gdb.TYPE_CODE_PTR and arg.type.target().name == "NimStringDesc": + if arg.type.code == gdb.TYPE_CODE_PTR and arg.type.target().name in NIM_STRING_TYPES: return arg - argTypeName = str(arg.type) - for func, arg_typ in DollarPrintFunction.dollar_functions: # this way of overload resolution cannot deal with type aliases, # therefore it won't find all overloads. @@ -163,7 +169,8 @@ class DollarPrintFunction (gdb.Function): func_value = gdb.lookup_global_symbol(func, gdb.SYMBOL_FUNCTIONS_DOMAIN).value() return func_value(arg.address) - printErrorOnce(argTypeName, "No suitable Nim $ operator found for type: " + argTypeName + "\n") + + debugPrint(f"No suitable Nim $ operator found for type: {getNimName(argTypeName)}\n") return None def invoke(self, arg): @@ -184,11 +191,11 @@ class NimStringEqFunction (gdb.Function): @staticmethod def invoke_static(arg1,arg2): - if arg1.type.code == gdb.TYPE_CODE_PTR and arg1.type.target().name == "NimStringDesc": + if arg1.type.code == gdb.TYPE_CODE_PTR and arg1.type.target().name in NIM_STRING_TYPES: str1 = NimStringPrinter(arg1).to_string() else: str1 = arg1.string() - if arg2.type.code == gdb.TYPE_CODE_PTR and arg2.type.target().name == "NimStringDesc": + if arg2.type.code == gdb.TYPE_CODE_PTR and arg2.type.target().name in NIM_STRING_TYPES: str2 = NimStringPrinter(arg1).to_string() else: str2 = arg2.string() @@ -216,7 +223,7 @@ class DollarPrintCmd (gdb.Command): strValue = DollarPrintFunction.invoke_static(param) if strValue: gdb.write( - NimStringPrinter(strValue).to_string() + "\n", + str(NimStringPrinter(strValue)) + "\n", gdb.STDOUT ) @@ -254,7 +261,6 @@ class KochCmd (gdb.Command): os.path.dirname(os.path.dirname(__file__)), "koch") def invoke(self, argument, from_tty): - import os subprocess.run([self.binary] + gdb.string_to_argv(argument)) KochCmd() @@ -308,8 +314,14 @@ class NimBoolPrinter: ################################################################################ +def strFromLazy(strVal): + if isinstance(strVal, str): + return strVal + else: + return strVal.value().string("utf-8") + class NimStringPrinter: - pattern = re.compile(r'^NimStringDesc \*$') + pattern = re.compile(r'^(NimStringDesc \*|NimStringV2)$') def __init__(self, val): self.val = val @@ -319,11 +331,19 @@ class NimStringPrinter: def to_string(self): if self.val: - l = int(self.val['Sup']['len']) - return self.val['data'].lazy_string(encoding="utf-8", length=l) + if self.val.type.name == "NimStringV2": + l = int(self.val["len"]) + data = self.val["p"]["data"] + else: + l = int(self.val['Sup']['len']) + data = self.val["data"] + return data.lazy_string(encoding="utf-8", length=l) else: return "" + def __str__(self): + return strFromLazy(self.to_string()) + class NimRopePrinter: pattern = re.compile(r'^tyObject_RopeObj__([A-Za-z0-9]*) \*$') @@ -345,39 +365,11 @@ class NimRopePrinter: ################################################################################ -# proc reprEnum(e: int, typ: PNimType): string {.compilerRtl.} = -# ## Return string representation for enumeration values -# var n = typ.node -# if ntfEnumHole notin typ.flags: -# let o = e - n.sons[0].offset -# if o >= 0 and o <% typ.node.len: -# return $n.sons[o].name -# else: -# # ugh we need a slow linear search: -# var s = n.sons -# for i in 0 .. n.len-1: -# if s[i].offset == e: -# return $s[i].name -# result = $e & " (invalid data!)" - def reprEnum(e, typ): - """ this is a port of the nim runtime function `reprEnum` to python """ + # Casts the value to the enum type and then calls the enum printer e = int(e) - n = typ["node"] - flags = int(typ["flags"]) - # 1 << 6 is {ntfEnumHole} - if ((1 << 6) & flags) == 0: - o = e - int(n["sons"][0]["offset"]) - if o >= 0 and 0 < int(n["len"]): - return n["sons"][o]["name"].string("utf-8", "ignore") - else: - # ugh we need a slow linear search: - s = n["sons"] - for i in range(0, int(n["len"])): - if int(s[i]["offset"]) == e: - return s[i]["name"].string("utf-8", "ignore") - - return str(e) + " (invalid data!)" + val = gdb.Value(e).cast(typ) + return strFromLazy(NimEnumPrinter(val).to_string()) def enumNti(typeNimName, idString): typeInfoName = "NTI" + typeNimName.lower() + "__" + idString + "_" @@ -389,6 +381,7 @@ def enumNti(typeNimName, idString): class NimEnumPrinter: pattern = re.compile(r'^tyEnum_([A-Za-z0-9]+)__([A-Za-z0-9]*)$') + enumReprProc = gdb.lookup_global_symbol("reprEnum", gdb.SYMBOL_FUNCTIONS_DOMAIN) def __init__(self, val): self.val = val @@ -397,14 +390,18 @@ class NimEnumPrinter: self.typeNimName = match.group(1) typeInfoName, self.nti = enumNti(self.typeNimName, match.group(2)) - if self.nti is None: - printErrorOnce(typeInfoName, f"NimEnumPrinter: lookup global symbol: '{typeInfoName}' failed for {typeName}.\n") - def to_string(self): - if self.nti: - arg0 = self.val - arg1 = self.nti.value(gdb.newest_frame()) - return reprEnum(arg0, arg1) + if NimEnumPrinter.enumReprProc and self.nti: + # Use the old runtimes enumRepr function. + # We call the Nim proc itself so that the implementation is correct + f = gdb.newest_frame() + # We need to strip the quotes so it looks like an enum instead of a string + reprProc = NimEnumPrinter.enumReprProc.value() + return str(reprProc(self.val, self.nti.value(f).address)).strip('"') + elif dollarResult := DollarPrintFunction.invoke_static(self.val): + # New runtime doesn't use enumRepr so we instead try and call the + # dollar function for it + return str(NimStringPrinter(dollarResult)) else: return self.typeNimName + "(" + str(int(self.val)) + ")" @@ -421,26 +418,20 @@ class NimSetPrinter: typeName = self.val.type.name match = self.pattern.match(typeName) self.typeNimName = match.group(1) - typeInfoName, self.nti = enumNti(self.typeNimName, match.group(2)) - - if self.nti is None: - printErrorOnce(typeInfoName, f"NimSetPrinter: lookup global symbol: '{typeInfoName}' failed for {typeName}.\n") def to_string(self): - if self.nti: - nti = self.nti.value(gdb.newest_frame()) - enumStrings = [] - val = int(self.val) - i = 0 - while val > 0: - if (val & 1) == 1: - enumStrings.append(reprEnum(i, nti)) - val = val >> 1 - i += 1 + # Remove the tySet from the type name + typ = gdb.lookup_type(self.val.type.name[6:]) + enumStrings = [] + val = int(self.val) + i = 0 + while val > 0: + if (val & 1) == 1: + enumStrings.append(reprEnum(i, typ)) + val = val >> 1 + i += 1 - return '{' + ', '.join(enumStrings) + '}' - else: - return str(int(self.val)) + return '{' + ', '.join(enumStrings) + '}' ################################################################################ @@ -472,41 +463,81 @@ class NimHashSetPrinter: ################################################################################ -class NimSeqPrinter: - # the pointer is explicity part of the type. So it is part of - # ``pattern``. - pattern = re.compile(r'^tySequence_\w* \*$') +class NimSeq: + # Wrapper around sequences. + # This handles the differences between old and new runtime def __init__(self, val): self.val = val + # new runtime has sequences on stack, old has them on heap + self.new = val.type.code != gdb.TYPE_CODE_PTR + if self.new: + # Some seqs are just the content and to save repeating ourselves we do + # handle them here. Only thing that needs to check this is the len/data getters + self.isContent = val.type.name.endswith("Content") + + def __bool__(self): + if self.new: + return self.val is not None + else: + return bool(self.val) + + def __len__(self): + if not self: + return 0 + if self.new: + if self.isContent: + return int(self.val["cap"]) + else: + return int(self.val["len"]) + else: + return self.val["Sup"]["len"] + + @property + def data(self): + if self.new: + if self.isContent: + return self.val["data"] + elif self.val["p"]: + return self.val["p"]["data"] + else: + return self.val["data"] + + @property + def cap(self): + if not self: + return 0 + if self.new: + if self.isContent: + return int(self.val["cap"]) + elif self.val["p"]: + return int(self.val["p"]["cap"]) + else: + return 0 + return int(self.val['Sup']['reserved']) + +class NimSeqPrinter: + pattern = re.compile(r'^tySequence_\w*\s?\*?$') + + def __init__(self, val): + self.val = NimSeq(val) + def display_hint(self): return 'array' def to_string(self): - len = 0 - cap = 0 - if self.val: - len = int(self.val['Sup']['len']) - cap = int(self.val['Sup']['reserved']) - - return 'seq({0}, {1})'.format(len, cap) + return f'seq({len(self.val)}, {self.val.cap})' def children(self): if self.val: val = self.val - valType = val.type - length = int(val['Sup']['len']) + length = len(val) if length <= 0: return - dataType = valType['data'].type - data = val['data'] - - if self.val.type.name is None: - dataType = valType['data'].type.target().pointer() - data = val['data'].cast(dataType) + data = val.data inaccessible = False for i in range(length): @@ -585,7 +616,7 @@ class NimTablePrinter: if self.val: counter = int(self.val['counter']) if self.val['data']: - capacity = int(self.val['data']['Sup']['len']) + capacity = NimSeq(self.val["data"]).cap return 'Table({0}, {1})'.format(counter, capacity) @@ -597,163 +628,6 @@ class NimTablePrinter: yield (idxStr + '.Field1', entry['Field1']) yield (idxStr + '.Field2', entry['Field2']) -################################################################ - -# this is untested, therefore disabled - -# class NimObjectPrinter: -# pattern = re.compile(r'^tyObject_([A-Za-z0-9]+)__(_?[A-Za-z0-9]*)(:? \*)?$') - -# def __init__(self, val): -# self.val = val -# self.valType = None -# self.valTypeNimName = None - -# def display_hint(self): -# return 'object' - -# def _determineValType(self): -# if self.valType is None: -# vt = self.val.type -# if vt.name is None: -# target = vt.target() -# self.valType = target.pointer() -# self.fields = target.fields() -# self.valTypeName = target.name -# self.isPointer = True -# else: -# self.valType = vt -# self.fields = vt.fields() -# self.valTypeName = vt.name -# self.isPointer = False - -# def to_string(self): -# if self.valTypeNimName is None: -# self._determineValType() -# match = self.pattern.match(self.valTypeName) -# self.valTypeNimName = match.group(1) - -# return self.valTypeNimName - -# def children(self): -# self._determineValType() -# if self.isPointer and int(self.val) == 0: -# return -# self.baseVal = self.val.referenced_value() if self.isPointer else self.val - -# for c in self.handleFields(self.baseVal, getNimRti(self.valTypeName)): -# yield c - -# def handleFields(self, currVal, rti, fields = None): -# rtiSons = None -# discField = (0, None) -# seenSup = False -# if fields is None: -# fields = self.fields -# try: # XXX: remove try after finished debugging this method -# for (i, field) in enumerate(fields): -# if field.name == "Sup": # inherited data -# seenSup = True -# baseRef = rti['base'] -# if baseRef: -# baseRti = baseRef.referenced_value() -# baseVal = currVal['Sup'] -# baseValType = baseVal.type -# if baseValType.name is None: -# baseValType = baseValType.target().pointer() -# baseValFields = baseValType.target().fields() -# else: -# baseValFields = baseValType.fields() - -# for c in self.handleFields(baseVal, baseRti, baseValFields): -# yield c -# else: -# if field.type.code == gdb.TYPE_CODE_UNION: -# # if not rtiSons: -# rtiNode = rti['node'].referenced_value() -# rtiSons = rtiNode['sons'] - -# if not rtiSons and int(rtiNode['len']) == 0 and str(rtiNode['name']) != "0x0": -# rtiSons = [rti['node']] # sons are dereferenced by the consumer - -# if not rtiSons: -# printErrorOnce(self.valTypeName, f"NimObjectPrinter: UNION field can't be displayed without RTI {self.valTypeName}, using fallback.\n") -# # yield (field.name, self.baseVal[field]) # XXX: this fallback seems wrong -# return # XXX: this should probably continue instead? - -# if int(rtiNode['len']) != 0 and str(rtiNode['name']) != "0x0": -# gdb.write(f"wtf IT HAPPENED {self.valTypeName}\n", gdb.STDERR) - -# discNode = rtiSons[discField[0]].referenced_value() -# if not discNode: -# raise ValueError("Can't find union discriminant field in object RTI") - -# discNodeLen = int(discNode['len']) -# discFieldVal = int(currVal[discField[1].name]) - -# unionNodeRef = None -# if discFieldVal < discNodeLen: -# unionNodeRef = discNode['sons'][discFieldVal] -# if not unionNodeRef: -# unionNodeRef = discNode['sons'][discNodeLen] - -# if not unionNodeRef: -# printErrorOnce(self.valTypeName + "no union node", f"wtf is up with sons {self.valTypeName} {unionNodeRef} {rtiNode['offset']} {discNode} {discFieldVal} {discNodeLen} {discField[1].name} {field.name} {field.type}\n") -# continue - -# unionNode = unionNodeRef.referenced_value() - -# fieldName = "" if field.name == None else field.name.lower() -# unionNodeName = "" if not unionNode['name'] else unionNode['name'].string("utf-8", "ignore") -# if not unionNodeName or unionNodeName.lower() != fieldName: -# unionFieldName = f"_{discField[1].name.lower()}_{int(rti['node'].referenced_value()['len'])}" -# gdb.write(f"wtf i: {i} union: {unionFieldName} field: {fieldName} type: {field.type.name} tag: {field.type.tag}\n", gdb.STDERR) -# else: -# unionFieldName = unionNodeName - -# if discNodeLen == 0: -# yield (unionFieldName, currVal[unionFieldName]) -# else: -# unionNodeLen = int(unionNode['len']) -# if unionNodeLen > 0: -# for u in range(unionNodeLen): -# un = unionNode['sons'][u].referenced_value()['name'].string("utf-8", "ignore") -# yield (un, currVal[unionFieldName][un]) -# else: -# yield(unionNodeName, currVal[unionFieldName]) -# else: -# discIndex = i - 1 if seenSup else i -# discField = (discIndex, field) # discriminant field is the last normal field -# yield (field.name, currVal[field.name]) -# except GeneratorExit: -# raise -# except: -# gdb.write(f"wtf {self.valTypeName} {i} fn: {field.name} df: {discField} rti: {rti} rtiNode: {rti['node'].referenced_value()} rtiSons: {rtiSons} {sys.exc_info()} {traceback.format_tb(sys.exc_info()[2], limit = 10)}\n", gdb.STDERR) -# gdb.write(f"wtf {self.valTypeName} {i} {field.name}\n", gdb.STDERR) - -# # seenSup = False -# # for (i, field) in enumerate(fields): -# # # if field.name: -# # # val = currVal[field.name] -# # # else: -# # # val = None -# # rtiNode = rti['node'].referenced_value() -# # rtiLen = int(rtiNode['len']) -# # if int(rtiNode['len']) > 0: -# # sons = rtiNode['sons'] -# # elif int(rti['len']) == 0 and str(rti['name']) != "0x0": -# # sons = [rti['node']] # sons are dereferenced by the consumer -# # sonsIdx = i - 1 if seenSup else i -# # s = sons[sonsIdx].referenced_value() -# # addr = int(currVal.address) -# # off = addr + int(rtiNode['offset']) -# # seenSup = seenSup or field.name == "Sup" - -# # gdb.write(f"wtf: i: {i} sonsIdx: {sonsIdx} field: {field.name} rtiLen: {rtiLen} rti: {rti} rtiNode: {rtiNode} isUnion: {field.type.code == gdb.TYPE_CODE_UNION} s: {s}\n", gdb.STDERR) - -# raise - - ################################################################################ class NimFrameFilter: From c73c76fdc62e76a9e36e2595b9854a6f0cb9e904 Mon Sep 17 00:00:00 2001 From: Century Systems Date: Tue, 21 Feb 2023 11:38:25 +0900 Subject: [PATCH 2413/2472] NuttX added supports getrlimit(RLIMIT_NOFILE), so remove NuttX specific codes. (#21385) async: NuttX added supports getrlimit(RLIMIT_NOFILE), so remove NuttX-specific codes. Signed-off-by: Takeyoshi Kikuchi --- lib/posix/posix_other_consts.nim | 3 --- lib/pure/asyncdispatch.nim | 3 --- lib/pure/selectors.nim | 3 --- 3 files changed, 9 deletions(-) diff --git a/lib/posix/posix_other_consts.nim b/lib/posix/posix_other_consts.nim index 4b1644f20..506c92158 100644 --- a/lib/posix/posix_other_consts.nim +++ b/lib/posix/posix_other_consts.nim @@ -473,9 +473,6 @@ var FD_SETSIZE* {.importc: "FD_SETSIZE", header: "".}: cint when defined(zephyr): # Zephyr specific hardcoded value var FD_MAX* {.importc: "CONFIG_POSIX_MAX_FDS ", header: "".}: cint -elif defined(nuttx): - # NuttX specific user configuration value - var NACTIVESOCKETS* {.importc: "CONFIG_NET_NACTIVESOCKETS", header: "".}: cint # var MSG_CTRUNC* {.importc: "MSG_CTRUNC", header: "".}: cint diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 64100afa7..2bfa8587c 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -2033,9 +2033,6 @@ when defined(linux) or defined(windows) or defined(macosx) or defined(bsd) or result = 16_700_000 elif defined(zephyr) or defined(freertos): result = FD_MAX - elif defined(nuttx): - # The maximum number of concurrently active UDP and TCP ports. - result = NACTIVESOCKETS else: var fdLim: RLimit if getrlimit(RLIMIT_NOFILE, fdLim) < 0: diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim index ef8736cdf..b15a25a1d 100644 --- a/lib/pure/selectors.nim +++ b/lib/pure/selectors.nim @@ -337,9 +337,6 @@ else: 16_700_000 elif defined(zephyr) or defined(freertos): FD_MAX - elif defined(nuttx): - # The maximum number of concurrently active UDP and TCP ports. - NACTIVESOCKETS else: var fdLim: RLimit var res = int(getrlimit(RLIMIT_NOFILE, fdLim)) From f7bd2088aa5fe970902d4cafc94933354fdec8a7 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Tue, 21 Feb 2023 11:27:42 +0800 Subject: [PATCH 2414/2472] corrects the linkerexe of riscv64 in config (#21292) --- config/nim.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/nim.cfg b/config/nim.cfg index f7b7c119a..973de077a 100644 --- a/config/nim.cfg +++ b/config/nim.cfg @@ -39,7 +39,7 @@ arm64.linux.gcc.linkerexe = "aarch64-linux-gnu-gcc" riscv32.linux.gcc.exe = "riscv64-linux-gnu-gcc" riscv32.linux.gcc.linkerexe = "riscv64-linux-gnu-gcc" riscv64.linux.gcc.exe = "riscv64-linux-gnu-gcc" -riscv64.linux.gcc.linkerexe = "arm-linux-gnueabihf-gcc" +riscv64.linux.gcc.linkerexe = "riscv64-linux-gnu-gcc" # For OpenWRT, you will also need to adjust PATH to point to your toolchain. mips.linux.gcc.exe = "mips-openwrt-linux-gcc" From ffd6bc87495ede82f9a8086d38c3348a40b11672 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Tue, 21 Feb 2023 15:58:45 +0800 Subject: [PATCH 2415/2472] csources_v2 can build the ORC-booted compiler; building now uses strict mode (#21411) * csources_v2 can build the ORC-booted compiler; building now uses strict mode * test booting in refc --- koch.nim | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/koch.nim b/koch.nim index fa498e017..52378e43f 100644 --- a/koch.nim +++ b/koch.nim @@ -530,8 +530,7 @@ proc runCI(cmd: string) = # boot without -d:nimHasLibFFI to make sure this still works # `--lib:lib` is needed for bootstrap on openbsd, for reasons described in # https://github.com/nim-lang/Nim/pull/14291 (`getAppFilename` bugsfor older nim on openbsd). - kochExecFold("Boot in release mode", "boot -d:release --gc:refc -d:nimStrictMode --lib:lib") - kochExecFold("Boot Nim ORC", "boot -d:release --lib:lib") + kochExecFold("Boot Nim ORC", "boot -d:release -d:nimStrictMode --lib:lib") when false: # debugging: when you need to run only 1 test in CI, use something like this: execFold("debugging test", "nim r tests/stdlib/tosproc.nim") @@ -587,6 +586,9 @@ proc runCI(cmd: string) = execFold("Run atlas tests", "nim c -r -d:atlasTests tools/atlas/atlas.nim clone https://github.com/disruptek/balls") + kochExecFold("Testing booting in refc", "boot -d:release --mm:refc -d:nimStrictMode --lib:lib") + + proc testUnixInstall(cmdLineRest: string) = csource("-d:danger" & cmdLineRest) xz(false, cmdLineRest) From bdc850916fb5d23fa4b9282309985e0fdfb0db36 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Tue, 21 Feb 2023 18:46:50 +0800 Subject: [PATCH 2416/2472] improve invalid module names errors (#21412) I have seen these questions: "Why I got an invalid module name errors?". To eliminate this kind of questions, We might improve th error messages. Though, the question might evolve into "What is a valid Nim identifier", which should be more searchable on the Internet. --- compiler/modules.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/modules.nim b/compiler/modules.nim index 7f6ff8622..838a89d83 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -38,7 +38,8 @@ proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym = name: getModuleIdent(graph, filename), info: newLineInfo(fileIdx, 1, 1)) if not isNimIdentifier(result.name.s): - rawMessage(graph.config, errGenerated, "invalid module name: " & result.name.s) + rawMessage(graph.config, errGenerated, "invalid module name: '" & result.name.s & + "'; a module name must be a valid Nim identifier.") partialInitModule(result, graph, fileIdx, filename) graph.registerModule(result) From e54d3cc418f9eab750563aa771070374d559fe57 Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Tue, 21 Feb 2023 22:02:42 +1100 Subject: [PATCH 2417/2472] Support tuples in nim-gdb (#21410) * Support for printing tuple types * Add test of printing a tuple * Add support for printing tuples in GDB * Forgot to [skip ci] --- .../untestable/gdb/gdb_pretty_printer_test.py | 3 ++- .../gdb/gdb_pretty_printer_test_program.nim | 5 +++- tools/nim-gdb.py | 25 ++++++++++++++++--- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/tests/untestable/gdb/gdb_pretty_printer_test.py b/tests/untestable/gdb/gdb_pretty_printer_test.py index d28d01a60..a96df9992 100644 --- a/tests/untestable/gdb/gdb_pretty_printer_test.py +++ b/tests/untestable/gdb/gdb_pretty_printer_test.py @@ -28,7 +28,8 @@ outputs = [ 'seq(3, 3) = {"one", "two", "three"}', 'Table(3, 64) = {[4] = "four", [5] = "five", [6] = "six"}', 'Table(3, 8) = {["two"] = 2, ["three"] = 3, ["one"] = 1}', - '{a = 1, b = "some string"}' + '{a = 1, b = "some string"}', + '("hello", 42)' ] argRegex = re.compile("^.* = (?:No suitable Nim \$ operator found for type: \w+\s*)*(.*)$") diff --git a/tests/untestable/gdb/gdb_pretty_printer_test_program.nim b/tests/untestable/gdb/gdb_pretty_printer_test_program.nim index b54fc1a7f..163c99860 100644 --- a/tests/untestable/gdb/gdb_pretty_printer_test_program.nim +++ b/tests/untestable/gdb/gdb_pretty_printer_test_program.nim @@ -80,7 +80,10 @@ proc testProc(): void = var obj = MyObj(a: 1, b: "some string") myDebug(obj) #15 - assert counter == 15 + var tup = ("hello", 42) + myDebug(tup) # 16 + + assert counter == 16 testProc() diff --git a/tools/nim-gdb.py b/tools/nim-gdb.py index e3af0dde6..1050197c9 100644 --- a/tools/nim-gdb.py +++ b/tools/nim-gdb.py @@ -112,6 +112,12 @@ class NimTypeRecognizer: result = self.type_map_static.get(tname, None) if result: return result + elif tname.startswith("tyEnum_"): + return getNimName(tname) + elif tname.startswith("tyTuple__"): + # We make the name be the field types (Just like in Nim) + fields = ", ".join([self.recognize(field.type) for field in type_obj.fields()]) + return f"({fields})" rti = getNimRti(tname) if rti: @@ -154,7 +160,7 @@ class DollarPrintFunction (gdb.Function): @staticmethod - def invoke_static(arg): + def invoke_static(arg, ignore_errors = False): if arg.type.code == gdb.TYPE_CODE_PTR and arg.type.target().name in NIM_STRING_TYPES: return arg argTypeName = str(arg.type) @@ -169,8 +175,8 @@ class DollarPrintFunction (gdb.Function): func_value = gdb.lookup_global_symbol(func, gdb.SYMBOL_FUNCTIONS_DOMAIN).value() return func_value(arg.address) - - debugPrint(f"No suitable Nim $ operator found for type: {getNimName(argTypeName)}\n") + if not ignore_errors: + debugPrint(f"No suitable Nim $ operator found for type: {getNimName(argTypeName)}\n") return None def invoke(self, arg): @@ -630,6 +636,19 @@ class NimTablePrinter: ################################################################################ +class NimTuplePrinter: + pattern = re.compile(r"^tyTuple__([A-Za-z0-9]*)") + + def __init__(self, val): + self.val = val + + def to_string(self): + # We don't have field names so just print out the tuple as if it was anonymous + tupleValues = [str(self.val[field.name]) for field in self.val.type.fields()] + return f"({', '.join(tupleValues)})" + +################################################################################ + class NimFrameFilter: def __init__(self): self.name = "nim-frame-filter" From 6d423f1856d086ea9e6b0995e3e5f831f4f89aa6 Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Tue, 21 Feb 2023 22:04:27 +1100 Subject: [PATCH 2418/2472] Make `Time` work with `std/strformat` (#21409) * Add test case * Remove formatValue template for `Time` It didn't handle empty specifier correctly which caused it to be blank with strformat --- lib/pure/times.nim | 6 +----- tests/stdlib/t21406.nim | 5 +++++ 2 files changed, 6 insertions(+), 5 deletions(-) create mode 100644 tests/stdlib/t21406.nim diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 3327b2510..70af466d9 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -2109,7 +2109,7 @@ proc format*(dt: DateTime, f: static[string]): string {.raises: [].} = const f2 = initTimeFormat(f) result = dt.format(f2) -proc formatValue*(result: var string; value: DateTime, specifier: string) = +proc formatValue*(result: var string; value: DateTime | Time, specifier: string) = ## adapter for strformat. Not intended to be called directly. result.add format(value, if specifier.len == 0: "yyyy-MM-dd'T'HH:mm:sszzz" else: specifier) @@ -2133,10 +2133,6 @@ proc format*(time: Time, f: static[string], zone: Timezone = local()): string const f2 = initTimeFormat(f) result = time.inZone(zone).format(f2) -template formatValue*(result: var string; value: Time, specifier: string) = - ## adapter for `strformat`. Not intended to be called directly. - result.add format(value, specifier) - proc parse*(input: string, f: TimeFormat, zone: Timezone = local(), loc: DateTimeLocale = DefaultLocale): DateTime {.raises: [TimeParseError, Defect].} = diff --git a/tests/stdlib/t21406.nim b/tests/stdlib/t21406.nim new file mode 100644 index 000000000..5b96227ce --- /dev/null +++ b/tests/stdlib/t21406.nim @@ -0,0 +1,5 @@ +import std/[times, strformat] +import std/assertions + +doAssert fmt"{getTime()}" == $getTime() +doAssert fmt"{now()}" == $now() From de65b380edb0fe256054e217a1e6e64a6bb97527 Mon Sep 17 00:00:00 2001 From: Ecorous <49562894+Ecorous@users.noreply.github.com> Date: Tue, 21 Feb 2023 11:14:41 +0000 Subject: [PATCH 2419/2472] Add `getDataDir` proc (#21408) * Add getDataDir() * Update lib/std/private/osappdirs.nim --------- Co-authored-by: Andreas Rumpf --- lib/std/private/osappdirs.nim | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/lib/std/private/osappdirs.nim b/lib/std/private/osappdirs.nim index 581b511ff..07a6809bb 100644 --- a/lib/std/private/osappdirs.nim +++ b/lib/std/private/osappdirs.nim @@ -12,6 +12,7 @@ proc getHomeDir*(): string {.rtl, extern: "nos$1", ## for the convenience of processing paths coming from user configuration files. ## ## See also: + ## * `getDataDir proc`_ ## * `getConfigDir proc`_ ## * `getTempDir proc`_ ## * `expandTilde proc`_ @@ -24,6 +25,30 @@ proc getHomeDir*(): string {.rtl, extern: "nos$1", when defined(windows): return getEnv("USERPROFILE") & "\\" else: return getEnv("HOME") & "/" +proc getDataDir*(): string {.rtl, extern: "nos$1" + tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the data directory of the current user for applications. + ## + ## On non-Windows OSs, this proc conforms to the XDG Base Directory + ## spec. Thus, this proc returns the value of the `XDG_DATA_HOME` environment + ## variable if it is set, otherwise it returns the default configuration + ## directory ("~/.local/share" or "~/Library/Application Support" on macOS). + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `expandTilde proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ + when defined(windows): + result = getEnv("APPDATA") + elif defined(macosx): + result = getEnv("XDG_DATA_HOME", getEnv("HOME") / "Library" / "Application Support") + else: + result = getEnv("XDG_DATA_HOME", getEnv("HOME") / ".local" / "share") + result.normalizePathEnd(trailingSep = true) + proc getConfigDir*(): string {.rtl, extern: "nos$1", tags: [ReadEnvEffect, ReadIOEffect].} = ## Returns the config directory of the current user for applications. @@ -38,6 +63,7 @@ proc getConfigDir*(): string {.rtl, extern: "nos$1", ## ## See also: ## * `getHomeDir proc`_ + ## * `getDataDir proc`_ ## * `getTempDir proc`_ ## * `expandTilde proc`_ ## * `getCurrentDir proc`_ @@ -63,6 +89,7 @@ proc getCacheDir*(): string = ## * `getHomeDir proc`_ ## * `getTempDir proc`_ ## * `getConfigDir proc`_ + ## * `getDataDir proc`_ # follows https://crates.io/crates/platform-dirs when defined(windows): result = getEnv("LOCALAPPDATA") From 0a45543cc1e8a2e8db59eaed864cd908726bf48d Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Tue, 21 Feb 2023 22:27:12 +1100 Subject: [PATCH 2420/2472] Specify that address is taken when converter takes a var parameter (#21391) * Add test case * closes #21247 Add the sfAddrTaken flag to var parameters in converters This allows the JS backend to properly pass the parameter as a fat pointer --- compiler/sigmatch.nim | 1 + tests/js/t21247.nim | 15 +++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 tests/js/t21247.nim diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 0dbd5e5f1..f8c7fa870 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -1980,6 +1980,7 @@ proc userConvMatch(c: PContext, m: var TCandidate, f, a: PType, param = implicitConv(nkHiddenSubConv, src, copyTree(arg), m, c) elif src.kind in {tyVar}: # Analyse the converter return type + arg.sym.flags.incl sfAddrTaken param = newNodeIT(nkHiddenAddr, arg.info, s.typ[1]) param.add copyTree(arg) else: diff --git a/tests/js/t21247.nim b/tests/js/t21247.nim new file mode 100644 index 000000000..5b38787b3 --- /dev/null +++ b/tests/js/t21247.nim @@ -0,0 +1,15 @@ +import std/typetraits + +type + QueryParams* = distinct seq[(string, string)] + +converter toBase*(params: var QueryParams): var seq[(string, string)] = + params.distinctBase + +proc foo(): QueryParams = + # Issue was that the implicit converter call didn't say that it took the + # address of the parameter it was converting. This led to the parameter not being + # passed as a fat pointer which toBase expected + result.add(("hello", "world")) + +assert foo().distinctBase() == @[("hello", "world")] From a3603c8a6ee4a8ae147d2f2099a18395e819a8a3 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Wed, 22 Feb 2023 06:42:28 +0800 Subject: [PATCH 2421/2472] saves one unnecessary compare which is also a small regression (#21413) saves one unnecessary compare which is also a sall regression follow up https://github.com/nim-lang/Nim/commit/d30c6419a051a815e3fdb354ac79522f17e55bda --- compiler/passes.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/passes.nim b/compiler/passes.nim index a8f67300c..c20763fc3 100644 --- a/compiler/passes.nim +++ b/compiler/passes.nim @@ -143,7 +143,7 @@ proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator; while true: openParser(p, fileIdx, s, graph.cache, graph.config) - if not belongsToStdlib(graph, module) or (belongsToStdlib(graph, module) and module.name.s == "distros"): + if (not belongsToStdlib(graph, module)) or module.name.s == "distros": # XXX what about caching? no processing then? what if I change the # modules to include between compilation runs? we'd need to track that # in ROD files. I think we should enable this feature only From 64a788cafb287ace8c63f5e86cb84c520eab3f2a Mon Sep 17 00:00:00 2001 From: Benji York Date: Wed, 22 Feb 2023 05:12:10 -0600 Subject: [PATCH 2422/2472] Fix a couple of small keyword issues. (#21416) In this section of the manual, "if" should be enclosed in backticks and "elif" should be lower case. --- doc/manual.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual.md b/doc/manual.md index 779e99186..ef790c362 100644 --- a/doc/manual.md +++ b/doc/manual.md @@ -3574,8 +3574,8 @@ Example: var y = if x > 8: 9 else: 10 ``` -An if expression always results in a value, so the `else` part is -required. `Elif` parts are also allowed. +An `if` expression always results in a value, so the `else` part is +required. `elif` parts are also allowed. When expression --------------- From ab1d4a5d58af8452b26f57f244bcd0d39b409c5e Mon Sep 17 00:00:00 2001 From: Century Systems Date: Thu, 23 Feb 2023 03:53:04 +0900 Subject: [PATCH 2423/2472] ioselectors_epoll: for NuttX, limit initial numFD to configured value. (#21421) ioselectors: ioselectors_epoll: for NuttX, limit initial numFD to configured value. In the NuttX build config, there is a setting called "FS_NEPOLL_DESCRIPTORS". -------- config FS_NEPOLL_DESCRIPTORS int "Maximum number of default epoll descriptors for epoll_create1(2)" default 8 ---help--- The maximum number of default epoll descriptors for epoll_create1(2) -------- For NuttX, change the number of fd arrays allocated by newSelector() to that value. Signed-off-by: Takeyoshi Kikuchi --- lib/posix/posix_other_consts.nim | 3 +++ lib/pure/ioselects/ioselectors_epoll.nim | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/posix/posix_other_consts.nim b/lib/posix/posix_other_consts.nim index 506c92158..08069fe9a 100644 --- a/lib/posix/posix_other_consts.nim +++ b/lib/posix/posix_other_consts.nim @@ -750,3 +750,6 @@ var SEEK_SET* {.importc: "SEEK_SET", header: "".}: cint var SEEK_CUR* {.importc: "SEEK_CUR", header: "".}: cint var SEEK_END* {.importc: "SEEK_END", header: "".}: cint +# +when defined(nuttx): + var NEPOLL_MAX* {.importc: "CONFIG_FS_NEPOLL_DESCRIPTORS", header: "".}: cint diff --git a/lib/pure/ioselects/ioselectors_epoll.nim b/lib/pure/ioselects/ioselectors_epoll.nim index 8526eb8a3..08cb6ed74 100644 --- a/lib/pure/ioselects/ioselectors_epoll.nim +++ b/lib/pure/ioselects/ioselectors_epoll.nim @@ -72,11 +72,16 @@ type SelectEvent* = ptr SelectEventImpl proc newSelector*[T](): Selector[T] = + proc initialNumFD(): int {.inline.} = + when defined(nuttx): + result = NEPOLL_MAX + else: + result = 1024 # Retrieve the maximum fd count (for current OS) via getrlimit() var maxFD = maxDescriptors() doAssert(maxFD > 0) # Start with a reasonable size, checkFd() will grow this on demand - const numFD = 1024 + let numFD = initialNumFD() var epollFD = epoll_create1(O_CLOEXEC) if epollFD < 0: From fdd75202576634f1fb8c7948d6031e34af9ea67e Mon Sep 17 00:00:00 2001 From: c-blake Date: Wed, 22 Feb 2023 13:54:04 -0500 Subject: [PATCH 2424/2472] Fix the TODO portion of recently added `posix_fallocate` on OS X. (#21387) --- lib/posix/posix.nim | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index aee9308a6..07652c64d 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -198,9 +198,24 @@ proc posix_fadvise*(a1: cint, a2, a3: Off, a4: cint): cint {. proc ftruncate*(a1: cint, a2: Off): cint {.importc, header: "".} when defined(osx): # 2001 POSIX evidently does not concern Apple + type FStore {.importc: "fstore_t", header: "", bycopy.} = object + fst_flags: uint32 ## IN: flags word + fst_posmode: cint ## IN: indicates offset field + fst_offset, ## IN: start of the region + fst_length, ## IN: size of the region + fst_bytesalloc: Off ## OUT: number of bytes allocated + var F_PEOFPOSMODE {.importc, header: "".}: cint + var F_ALLOCATEALL {.importc, header: "".}: uint32 + var F_PREALLOCATE {.importc, header: "".}: cint proc posix_fallocate*(a1: cint, a2, a3: Off): cint = - ftruncate(a1, a2 + a3) # Set size to off + len, max offset -else: # TODO: Use fcntl(fd, F_PREALLOCATE, ..) above + var fst = FStore(fst_flags: F_ALLOCATEALL, fst_posmode: F_PEOFPOSMODE, + fst_offset: a2, fst_length: a3) + # Must also call ftruncate to match what POSIX does. Unlike posix_fallocate, + # this can shrink files. Could guard w/getFileSize, but caller likely knows + # present size & has no good reason to call this unless it is growing. + if fcntl(a1, F_PREALLOCATE, fst.addr) != cint(-1): ftruncate(a1, a2 + a3) + else: cint(-1) +else: proc posix_fallocate*(a1: cint, a2, a3: Off): cint {. importc, header: "".} From 38f876dd484fc1a3ce44788b1ed9601eea8619f6 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 23 Feb 2023 03:34:20 +0800 Subject: [PATCH 2425/2472] fixes #19795; fixes #11852; fixes #19974; remove parsing pipeline, Nim now parses the whole module at one time (#21379) * fixes #19795; remove parse pipeline * isScript * fixes nimscriptapi * don't touch reorder * check script * fixes tests * it seems implicit imports of system cause troubles * access the first child of `nkStmtList` * ignore comments * minor messages * perhaps increases hloLoopDetector * the module is a stmtList, which changes the errors * fixes nimdoc * fixes tlinter * fixes nim secret tests * fixes arc_misc * fixes nim secret tests again * safe; fixes one more test * GlobalError is the root cause too * fixes parsing errors * put emit types to the cfsForwardTypes section * fixes #11852; `{.push checks:off}` now works in procs * disable navigator * fixes nimdoc * add tests for JS * fixes nimsuggest --- compiler/backendpragmas.nim | 25 ++++++ compiler/ccgstmts.nim | 10 ++- compiler/cgen.nim | 2 +- compiler/cgendata.nim | 1 + compiler/jsgen.nim | 10 ++- compiler/passes.nim | 7 +- compiler/pragmas.nim | 2 +- compiler/sem.nim | 6 ++ .../expected/subdir/subdir_b/utils.idx | 2 +- nimsuggest/tests/tchk1.nim | 2 +- tests/arc/tarcmisc.nim | 4 +- .../tdont_return_unowned_from_owned.nim | 35 ++++---- tests/errmsgs/t8064.nim | 8 +- tests/misc/tnoforward.nim | 1 + tests/misc/trunner.nim | 4 +- tests/modules/t8665.nim | 1 + tests/modules/treorder.nim | 2 +- tests/navigator/tincludefile.nim | 2 +- tests/navigator/tnav1.nim | 2 +- tests/nimdoc/trunnableexamples.nim | 4 +- .../{overflw => overflow}/tdistinct_range.nim | 0 tests/overflow/toverflow.nim | 82 +++++++++++++++++++ .../toverflw2.nim => overflow/toverflow2.nim} | 0 .../toverflow_reorder.nim} | 4 +- tests/{overflw => overflow}/tovfint.nim | 0 tests/parser/t20922.nim | 29 ++----- tests/pragmas/tpush.nim | 12 +++ tests/pragmas/treorder.nim | 1 + tests/proc/t19795.nim | 19 +++++ tests/tools/tlinter.nim | 2 +- tests/trmacros/tor.nim | 20 +++-- 31 files changed, 227 insertions(+), 72 deletions(-) create mode 100644 compiler/backendpragmas.nim rename tests/{overflw => overflow}/tdistinct_range.nim (100%) create mode 100644 tests/overflow/toverflow.nim rename tests/{overflw/toverflw2.nim => overflow/toverflow2.nim} (100%) rename tests/{overflw/toverflw.nim => overflow/toverflow_reorder.nim} (96%) rename tests/{overflw => overflow}/tovfint.nim (100%) create mode 100644 tests/proc/t19795.nim diff --git a/compiler/backendpragmas.nim b/compiler/backendpragmas.nim new file mode 100644 index 000000000..b18644810 --- /dev/null +++ b/compiler/backendpragmas.nim @@ -0,0 +1,25 @@ +import pragmas, options, ast, trees + +proc pushBackendOption(optionsStack: var seq[TOptions], options: var TOptions) = + optionsStack.add options + +proc popBackendOption(optionsStack: var seq[TOptions], options: var TOptions) = + options = optionsStack[^1] + optionsStack.setLen(optionsStack.len-1) + +proc processPushBackendOption*(optionsStack: var seq[TOptions], options: var TOptions, + n: PNode, start: int) = + pushBackendOption(optionsStack, options) + for i in start..= 1 and n[0].kind in {nkStrLit..nkTripleStrLit}: let sec = n[0].strVal - if sec.startsWith("/*TYPESECTION*/"): result = cfsTypes + if sec.startsWith("/*TYPESECTION*/"): result = cfsForwardTypes # TODO WORKAROUND elif sec.startsWith("/*VARSECTION*/"): result = cfsVars elif sec.startsWith("/*INCLUDESECTION*/"): result = cfsHeaders @@ -1555,9 +1554,14 @@ proc genEmit(p: BProc, t: PNode) = line(p, cpsStmts, s) proc genPragma(p: BProc, n: PNode) = - for it in n.sons: + for i in 0.. 0 if we are within a loop splitDecls*: int # > 0 if we are in some context for C++ that diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 60399d3cd..572d19b13 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -33,7 +33,7 @@ import nversion, msgs, idents, types, ropes, passes, ccgutils, wordrecg, renderer, cgmeth, lowerings, sighashes, modulegraphs, lineinfos, rodutils, - transf, injectdestructors, sourcemap, astmsgs + transf, injectdestructors, sourcemap, astmsgs, backendpragmas import json, sets, math, tables, intsets import strutils except addf @@ -98,6 +98,7 @@ type prc: PSym globals, locals, body: Rope options: TOptions + optionsStack: seq[TOptions] module: BModule g: PGlobals generatedParamCopies: IntSet @@ -2557,9 +2558,14 @@ proc genStmt(p: PProc, n: PNode) = if r.res != "": lineF(p, "$#;$n", [r.res]) proc genPragma(p: PProc, n: PNode) = - for it in n.sons: + for i in 0..> (toplevel): import(dirty): tests/tchk1.nim [Processing]";;0 chk;;skUnknown;;;;Error;;$file;;12;;0;;"identifier expected, but got \'keyword template\'";;0 chk;;skUnknown;;;;Error;;$file;;14;;0;;"nestable statement requires indentation";;0 -chk;;skUnknown;;;;Error;;$file;;12;;0;;"implementation of \'foo\' expected";;0 chk;;skUnknown;;;;Error;;$file;;17;;0;;"invalid indentation";;0 +chk;;skUnknown;;;;Error;;$file;;12;;0;;"implementation of \'foo\' expected";;0 chk;;skUnknown;;;;Hint;;$file;;12;;9;;"\'foo\' is declared but not used [XDeclaredButNotUsed]";;0 chk;;skUnknown;;;;Hint;;$file;;14;;5;;"\'main\' is declared but not used [XDeclaredButNotUsed]";;0 """ diff --git a/tests/arc/tarcmisc.nim b/tests/arc/tarcmisc.nim index 6435f5e94..a03da6211 100644 --- a/tests/arc/tarcmisc.nim +++ b/tests/arc/tarcmisc.nim @@ -27,14 +27,14 @@ new line after - @['a'] finalizer aaaaa hello -ok true copying 123 42 -closed +ok destroying variable: 20 destroying variable: 10 +closed ''' cmd: "nim c --gc:arc --deepcopy:on -d:nimAllocPagesViaMalloc $file" """ diff --git a/tests/destructor/tdont_return_unowned_from_owned.nim b/tests/destructor/tdont_return_unowned_from_owned.nim index d27626dea..ffe87cd76 100644 --- a/tests/destructor/tdont_return_unowned_from_owned.nim +++ b/tests/destructor/tdont_return_unowned_from_owned.nim @@ -1,9 +1,12 @@ discard """ cmd: "nim check --newruntime --hints:off $file" nimout: ''' -tdont_return_unowned_from_owned.nim(36, 10) Error: cannot return an owned pointer as an unowned pointer; use 'owned(Obj)' as the return type -tdont_return_unowned_from_owned.nim(39, 10) Error: cannot return an owned pointer as an unowned pointer; use 'owned(Obj)' as the return type -tdont_return_unowned_from_owned.nim(42, 6) Error: type mismatch: got +tdont_return_unowned_from_owned.nim(26, 13) Error: assignment produces a dangling ref: the unowned ref lives longer than the owned ref +tdont_return_unowned_from_owned.nim(27, 13) Error: assignment produces a dangling ref: the unowned ref lives longer than the owned ref +tdont_return_unowned_from_owned.nim(31, 10) Error: cannot return an owned pointer as an unowned pointer; use 'owned(RootRef)' as the return type +tdont_return_unowned_from_owned.nim(43, 10) Error: cannot return an owned pointer as an unowned pointer; use 'owned(Obj)' as the return type +tdont_return_unowned_from_owned.nim(46, 10) Error: cannot return an owned pointer as an unowned pointer; use 'owned(Obj)' as the return type +tdont_return_unowned_from_owned.nim(49, 6) Error: type mismatch: got but expected one of: proc new[T](a: var ref T; finalizer: proc (x: ref T) {.nimcall.}) first type mismatch at position: 2 @@ -11,17 +14,21 @@ proc new[T](a: var ref T; finalizer: proc (x: ref T) {.nimcall.}) 2 other mismatching symbols have been suppressed; compile with --showAllMismatches:on to see them expression: new(result) -tdont_return_unowned_from_owned.nim(42, 6) Error: illformed AST: -tdont_return_unowned_from_owned.nim(50, 13) Error: assignment produces a dangling ref: the unowned ref lives longer than the owned ref -tdont_return_unowned_from_owned.nim(51, 13) Error: assignment produces a dangling ref: the unowned ref lives longer than the owned ref -tdont_return_unowned_from_owned.nim(55, 10) Error: cannot return an owned pointer as an unowned pointer; use 'owned(RootRef)' as the return type +tdont_return_unowned_from_owned.nim(49, 6) Error: illformed AST: ''' - errormsg: "cannot return an owned pointer as an unowned pointer; use 'owned(RootRef)' as the return type" + errormsg: "illformed AST:" """ +proc testA(result: var (RootRef, RootRef)) = + let r: owned RootRef = RootRef() + result[0] = r + result[1] = RootRef() +proc testB(): RootRef = + let r: owned RootRef = RootRef() + result = r @@ -39,17 +46,11 @@ proc newObjB(): Obj = result = Obj() proc newObjC(): Obj = - new(result) + new(result) # illFormedAst raises GlobalError, + # without pipeline parsing, it needs to placed at the end + # in case that it disturbs other errors let a = newObjA() let b = newObjB() let c = newObjC() -proc testA(result: var (RootRef, RootRef)) = - let r: owned RootRef = RootRef() - result[0] = r - result[1] = RootRef() - -proc testB(): RootRef = - let r: owned RootRef = RootRef() - result = r diff --git a/tests/errmsgs/t8064.nim b/tests/errmsgs/t8064.nim index 10bb86299..e7941e36a 100644 --- a/tests/errmsgs/t8064.nim +++ b/tests/errmsgs/t8064.nim @@ -1,6 +1,8 @@ +import tables + +values + + discard """ errormsg: "expression has no type: values" """ -import tables - -values \ No newline at end of file diff --git a/tests/misc/tnoforward.nim b/tests/misc/tnoforward.nim index b6a71897a..0d9b40c83 100644 --- a/tests/misc/tnoforward.nim +++ b/tests/misc/tnoforward.nim @@ -1,4 +1,5 @@ discard """ + matrix: "--experimental:codeReordering" output: "10" """ diff --git a/tests/misc/trunner.nim b/tests/misc/trunner.nim index 7fb437ce3..aca19f72d 100644 --- a/tests/misc/trunner.nim +++ b/tests/misc/trunner.nim @@ -342,8 +342,8 @@ tests/newconfig/bar/mfoo.nims""".splitLines when not defined(windows): check3 lines.len == 5 check3 lines[0].isDots - check3 lines[1].dup(removePrefix(">>> ")) == "3" # prompt depends on `nimUseLinenoise` - check3 lines[2].isDots + # check3 lines[1].isDots # todo nim secret might use parsing pipeline + check3 lines[2].dup(removePrefix(">>> ")) == "3" # prompt depends on `nimUseLinenoise` check3 lines[3] == "ab" check3 lines[4] == "" else: diff --git a/tests/modules/t8665.nim b/tests/modules/t8665.nim index 74d31452f..7cfdbdb00 100644 --- a/tests/modules/t8665.nim +++ b/tests/modules/t8665.nim @@ -1,4 +1,5 @@ discard """ + matrix: "--experimental:codeReordering" action: compile """ diff --git a/tests/modules/treorder.nim b/tests/modules/treorder.nim index c81715cd8..626d9684b 100644 --- a/tests/modules/treorder.nim +++ b/tests/modules/treorder.nim @@ -1,5 +1,5 @@ discard """ - cmd: "nim -d:testdef $target $file" + matrix: "--experimental:codeReordering -d:testdef" output: '''works 34 34 defined diff --git a/tests/navigator/tincludefile.nim b/tests/navigator/tincludefile.nim index f35ab2ec9..a913d0736 100644 --- a/tests/navigator/tincludefile.nim +++ b/tests/navigator/tincludefile.nim @@ -1,4 +1,5 @@ discard """ + disabled: true cmd: "nim check $options --defusages:$file,12,7 $file" nimout: '''def tincludefile_temp.nim(11, 10) usage tincludefile_temp.nim(12, 8) @@ -7,7 +8,6 @@ usage tincludefile_temp.nim(12, 8) - proc foo(x: int) = echo x diff --git a/tests/navigator/tnav1.nim b/tests/navigator/tnav1.nim index b6bbdbf19..e76c921f3 100644 --- a/tests/navigator/tnav1.nim +++ b/tests/navigator/tnav1.nim @@ -1,11 +1,11 @@ discard """ + disabled: true cmd: "nim check $options --defusages:$file,12,7 $file" nimout: '''def tnav1_temp.nim(11, 10) usage tnav1_temp.nim(12, 8) ''' """ - import std / [times] proc foo(x: int) = diff --git a/tests/nimdoc/trunnableexamples.nim b/tests/nimdoc/trunnableexamples.nim index c88d8fa3f..1188cdd1b 100644 --- a/tests/nimdoc/trunnableexamples.nim +++ b/tests/nimdoc/trunnableexamples.nim @@ -7,7 +7,6 @@ foo1 foo2 foo3 foo5 -foo6 foo7 in examplesInTemplate1 doc in outer @@ -15,6 +14,7 @@ doc in inner1 doc in inner2 foo8 foo9 +foo6 ''' joinable: false """ @@ -43,7 +43,7 @@ proc fun*() = proc fun*()=echo "foo5" fun() - runnableExamples: + runnableExamples("--experimental:codeReordering --warnings:off"): # `codeReordering` only allowed at top level {.experimental: "codeReordering".} proc fun1() = fun2() diff --git a/tests/overflw/tdistinct_range.nim b/tests/overflow/tdistinct_range.nim similarity index 100% rename from tests/overflw/tdistinct_range.nim rename to tests/overflow/tdistinct_range.nim diff --git a/tests/overflow/toverflow.nim b/tests/overflow/toverflow.nim new file mode 100644 index 000000000..972f929c6 --- /dev/null +++ b/tests/overflow/toverflow.nim @@ -0,0 +1,82 @@ +discard """ + output: "ok" + matrix: "--overflowChecks:off; --overflowChecks:off --b:js" +""" +# Tests nim's ability to detect overflows + +{.push overflowChecks: on.} + +var + a = high(int) + b = -2 + overflowDetected = false + +try: + echo(b - a) +except OverflowDefect: + overflowDetected = true + +{.pop.} # overflow check + +doAssert(overflowDetected) + +block: # Overflow checks in a proc + var + a = high(int) + b = -2 + overflowDetected = false + + {.push overflowChecks: on.} + proc foo() = + let c = b - a + {.pop.} + + try: + foo() + except OverflowDefect: + overflowDetected = true + + doAssert(overflowDetected) + +block: # Overflow checks in a forward declared proc + var + a = high(int) + b = -2 + overflowDetected = false + + proc foo() + + {.push overflowChecks: on.} + proc foo() = + let c = b - a + {.pop.} + + try: + foo() + except OverflowDefect: + overflowDetected = true + + doAssert(overflowDetected) + +block: # Overflow checks doesn't affect fwd declaration + var + a = high(int) + b = -2 + overflowDetected = false + + {.push overflowChecks: on.} + proc foo() + {.pop.} + + proc foo() = + let c = b - a + + try: + foo() + except OverflowDefect: + overflowDetected = true + + doAssert(not overflowDetected) + + +echo "ok" diff --git a/tests/overflw/toverflw2.nim b/tests/overflow/toverflow2.nim similarity index 100% rename from tests/overflw/toverflw2.nim rename to tests/overflow/toverflow2.nim diff --git a/tests/overflw/toverflw.nim b/tests/overflow/toverflow_reorder.nim similarity index 96% rename from tests/overflw/toverflw.nim rename to tests/overflow/toverflow_reorder.nim index 164e16e5c..fcf7b0c82 100644 --- a/tests/overflw/toverflw.nim +++ b/tests/overflow/toverflow_reorder.nim @@ -1,3 +1,5 @@ +{.experimental: "codeReordering".} + discard """ output: "ok" cmd: "nim $target --overflowChecks:off $options $file" @@ -12,7 +14,7 @@ var overflowDetected = false try: - writeLine(stdout, b - a) + echo b - a except OverflowDefect: overflowDetected = true diff --git a/tests/overflw/tovfint.nim b/tests/overflow/tovfint.nim similarity index 100% rename from tests/overflw/tovfint.nim rename to tests/overflow/tovfint.nim diff --git a/tests/parser/t20922.nim b/tests/parser/t20922.nim index 01af9868f..110610fb1 100644 --- a/tests/parser/t20922.nim +++ b/tests/parser/t20922.nim @@ -1,28 +1,17 @@ discard """ - cmd: "nim check $options --verbosity:0 $file" + cmd: "nim check $options --verbosity:0 --hints:off $file" action: "reject" nimout: ''' -t20922.nim(37, 5) Error: expression expected, but found ':' +t20922.nim(26, 5) Error: expression expected, but found ':' +t20922.nim(34, 7) Error: expression expected, but found ':' +t20922.nim(35, 5) Error: ':' or '=' expected, but got 'keyword of' Error: in expression ' '+'': identifier expected, but found '' -t20922.nim(37, 7) Error: attempting to call undeclared routine: '' +t20922.nim(26, 7) Error: attempting to call undeclared routine: '' Error: in expression ' '+'': identifier expected, but found '' -t20922.nim(37, 7) Error: attempting to call undeclared routine: '' -t20922.nim(37, 7) Error: expression '' cannot be called -t20922.nim(37, 7) Error: expression '' has no type (or is ambiguous) -t20922.nim(37, 7) Error: VM problem: dest register is not set -t20922.nim(45, 7) Error: expression expected, but found ':' -t20922.nim(46, 5) Error: ':' or '=' expected, but got 'keyword of' -t20922.nim(45, 9) Error: undeclared identifier: 'x' -t20922.nim(45, 9) Error: expression 'x' has no type (or is ambiguous) -Error: in expression ' x': identifier expected, but found '' -t20922.nim(45, 9) Error: attempting to call undeclared routine: '' -Error: in expression ' x': identifier expected, but found '' -t20922.nim(45, 9) Error: attempting to call undeclared routine: '' -t20922.nim(45, 9) Error: expression '' cannot be called -t20922.nim(45, 9) Error: expression '' has no type (or is ambiguous) -t20922.nim(45, 9) Error: VM problem: dest register is not set -t20922.nim(33, 6) Hint: 'mapInstrToToken' is declared but not used [XDeclaredButNotUsed] -t20922.nim(43, 3) Hint: 'Foo' is declared but not used [XDeclaredButNotUsed] +t20922.nim(26, 7) Error: attempting to call undeclared routine: '' +t20922.nim(26, 7) Error: expression '' cannot be called +t20922.nim(26, 7) Error: expression '' has no type (or is ambiguous) +t20922.nim(26, 7) Error: VM problem: dest register is not set ''' """ # original test case issue #20922 diff --git a/tests/pragmas/tpush.nim b/tests/pragmas/tpush.nim index 5ecfe9704..f2779ea70 100644 --- a/tests/pragmas/tpush.nim +++ b/tests/pragmas/tpush.nim @@ -13,3 +13,15 @@ proc WarnMe() = x: int echo(x) +# bug #11852 +proc foo(x: string, y: int, res: int) = + {.push checks: off} + var a: ptr char = unsafeAddr(x[y]) + {.pop.} + if x.len > y: + doAssert ord(a[]) == 51 + else: + doAssert x.len + 48 == res + +foo("", 0, 48) +foo("abc", 40, 51) diff --git a/tests/pragmas/treorder.nim b/tests/pragmas/treorder.nim index 09a98ef6a..72e8808b3 100644 --- a/tests/pragmas/treorder.nim +++ b/tests/pragmas/treorder.nim @@ -1,4 +1,5 @@ discard """ +matrix: "--experimental:codeReordering" output:'''0 1 2 diff --git a/tests/proc/t19795.nim b/tests/proc/t19795.nim new file mode 100644 index 000000000..677ec0a63 --- /dev/null +++ b/tests/proc/t19795.nim @@ -0,0 +1,19 @@ +discard """ + matrix: "--mm:arc" +""" + +# bug #19795 +# bug #21085 + +type Vector = seq[int] + +var vect: Vector = newSeq[int](5) +doAssert vect == @[0, 0, 0, 0, 0] + +# Needed to get the problem. Could also use "var". +let vectCopy = vect + +# Then some procedure definition is needed to get the problem. +proc p(): int = 3 + +doAssert vect == @[0, 0, 0, 0, 0] \ No newline at end of file diff --git a/tests/tools/tlinter.nim b/tests/tools/tlinter.nim index a6d45ab3b..16f67905e 100644 --- a/tests/tools/tlinter.nim +++ b/tests/tools/tlinter.nim @@ -1,10 +1,10 @@ discard """ cmd: '''nim c --styleCheck:hint $file''' nimout: ''' +tlinter.nim(25, 1) Hint: 'tyPE' should be: 'type' [Name] tlinter.nim(21, 14) Hint: 'nosideeffect' should be: 'noSideEffect' [Name] tlinter.nim(21, 28) Hint: 'myown' should be: 'myOwn' [template declared in tlinter.nim(19, 9)] [Name] tlinter.nim(21, 35) Hint: 'inLine' should be: 'inline' [Name] -tlinter.nim(25, 1) Hint: 'tyPE' should be: 'type' [Name] tlinter.nim(23, 1) Hint: 'foO' should be: 'foo' [proc declared in tlinter.nim(21, 6)] [Name] tlinter.nim(27, 14) Hint: 'Foo_bar' should be: 'FooBar' [type declared in tlinter.nim(25, 6)] [Name] tlinter.nim(29, 6) Hint: 'someVAR' should be: 'someVar' [var declared in tlinter.nim(27, 5)] [Name] diff --git a/tests/trmacros/tor.nim b/tests/trmacros/tor.nim index 087dc0d68..6b4e71216 100644 --- a/tests/trmacros/tor.nim +++ b/tests/trmacros/tor.nim @@ -1,9 +1,19 @@ discard """ - output: '''0 + output: ''' +3 +0 true -3''' +''' """ + +# bug #798 +template t012{(0|1|2){x}}(x: untyped): untyped = x+1 +let z = 1 +# outputs 3 thanks to fixpoint iteration: +echo z + + template arithOps: untyped = (`+` | `-` | `*`) template testOr{ (arithOps{f})(a, b) }(a, b, f: untyped): untyped = f(a mod 10, b) @@ -20,9 +30,3 @@ var c = false a = b and a echo a - -# bug #798 -template t012{(0|1|2){x}}(x: untyped): untyped = x+1 -let z = 1 -# outputs 3 thanks to fixpoint iteration: -echo z From 7dc80a7fefb0d2cecbdc2696c71ea2fa94d6aa76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francis=20Th=C3=A9rien?= Date: Wed, 22 Feb 2023 15:10:02 -0500 Subject: [PATCH 2426/2472] Mitigate issues related to compiler options when cross-compiling (#21330) * Document C compiler options config when cross-compiling * Allow empty string to override default --- compiler/extccomp.nim | 4 +++- doc/nimc.md | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index db97de65c..45a531852 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -328,7 +328,9 @@ proc getConfigVar(conf: ConfigRef; c: TSystemCC, suffix: string): string = platform.OS[conf.target.targetOS].name & '.' & CC[c].name & fullSuffix result = getConfigVar(conf, fullCCname) - if result.len == 0: + if existsConfigVar(conf, fullCCname): + result = getConfigVar(conf, fullCCname) + else: # not overridden for this cross compilation setting? result = getConfigVar(conf, CC[c].name & fullSuffix) else: diff --git a/doc/nimc.md b/doc/nimc.md index 81a060d8a..7b0e99e06 100644 --- a/doc/nimc.md +++ b/doc/nimc.md @@ -316,14 +316,16 @@ Another way is to make Nim invoke a cross compiler toolchain: nim c --cpu:arm --os:linux myproject.nim ``` -For cross compilation, the compiler invokes a C compiler named -like `$cpu.$os.$cc` (for example arm.linux.gcc) and the configuration -system is used to provide meaningful defaults. For example for `ARM` your +For cross compilation, the compiler invokes a C compiler named like +`$cpu.$os.$cc` (for example `arm.linux.gcc`) with options defined in +`$cpu.$os.$cc.options.always`. The configuration system is used to provide +meaningful defaults. For example, for Linux on a 32-bit ARM CPU, your configuration file should contain something like: arm.linux.gcc.path = "/usr/bin" arm.linux.gcc.exe = "arm-linux-gcc" arm.linux.gcc.linkerexe = "arm-linux-gcc" + arm.linux.gcc.options.always = "-w -fmax-errors=3" Cross-compilation for Windows ============================= From 8a19ac20708c66fb3541533d55b60624cee56ab7 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 24 Feb 2023 09:02:15 +0100 Subject: [PATCH 2427/2472] fixes #21393 and misc style changes (#21419) * fixes #21393 and misc style changes * progress --------- Co-authored-by: ringabout <43030857+ringabout@users.noreply.github.com> --- compiler/int128.nim | 39 +++++++++++++++-------------- compiler/sempass2.nim | 19 +++++++------- lib/pure/collections/tables.nim | 31 +++++++++++++++-------- lib/pure/hashes.nim | 5 +++- lib/pure/osproc.nim | 8 ++---- lib/std/private/digitsutils.nim | 4 +-- lib/std/private/ospaths2.nim | 2 +- lib/system/seqs_v2.nim | 3 ++- lib/system/strmantle.nim | 2 +- lib/windows/winlean.nim | 16 ++++++------ tests/effects/tstrictfuncs_misc.nim | 2 +- 11 files changed, 70 insertions(+), 61 deletions(-) diff --git a/compiler/int128.nim b/compiler/int128.nim index e357706b9..b0341eb37 100644 --- a/compiler/int128.nim +++ b/compiler/int128.nim @@ -33,26 +33,27 @@ template high*(t: typedesc[Int128]): Int128 = Max proc `$`*(a: Int128): string proc toInt128*[T: SomeInteger | bool](arg: T): Int128 = - when T is bool: result.sdata(0) = int32(arg) - elif T is SomeUnsignedInt: - when sizeof(arg) <= 4: - result.udata[0] = uint32(arg) + {.noSideEffect.}: + when T is bool: result.sdata(0) = int32(arg) + elif T is SomeUnsignedInt: + when sizeof(arg) <= 4: + result.udata[0] = uint32(arg) + else: + result.udata[0] = uint32(arg and T(0xffffffff)) + result.udata[1] = uint32(arg shr 32) + elif sizeof(arg) <= 4: + result.sdata(0) = int32(arg) + if arg < 0: # sign extend + result.sdata(1) = -1 + result.sdata(2) = -1 + result.sdata(3) = -1 else: - result.udata[0] = uint32(arg and T(0xffffffff)) - result.udata[1] = uint32(arg shr 32) - elif sizeof(arg) <= 4: - result.sdata(0) = int32(arg) - if arg < 0: # sign extend - result.sdata(1) = -1 - result.sdata(2) = -1 - result.sdata(3) = -1 - else: - let tmp = int64(arg) - result.udata[0] = uint32(tmp and 0xffffffff) - result.sdata(1) = int32(tmp shr 32) - if arg < 0: # sign extend - result.sdata(2) = -1 - result.sdata(3) = -1 + let tmp = int64(arg) + result.udata[0] = uint32(tmp and 0xffffffff) + result.sdata(1) = int32(tmp shr 32) + if arg < 0: # sign extend + result.sdata(2) = -1 + result.sdata(3) = -1 template isNegative(arg: Int128): bool = arg.sdata(3) < 0 diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index e989eb4a6..396ea6606 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -822,9 +822,6 @@ proc checkForSink(tracked: PEffects; n: PNode) = if tracked.inIfStmt == 0 and optSinkInference in tracked.config.options: checkForSink(tracked.config, tracked.c.idgen, tracked.owner, n) -proc strictFuncsActive(tracked: PEffects): bool {.inline.} = - sfNoSideEffect in tracked.owner.flags and strictFuncs in tracked.c.features and not tracked.inEnforcedNoSideEffects - proc trackCall(tracked: PEffects; n: PNode) = template gcsafeAndSideeffectCheck() = if notGcSafe(op) and not importedFromC(a): @@ -934,9 +931,11 @@ proc trackCall(tracked: PEffects; n: PNode) = # initialized until after the call. Since we do this after we analysed the # call, this is fine. initVar(tracked, n[i].skipAddr, false) - if tracked.strictFuncsActive and isDangerousLocation(n[i].skipAddr, tracked.owner): - localError(tracked.config, n[i].info, - "cannot pass $1 to `var T` parameter within a strict func" % renderTree(n[i])) + if strictFuncs in tracked.c.features and not tracked.inEnforcedNoSideEffects and + isDangerousLocation(n[i].skipAddr, tracked.owner): + if sfNoSideEffect in tracked.owner.flags: + localError(tracked.config, n[i].info, + "cannot pass $1 to `var T` parameter within a strict func" % renderTree(n[i])) tracked.hasSideEffect = true else: discard @@ -1090,10 +1089,12 @@ proc track(tracked: PEffects, n: PNode) = createTypeBoundOps(tracked, n[0].typ, n.info) if n[0].kind != nkSym or not isLocalSym(tracked, n[0].sym): checkForSink(tracked, n[1]) - if tracked.strictFuncsActive and isDangerousLocation(n[0], tracked.owner): + if strictFuncs in tracked.c.features and not tracked.inEnforcedNoSideEffects and + isDangerousLocation(n[0], tracked.owner): tracked.hasSideEffect = true - localError(tracked.config, n[0].info, - "cannot mutate location $1 within a strict func" % renderTree(n[0])) + if sfNoSideEffect in tracked.owner.flags: + localError(tracked.config, n[0].info, + "cannot mutate location $1 within a strict func" % renderTree(n[0])) of nkVarSection, nkLetSection: for child in n: let last = lastSon(child) diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 57a0107fd..4737fa478 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -447,7 +447,7 @@ proc mgetOrPut*[A, B](t: var Table[A, B], key: A, val: B): var B = ## ## ## Note that while the value returned is of type `var B`, - ## it is easy to accidentally create an copy of the value at `t[key]`. + ## it is easy to accidentally create a copy of the value at `t[key]`. ## Remember that seqs and strings are value types, and therefore ## cannot be copied into a separate variable for modification. ## See the example below. @@ -828,7 +828,8 @@ proc newTable*[A, B](initialSize = defaultInitialSize): TableRef[A, B] = b = newTable[char, seq[int]]() new(result) - result[] = initTable[A, B](initialSize) + {.noSideEffect.}: + result[] = initTable[A, B](initialSize) proc newTable*[A, B](pairs: openArray[(A, B)]): TableRef[A, B] = ## Creates a new ref hash table that contains the given `pairs`. @@ -844,14 +845,16 @@ proc newTable*[A, B](pairs: openArray[(A, B)]): TableRef[A, B] = assert b == {'a': 5, 'b': 9}.newTable new(result) - result[] = toTable[A, B](pairs) + {.noSideEffect.}: + result[] = toTable[A, B](pairs) proc newTableFrom*[A, B, C](collection: A, index: proc(x: B): C): TableRef[C, B] = ## Index the collection with the proc provided. # TODO: As soon as supported, change collection: A to collection: A[B] result = newTable[C, B]() - for item in collection: - result[index(item)] = item + {.noSideEffect.}: + for item in collection: + result[index(item)] = item proc `[]`*[A, B](t: TableRef[A, B], key: A): var B = ## Retrieves the value at `t[key]`. @@ -1825,7 +1828,8 @@ proc newOrderedTable*[A, B](initialSize = defaultInitialSize): OrderedTableRef[A a = newOrderedTable[int, string]() b = newOrderedTable[char, seq[int]]() new(result) - result[] = initOrderedTable[A, B](initialSize) + {.noSideEffect.}: + result[] = initOrderedTable[A, B](initialSize) proc newOrderedTable*[A, B](pairs: openArray[(A, B)]): OrderedTableRef[A, B] = ## Creates a new ordered ref hash table that contains the given `pairs`. @@ -1842,7 +1846,8 @@ proc newOrderedTable*[A, B](pairs: openArray[(A, B)]): OrderedTableRef[A, B] = assert b == {'a': 5, 'b': 9}.newOrderedTable result = newOrderedTable[A, B](pairs.len) - for key, val in items(pairs): result[key] = val + {.noSideEffect.}: + for key, val in items(pairs): result[key] = val proc `[]`*[A, B](t: OrderedTableRef[A, B], key: A): var B = @@ -2641,13 +2646,15 @@ proc newCountTable*[A](initialSize = defaultInitialSize): CountTableRef[A] = ## * `initCountTable proc<#initCountTable>`_ for creating a ## `CountTable` new(result) - result[] = initCountTable[A](initialSize) + {.noSideEffect.}: + result[] = initCountTable[A](initialSize) proc newCountTable*[A](keys: openArray[A]): CountTableRef[A] = ## Creates a new ref count table with every member of a container `keys` ## having a count of how many times it occurs in that container. result = newCountTable[A](keys.len) - for key in items(keys): result.inc(key) + {.noSideEffect.}: + for key in items(keys): result.inc(key) proc `[]`*[A](t: CountTableRef[A], key: A): int = ## Retrieves the value at `t[key]` if `key` is in `t`. @@ -2671,7 +2678,8 @@ proc `[]=`*[A](t: CountTableRef[A], key: A, val: int) = ## * `inc proc<#inc,CountTableRef[A],A,int>`_ for incrementing a ## value of a key assert val > 0 - t[][key] = val + {.noSideEffect.}: + t[][key] = val proc inc*[A](t: CountTableRef[A], key: A, val = 1) = ## Increments `t[key]` by `val` (default: 1). @@ -2680,7 +2688,8 @@ proc inc*[A](t: CountTableRef[A], key: A, val = 1) = a.inc('a') a.inc('b', 10) doAssert a == newCountTable("aaabbbbbbbbbbb") - t[].inc(key, val) + {.noSideEffect.}: + t[].inc(key, val) proc smallest*[A](t: CountTableRef[A]): tuple[key: A, val: int] = ## Returns the `(key, value)` pair with the smallest `val`. Efficiency: O(n) diff --git a/lib/pure/hashes.nim b/lib/pure/hashes.nim index 09e072812..4ae4938c6 100644 --- a/lib/pure/hashes.nim +++ b/lib/pure/hashes.nim @@ -247,7 +247,7 @@ proc hash*[T](x: ptr[T]): Hash {.inline.} = when defined(nimPreviewHashRef) or defined(nimdoc): proc hash*[T](x: ref[T]): Hash {.inline.} = ## Efficient `hash` overload. - ## + ## ## .. important:: Use `-d:nimPreviewHashRef` to ## enable hashing `ref`s. It is expected that this behavior ## becomes the new default in upcoming versions. @@ -536,6 +536,7 @@ proc hash*[T: tuple | object | proc](x: T): Hash = elif T is (proc): result = hash(pointer(x)) else: + result = 0 for f in fields(x): result = result !& hash(f) result = !$result @@ -551,6 +552,7 @@ proc hash*[A](x: openArray[A]): Hash = else: result = murmurHash(toOpenArrayByte(x, 0, x.high)) else: + result = 0 for a in x: result = result !& hash(a) result = !$result @@ -583,6 +585,7 @@ proc hash*[A](aBuf: openArray[A], sPos, ePos: int): Hash = proc hash*[A](x: set[A]): Hash = ## Efficient hashing of sets. ## There must be a `hash` proc defined for the element type `A`. + result = 0 for it in items(x): result = result !& hash(it) result = !$result diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 9a0e5895d..bb6abc571 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -570,12 +570,8 @@ when defined(windows) and not defined(useNimRtl): if a == 0: raiseOSError(osLastError()) proc newFileHandleStream(handle: Handle): owned FileHandleStream = - new(result) - result.handle = handle - result.closeImpl = hsClose - result.atEndImpl = hsAtEnd - result.readDataImpl = hsReadData - result.writeDataImpl = hsWriteData + result = FileHandleStream(handle: handle, closeImpl: hsClose, atEndImpl: hsAtEnd, + readDataImpl: hsReadData, writeDataImpl: hsWriteData) proc buildCommandLine(a: string, args: openArray[string]): string = result = quoteShell(a) diff --git a/lib/std/private/digitsutils.nim b/lib/std/private/digitsutils.nim index b3dc5d14f..e051e5218 100644 --- a/lib/std/private/digitsutils.nim +++ b/lib/std/private/digitsutils.nim @@ -101,9 +101,7 @@ proc addInt*(result: var string; x: int64) {.enforceNoRaises.} = num = cast[uint64](x) else: num = uint64(-x) - let base = result.len - setLen(result, base + 1) - result[base] = '-' + result.add '-' else: num = uint64(x) addInt(result, num) diff --git a/lib/std/private/ospaths2.nim b/lib/std/private/ospaths2.nim index 5e3bece68..78a806675 100644 --- a/lib/std/private/ospaths2.nim +++ b/lib/std/private/ospaths2.nim @@ -571,7 +571,7 @@ proc normExt(ext: string): string = proc searchExtPos*(path: string): int = ## Returns index of the `'.'` char in `path` if it signifies the beginning - ## of extension. Returns -1 otherwise. + ## of the file extension. Returns -1 otherwise. ## ## See also: ## * `splitFile proc`_ diff --git a/lib/system/seqs_v2.nim b/lib/system/seqs_v2.nim index 50f23111c..d7ace446d 100644 --- a/lib/system/seqs_v2.nim +++ b/lib/system/seqs_v2.nim @@ -83,7 +83,8 @@ proc shrink*[T](x: var seq[T]; newLen: Natural) {.tags: [], raises: [].} = for i in countdown(x.len - 1, newLen): reset x[i] # XXX This is wrong for const seqs that were moved into 'x'! - cast[ptr NimSeqV2[T]](addr x).len = newLen + {.noSideEffect.}: + cast[ptr NimSeqV2[T]](addr x).len = newLen proc grow*[T](x: var seq[T]; newLen: Natural; value: T) = let oldLen = x.len diff --git a/lib/system/strmantle.nim b/lib/system/strmantle.nim index f4733ed21..ab158d6b7 100644 --- a/lib/system/strmantle.nim +++ b/lib/system/strmantle.nim @@ -33,7 +33,7 @@ proc eqStrings(a, b: string): bool {.inline, compilerproc.} = proc hashString(s: string): int {.compilerproc.} = # the compiler needs exactly the same hash function! # this used to be used for efficient generation of string case statements - var h : uint = 0 + var h = 0'u for i in 0..len(s)-1: h = h + uint(s[i]) h = h + h shl 10 diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index 93ab3dd2e..230f963f0 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -48,12 +48,12 @@ type HGLRC* = Handle BYTE* = uint8 - SECURITY_ATTRIBUTES* {.final, pure.} = object + SECURITY_ATTRIBUTES* = object nLength*: int32 lpSecurityDescriptor*: pointer bInheritHandle*: WINBOOL - STARTUPINFO* {.final, pure.} = object + STARTUPINFO* = object cb*: int32 lpReserved*: cstring lpDesktop*: cstring @@ -73,17 +73,17 @@ type hStdOutput*: Handle hStdError*: Handle - PROCESS_INFORMATION* {.final, pure.} = object + PROCESS_INFORMATION* = object hProcess*: Handle hThread*: Handle dwProcessId*: int32 dwThreadId*: int32 - FILETIME* {.final, pure.} = object ## CANNOT BE int64 BECAUSE OF ALIGNMENT + FILETIME* = object ## CANNOT BE int64 BECAUSE OF ALIGNMENT dwLowDateTime*: DWORD dwHighDateTime*: DWORD - BY_HANDLE_FILE_INFORMATION* {.final, pure.} = object + BY_HANDLE_FILE_INFORMATION* = object dwFileAttributes*: DWORD ftCreationTime*: FILETIME ftLastAccessTime*: FILETIME @@ -95,7 +95,7 @@ type nFileIndexHigh*: DWORD nFileIndexLow*: DWORD - OSVERSIONINFO* {.final, pure.} = object + OSVERSIONINFO* = object dwOSVersionInfoSize*: DWORD dwMajorVersion*: DWORD dwMinorVersion*: DWORD @@ -820,7 +820,7 @@ type POVERLAPPED_COMPLETION_ROUTINE* = proc (para1: DWORD, para2: DWORD, para3: POVERLAPPED){.stdcall.} - GUID* {.final, pure.} = object + GUID* = object D1*: int32 D2*: int16 D3*: int16 @@ -1101,7 +1101,7 @@ proc wsaResetEvent*(hEvent: Handle): bool {.stdcall, importc: "WSAResetEvent", dynlib: "ws2_32.dll".} type - KEY_EVENT_RECORD* {.final, pure.} = object + KEY_EVENT_RECORD* = object eventType*: int16 bKeyDown*: WINBOOL wRepeatCount*: int16 diff --git a/tests/effects/tstrictfuncs_misc.nim b/tests/effects/tstrictfuncs_misc.nim index 59d850763..8c573bb3a 100644 --- a/tests/effects/tstrictfuncs_misc.nim +++ b/tests/effects/tstrictfuncs_misc.nim @@ -53,7 +53,7 @@ type JsonNode3 = ref object fields: MyTable -proc `[]`(t: var MyTable, key: string): var int = +proc `[]`(t: MyTable, key: string): int = result = t.data[0] proc `[]`(x: JsonNode3, key: string): int = From bbb6d2c69d1b2bca9987c708ebc6ffd2479bb822 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 24 Feb 2023 16:02:44 +0800 Subject: [PATCH 2428/2472] fixes #20695; fixes object with distinct defaults and tables (#21428) --- compiler/semtypes.nim | 4 ++- tests/objects/tobject_default_value.nim | 39 ++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index aaffa1ea7..26493703d 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -222,9 +222,11 @@ proc isRecursiveType(t: PType, cycleDetector: var IntSet): bool = proc fitDefaultNode(c: PContext, n: PNode): PType = let expectedType = if n[^2].kind != nkEmpty: semTypeNode(c, n[^2], nil) else: nil + let oldType = n[^1].typ n[^1] = semConstExpr(c, n[^1], expectedType = expectedType) + n[^1].flags.incl nfSem if n[^2].kind != nkEmpty: - if expectedType != nil: + if expectedType != nil and oldType != expectedType: n[^1] = fitNodeConsiderViewType(c, expectedType, n[^1], n[^1].info) result = n[^1].typ else: diff --git a/tests/objects/tobject_default_value.nim b/tests/objects/tobject_default_value.nim index cdc6016e0..150fb0876 100644 --- a/tests/objects/tobject_default_value.nim +++ b/tests/objects/tobject_default_value.nim @@ -3,7 +3,7 @@ discard """ targets: "c cpp js" """ -import std/[times, macros] +import std/[times, macros, tables] type Guess = object @@ -236,6 +236,7 @@ template main {.dirty.} = doAssert x.obj.name.scale == 1 when nimvm: + # todo discard "fixme" else: when defined(gcArc) or defined(gcOrc): @@ -523,5 +524,41 @@ template main {.dirty.} = discard oToEither(O()) + block: # bug #20695 + type + Default = object + tabs: Table[string, int] = initTable[string, int]() + + let d = default(Default) + doAssert d.tabs.len == 0 + + block: + type + Default = object + tabs: Table[string, int] = Table[string, int]() + + let d = default(Default) + doAssert d.tabs.len == 0 + + + block: + type DjangoDateTime = distinct DateTime + + type Default = object + data: DjangoDateTime = DjangoDateTime(DateTime()) + + let x = default(Default) + doAssert x.data is DjangoDateTime + + block: + type DjangoDateTime = distinct DateTime + + type Default = object + data = DjangoDateTime(DateTime()) + + let x = default(Default) + doAssert x.data is DjangoDateTime + + static: main() main() From 7837e57e93b1cb89f5a80a779e853bad070de777 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francis=20Th=C3=A9rien?= Date: Fri, 24 Feb 2023 03:07:35 -0500 Subject: [PATCH 2429/2472] Disable threads for os:any (#21425) --- config/nim.cfg | 5 +++++ lib/std/typedthreads.nim | 2 ++ 2 files changed, 7 insertions(+) diff --git a/config/nim.cfg b/config/nim.cfg index 973de077a..ef93ec99e 100644 --- a/config/nim.cfg +++ b/config/nim.cfg @@ -100,6 +100,11 @@ nimblepath="$home/.nimble/pkgs/" gcc.options.always %= "${gcc.options.always} -fsanitize=null -fsanitize-undefined-trap-on-error" @end +# Turn off threads support when compiling for bare-metal targets (--os:any) +@if any: + threads:off +@end + @if unix and mingw: # Cross compile for Windows from Linux/OSX using MinGW i386.windows.gcc.exe = "i686-w64-mingw32-gcc" diff --git a/lib/std/typedthreads.nim b/lib/std/typedthreads.nim index b45ce8a61..8359ca353 100644 --- a/lib/std/typedthreads.nim +++ b/lib/std/typedthreads.nim @@ -45,6 +45,8 @@ when defined(nimPreviewSlimSystem): when defined(genode): import genode/env +when hostOS == "any": + {.error: "Threads not implemented for os:any. Please compile with --threads:off.".} when hasAllocStack or defined(zephyr) or defined(freertos) or defined(nuttx) or defined(cpu16) or defined(cpu8): From d4782c9e42ff6cee9f674a376b1595583e08c2a3 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sat, 25 Feb 2023 00:53:04 +0800 Subject: [PATCH 2430/2472] closes #17864; add a test case (#21434) --- tests/vm/tmisc_vm.nim | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/vm/tmisc_vm.nim b/tests/vm/tmisc_vm.nim index af09e8871..1ad830b5f 100644 --- a/tests/vm/tmisc_vm.nim +++ b/tests/vm/tmisc_vm.nim @@ -445,3 +445,15 @@ static: needSecondIdentifier.mgetOrPut(firstPart, @[]).add((name, pattern)) doAssert needSecondIdentifier[0] == @[("aaaa", "bbbb"), ("aaaaa", "bbbbb"), ("aaaaaa", "bbbbbb"), ("aaaaaaa", "bbbbbbb"), ("aaaaaaaa", "bbbbb")] + +# bug #17864 +macro transform*(fn: typed) = + quote do: + `fn` + +var map: Table[string, HashSet[string]] +proc publish*(): void {.transform.} = + map["k"] = init_hash_set[string]() + map["k"].incl "d" + +publish() From b2edfe7a02f49f07eb9fcf71cd500215725c217e Mon Sep 17 00:00:00 2001 From: Dmitry Arkhipenko <36101416+dkgitdev@users.noreply.github.com> Date: Sat, 25 Feb 2023 13:47:19 +0300 Subject: [PATCH 2431/2472] Fix: nintendoswitch compilation (#21368) * Fix: make nintendoswitch someGcc, remove symlink support for nintendoswitch, add getAppFilename for nintendoswitch * Fix: use getApplHeuristic on nintendoswitch --- lib/posix/posix.nim | 8 ++++++-- lib/pure/os.nim | 4 +++- lib/std/private/ossymlinks.nim | 2 +- lib/std/sysatomics.nim | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 07652c64d..12fc8fd57 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -584,7 +584,8 @@ proc pread*(a1: cint, a2: pointer, a3: int, a4: Off): int {. proc pwrite*(a1: cint, a2: pointer, a3: int, a4: Off): int {. importc, header: "".} proc read*(a1: cint, a2: pointer, a3: int): int {.importc, header: "".} -proc readlink*(a1, a2: cstring, a3: int): int {.importc, header: "".} +when not defined(nintendoswitch): + proc readlink*(a1, a2: cstring, a3: int): int {.importc, header: "".} proc ioctl*(f: FileHandle, device: uint): int {.importc: "ioctl", header: "", varargs, tags: [WriteIOEffect].} ## A system call for device-specific input/output operations and other @@ -603,7 +604,10 @@ proc setsid*(): Pid {.importc, header: "".} proc setuid*(a1: Uid): cint {.importc, header: "".} proc sleep*(a1: cint): cint {.importc, header: "".} proc swab*(a1, a2: pointer, a3: int) {.importc, header: "".} -proc symlink*(a1, a2: cstring): cint {.importc, header: "".} +when not defined(nintendoswitch): + proc symlink*(a1, a2: cstring): cint {.importc, header: "".} +else: + proc symlink*(a1, a2: cstring): cint = -1 proc sync*() {.importc, header: "".} proc sysconf*(a1: cint): int {.importc, header: "".} proc tcgetpgrp*(a1: cint): Pid {.importc, header: "".} diff --git a/lib/pure/os.nim b/lib/pure/os.nim index ecceed671..3285c07df 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -254,7 +254,7 @@ proc findExe*(exe: string, followSymlinks: bool = true; for ext in extensions: var x = addFileExt(x, ext) if fileExists(x): - when not defined(windows): + when not (defined(windows) or defined(nintendoswitch)): while followSymlinks: # doubles as if here if x.symlinkExists: var r = newString(maxSymlinkLen) @@ -702,6 +702,8 @@ proc getAppFilename*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noW result = getApplHaiku() elif defined(openbsd): result = getApplOpenBsd() + elif defined(nintendoswitch): + result = "" # little heuristic that may work on other POSIX-like systems: if result.len == 0: diff --git a/lib/std/private/ossymlinks.nim b/lib/std/private/ossymlinks.nim index 53f59f939..6b2de6237 100644 --- a/lib/std/private/ossymlinks.nim +++ b/lib/std/private/ossymlinks.nim @@ -67,7 +67,7 @@ proc expandSymlink*(symlinkPath: string): string {.noWeirdTarget.} = ## ## See also: ## * `createSymlink proc`_ - when defined(windows): + when defined(windows) or defined(nintendoswitch): result = symlinkPath else: result = newString(maxSymlinkLen) diff --git a/lib/std/sysatomics.nim b/lib/std/sysatomics.nim index b7ccb4092..36a4e5537 100644 --- a/lib/std/sysatomics.nim +++ b/lib/std/sysatomics.nim @@ -15,7 +15,7 @@ when defined(nimPreviewSlimSystem): const hasThreadSupport = compileOption("threads") and not defined(nimscript) -const someGcc = defined(gcc) or defined(llvm_gcc) or defined(clang) +const someGcc = defined(gcc) or defined(llvm_gcc) or defined(clang) or defined(nintendoswitch) const someVcc = defined(vcc) or defined(clang_cl) type From 425225119a383511ac6f84ba5c02a691e6c51aea Mon Sep 17 00:00:00 2001 From: PhilippMDoerner Date: Sun, 26 Feb 2023 09:57:51 +0100 Subject: [PATCH 2432/2472] Add mention of breaking change about `[_]` in generics to changelog (#21437) Add mention of breaking change to changelog As described by #21435 , generic proc declarations making use of `_` as generic parameter are no longer possible. This is a change that was introduced by #21192 . During the debate over if this should be part of the intended spec in #21435 we concluded that it is and thus should be mentioned as a breaking change in the changelogs. --- changelogs/changelog_2_0_0.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/changelogs/changelog_2_0_0.md b/changelogs/changelog_2_0_0.md index c8427cc2a..e081865be 100644 --- a/changelogs/changelog_2_0_0.md +++ b/changelogs/changelog_2_0_0.md @@ -149,6 +149,13 @@ proc foo(_, _: int): int = 123 echo foo(1, 2) ``` +- Underscores (`_`) as generic parameters are not supported and cannot be used. + Generics that use `_` as parameters will no longer compile requires you to replace `_` with something else: + + ```nim + proc foo[_](t: typedesc[_]): string = "BAR" # Can not compile + proc foo[T](t: typedesc[T]): string = "BAR" # Can compile + ``` - - Added the `--legacy:verboseTypeMismatch` switch to get legacy type mismatch error messages. From 4ae598762e7dac1886b481a48ae0875843e5153f Mon Sep 17 00:00:00 2001 From: Constantine Molchanov Date: Mon, 27 Feb 2023 02:56:43 +0300 Subject: [PATCH 2433/2472] fixes #21439; Add tyOpenArray to genTypeInfo. (#21440) * fixes #21439; Add tyOpenArray to genTypeInfo. * Add test. --- compiler/jstypes.nim | 2 +- tests/js/t21439.nim | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 tests/js/t21439.nim diff --git a/compiler/jstypes.nim b/compiler/jstypes.nim index 56d075af7..4b4ca9fe7 100644 --- a/compiler/jstypes.nim +++ b/compiler/jstypes.nim @@ -133,7 +133,7 @@ proc genTypeInfo(p: PProc, typ: PType): Rope = "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" % [result, rope(ord(t.kind))] prepend(p.g.typeInfo, s) - of tyVar, tyLent, tyRef, tyPtr, tySequence, tyRange, tySet: + of tyVar, tyLent, tyRef, tyPtr, tySequence, tyRange, tySet, tyOpenArray: var s = "var $1 = {size: 0, kind: $2, base: null, node: null, finalizer: null};$n" % [result, rope(ord(t.kind))] diff --git a/tests/js/t21439.nim b/tests/js/t21439.nim new file mode 100644 index 000000000..972356cd0 --- /dev/null +++ b/tests/js/t21439.nim @@ -0,0 +1,14 @@ +discard """ + action: "compile" +""" + +proc test(a: openArray[string]): proc = + result = proc = + for i in a: + discard i + + +const a = ["t1", "t2"] + +discard test(a) + From 89a60939f8b0c60e8abf63b698491a8b138da772 Mon Sep 17 00:00:00 2001 From: Ikko Eltociear Ashimine Date: Mon, 27 Feb 2023 08:57:02 +0900 Subject: [PATCH 2434/2472] Fix typo in sourcemap.nim (#21438) seperated -> separated --- compiler/sourcemap.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/sourcemap.nim b/compiler/sourcemap.nim index 2245250ac..2fcc50bbe 100644 --- a/compiler/sourcemap.nim +++ b/compiler/sourcemap.nim @@ -164,7 +164,7 @@ func toSourceMap*(info: SourceInfo, file: string): SourceMap {.raises: [].} = result.names = info.names # Convert nodes into mappings. # Mappings are split into blocks where each block referes to a line in the outputted JS. - # Blocks can be seperated into statements which refere to tokens on the line. + # Blocks can be separated into statements which refere to tokens on the line. # Since the mappings depend on previous values we need to # keep track of previous file, name, etc var From 6fea221d65c999ef7923bf51bbdaa8421cb2e9df Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Mon, 27 Feb 2023 07:57:36 +0800 Subject: [PATCH 2435/2472] Overrides `=copy` for `PackedSets` (#21417) --- lib/std/packedsets.nim | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/lib/std/packedsets.nim b/lib/std/packedsets.nim index 1e2892658..e8e03af9e 100644 --- a/lib/std/packedsets.nim +++ b/lib/std/packedsets.nim @@ -12,11 +12,6 @@ ## ## Supports any Ordinal type. ## -## .. note:: Currently the assignment operator `=` for `PackedSet[A]` -## performs some rather meaningless shallow copy. Since Nim currently does -## not allow the assignment operator to be overloaded, use the `assign proc -## <#assign,PackedSet[A],PackedSet[A]>`_ to get a deep copy. -## ## See also ## ======== ## * `sets module `_ for more general hash sets @@ -412,18 +407,9 @@ proc isNil*[A](x: PackedSet[A]): bool {.inline.} = x.head.isNil and x.elems == 0 -proc assign*[A](dest: var PackedSet[A], src: PackedSet[A]) = +proc `=copy`*[A](dest: var PackedSet[A], src: PackedSet[A]) = ## Copies `src` to `dest`. ## `dest` does not need to be initialized by the `initPackedSet proc <#initPackedSet>`_. - runnableExamples: - var - a = initPackedSet[int]() - b = initPackedSet[int]() - b.incl(5) - b.incl(7) - a.assign(b) - assert len(a) == 2 - if src.elems <= src.a.len: dest.data = @[] dest.max = 0 @@ -452,6 +438,19 @@ proc assign*[A](dest: var PackedSet[A], src: PackedSet[A]) = dest.data[h] = n it = it.next +proc assign*[A](dest: var PackedSet[A], src: PackedSet[A]) {.inline, deprecated.} = + ## Copies `src` to `dest`. + ## `dest` does not need to be initialized by the `initPackedSet proc <#initPackedSet>`_. + runnableExamples: + var + a = initPackedSet[int]() + b = initPackedSet[int]() + b.incl(5) + b.incl(7) + a.assign(b) + assert len(a) == 2 + `=copy`(dest, src) + proc union*[A](s1, s2: PackedSet[A]): PackedSet[A] = ## Returns the union of the sets `s1` and `s2`. ## From 83e33207250643dedc541e44ecf32d58f3edd097 Mon Sep 17 00:00:00 2001 From: Antonis Geralis <43617260+planetis-m@users.noreply.github.com> Date: Tue, 28 Feb 2023 15:17:21 +0200 Subject: [PATCH 2436/2472] Use a cast to suppress KeyError raises (#21451) --- lib/pure/json.nim | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index e881fbff7..d91da3545 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -465,11 +465,12 @@ proc `==`*(a, b: JsonNode): bool {.noSideEffect.} = if a.fields.len != b.fields.len: return false for key, val in a.fields: if not b.fields.hasKey(key): return false - when defined(nimHasEffectsOf): - {.noSideEffect.}: + {.cast(raises: []).}: + when defined(nimHasEffectsOf): + {.noSideEffect.}: + if b.fields[key] != val: return false + else: if b.fields[key] != val: return false - else: - if b.fields[key] != val: return false result = true proc hash*(n: OrderedTable[string, JsonNode]): Hash {.noSideEffect.} From dd629c8f450aa6bc871b54e4e549ad5fea2979a3 Mon Sep 17 00:00:00 2001 From: Century Systems Date: Tue, 28 Feb 2023 22:38:50 +0900 Subject: [PATCH 2437/2472] asyncdispatch: for NuttX, add destructor to clear global dispatcher. (#21432) * asyncdispatch: for NuttX, add destructor to clear global dispatcher using atexit(). Signed-off-by: Takeyoshi Kikuchi * std: exitprocs: remove "when defined(nuttx)" block. Signed-off-by: Takeyoshi Kikuchi --------- Signed-off-by: Takeyoshi Kikuchi --- lib/pure/asyncdispatch.nim | 13 ++++++++++++- lib/std/exitprocs.nim | 1 + 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 2bfa8587c..63062d8da 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -1216,6 +1216,15 @@ else: var gDisp{.threadvar.}: owned PDispatcher ## Global dispatcher + when defined(nuttx): + import std/exitprocs + + proc cleanDispatcher() {.noconv.} = + gDisp = nil + + proc addFinalyzer() = + addExitProc(cleanDispatcher) + proc setGlobalDispatcher*(disp: owned PDispatcher) = if not gDisp.isNil: assert gDisp.callbacks.len == 0 @@ -1225,6 +1234,8 @@ else: proc getGlobalDispatcher*(): PDispatcher = if gDisp.isNil: setGlobalDispatcher(newDispatcher()) + when defined(nuttx): + addFinalyzer() result = gDisp proc getIoHandler*(disp: PDispatcher): Selector[AsyncData] = @@ -1591,7 +1602,7 @@ else: owned(Future[tuple[address: string, client: AsyncFD]]) = var retFuture = newFuture[tuple[address: string, client: AsyncFD]]("acceptAddr") - proc cb(sock: AsyncFD): bool = + proc cb(sock: AsyncFD): bool {.gcsafe.} = result = true var sockAddress: Sockaddr_storage var addrLen = sizeof(sockAddress).SockLen diff --git a/lib/std/exitprocs.nim b/lib/std/exitprocs.nim index d63c1abc5..48b4fca7f 100644 --- a/lib/std/exitprocs.nim +++ b/lib/std/exitprocs.nim @@ -45,6 +45,7 @@ proc callClosures() {.noconv.} = case fun.kind of kClosure: fun.fun1() of kNoconv: fun.fun2() + gFuns.setLen(0) template fun() = if gFuns.len == 0: From 39d0a93d0e0bc2ebf24a9801fe49cdb71c085527 Mon Sep 17 00:00:00 2001 From: Constantine Molchanov Date: Wed, 1 Mar 2023 05:30:38 +0300 Subject: [PATCH 2438/2472] Fix #21452; enable Norm in important packages. (#21455) * fix #21452; enable Norm in important packages. * Run test that doesn't require SQLite. --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index b3a4bfb82..0af0eb93e 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -119,7 +119,7 @@ pkg "nimterop", "nimble minitest" pkg "nimwc", "nim c nimwc.nim", allowFailure = true pkg "nimx", "nim c --threads:on test/main.nim", allowFailure = true pkg "nitter", "nim c src/nitter.nim", "https://github.com/zedeus/nitter" -pkg "norm", "testament r tests/sqlite/trows.nim", allowFailure = true +pkg "norm", "testament r tests/common/tmodel.nim" pkg "npeg", "nimble testarc" pkg "numericalnim", "nimble nimCI" pkg "optionsutils" From c4dd0c43016faadbdaaddb2decf95dc881eb772e Mon Sep 17 00:00:00 2001 From: heterodoxic <122719743+heterodoxic@users.noreply.github.com> Date: Wed, 1 Mar 2023 17:17:05 +0100 Subject: [PATCH 2439/2472] suggestion for a simple fix for #21279 (#21378) --- compiler/cgen.nim | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 60623b72d..c346a0b78 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -480,6 +480,9 @@ proc resetLoc(p: BProc, loc: var TLoc) = # on the bytes following the m_type field? genObjectInit(p, cpsStmts, loc.t, loc, constructObj) +proc isOrHasImportedCppType(typ: PType): bool = + searchTypeFor(typ.skipTypes({tyRef}), isImportedCppType) + proc constructLoc(p: BProc, loc: var TLoc, isTemp = false) = let typ = loc.t if optSeqDestructors in p.config.globalOptions and skipTypes(typ, abstractInst + {tyStatic}).kind in {tyString, tySequence}: @@ -497,7 +500,7 @@ proc constructLoc(p: BProc, loc: var TLoc, isTemp = false) = if not isTemp or containsGarbageCollectedRef(loc.t): # don't use nimZeroMem for temporary values for performance if we can # avoid it: - if not isImportedCppType(typ): + if not isOrHasImportedCppType(typ): linefmt(p, cpsStmts, "#nimZeroMem((void*)$1, sizeof($2));$n", [addrLoc(p.config, loc), getTypeDesc(p.module, typ, mapTypeChooser(loc))]) genObjectInit(p, cpsStmts, loc.t, loc, constructObj) @@ -517,7 +520,10 @@ proc initLocalVar(p: BProc, v: PSym, immediateAsgn: bool) = proc getTemp(p: BProc, t: PType, result: var TLoc; needsInit=false) = inc(p.labels) result.r = "T" & rope(p.labels) & "_" - linefmt(p, cpsLocals, "$1 $2;$n", [getTypeDesc(p.module, t, skVar), result.r]) + if p.module.compileToCpp and isOrHasImportedCppType(t): + linefmt(p, cpsLocals, "$1 $2{};$n", [getTypeDesc(p.module, t, skVar), result.r]) + else: + linefmt(p, cpsLocals, "$1 $2;$n", [getTypeDesc(p.module, t, skVar), result.r]) result.k = locTemp result.lode = lodeTyp t result.storage = OnStack @@ -575,7 +581,7 @@ proc assignLocalVar(p: BProc, n: PNode) = # this need not be fulfilled for inline procs; they are regenerated # for each module that uses them! let nl = if optLineDir in p.config.options: "" else: "\L" - let decl = localVarDecl(p, n) & ";" & nl + let decl = localVarDecl(p, n) & (if p.module.compileToCpp and isOrHasImportedCppType(n.typ): "{};" else: ";") & nl line(p, cpsLocals, decl) include ccgthreadvars From 1b1412f3d148b02fb553f37d84505745cf3fb435 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 2 Mar 2023 00:18:09 +0800 Subject: [PATCH 2440/2472] fixes #10938; fixes #13312; fixes #13918; fixes #20985; always initializes global variables with null values in VM (#21351) * fixes #10938; always initialize global variable in VM * fixes importc vars * there is a pre-existing issue regarding closure types in the VM * add tests --- compiler/vmgen.nim | 12 ++++++++++++ tests/vm/tvmmisc.nim | 46 ++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index bdf7b679e..be7938e53 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1909,6 +1909,18 @@ proc genVarSection(c: PCtx; n: PNode) = c.genAdditionalCopy(a[2], opcWrDeref, tmp, 0, val) c.freeTemp(val) c.freeTemp(tmp) + elif not importcCondVar(s) and not (s.typ.kind == tyProc and s.typ.callConv == ccClosure): # fixes #10938 + # there is a pre-existing issue with closure types in VM + # if `(var s: proc () = default(proc ()); doAssert s == nil)` works for you; + # you might remove the second condition. + # the problem is that closure types are tuples in VM, but the types of its children + # shouldn't have the same type as closure types. + let tmp = c.genx(a[0], {gfNodeAddr}) + let sa = getNullValue(s.typ, a.info, c.config) + let val = c.genx(sa) + c.genAdditionalCopy(sa, opcWrDeref, tmp, 0, val) + c.freeTemp(val) + c.freeTemp(tmp) else: setSlot(c, s) if a[2].kind == nkEmpty: diff --git a/tests/vm/tvmmisc.nim b/tests/vm/tvmmisc.nim index 2ce4703b0..673e3e965 100644 --- a/tests/vm/tvmmisc.nim +++ b/tests/vm/tvmmisc.nim @@ -564,14 +564,14 @@ block: block: static: let x = int 1 - echo x.type # Foo + doAssert $(x.type) == "Foo" # Foo block: static: let x = int 1 let y = x + 1 # Error: unhandled exception: value out of range: -8 notin 0 .. 65535 [RangeDefect] - echo y + doAssert y == 2 type Atom* = object @@ -601,3 +601,45 @@ proc foo(s: sink string) = doAssert s.len == 3 static: foo("abc") + + +static: + for i in '1' .. '2': # bug #10938 + var s: set[char] + doAssert s == {} + incl(s, i) + + for _ in 0 ..< 3: # bug #13312 + var s: string + s.add("foo") + doAssert s == "foo" + + for i in 1 .. 5: # bug #13918 + var arr: array[3, int] + var val: int + doAssert arr == [0, 0, 0] and val == 0 + for j in 0 ..< len(arr): + arr[j] = i + val = i + +# bug #20985 +let a = block: + var groups: seq[seq[int]] + for i in 0 ..< 3: + var group: seq[int] + for j in 0 ..< 3: + group.add j + groups.add group + groups + +const b = block: + var groups: seq[seq[int]] + for i in 0 ..< 3: + var group: seq[int] + for j in 0 ..< 3: + group.add j + groups.add group + groups + +doAssert a == @[@[0, 1, 2], @[0, 1, 2], @[0, 1, 2]] +doAssert b == @[@[0, 1, 2], @[0, 1, 2], @[0, 1, 2]] From 89d8c0b24678a0b65820692069adffc7bec38ae7 Mon Sep 17 00:00:00 2001 From: Anna Date: Wed, 1 Mar 2023 22:00:58 +0500 Subject: [PATCH 2441/2472] tests/stylecheck: make sure necessary hints are enabled (#21240) --- tests/stylecheck/t20397_1.nim | 4 ++-- tests/stylecheck/treject.nim | 4 ++-- tests/stylecheck/tusages.nim | 6 ++---- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/stylecheck/t20397_1.nim b/tests/stylecheck/t20397_1.nim index 24f5791f8..76c03dca1 100644 --- a/tests/stylecheck/t20397_1.nim +++ b/tests/stylecheck/t20397_1.nim @@ -1,8 +1,8 @@ discard """ - matrix: "--styleCheck:off" + matrix: "--styleCheck:off --hint:Name:on" """ {.hintAsError[Name]:on.} var a_b = 1 discard a_b -{.hintAsError[Name]:off.} \ No newline at end of file +{.hintAsError[Name]:off.} diff --git a/tests/stylecheck/treject.nim b/tests/stylecheck/treject.nim index b9d97a58d..458a2d039 100644 --- a/tests/stylecheck/treject.nim +++ b/tests/stylecheck/treject.nim @@ -1,7 +1,7 @@ discard """ action: reject nimout: '''treject.nim(14, 13) Error: 'iD' should be: 'id' [field declared in treject.nim(9, 5)]''' - matrix: "--styleCheck:error --styleCheck:usages" + matrix: "--styleCheck:error --styleCheck:usages --hint:Name:on" """ type @@ -14,4 +14,4 @@ template hello = echo name.iD echo iD -hello() \ No newline at end of file +hello() diff --git a/tests/stylecheck/tusages.nim b/tests/stylecheck/tusages.nim index 2f99c70c5..b689dabb3 100644 --- a/tests/stylecheck/tusages.nim +++ b/tests/stylecheck/tusages.nim @@ -1,11 +1,9 @@ discard """ action: reject - nimout: '''tusages.nim(22, 5) Error: 'BAD_STYLE' should be: 'BADSTYLE' [proc declared in tusages.nim(11, 6)]''' - matrix: "--styleCheck:error --styleCheck:usages" + nimout: '''tusages.nim(20, 5) Error: 'BAD_STYLE' should be: 'BADSTYLE' [proc declared in tusages.nim(9, 6)]''' + matrix: "--styleCheck:error --styleCheck:usages --hint:all:off --hint:Name:on" """ - - import strutils proc BADSTYLE(c: char) = discard From cf083af262d03f62af302a28920f7bfa4b0e6233 Mon Sep 17 00:00:00 2001 From: Anna Date: Wed, 1 Mar 2023 22:01:58 +0500 Subject: [PATCH 2442/2472] tests: explicitly enable stack traces where needed (#21236) * tests/assert/tassert_c.nim: explicitly enable stack traces * tests/errmsgs: explicitly enable stack traces --- tests/assert/tassert_c.nim | 2 +- tests/errmsgs/tproper_stacktrace.nim | 31 ++++++++++++++------------- tests/errmsgs/tproper_stacktrace2.nim | 3 ++- tests/errmsgs/tproper_stacktrace3.nim | 3 ++- 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/tests/assert/tassert_c.nim b/tests/assert/tassert_c.nim index 4d49a6035..e3e3b8147 100644 --- a/tests/assert/tassert_c.nim +++ b/tests/assert/tassert_c.nim @@ -1,5 +1,5 @@ discard """ - cmd: "nim $target $options -d:nimPreviewSlimSystem --excessiveStackTrace:off $file" + matrix: "-d:nimPreviewSlimSystem --stackTrace:on --excessiveStackTrace:off" output: '''true''' """ import std/assertions diff --git a/tests/errmsgs/tproper_stacktrace.nim b/tests/errmsgs/tproper_stacktrace.nim index 8617984fb..b0a008840 100644 --- a/tests/errmsgs/tproper_stacktrace.nim +++ b/tests/errmsgs/tproper_stacktrace.nim @@ -1,4 +1,5 @@ discard """ + matrix: "--stackTrace:on --hint:all:off --warnings:off" output: '''ok''' """ import strscans, strutils @@ -76,10 +77,10 @@ when true: bar() const expectedStackTrace = """ - tproper_stacktrace.nim(86) tproper_stacktrace - tproper_stacktrace.nim(76) foo - tproper_stacktrace.nim(73) bar - tproper_stacktrace.nim(7) raiseTestException + tproper_stacktrace.nim(87) tproper_stacktrace + tproper_stacktrace.nim(77) foo + tproper_stacktrace.nim(74) bar + tproper_stacktrace.nim(8) raiseTestException """ verifyStackTrace expectedStackTrace: @@ -93,9 +94,9 @@ when true: bar(x) const expectedStackTrace = """ - tproper_stacktrace.nim(103) tproper_stacktrace - tproper_stacktrace.nim(90) bar - tproper_stacktrace.nim(7) raiseTestException + tproper_stacktrace.nim(104) tproper_stacktrace + tproper_stacktrace.nim(91) bar + tproper_stacktrace.nim(8) raiseTestException """ verifyStackTrace expectedStackTrace: @@ -110,10 +111,10 @@ when true: bar() const expectedStackTrace = """ - tproper_stacktrace.nim(120) tproper_stacktrace - tproper_stacktrace.nim(110) foo - tproper_stacktrace.nim(107) bar - tproper_stacktrace.nim(7) raiseTestException + tproper_stacktrace.nim(121) tproper_stacktrace + tproper_stacktrace.nim(111) foo + tproper_stacktrace.nim(108) bar + tproper_stacktrace.nim(8) raiseTestException """ verifyStackTrace expectedStackTrace: @@ -129,10 +130,10 @@ when true: bar() const expectedStackTrace = """ - tproper_stacktrace.nim(139) tproper_stacktrace - tproper_stacktrace.nim(129) foo - tproper_stacktrace.nim(125) baz - tproper_stacktrace.nim(7) raiseTestException + tproper_stacktrace.nim(140) tproper_stacktrace + tproper_stacktrace.nim(130) foo + tproper_stacktrace.nim(126) baz + tproper_stacktrace.nim(8) raiseTestException """ verifyStackTrace expectedStackTrace: diff --git a/tests/errmsgs/tproper_stacktrace2.nim b/tests/errmsgs/tproper_stacktrace2.nim index 44b208c87..5a6ca3fcb 100644 --- a/tests/errmsgs/tproper_stacktrace2.nim +++ b/tests/errmsgs/tproper_stacktrace2.nim @@ -1,5 +1,6 @@ discard """ - outputsub: '''tproper_stacktrace2.nim(20) main''' + matrix: "--stackTrace:on" + outputsub: '''tproper_stacktrace2.nim(21) main''' exitcode: 1 """ diff --git a/tests/errmsgs/tproper_stacktrace3.nim b/tests/errmsgs/tproper_stacktrace3.nim index c292fa092..97d63e6ec 100644 --- a/tests/errmsgs/tproper_stacktrace3.nim +++ b/tests/errmsgs/tproper_stacktrace3.nim @@ -1,5 +1,6 @@ discard """ - outputsub: '''tproper_stacktrace3.nim(21) main''' + matrix: "--stackTrace:on" + outputsub: '''tproper_stacktrace3.nim(22) main''' exitcode: 1 """ From 070938720ad1a06e3273c164db53edb470bdb9a8 Mon Sep 17 00:00:00 2001 From: Anna Date: Thu, 2 Mar 2023 01:29:23 +0500 Subject: [PATCH 2443/2472] tests/errmsgs/tcall_with_default_arg.nim: sync (#21237) --- tests/errmsgs/tcall_with_default_arg.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/errmsgs/tcall_with_default_arg.nim b/tests/errmsgs/tcall_with_default_arg.nim index 1cc86638f..44752f1ee 100644 --- a/tests/errmsgs/tcall_with_default_arg.nim +++ b/tests/errmsgs/tcall_with_default_arg.nim @@ -1,5 +1,5 @@ discard """ -outputsub: '''tcall_with_default_arg.nim(16) anotherFoo''' +outputsub: '''tcall_with_default_arg.nim(8) fail''' exitcode: 1 """ # issue: #5604 From 9948fed919389229a48347aa9fa5adce9b7e0a98 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 2 Mar 2023 12:25:55 +0800 Subject: [PATCH 2444/2472] fixes #21326; fixes #7375; fixes #11986; fixes #9607; rework quote do; `getAst` uses type info to annotate the type of quoted variables; no more type erasures for quoted variables (#21433) * fixes #21326; getAst uses type info to annotateType quoted variables * simplify logics; sem types first * fixes important packages * add testcases * tiny --- compiler/semexprs.nim | 11 ++- compiler/vm.nim | 5 +- testament/important_packages.nim | 6 +- tests/stdlib/tmacros.nim | 136 +++++++++++++++++++++++++++++++ 4 files changed, 150 insertions(+), 8 deletions(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 7454c7f8e..9cf206b4c 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -2203,10 +2203,13 @@ proc semQuoteAst(c: PContext, n: PNode): PNode = if ids.len > 0: dummyTemplate[paramsPos] = newNodeI(nkFormalParams, n.info) dummyTemplate[paramsPos].add getSysSym(c.graph, n.info, "untyped").newSymNode # return type - ids.add getSysSym(c.graph, n.info, "untyped").newSymNode # params type - ids.add c.graph.emptyNode # no default value - dummyTemplate[paramsPos].add newTreeI(nkIdentDefs, n.info, ids) - + dummyTemplate[paramsPos].add newTreeI(nkIdentDefs, n.info, ids[0], getSysSym(c.graph, n.info, "typed").newSymNode, c.graph.emptyNode) + for i in 1.. Date: Thu, 2 Mar 2023 12:29:40 +0800 Subject: [PATCH 2445/2472] fixes #19291; implements `wasMoved` hook (#21303) * fixes #19291; implements `wasMoved` hook * basics * checkpoint * finish `wasMoved` * add a test for #19291 * add documentation and changelog * work `attachedWasMoved` with generics * fixes optimizer * register `=wasMoved` * handle wasMoved magcis * check another round * some patches * try `op == nil` * nicer * generate `wasMoved` before `destroy` * try again * fixes tests * default wasMoved * Update tests/destructor/tv2_cast.nim * Update tests/destructor/tv2_cast.nim * Update tests/arc/topt_refcursors.nim --- changelogs/changelog_2_0_0.md | 2 ++ compiler/ast.nim | 3 +- compiler/injectdestructors.nim | 17 +++++++--- compiler/liftdestructors.nim | 41 ++++++++++++++++++++--- compiler/optimizer.nim | 9 +++-- compiler/semstmts.nim | 5 ++- doc/manual.md | 2 +- tests/arc/topt_no_cursor.nim | 19 ++++++----- tests/arc/topt_refcursors.nim | 8 +++-- tests/arc/topt_wasmoved_destroy_pairs.nim | 13 ++++--- tests/destructor/tv2_cast.nim | 14 ++++---- tests/destructor/twasmoved_error.nim | 37 ++++++++++++++++++++ 12 files changed, 131 insertions(+), 39 deletions(-) create mode 100644 tests/destructor/twasmoved_error.nim diff --git a/changelogs/changelog_2_0_0.md b/changelogs/changelog_2_0_0.md index e081865be..cbe755478 100644 --- a/changelogs/changelog_2_0_0.md +++ b/changelogs/changelog_2_0_0.md @@ -341,6 +341,8 @@ - IBM Z architecture and macOS m1 arm64 architecture are supported. +- `=wasMoved` can be overridden by users. + ## Compiler changes - The `gc` switch has been renamed to `mm` ("memory management") in order to reflect the diff --git a/compiler/ast.nim b/compiler/ast.nim index aa70560ed..f93c8d910 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -935,6 +935,7 @@ type TTypeSeq* = seq[PType] TTypeAttachedOp* = enum ## as usual, order is important here + attachedWasMoved, attachedDestructor, attachedAsgn, attachedSink, @@ -1502,7 +1503,7 @@ proc newProcNode*(kind: TNodeKind, info: TLineInfo, body: PNode, const AttachedOpToStr*: array[TTypeAttachedOp, string] = [ - "=destroy", "=copy", "=sink", "=trace", "=deepcopy"] + "=wasMoved", "=destroy", "=copy", "=sink", "=trace", "=deepcopy"] proc `$`*(s: PSym): string = if s != nil: diff --git a/compiler/injectdestructors.nim b/compiler/injectdestructors.nim index 20e790202..13141b765 100644 --- a/compiler/injectdestructors.nim +++ b/compiler/injectdestructors.nim @@ -354,11 +354,18 @@ It is best to factor out piece of object that needs custom destructor into separ result.add newTree(nkFastAsgn, le, tmp) proc genWasMoved(c: var Con, n: PNode): PNode = - result = newNodeI(nkCall, n.info) - result.add(newSymNode(createMagic(c.graph, c.idgen, "wasMoved", mWasMoved))) - result.add copyTree(n) #mWasMoved does not take the address - #if n.kind != nkSym: - # message(c.graph.config, n.info, warnUser, "wasMoved(" & $n & ")") + let typ = n.typ.skipTypes({tyGenericInst, tyAlias, tySink}) + let op = getAttachedOp(c.graph, n.typ, attachedWasMoved) + if op != nil: + if sfError in op.flags: + c.checkForErrorPragma(n.typ, n, "=wasMoved") + result = genOp(c, op, n) + else: + result = newNodeI(nkCall, n.info) + result.add(newSymNode(createMagic(c.graph, c.idgen, "wasMoved", mWasMoved))) + result.add copyTree(n) #mWasMoved does not take the address + #if n.kind != nkSym: + # message(c.graph.config, n.info, warnUser, "wasMoved(" & $n & ")") proc genDefaultCall(t: PType; c: Con; info: TLineInfo): PNode = result = newNodeI(nkCall, info) diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim index 738f659b7..c7464d39e 100644 --- a/compiler/liftdestructors.nim +++ b/compiler/liftdestructors.nim @@ -88,6 +88,8 @@ proc defaultOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = let call = genBuiltin(c, mDefault, "default", x) call.typ = t body.add newAsgnStmt(x, call) + elif c.kind == attachedWasMoved: + body.add genBuiltin(c, mWasMoved, "wasMoved", x) proc genAddr(c: var TLiftCtx; x: PNode): PNode = if x.kind == nkHiddenDeref: @@ -145,6 +147,11 @@ proc destructorCall(c: var TLiftCtx; op: PSym; x: PNode): PNode = else: result = destroy +proc genWasMovedCall(c: var TLiftCtx; op: PSym; x: PNode): PNode = + result = newNodeIT(nkCall, x.info, op.typ[0]) + result.add(newSymNode(op)) + result.add genAddr(c, x) + proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool) = case n.kind of nkSym: @@ -442,6 +449,20 @@ proc considerUserDefinedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool = body.add newDeepCopyCall(c, op, x, y) result = true + of attachedWasMoved: + var op = getAttachedOp(c.g, t, attachedWasMoved) + if op != nil and sfOverriden in op.flags: + + if op.ast.isGenericRoutine: + # patch generic destructor: + op = instantiateGeneric(c, op, t, t.typeInst) + setAttachedOp(c.g, c.idgen.module, t, attachedWasMoved, op) + + #markUsed(c.g.config, c.info, op, c.g.usageSym) + onUse(c.info, op) + body.add genWasMovedCall(c, op, x) + result = true + proc declareCounter(c: var TLiftCtx; body: PNode; first: BiggestInt): PNode = var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), nextSymId(c.idgen), c.fn, c.info) temp.typ = getSysType(c.g, body.info, tyInt) @@ -524,6 +545,7 @@ proc fillSeqOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = if canFormAcycle(t.elemType): # follow all elements: forallElements(c, t, body, x, y) + of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "wasMoved", x) proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = createTypeBoundOps(c.g, c.c, t, body.info, c.idgen) @@ -561,6 +583,7 @@ proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = if op == nil: return # protect from recursion body.add newHookCall(c, op, x, y) + of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "wasMoved", x) proc fillStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = case c.kind @@ -576,6 +599,7 @@ proc fillStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = body.add genBuiltin(c, mDestroy, "destroy", x) of attachedTrace: discard "strings are atomic and have no inner elements that are to trace" + of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "wasMoved", x) proc cyclicType*(t: PType): bool = case t.kind @@ -674,6 +698,8 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = # If the ref is polymorphic we have to account for this body.add callCodegenProc(c.g, "nimTraceRefDyn", c.info, genAddrOf(x, c.idgen), y) #echo "can follow ", elemType, " static ", isFinal(elemType) + of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "wasMoved", x) + proc atomicClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = ## Closures are really like refs except they always use a virtual destructor @@ -722,6 +748,7 @@ proc atomicClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = of attachedDeepCopy: assert(false, "cannot happen") of attachedTrace: body.add callCodegenProc(c.g, "nimTraceRefDyn", c.info, genAddrOf(xenv, c.idgen), y) + of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "wasMoved", x) proc weakrefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = case c.kind @@ -746,6 +773,7 @@ proc weakrefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = body.sons.insert(des, 0) of attachedDeepCopy: assert(false, "cannot happen") of attachedTrace: discard + of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "wasMoved", x) proc ownedRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = var actions = newNodeI(nkStmtList, c.info) @@ -771,6 +799,7 @@ proc ownedRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = body.add genIf(c, x, actions) of attachedDeepCopy: assert(false, "cannot happen") of attachedTrace: discard + of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "wasMoved", x) proc closureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = if c.kind == attachedDeepCopy: @@ -805,6 +834,7 @@ proc closureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = body.sons.insert(des, 0) of attachedDeepCopy: assert(false, "cannot happen") of attachedTrace: discard + of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "wasMoved", x) proc ownedClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = let xx = genBuiltin(c, mAccessEnv, "accessEnv", x) @@ -820,6 +850,7 @@ proc ownedClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = body.add genIf(c, xx, actions) of attachedDeepCopy: assert(false, "cannot happen") of attachedTrace: discard + of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "wasMoved", x) proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode) = case t.kind @@ -935,7 +966,7 @@ proc symPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttachedOp result.typ = newProcType(info, nextTypeId(idgen), owner) result.typ.addParam dest - if kind != attachedDestructor: + if kind notin {attachedDestructor, attachedWasMoved}: result.typ.addParam src if kind == attachedAsgn and g.config.selectedGC == gcOrc and @@ -975,7 +1006,7 @@ proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp; let dest = result.typ.n[1].sym let d = newDeref(newSymNode(dest)) - let src = if kind == attachedDestructor: newNodeIT(nkSym, info, getSysType(g, info, tyPointer)) + let src = if kind in {attachedDestructor, attachedWasMoved}: newNodeIT(nkSym, info, getSysType(g, info, tyPointer)) else: newSymNode(result.typ.n[2].sym) # register this operation already: @@ -1103,15 +1134,15 @@ proc createTypeBoundOps(g: ModuleGraph; c: PContext; orig: PType; info: TLineInf # bug #15122: We need to produce all prototypes before entering the # mind boggling recursion. Hacks like these imply we should rewrite # this module. - var generics: array[attachedDestructor..attachedTrace, bool] - for k in attachedDestructor..lastAttached: + var generics: array[attachedWasMoved..attachedTrace, bool] + for k in attachedWasMoved..lastAttached: generics[k] = getAttachedOp(g, canon, k) != nil if not generics[k]: setAttachedOp(g, idgen.module, canon, k, symPrototype(g, canon, canon.owner, k, info, idgen)) # we generate the destructor first so that other operators can depend on it: - for k in attachedDestructor..lastAttached: + for k in attachedWasMoved..lastAttached: if not generics[k]: discard produceSym(g, c, canon, k, info, idgen) else: diff --git a/compiler/optimizer.nim b/compiler/optimizer.nim index 98939f80d..0b26e8d34 100644 --- a/compiler/optimizer.nim +++ b/compiler/optimizer.nim @@ -16,6 +16,8 @@ import from trees import exprStructuralEquivalent +import std/strutils + const nfMarkForDeletion = nfNone # faster than a lookup table @@ -110,16 +112,17 @@ proc analyse(c: var Con; b: var BasicBlock; n: PNode) = var reverse = false if n[0].kind == nkSym: let s = n[0].sym - if s.magic == mWasMoved: + let name = s.name.s.normalize + if s.magic == mWasMoved or name == "=wasmoved": b.wasMovedLocs.add n special = true - elif s.name.s == "=destroy": + elif name == "=destroy": if c.inFinally > 0 and (b.hasReturn or b.hasBreak): discard "cannot optimize away the destructor" else: c.wasMovedDestroyPair b, n special = true - elif s.name.s == "=sink": + elif name == "=sink": reverse = true if not special: diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 7dc976c19..6237e6eb0 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -1779,7 +1779,7 @@ proc whereToBindTypeHook(c: PContext; t: PType): PType = proc bindTypeHook(c: PContext; s: PSym; n: PNode; op: TTypeAttachedOp) = let t = s.typ var noError = false - let cond = if op == attachedDestructor: + let cond = if op in {attachedDestructor, attachedWasMoved}: t.len == 2 and t[0] == nil and t[1].kind == tyVar elif op == attachedTrace: t.len == 3 and t[0] == nil and t[1].kind == tyVar and t[2].kind == tyPointer @@ -1894,6 +1894,9 @@ proc semOverride(c: PContext, s: PSym, n: PNode) = of "=trace": if s.magic != mTrace: bindTypeHook(c, s, n, attachedTrace) + of "=wasmoved": + if s.magic != mWasMoved: + bindTypeHook(c, s, n, attachedWasMoved) else: if sfOverriden in s.flags: localError(c.config, n.info, errGenerated, diff --git a/doc/manual.md b/doc/manual.md index ef790c362..31c620ca7 100644 --- a/doc/manual.md +++ b/doc/manual.md @@ -4122,7 +4122,7 @@ the operator is in scope (including if it is private). ``` Type bound operators are: -`=destroy`, `=copy`, `=sink`, `=trace`, `=deepcopy`. +`=destroy`, `=copy`, `=sink`, `=trace`, `=deepcopy`, `=wasMoved`. These operations can be *overridden* instead of *overloaded*. This means that the implementation is automatically lifted to structured types. For instance, diff --git a/tests/arc/topt_no_cursor.nim b/tests/arc/topt_no_cursor.nim index 30d4c316c..50dfa26ac 100644 --- a/tests/arc/topt_no_cursor.nim +++ b/tests/arc/topt_no_cursor.nim @@ -1,7 +1,8 @@ discard """ nimoutFull: true cmd: '''nim c -r --warnings:off --hints:off --gc:arc --expandArc:newTarget --expandArc:delete --expandArc:p1 --expandArc:tt --hint:Performance:off --assertions:off --expandArc:extractConfig --expandArc:mergeShadowScope --expandArc:check $file''' - nimout: '''--expandArc: newTarget + nimout: ''' +--expandArc: newTarget var splat @@ -11,9 +12,9 @@ splat = splitDrive do: let blitTmp = path blitTmp :tmp = splat.drive -wasMoved(splat.drive) +`=wasMoved`(splat.drive) :tmp_1 = splat.path_1 -wasMoved(splat.path_1) +`=wasMoved`(splat.path_1) result = ( let blitTmp_1 = :tmp blitTmp_1, @@ -60,10 +61,10 @@ var try: it_cursor = x a = ( - wasMoved(:tmpD) + `=wasMoved`(:tmpD) `=copy`(:tmpD, it_cursor.key) :tmpD, - wasMoved(:tmpD_1) + `=wasMoved`(:tmpD_1) `=copy`(:tmpD_1, it_cursor.val) :tmpD_1) echo [ @@ -112,7 +113,7 @@ block :tmp: var :tmpD sym = shadowScope.symbols[i] addInterfaceDecl(c): - wasMoved(:tmpD) + `=wasMoved`(:tmpD) `=copy_1`(:tmpD, sym) :tmpD inc(i, 1) @@ -125,7 +126,7 @@ this.isValid = fileExists(this.value) if dirExists(this.value): var :tmpD par = (dir: - wasMoved(:tmpD) + `=wasMoved`(:tmpD) `=copy`(:tmpD, this.value) :tmpD, front: "") else: var @@ -133,10 +134,10 @@ if dirExists(this.value): :tmpD_2 :tmpD_3 par = (dir_1: parentDir(this.value), front_1: - wasMoved(:tmpD_1) + `=wasMoved`(:tmpD_1) `=copy`(:tmpD_1, :tmpD_3 = splitDrive do: - wasMoved(:tmpD_2) + `=wasMoved`(:tmpD_2) `=copy`(:tmpD_2, this.value) :tmpD_2 :tmpD_3.path) diff --git a/tests/arc/topt_refcursors.nim b/tests/arc/topt_refcursors.nim index c13d81bad..8c638a4a1 100644 --- a/tests/arc/topt_refcursors.nim +++ b/tests/arc/topt_refcursors.nim @@ -1,7 +1,8 @@ discard """ output: '''''' cmd: '''nim c --gc:arc --expandArc:traverse --hint:Performance:off $file''' - nimout: '''--expandArc: traverse + nimout: ''' +--expandArc: traverse var it_cursor @@ -22,12 +23,13 @@ try: `=copy`(ri_1, jt.ri) echo [jt.s] `=sink`(jt, ri_1) - wasMoved(ri_1) + `=wasMoved`(ri_1) finally: `=destroy`(ri_1) finally: `=destroy`(jt) --- end of expandArc ------------------------''' +-- end of expandArc ------------------------ +''' """ type diff --git a/tests/arc/topt_wasmoved_destroy_pairs.nim b/tests/arc/topt_wasmoved_destroy_pairs.nim index 2f971f112..6577d6787 100644 --- a/tests/arc/topt_wasmoved_destroy_pairs.nim +++ b/tests/arc/topt_wasmoved_destroy_pairs.nim @@ -1,7 +1,8 @@ discard """ output: '''''' cmd: '''nim c --gc:arc --expandArc:main --expandArc:tfor --hint:Performance:off $file''' - nimout: '''--expandArc: main + nimout: ''' +--expandArc: main var a @@ -29,6 +30,7 @@ try: x = f() block :tmp: var i_cursor + mixin inc var i_1 = 0 block :tmp_1: while i_1 < 4: @@ -37,25 +39,26 @@ try: if i_cursor == 2: return add(a): - wasMoved(:tmpD) + `=wasMoved`(:tmpD) `=copy`(:tmpD, x) :tmpD inc i_1, 1 if cond: add(a): let blitTmp = x - wasMoved(x) + `=wasMoved`(x) blitTmp else: add(b): let blitTmp_1 = x - wasMoved(x) + `=wasMoved`(x) blitTmp_1 finally: `=destroy`(x) `=destroy_1`(b) `=destroy_1`(a) --- end of expandArc ------------------------''' +-- end of expandArc ------------------------ +''' """ proc f(): seq[int] = diff --git a/tests/destructor/tv2_cast.nim b/tests/destructor/tv2_cast.nim index 917cf0eb3..6fa419996 100644 --- a/tests/destructor/tv2_cast.nim +++ b/tests/destructor/tv2_cast.nim @@ -4,7 +4,8 @@ discard """ @[1953719668, 875770417] destroying O1''' cmd: '''nim c --gc:arc --expandArc:main --expandArc:main1 --expandArc:main2 --expandArc:main3 --hints:off --assertions:off $file''' - nimout: '''--expandArc: main + nimout: ''' +--expandArc: main var data @@ -12,7 +13,7 @@ var :tmpD_1 :tmpD_2 data = - wasMoved(:tmpD) + `=wasMoved`(:tmpD) `=copy`(:tmpD, cast[string]( :tmpD_2 = encode(cast[seq[byte]]( :tmpD_1 = newString(100) @@ -32,7 +33,7 @@ var :tmpD_1 s = newString(100) data = - wasMoved(:tmpD) + `=wasMoved`(:tmpD) `=copy`(:tmpD, cast[string]( :tmpD_1 = encode(toOpenArrayByte(s, 0, len(s) - 1)) :tmpD_1)) @@ -50,7 +51,7 @@ var :tmpD_1 s = newSeq(100) data = - wasMoved(:tmpD) + `=wasMoved`(:tmpD) `=copy`(:tmpD, cast[string]( :tmpD_1 = encode(s) :tmpD_1)) @@ -67,7 +68,7 @@ var :tmpD_1 :tmpD_2 data = - wasMoved(:tmpD) + `=wasMoved`(:tmpD) `=copy`(:tmpD, cast[string]( :tmpD_2 = encode do: :tmpD_1 = newSeq(100) @@ -77,7 +78,8 @@ data = `=destroy`(:tmpD_2) `=destroy`(:tmpD_1) `=destroy_1`(data) --- end of expandArc ------------------------''' +-- end of expandArc ------------------------ +''' """ func encode*(src: openArray[byte]): seq[byte] = diff --git a/tests/destructor/twasmoved_error.nim b/tests/destructor/twasmoved_error.nim new file mode 100644 index 000000000..1cd57e3df --- /dev/null +++ b/tests/destructor/twasmoved_error.nim @@ -0,0 +1,37 @@ +discard """ + cmd: '''nim c --mm:arc $file''' + errormsg: "'=wasMoved' is not available for type ; routine: main" +""" + +# bug #19291 + +const + screenWidth = 800 + screenHeight = 450 + +var + ready = false +type + Game = object + +proc `=destroy`(x: var Game) = + assert ready, "Window is already opened" + ready = false + +proc `=sink`(x: var Game; y: Game) {.error.} +proc `=copy`(x: var Game; y: Game) {.error.} +proc `=wasMoved`(x: var Game) {.error.} + +proc initGame(width, height: int32, title: string): Game = + assert not ready, "Window is already closed" + ready = true + +proc update(x: Game) = discard + +proc main = + var g = initGame(screenWidth, screenHeight, "Tetris raylib") + g.update() + var g2 = g + echo "hello" + +main() From 9b5ae2b2eb2ba5aca219de109b03f1bb5637a777 Mon Sep 17 00:00:00 2001 From: Ivan Yonchovski Date: Thu, 2 Mar 2023 06:30:55 +0200 Subject: [PATCH 2446/2472] Define the version of nim package without using system module (#21415) This is follow up from https://github.com/nim-lang/Nim/pull/21313 --- nim.nimble | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nim.nimble b/nim.nimble index 713fc9256..380ffbce8 100644 --- a/nim.nimble +++ b/nim.nimble @@ -1,4 +1,5 @@ -version = system.NimVersion +include "lib/system/compilation.nim" +version = $NimMajor & "." & $NimMinor & "." & $NimPatch author = "Andreas Rumpf" description = "Compiler package providing the compiler sources as a library." license = "MIT" From 612abda4f40b2a0fd8dd0e4ad119c4415b9c34cb Mon Sep 17 00:00:00 2001 From: tersec Date: Thu, 2 Mar 2023 04:51:54 +0000 Subject: [PATCH 2447/2472] remove decades-deprecated Win32 API *A function support (#21315) --- lib/pure/asyncfile.nim | 11 +- lib/pure/browsers.nim | 2 +- lib/pure/memfiles.nim | 7 +- lib/pure/os.nim | 93 ++++-------- lib/pure/osproc.nim | 35 ++--- lib/std/cmdline.nim | 5 +- lib/std/private/oscommon.nim | 65 +++------ lib/std/private/osdirs.nim | 17 +-- lib/std/private/osfiles.nim | 43 ++---- lib/std/private/ospaths2.nim | 35 ++--- lib/std/private/ossymlinks.nim | 12 +- lib/std/syncio.nim | 4 +- lib/windows/winlean.nim | 258 ++++++++++----------------------- 13 files changed, 173 insertions(+), 414 deletions(-) diff --git a/lib/pure/asyncfile.nim b/lib/pure/asyncfile.nim index 88fa90406..96d5e6d2e 100644 --- a/lib/pure/asyncfile.nim +++ b/lib/pure/asyncfile.nim @@ -102,14 +102,9 @@ proc openAsync*(filename: string, mode = fmRead): AsyncFile = let flags = FILE_FLAG_OVERLAPPED or FILE_ATTRIBUTE_NORMAL let desiredAccess = getDesiredAccess(mode) let creationDisposition = getCreationDisposition(mode, filename) - when useWinUnicode: - let fd = createFileW(newWideCString(filename), desiredAccess, - FILE_SHARE_READ, - nil, creationDisposition, flags, 0) - else: - let fd = createFileA(filename, desiredAccess, - FILE_SHARE_READ, - nil, creationDisposition, flags, 0) + let fd = createFileW(newWideCString(filename), desiredAccess, + FILE_SHARE_READ, + nil, creationDisposition, flags, 0) if fd == INVALID_HANDLE_VALUE: raiseOSError(osLastError()) diff --git a/lib/pure/browsers.nim b/lib/pure/browsers.nim index b78034fe3..ab248db36 100644 --- a/lib/pure/browsers.nim +++ b/lib/pure/browsers.nim @@ -21,7 +21,7 @@ when defined(nimPreviewSlimSystem): when defined(windows): import winlean - when useWinUnicode and defined(nimPreviewSlimSystem): + when defined(nimPreviewSlimSystem): import std/widestrs from os import absolutePath else: diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim index 58c4888f1..74065bc2e 100644 --- a/lib/pure/memfiles.nim +++ b/lib/pure/memfiles.nim @@ -17,7 +17,7 @@ when defined(windows): import winlean - when useWinUnicode and defined(nimPreviewSlimSystem): + when defined(nimPreviewSlimSystem): import std/widestrs elif defined(posix): import posix @@ -199,10 +199,7 @@ proc open*(filename: string, mode: FileMode = fmRead, else: FILE_ATTRIBUTE_NORMAL or flags, 0) - when useWinUnicode: - result.fHandle = callCreateFile(createFileW, newWideCString(filename)) - else: - result.fHandle = callCreateFile(createFileA, filename) + result.fHandle = callCreateFile(createFileW, newWideCString(filename)) if result.fHandle == INVALID_HANDLE_VALUE: fail(osLastError(), "error opening file") diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 3285c07df..0569e6cab 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -422,32 +422,18 @@ proc expandFilename*(filename: string): string {.rtl, extern: "nos$1", ## Raises `OSError` in case of an error. Follows symlinks. when defined(windows): var bufsize = MAX_PATH.int32 - when useWinUnicode: - var unused: WideCString = nil - var res = newWideCString("", bufsize) - while true: - var L = getFullPathNameW(newWideCString(filename), bufsize, res, unused) - if L == 0'i32: - raiseOSError(osLastError(), filename) - elif L > bufsize: - res = newWideCString("", L) - bufsize = L - else: - result = res$L - break - else: - var unused: cstring = nil - result = newString(bufsize) - while true: - var L = getFullPathNameA(filename, bufsize, result, unused) - if L == 0'i32: - raiseOSError(osLastError(), filename) - elif L > bufsize: - result = newString(L) - bufsize = L - else: - setLen(result, L) - break + var unused: WideCString = nil + var res = newWideCString("", bufsize) + while true: + var L = getFullPathNameW(newWideCString(filename), bufsize, res, unused) + if L == 0'i32: + raiseOSError(osLastError(), filename) + elif L > bufsize: + res = newWideCString("", L) + bufsize = L + else: + result = res$L + break # getFullPathName doesn't do case corrections, so we have to use this convoluted # way of retrieving the true filename for x in walkFiles(result): @@ -483,14 +469,10 @@ proc createHardlink*(src, dest: string) {.noWeirdTarget.} = ## See also: ## * `createSymlink proc`_ when defined(windows): - when useWinUnicode: - var wSrc = newWideCString(src) - var wDst = newWideCString(dest) - if createHardLinkW(wDst, wSrc, nil) == 0: - raiseOSError(osLastError(), $(src, dest)) - else: - if createHardLinkA(dest, src, nil) == 0: - raiseOSError(osLastError(), $(src, dest)) + var wSrc = newWideCString(src) + var wDst = newWideCString(dest) + if createHardLinkW(wDst, wSrc, nil) == 0: + raiseOSError(osLastError(), $(src, dest)) else: if link(src, dest) != 0: raiseOSError(osLastError(), $(src, dest)) @@ -655,32 +637,18 @@ proc getAppFilename*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noW # /proc//path/a.out (complete pathname) when defined(windows): var bufsize = int32(MAX_PATH) - when useWinUnicode: - var buf = newWideCString("", bufsize) - while true: - var L = getModuleFileNameW(0, buf, bufsize) - if L == 0'i32: - result = "" # error! - break - elif L > bufsize: - buf = newWideCString("", L) - bufsize = L - else: - result = buf$L - break - else: - result = newString(bufsize) - while true: - var L = getModuleFileNameA(0, result, bufsize) - if L == 0'i32: - result = "" # error! - break - elif L > bufsize: - result = newString(L) - bufsize = L - else: - setLen(result, L) - break + var buf = newWideCString("", bufsize) + while true: + var L = getModuleFileNameW(0, buf, bufsize) + if L == 0'i32: + result = "" # error! + break + elif L > bufsize: + buf = newWideCString("", L) + bufsize = L + else: + result = buf$L + break elif defined(macosx): var size = cuint32(0) getExecPath1(nil, size) @@ -977,10 +945,7 @@ proc isHidden*(path: string): bool {.noWeirdTarget.} = assert ".foo/".isHidden when defined(windows): - when useWinUnicode: - wrapUnary(attributes, getFileAttributesW, path) - else: - var attributes = getFileAttributesA(path) + wrapUnary(attributes, getFileAttributesW, path) if attributes != -1'i32: result = (attributes and FILE_ATTRIBUTE_HIDDEN) != 0'i32 else: diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index bb6abc571..4cdd98c0d 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -714,22 +714,15 @@ when defined(windows) and not defined(useNimRtl): if len(workingDir) > 0: wd = workingDir if env != nil: e = buildEnv(env) if poEchoCmd in options: echo($cmdl) - when useWinUnicode: - var tmp = newWideCString(cmdl) - var ee = - if e.str.isNil: newWideCString(cstring(nil)) - else: newWideCString(e.str, e.len) - var wwd = newWideCString(wd) - var flags = NORMAL_PRIORITY_CLASS or CREATE_UNICODE_ENVIRONMENT - if poDaemon in options: flags = flags or CREATE_NO_WINDOW - success = winlean.createProcessW(nil, tmp, nil, nil, 1, flags, - ee, wwd, si, procInfo) - else: - var ee = - if e.str.isNil: cstring(nil) - else: cstring(e.str) - success = winlean.createProcessA(nil, - cmdl, nil, nil, 1, NORMAL_PRIORITY_CLASS, ee, wd, si, procInfo) + var tmp = newWideCString(cmdl) + var ee = + if e.str.isNil: newWideCString(cstring(nil)) + else: newWideCString(e.str, e.len) + var wwd = newWideCString(wd) + var flags = NORMAL_PRIORITY_CLASS or CREATE_UNICODE_ENVIRONMENT + if poDaemon in options: flags = flags or CREATE_NO_WINDOW + success = winlean.createProcessW(nil, tmp, nil, nil, 1, flags, + ee, wwd, si, procInfo) let lastError = osLastError() if poParentStreams notin options: @@ -870,13 +863,9 @@ when defined(windows) and not defined(useNimRtl): si.hStdError = getStdHandle(STD_ERROR_HANDLE) si.hStdInput = getStdHandle(STD_INPUT_HANDLE) si.hStdOutput = getStdHandle(STD_OUTPUT_HANDLE) - when useWinUnicode: - var c = newWideCString(command) - var res = winlean.createProcessW(nil, c, nil, nil, 0, - NORMAL_PRIORITY_CLASS, nil, nil, si, procInfo) - else: - var res = winlean.createProcessA(nil, command, nil, nil, 0, - NORMAL_PRIORITY_CLASS, nil, nil, si, procInfo) + var c = newWideCString(command) + var res = winlean.createProcessW(nil, c, nil, nil, 0, + NORMAL_PRIORITY_CLASS, nil, nil, si, procInfo) if res == 0: raiseOSError(osLastError()) else: diff --git a/lib/std/cmdline.nim b/lib/std/cmdline.nim index 3208da1b8..6788dacde 100644 --- a/lib/std/cmdline.nim +++ b/lib/std/cmdline.nim @@ -40,10 +40,7 @@ else: # Needed by windows in order to obtain the command line for targets # other than command line targets when defined(windows) and not weirdTarget: - when useWinUnicode: - template getCommandLine*(): untyped = getCommandLineW() - else: - template getCommandLine*(): untyped = getCommandLineA() + template getCommandLine*(): untyped = getCommandLineW() proc parseCmdLine*(c: string): seq[string] {. diff --git a/lib/std/private/oscommon.nim b/lib/std/private/oscommon.nim index 3aac58636..b747e33f1 100644 --- a/lib/std/private/oscommon.nim +++ b/lib/std/private/oscommon.nim @@ -47,23 +47,17 @@ else: when defined(windows) and not weirdTarget: - when useWinUnicode: - template wrapUnary*(varname, winApiProc, arg: untyped) = - var varname = winApiProc(newWideCString(arg)) + template wrapUnary*(varname, winApiProc, arg: untyped) = + var varname = winApiProc(newWideCString(arg)) - template wrapBinary*(varname, winApiProc, arg, arg2: untyped) = - var varname = winApiProc(newWideCString(arg), arg2) - proc findFirstFile*(a: string, b: var WIN32_FIND_DATA): Handle = - result = findFirstFileW(newWideCString(a), b) - template findNextFile*(a, b: untyped): untyped = findNextFileW(a, b) + template wrapBinary*(varname, winApiProc, arg, arg2: untyped) = + var varname = winApiProc(newWideCString(arg), arg2) + proc findFirstFile*(a: string, b: var WIN32_FIND_DATA): Handle = + result = findFirstFileW(newWideCString(a), b) + template findNextFile*(a, b: untyped): untyped = findNextFileW(a, b) - template getFilename*(f: untyped): untyped = - $cast[WideCString](addr(f.cFileName[0])) - else: - template findFirstFile*(a, b: untyped): untyped = findFirstFileA(a, b) - template findNextFile*(a, b: untyped): untyped = findNextFileA(a, b) - - template getFilename*(f: untyped): untyped = $cast[cstring](addr f.cFileName) + template getFilename*(f: untyped): untyped = + $cast[WideCString](addr(f.cFileName[0])) proc skipFindData*(f: WIN32_FIND_DATA): bool {.inline.} = # Note - takes advantage of null delimiter in the cstring @@ -104,12 +98,9 @@ proc tryMoveFSObject*(source, dest: string, isDir: bool): bool {.noWeirdTarget.} ## In case of other errors `OSError` is raised. ## Returns true in case of success. when defined(windows): - when useWinUnicode: - let s = newWideCString(source) - let d = newWideCString(dest) - result = moveFileExW(s, d, MOVEFILE_COPY_ALLOWED or MOVEFILE_REPLACE_EXISTING) != 0'i32 - else: - result = moveFileExA(source, dest, MOVEFILE_COPY_ALLOWED or MOVEFILE_REPLACE_EXISTING) != 0'i32 + let s = newWideCString(source) + let d = newWideCString(dest) + result = moveFileExW(s, d, MOVEFILE_COPY_ALLOWED or MOVEFILE_REPLACE_EXISTING) != 0'i32 else: result = c_rename(source, dest) == 0'i32 @@ -137,10 +128,7 @@ proc fileExists*(filename: string): bool {.rtl, extern: "nos$1", ## * `dirExists proc`_ ## * `symlinkExists proc`_ when defined(windows): - when useWinUnicode: - wrapUnary(a, getFileAttributesW, filename) - else: - var a = getFileAttributesA(filename) + wrapUnary(a, getFileAttributesW, filename) if a != -1'i32: result = (a and FILE_ATTRIBUTE_DIRECTORY) == 0'i32 else: @@ -157,10 +145,7 @@ proc dirExists*(dir: string): bool {.rtl, extern: "nos$1", tags: [ReadDirEffect] ## * `fileExists proc`_ ## * `symlinkExists proc`_ when defined(windows): - when useWinUnicode: - wrapUnary(a, getFileAttributesW, dir) - else: - var a = getFileAttributesA(dir) + wrapUnary(a, getFileAttributesW, dir) if a != -1'i32: result = (a and FILE_ATTRIBUTE_DIRECTORY) != 0'i32 else: @@ -178,10 +163,7 @@ proc symlinkExists*(link: string): bool {.rtl, extern: "nos$1", ## * `fileExists proc`_ ## * `dirExists proc`_ when defined(windows): - when useWinUnicode: - wrapUnary(a, getFileAttributesW, link) - else: - var a = getFileAttributesA(link) + wrapUnary(a, getFileAttributesW, link) if a != -1'i32: # xxx see: bug #16784 (bug9); checking `IO_REPARSE_TAG_SYMLINK` # may also be needed. @@ -197,15 +179,8 @@ when defined(windows) and not weirdTarget: flags = flags or FILE_FLAG_OPEN_REPARSE_POINT let access = if writeAccess: GENERIC_WRITE else: 0'i32 - when useWinUnicode: - result = createFileW( - newWideCString(path), access, - FILE_SHARE_DELETE or FILE_SHARE_READ or FILE_SHARE_WRITE, - nil, OPEN_EXISTING, flags, 0 - ) - else: - result = createFileA( - path, access, - FILE_SHARE_DELETE or FILE_SHARE_READ or FILE_SHARE_WRITE, - nil, OPEN_EXISTING, flags, 0 - ) + result = createFileW( + newWideCString(path), access, + FILE_SHARE_DELETE or FILE_SHARE_READ or FILE_SHARE_WRITE, + nil, OPEN_EXISTING, flags, 0 + ) diff --git a/lib/std/private/osdirs.nim b/lib/std/private/osdirs.nim index 4af418ead..a4318367d 100644 --- a/lib/std/private/osdirs.nim +++ b/lib/std/private/osdirs.nim @@ -328,10 +328,7 @@ iterator walkDirRec*(dir: string, proc rawRemoveDir(dir: string) {.noWeirdTarget.} = when defined(windows): - when useWinUnicode: - wrapUnary(res, removeDirectoryW, dir) - else: - var res = removeDirectoryA(dir) + wrapUnary(res, removeDirectoryW, dir) let lastError = osLastError() if res == 0'i32 and lastError.int32 != 3'i32 and lastError.int32 != 18'i32 and lastError.int32 != 2'i32: @@ -396,10 +393,7 @@ proc rawCreateDir(dir: string): bool {.noWeirdTarget.} = #echo res raiseOSError(osLastError(), dir) else: - when useWinUnicode: - wrapUnary(res, createDirectoryW, dir) - else: - let res = createDirectoryA(dir) + wrapUnary(res, createDirectoryW, dir) if res != 0'i32: result = true @@ -561,10 +555,7 @@ proc setCurrentDir*(newDir: string) {.inline, tags: [], noWeirdTarget.} = ## * `getTempDir proc`_ ## * `getCurrentDir proc`_ when defined(windows): - when useWinUnicode: - if setCurrentDirectoryW(newWideCString(newDir)) == 0'i32: - raiseOSError(osLastError(), newDir) - else: - if setCurrentDirectoryA(newDir) == 0'i32: raiseOSError(osLastError(), newDir) + if setCurrentDirectoryW(newWideCString(newDir)) == 0'i32: + raiseOSError(osLastError(), newDir) else: if chdir(newDir) != 0'i32: raiseOSError(osLastError(), newDir) diff --git a/lib/std/private/osfiles.nim b/lib/std/private/osfiles.nim index a7a595d97..7f822ffcc 100644 --- a/lib/std/private/osfiles.nim +++ b/lib/std/private/osfiles.nim @@ -84,10 +84,7 @@ proc getFilePermissions*(filename: string): set[FilePermission] {. if (a.st_mode and S_IWOTH.Mode) != 0.Mode: result.incl(fpOthersWrite) if (a.st_mode and S_IXOTH.Mode) != 0.Mode: result.incl(fpOthersExec) else: - when useWinUnicode: - wrapUnary(res, getFileAttributesW, filename) - else: - var res = getFileAttributesA(filename) + wrapUnary(res, getFileAttributesW, filename) if res == -1'i32: raiseOSError(osLastError(), filename) if (res and FILE_ATTRIBUTE_READONLY) != 0'i32: result = {fpUserExec, fpUserRead, fpGroupExec, fpGroupRead, @@ -136,19 +133,13 @@ proc setFilePermissions*(filename: string, permissions: set[FilePermission], if chmod(filename, cast[Mode](p)) != 0: raiseOSError(osLastError(), $(filename, permissions)) else: - when useWinUnicode: - wrapUnary(res, getFileAttributesW, filename) - else: - var res = getFileAttributesA(filename) + wrapUnary(res, getFileAttributesW, filename) if res == -1'i32: raiseOSError(osLastError(), filename) if fpUserWrite in permissions: res = res and not FILE_ATTRIBUTE_READONLY else: res = res or FILE_ATTRIBUTE_READONLY - when useWinUnicode: - wrapBinary(res2, setFileAttributesW, filename, res) - else: - var res2 = setFileAttributesA(filename, res) + wrapBinary(res2, setFileAttributesW, filename, res) if res2 == - 1'i32: raiseOSError(osLastError(), $(filename, permissions)) @@ -221,14 +212,10 @@ proc copyFile*(source, dest: string, options = {cfSymlinkFollow}) {.rtl, if isSymlink and (cfSymlinkIgnore in options or defined(windows)): return when defined(windows): - when useWinUnicode: - let s = newWideCString(source) - let d = newWideCString(dest) - if copyFileW(s, d, 0'i32) == 0'i32: - raiseOSError(osLastError(), $(source, dest)) - else: - if copyFileA(source, dest, 0'i32) == 0'i32: - raiseOSError(osLastError(), $(source, dest)) + let s = newWideCString(source) + let d = newWideCString(dest) + if copyFileW(s, d, 0'i32) == 0'i32: + raiseOSError(osLastError(), $(source, dest)) else: if isSymlink and cfSymlinkAsIs in options: createSymlink(expandSymlink(source), dest) @@ -334,14 +321,9 @@ when not declared(ENOENT) and not defined(windows): var ENOENT {.importc, header: "".}: cint when defined(windows) and not weirdTarget: - when useWinUnicode: - template deleteFile(file: untyped): untyped = deleteFileW(file) - template setFileAttributes(file, attrs: untyped): untyped = - setFileAttributesW(file, attrs) - else: - template deleteFile(file: untyped): untyped = deleteFileA(file) - template setFileAttributes(file, attrs: untyped): untyped = - setFileAttributesA(file, attrs) + template deleteFile(file: untyped): untyped = deleteFileW(file) + template setFileAttributes(file, attrs: untyped): untyped = + setFileAttributesW(file, attrs) proc tryRemoveFile*(file: string): bool {.rtl, extern: "nos$1", tags: [WriteDirEffect], noWeirdTarget.} = ## Removes the `file`. @@ -358,10 +340,7 @@ proc tryRemoveFile*(file: string): bool {.rtl, extern: "nos$1", tags: [WriteDirE ## * `moveFile proc`_ result = true when defined(windows): - when useWinUnicode: - let f = newWideCString(file) - else: - let f = file + let f = newWideCString(file) if deleteFile(f) == 0: result = false let err = getLastError() diff --git a/lib/std/private/ospaths2.nim b/lib/std/private/ospaths2.nim index 78a806675..612003023 100644 --- a/lib/std/private/ospaths2.nim +++ b/lib/std/private/ospaths2.nim @@ -849,30 +849,17 @@ when not defined(nimscript): doAssert false, "use -d:nodejs to have `getCurrentDir` defined" elif defined(windows): var bufsize = MAX_PATH.int32 - when useWinUnicode: - var res = newWideCString("", bufsize) - while true: - var L = getCurrentDirectoryW(bufsize, res) - if L == 0'i32: - raiseOSError(osLastError()) - elif L > bufsize: - res = newWideCString("", L) - bufsize = L - else: - result = res$L - break - else: - result = newString(bufsize) - while true: - var L = getCurrentDirectoryA(bufsize, result) - if L == 0'i32: - raiseOSError(osLastError()) - elif L > bufsize: - result = newString(L) - bufsize = L - else: - setLen(result, L) - break + var res = newWideCString("", bufsize) + while true: + var L = getCurrentDirectoryW(bufsize, res) + if L == 0'i32: + raiseOSError(osLastError()) + elif L > bufsize: + res = newWideCString("", L) + bufsize = L + else: + result = res$L + break else: var bufsize = 1024 # should be enough result = newString(bufsize) diff --git a/lib/std/private/ossymlinks.nim b/lib/std/private/ossymlinks.nim index 6b2de6237..18737b8b5 100644 --- a/lib/std/private/ossymlinks.nim +++ b/lib/std/private/ossymlinks.nim @@ -48,14 +48,10 @@ proc createSymlink*(src, dest: string) {.noWeirdTarget.} = const SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE = 2 # allows anyone with developer mode on to create a link let flag = dirExists(src).int32 or SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE - when useWinUnicode: - var wSrc = newWideCString(src) - var wDst = newWideCString(dest) - if createSymbolicLinkW(wDst, wSrc, flag) == 0 or getLastError() != 0: - raiseOSError(osLastError(), $(src, dest)) - else: - if createSymbolicLinkA(dest, src, flag) == 0 or getLastError() != 0: - raiseOSError(osLastError(), $(src, dest)) + var wSrc = newWideCString(src) + var wDst = newWideCString(dest) + if createSymbolicLinkW(wDst, wSrc, flag) == 0 or getLastError() != 0: + raiseOSError(osLastError(), $(src, dest)) else: if symlink(src, dest) != 0: raiseOSError(osLastError(), $(src, dest)) diff --git a/lib/std/syncio.nim b/lib/std/syncio.nim index 83b2d6d13..77d4fa04c 100644 --- a/lib/std/syncio.nim +++ b/lib/std/syncio.nim @@ -390,7 +390,7 @@ proc readLine*(f: File, line: var string): bool {.tags: [ReadIOEffect], proc c_memchr(s: pointer, c: cint, n: csize_t): pointer {. importc: "memchr", header: "".} - when defined(windows) and not defined(useWinAnsi): + when defined(windows): proc readConsole(hConsoleInput: FileHandle, lpBuffer: pointer, nNumberOfCharsToRead: int32, lpNumberOfCharsRead: ptr int32, @@ -612,7 +612,7 @@ proc writeLine*[Ty](f: File, x: varargs[Ty, `$`]) {.inline, # interface to the C procs: -when defined(windows) and not defined(useWinAnsi): +when defined(windows): when defined(cpp): proc wfopen(filename, mode: WideCString): pointer {. importcpp: "_wfopen((const wchar_t*)#, (const wchar_t*)#)", nodecl.} diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index 230f963f0..f8c6ce021 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -21,13 +21,7 @@ when defined(nimPreviewSlimSystem): from std/syncio import FileHandle import std/widestrs -const - useWinUnicode* = not defined(useWinAnsi) - -when useWinUnicode: - type WinChar* = Utf16Char -else: - type WinChar* = char +type WinChar* = Utf16Char # See https://docs.microsoft.com/en-us/windows/win32/winprog/windows-data-types type @@ -184,26 +178,14 @@ proc peekNamedPipe*(hNamedPipe: Handle, lpBuffer: pointer=nil, lpBytesLeftThisMessage: ptr int32 = nil): bool {. stdcall, dynlib: "kernel32", importc: "PeekNamedPipe".} -when useWinUnicode: - proc createProcessW*(lpApplicationName, lpCommandLine: WideCString, - lpProcessAttributes: ptr SECURITY_ATTRIBUTES, - lpThreadAttributes: ptr SECURITY_ATTRIBUTES, - bInheritHandles: WINBOOL, dwCreationFlags: int32, - lpEnvironment, lpCurrentDirectory: WideCString, - lpStartupInfo: var STARTUPINFO, - lpProcessInformation: var PROCESS_INFORMATION): WINBOOL{. - stdcall, dynlib: "kernel32", importc: "CreateProcessW", sideEffect.} - -else: - proc createProcessA*(lpApplicationName, lpCommandLine: cstring, - lpProcessAttributes: ptr SECURITY_ATTRIBUTES, - lpThreadAttributes: ptr SECURITY_ATTRIBUTES, - bInheritHandles: WINBOOL, dwCreationFlags: int32, - lpEnvironment: pointer, lpCurrentDirectory: cstring, - lpStartupInfo: var STARTUPINFO, - lpProcessInformation: var PROCESS_INFORMATION): WINBOOL{. - stdcall, dynlib: "kernel32", importc: "CreateProcessA", sideEffect.} - +proc createProcessW*(lpApplicationName, lpCommandLine: WideCString, + lpProcessAttributes: ptr SECURITY_ATTRIBUTES, + lpThreadAttributes: ptr SECURITY_ATTRIBUTES, + bInheritHandles: WINBOOL, dwCreationFlags: int32, + lpEnvironment, lpCurrentDirectory: WideCString, + lpStartupInfo: var STARTUPINFO, + lpProcessInformation: var PROCESS_INFORMATION): WINBOOL{. + stdcall, dynlib: "kernel32", importc: "CreateProcessW", sideEffect.} proc suspendThread*(hThread: Handle): int32 {.stdcall, dynlib: "kernel32", importc: "SuspendThread", sideEffect.} @@ -232,67 +214,37 @@ proc getLastError*(): int32 {.importc: "GetLastError", proc setLastError*(error: int32) {.importc: "SetLastError", stdcall, dynlib: "kernel32", sideEffect.} -when useWinUnicode: - proc formatMessageW*(dwFlags: int32, lpSource: pointer, - dwMessageId, dwLanguageId: int32, - lpBuffer: pointer, nSize: int32, - arguments: pointer): int32 {. - importc: "FormatMessageW", stdcall, dynlib: "kernel32".} -else: - proc formatMessageA*(dwFlags: int32, lpSource: pointer, +proc formatMessageW*(dwFlags: int32, lpSource: pointer, dwMessageId, dwLanguageId: int32, lpBuffer: pointer, nSize: int32, arguments: pointer): int32 {. - importc: "FormatMessageA", stdcall, dynlib: "kernel32".} + importc: "FormatMessageW", stdcall, dynlib: "kernel32".} proc localFree*(p: pointer) {. importc: "LocalFree", stdcall, dynlib: "kernel32".} -when useWinUnicode: - proc getCurrentDirectoryW*(nBufferLength: int32, - lpBuffer: WideCString): int32 {. - importc: "GetCurrentDirectoryW", dynlib: "kernel32", stdcall, sideEffect.} - proc setCurrentDirectoryW*(lpPathName: WideCString): int32 {. - importc: "SetCurrentDirectoryW", dynlib: "kernel32", stdcall, sideEffect.} - proc createDirectoryW*(pathName: WideCString, security: pointer=nil): int32 {. - importc: "CreateDirectoryW", dynlib: "kernel32", stdcall, sideEffect.} - proc removeDirectoryW*(lpPathName: WideCString): int32 {. - importc: "RemoveDirectoryW", dynlib: "kernel32", stdcall, sideEffect.} - proc setEnvironmentVariableW*(lpName, lpValue: WideCString): int32 {. - stdcall, dynlib: "kernel32", importc: "SetEnvironmentVariableW", sideEffect.} +proc getCurrentDirectoryW*(nBufferLength: int32, + lpBuffer: WideCString): int32 {. + importc: "GetCurrentDirectoryW", dynlib: "kernel32", stdcall, sideEffect.} +proc setCurrentDirectoryW*(lpPathName: WideCString): int32 {. + importc: "SetCurrentDirectoryW", dynlib: "kernel32", stdcall, sideEffect.} +proc createDirectoryW*(pathName: WideCString, security: pointer=nil): int32 {. + importc: "CreateDirectoryW", dynlib: "kernel32", stdcall, sideEffect.} +proc removeDirectoryW*(lpPathName: WideCString): int32 {. + importc: "RemoveDirectoryW", dynlib: "kernel32", stdcall, sideEffect.} +proc setEnvironmentVariableW*(lpName, lpValue: WideCString): int32 {. + stdcall, dynlib: "kernel32", importc: "SetEnvironmentVariableW", sideEffect.} - proc getModuleFileNameW*(handle: Handle, buf: WideCString, - size: int32): int32 {.importc: "GetModuleFileNameW", - dynlib: "kernel32", stdcall.} -else: - proc getCurrentDirectoryA*(nBufferLength: int32, lpBuffer: cstring): int32 {. - importc: "GetCurrentDirectoryA", dynlib: "kernel32", stdcall, sideEffect.} - proc setCurrentDirectoryA*(lpPathName: cstring): int32 {. - importc: "SetCurrentDirectoryA", dynlib: "kernel32", stdcall, sideEffect.} - proc createDirectoryA*(pathName: cstring, security: pointer=nil): int32 {. - importc: "CreateDirectoryA", dynlib: "kernel32", stdcall, sideEffect.} - proc removeDirectoryA*(lpPathName: cstring): int32 {. - importc: "RemoveDirectoryA", dynlib: "kernel32", stdcall, sideEffect.} - proc setEnvironmentVariableA*(lpName, lpValue: cstring): int32 {. - stdcall, dynlib: "kernel32", importc: "SetEnvironmentVariableA", sideEffect.} +proc getModuleFileNameW*(handle: Handle, buf: WideCString, + size: int32): int32 {.importc: "GetModuleFileNameW", + dynlib: "kernel32", stdcall.} - proc getModuleFileNameA*(handle: Handle, buf: cstring, size: int32): int32 {. - importc: "GetModuleFileNameA", dynlib: "kernel32", stdcall.} - -when useWinUnicode: - proc createSymbolicLinkW*(lpSymlinkFileName, lpTargetFileName: WideCString, - flags: DWORD): int32 {. - importc:"CreateSymbolicLinkW", dynlib: "kernel32", stdcall, sideEffect.} - proc createHardLinkW*(lpFileName, lpExistingFileName: WideCString, - security: pointer=nil): int32 {. - importc:"CreateHardLinkW", dynlib: "kernel32", stdcall, sideEffect.} -else: - proc createSymbolicLinkA*(lpSymlinkFileName, lpTargetFileName: cstring, - flags: DWORD): int32 {. - importc:"CreateSymbolicLinkA", dynlib: "kernel32", stdcall, sideEffect.} - proc createHardLinkA*(lpFileName, lpExistingFileName: cstring, - security: pointer=nil): int32 {. - importc:"CreateHardLinkA", dynlib: "kernel32", stdcall, sideEffect.} +proc createSymbolicLinkW*(lpSymlinkFileName, lpTargetFileName: WideCString, + flags: DWORD): int32 {. + importc:"CreateSymbolicLinkW", dynlib: "kernel32", stdcall, sideEffect.} +proc createHardLinkW*(lpFileName, lpExistingFileName: WideCString, + security: pointer=nil): int32 {. + importc:"CreateHardLinkW", dynlib: "kernel32", stdcall, sideEffect.} const FILE_ATTRIBUTE_READONLY* = 0x00000001'i32 @@ -343,84 +295,45 @@ type cFileName*: array[0..(MAX_PATH) - 1, WinChar] cAlternateFileName*: array[0..13, WinChar] -when useWinUnicode: - proc findFirstFileW*(lpFileName: WideCString, - lpFindFileData: var WIN32_FIND_DATA): Handle {. - stdcall, dynlib: "kernel32", importc: "FindFirstFileW", sideEffect.} - proc findNextFileW*(hFindFile: Handle, - lpFindFileData: var WIN32_FIND_DATA): int32 {. - stdcall, dynlib: "kernel32", importc: "FindNextFileW", sideEffect.} -else: - proc findFirstFileA*(lpFileName: cstring, - lpFindFileData: var WIN32_FIND_DATA): Handle {. - stdcall, dynlib: "kernel32", importc: "FindFirstFileA", sideEffect.} - proc findNextFileA*(hFindFile: Handle, - lpFindFileData: var WIN32_FIND_DATA): int32 {. - stdcall, dynlib: "kernel32", importc: "FindNextFileA", sideEffect.} +proc findFirstFileW*(lpFileName: WideCString, + lpFindFileData: var WIN32_FIND_DATA): Handle {. + stdcall, dynlib: "kernel32", importc: "FindFirstFileW", sideEffect.} +proc findNextFileW*(hFindFile: Handle, + lpFindFileData: var WIN32_FIND_DATA): int32 {. + stdcall, dynlib: "kernel32", importc: "FindNextFileW", sideEffect.} proc findClose*(hFindFile: Handle) {.stdcall, dynlib: "kernel32", importc: "FindClose".} -when useWinUnicode: - proc getFullPathNameW*(lpFileName: WideCString, nBufferLength: int32, - lpBuffer: WideCString, - lpFilePart: var WideCString): int32 {. +proc getFullPathNameW*(lpFileName: WideCString, nBufferLength: int32, + lpBuffer: WideCString, + lpFilePart: var WideCString): int32 {. + stdcall, dynlib: "kernel32", + importc: "GetFullPathNameW", sideEffect.} +proc getFileAttributesW*(lpFileName: WideCString): int32 {. stdcall, dynlib: "kernel32", - importc: "GetFullPathNameW", sideEffect.} - proc getFileAttributesW*(lpFileName: WideCString): int32 {. - stdcall, dynlib: "kernel32", - importc: "GetFileAttributesW", sideEffect.} - proc setFileAttributesW*(lpFileName: WideCString, - dwFileAttributes: int32): WINBOOL {. - stdcall, dynlib: "kernel32", importc: "SetFileAttributesW", sideEffect.} + importc: "GetFileAttributesW", sideEffect.} +proc setFileAttributesW*(lpFileName: WideCString, + dwFileAttributes: int32): WINBOOL {. + stdcall, dynlib: "kernel32", importc: "SetFileAttributesW", sideEffect.} - proc copyFileW*(lpExistingFileName, lpNewFileName: WideCString, - bFailIfExists: WINBOOL): WINBOOL {. - importc: "CopyFileW", stdcall, dynlib: "kernel32", sideEffect.} +proc copyFileW*(lpExistingFileName, lpNewFileName: WideCString, + bFailIfExists: WINBOOL): WINBOOL {. + importc: "CopyFileW", stdcall, dynlib: "kernel32", sideEffect.} - proc moveFileW*(lpExistingFileName, lpNewFileName: WideCString): WINBOOL {. - importc: "MoveFileW", stdcall, dynlib: "kernel32", sideEffect.} - proc moveFileExW*(lpExistingFileName, lpNewFileName: WideCString, - flags: DWORD): WINBOOL {. - importc: "MoveFileExW", stdcall, dynlib: "kernel32", sideEffect.} +proc moveFileW*(lpExistingFileName, lpNewFileName: WideCString): WINBOOL {. + importc: "MoveFileW", stdcall, dynlib: "kernel32", sideEffect.} +proc moveFileExW*(lpExistingFileName, lpNewFileName: WideCString, + flags: DWORD): WINBOOL {. + importc: "MoveFileExW", stdcall, dynlib: "kernel32", sideEffect.} - proc getEnvironmentStringsW*(): WideCString {. - stdcall, dynlib: "kernel32", importc: "GetEnvironmentStringsW", sideEffect.} - proc freeEnvironmentStringsW*(para1: WideCString): int32 {. - stdcall, dynlib: "kernel32", importc: "FreeEnvironmentStringsW", sideEffect.} +proc getEnvironmentStringsW*(): WideCString {. + stdcall, dynlib: "kernel32", importc: "GetEnvironmentStringsW", sideEffect.} +proc freeEnvironmentStringsW*(para1: WideCString): int32 {. + stdcall, dynlib: "kernel32", importc: "FreeEnvironmentStringsW", sideEffect.} - proc getCommandLineW*(): WideCString {.importc: "GetCommandLineW", - stdcall, dynlib: "kernel32", sideEffect.} - -else: - proc getFullPathNameA*(lpFileName: cstring, nBufferLength: int32, - lpBuffer: cstring, lpFilePart: var cstring): int32 {. - stdcall, dynlib: "kernel32", - importc: "GetFullPathNameA", sideEffect.} - proc getFileAttributesA*(lpFileName: cstring): int32 {. - stdcall, dynlib: "kernel32", - importc: "GetFileAttributesA", sideEffect.} - proc setFileAttributesA*(lpFileName: cstring, - dwFileAttributes: int32): WINBOOL {. - stdcall, dynlib: "kernel32", importc: "SetFileAttributesA", sideEffect.} - - proc copyFileA*(lpExistingFileName, lpNewFileName: cstring, - bFailIfExists: cint): cint {. - importc: "CopyFileA", stdcall, dynlib: "kernel32", sideEffect.} - - proc moveFileA*(lpExistingFileName, lpNewFileName: cstring): WINBOOL {. - importc: "MoveFileA", stdcall, dynlib: "kernel32", sideEffect.} - proc moveFileExA*(lpExistingFileName, lpNewFileName: cstring, - flags: DWORD): WINBOOL {. - importc: "MoveFileExA", stdcall, dynlib: "kernel32", sideEffect.} - - proc getEnvironmentStringsA*(): cstring {. - stdcall, dynlib: "kernel32", importc: "GetEnvironmentStringsA", sideEffect.} - proc freeEnvironmentStringsA*(para1: cstring): int32 {. - stdcall, dynlib: "kernel32", importc: "FreeEnvironmentStringsA", sideEffect.} - - proc getCommandLineA*(): cstring {. - importc: "GetCommandLineA", stdcall, dynlib: "kernel32", sideEffect.} +proc getCommandLineW*(): WideCString {.importc: "GetCommandLineW", + stdcall, dynlib: "kernel32", sideEffect.} proc rdFileTime*(f: FILETIME): int64 = result = int64(cast[uint32](f.dwLowDateTime)) or (int64(cast[uint32](f.dwHighDateTime)) shl 32) @@ -434,17 +347,10 @@ proc getSystemTimeAsFileTime*(lpSystemTimeAsFileTime: var FILETIME) {. proc sleep*(dwMilliseconds: int32){.stdcall, dynlib: "kernel32", importc: "Sleep", sideEffect.} -when useWinUnicode: - proc shellExecuteW*(hwnd: Handle, lpOperation, lpFile, - lpParameters, lpDirectory: WideCString, - nShowCmd: int32): Handle{. - stdcall, dynlib: "shell32.dll", importc: "ShellExecuteW", sideEffect.} - -else: - proc shellExecuteA*(hwnd: Handle, lpOperation, lpFile, - lpParameters, lpDirectory: cstring, - nShowCmd: int32): Handle{. - stdcall, dynlib: "shell32.dll", importc: "ShellExecuteA", sideEffect.} +proc shellExecuteW*(hwnd: Handle, lpOperation, lpFile, + lpParameters, lpDirectory: WideCString, + nShowCmd: int32): Handle{. + stdcall, dynlib: "shell32.dll", importc: "ShellExecuteW", sideEffect.} proc getFileInformationByHandle*(hFile: Handle, lpFileInformation: ptr BY_HANDLE_FILE_INFORMATION): WINBOOL{. @@ -794,13 +700,6 @@ proc createFileMappingW*(hFile: Handle, lpName: pointer): Handle {. stdcall, dynlib: "kernel32", importc: "CreateFileMappingW".} -when not useWinUnicode: - proc createFileMappingA*(hFile: Handle, - lpFileMappingAttributes: pointer, - flProtect, dwMaximumSizeHigh: DWORD, - dwMaximumSizeLow: DWORD, lpName: cstring): Handle {. - stdcall, dynlib: "kernel32", importc: "CreateFileMappingA".} - proc unmapViewOfFile*(lpBaseAddress: pointer): WINBOOL {.stdcall, dynlib: "kernel32", importc: "UnmapViewOfFile".} @@ -976,7 +875,7 @@ proc inet_ntop*(family: cint, paddr: pointer, pStringBuffer: cstring, stringBufSize: int32): cstring {.stdcall.} = var ver: OSVERSIONINFO ver.dwOSVersionInfoSize = sizeof(ver).DWORD - let res = when useWinUnicode: getVersionExW(ver.addr) else: getVersionExA(ver.addr) + let res = getVersionExW(ver.addr) if res == 0: result = nil elif ver.dwMajorVersion >= 6: @@ -1060,16 +959,10 @@ proc openProcess*(dwDesiredAccess: DWORD, bInheritHandle: WINBOOL, dwProcessId: DWORD): Handle {.stdcall, dynlib: "kernel32", importc: "OpenProcess".} -when defined(useWinAnsi): - proc createEvent*(lpEventAttributes: ptr SECURITY_ATTRIBUTES, - bManualReset: DWORD, bInitialState: DWORD, - lpName: cstring): Handle - {.stdcall, dynlib: "kernel32", importc: "CreateEventA".} -else: - proc createEvent*(lpEventAttributes: ptr SECURITY_ATTRIBUTES, - bManualReset: DWORD, bInitialState: DWORD, - lpName: ptr Utf16Char): Handle - {.stdcall, dynlib: "kernel32", importc: "CreateEventW".} +proc createEvent*(lpEventAttributes: ptr SECURITY_ATTRIBUTES, + bManualReset: DWORD, bInitialState: DWORD, + lpName: ptr Utf16Char): Handle + {.stdcall, dynlib: "kernel32", importc: "CreateEventW".} proc setEvent*(hEvent: Handle): cint {.stdcall, dynlib: "kernel32", importc: "SetEvent".} @@ -1110,14 +1003,9 @@ type uChar*: int16 dwControlKeyState*: DWORD -when defined(useWinAnsi): - proc readConsoleInput*(hConsoleInput: Handle, lpBuffer: pointer, nLength: cint, - lpNumberOfEventsRead: ptr cint): cint - {.stdcall, dynlib: "kernel32", importc: "ReadConsoleInputA".} -else: - proc readConsoleInput*(hConsoleInput: Handle, lpBuffer: pointer, nLength: cint, - lpNumberOfEventsRead: ptr cint): cint - {.stdcall, dynlib: "kernel32", importc: "ReadConsoleInputW".} +proc readConsoleInput*(hConsoleInput: Handle, lpBuffer: pointer, nLength: cint, + lpNumberOfEventsRead: ptr cint): cint + {.stdcall, dynlib: "kernel32", importc: "ReadConsoleInputW".} type LPFIBER_START_ROUTINE* = proc (param: pointer) {.stdcall.} From 50baf21eacfaf1c9d6949bf2b2e9f931b0e1509c Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 2 Mar 2023 08:36:02 +0100 Subject: [PATCH 2448/2472] =?UTF-8?q?fixes=20#20422;=20emit=20nimPrepareSt?= =?UTF-8?q?rMutationV2=20for=20toOpenArray=20to=20keep=20th=E2=80=A6=20(#2?= =?UTF-8?q?1459)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fixes #20422; emit nimPrepareStrMutationV2 for toOpenArray to keep the abstraction of mutable strings which have immutable string literals --- compiler/ccgcalls.nim | 4 +++- compiler/ccgexprs.nim | 5 ++++- tests/arc/topenarray.nim | 12 ++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index 9993322fb..48e7fd290 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -156,7 +156,7 @@ proc reifiedOpenArray(n: PNode): bool {.inline.} = else: result = true -proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType): (Rope, Rope) = +proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType; prepareForMutation = false): (Rope, Rope) = var a, b, c: TLoc initLocExpr(p, q[1], a) initLocExpr(p, q[2], b) @@ -164,6 +164,8 @@ proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType): (Rope, # but first produce the required index checks: if optBoundsCheck in p.options: genBoundsCheck(p, a, b, c) + if prepareForMutation: + linefmt(p, cpsStmts, "#nimPrepareStrMutationV2($1);$n", [byRefLoc(p, a)]) let ty = skipTypes(a.t, abstractVar+{tyPtr}) let dest = getTypeDesc(p.module, destType) let lengthExpr = "($1)-($2)+1" % [rdLoc(c), rdLoc(b)] diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index fe7cb252e..f428324b7 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -2408,7 +2408,10 @@ proc genDispose(p: BProc; n: PNode) = lineCg(p, cpsStmts, ["#nimDestroyAndDispose($#)", rdLoc(a)]) proc genSlice(p: BProc; e: PNode; d: var TLoc) = - let (x, y) = genOpenArraySlice(p, e, e.typ, e.typ.lastSon) + let (x, y) = genOpenArraySlice(p, e, e.typ, e.typ.lastSon, + prepareForMutation = e[1].kind == nkHiddenDeref and + e[1].typ.skipTypes(abstractInst).kind == tyString and + p.config.selectedGC in {gcArc, gcOrc}) if d.k == locNone: getTemp(p, e.typ, d) linefmt(p, cpsStmts, "$1.Field0 = $2; $1.Field1 = $3;$n", [rdLoc(d), x, y]) when false: diff --git a/tests/arc/topenarray.nim b/tests/arc/topenarray.nim index 47c26a11f..0e45f3ec7 100644 --- a/tests/arc/topenarray.nim +++ b/tests/arc/topenarray.nim @@ -6,6 +6,8 @@ Nim ''' matrix: "--gc:arc -d:useMalloc; --gc:arc" """ +{.experimental: "views".} + block: # bug 18627 proc setPosition(params: openArray[string]) = for i in params.toOpenArray(0, params.len - 1): @@ -38,3 +40,13 @@ block: # bug #20954 var v: seq[int] echo len(toOpenArray(v, 20, 30)) + +# bug #20422 + +proc f(a: var string) = + var v = a.toOpenArray(1, 3) + v[0] = 'a' + +var a = "Hello" +f(a) +doAssert a == "Hallo" From 38d299dfc0c9f1315cf412aa76cb041642d62f07 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 2 Mar 2023 16:16:28 +0800 Subject: [PATCH 2449/2472] fixes #20139; hash types based on its path relative to its package path (#21274) [backport:1.6] * fixes #20139; hash types based on its path relative its project * add a test case * fixes procs * better implementation and test case --------- Co-authored-by: Andreas Rumpf --- compiler/ccgexprs.nim | 4 +- compiler/ccgstmts.nim | 6 +-- compiler/ccgtypes.nim | 38 +++++++-------- compiler/cgen.nim | 2 +- compiler/injectdestructors.nim | 2 +- compiler/jsgen.nim | 4 +- compiler/liftdestructors.nim | 4 +- compiler/pathutils.nim | 51 +++++++++++++++++++- compiler/sighashes.nim | 86 +++++++++++++++++----------------- compiler/vm.nim | 2 +- tests/ccgbugs/m1/defs.nim | 4 ++ tests/ccgbugs/m2/defs.nim | 4 ++ tests/ccgbugs/t20139.nim | 10 ++++ 13 files changed, 143 insertions(+), 74 deletions(-) create mode 100644 tests/ccgbugs/m1/defs.nim create mode 100644 tests/ccgbugs/m2/defs.nim create mode 100644 tests/ccgbugs/t20139.nim diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index f428324b7..a64a42219 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1704,7 +1704,7 @@ proc genNewFinalize(p: BProc, e: PNode) = proc genOfHelper(p: BProc; dest: PType; a: Rope; info: TLineInfo; result: var Rope) = if optTinyRtti in p.config.globalOptions: - let token = $genDisplayElem(MD5Digest(hashType(dest))) + let token = $genDisplayElem(MD5Digest(hashType(dest, p.config))) appcg(p.module, result, "#isObjDisplayCheck($#.m_type, $#, $#)", [a, getObjDepth(dest), token]) else: # unfortunately 'genTypeInfoV1' sets tfObjHasKids as a side effect, so we @@ -2779,7 +2779,7 @@ proc upConv(p: BProc, n: PNode, d: var TLoc) = rdMType(p, a, nilCheck, r) if optTinyRtti in p.config.globalOptions: let checkFor = $getObjDepth(dest) - let token = $genDisplayElem(MD5Digest(hashType(dest))) + let token = $genDisplayElem(MD5Digest(hashType(dest, p.config))) if nilCheck != "": linefmt(p, cpsStmts, "if ($1 && !#isObjDisplayCheck($2, $3, $4)){ #raiseObjectConversionError(); ", [nilCheck, r, checkFor, token]) diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index db839b21b..7ade85b42 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -1081,7 +1081,7 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) = let memberName = if p.module.compileToCpp: "m_type" else: "Sup.m_type" if optTinyRtti in p.config.globalOptions: let checkFor = $getObjDepth(typeNode.typ) - appcg(p.module, orExpr, "#isObjDisplayCheck(#nimBorrowCurrentException()->$1, $2, $3)", [memberName, checkFor, $genDisplayElem(MD5Digest(hashType(typeNode.typ)))]) + appcg(p.module, orExpr, "#isObjDisplayCheck(#nimBorrowCurrentException()->$1, $2, $3)", [memberName, checkFor, $genDisplayElem(MD5Digest(hashType(typeNode.typ, p.config)))]) else: let checkFor = genTypeInfoV1(p.module, typeNode.typ, typeNode.info) appcg(p.module, orExpr, "#isObj(#nimBorrowCurrentException()->$1, $2)", [memberName, checkFor]) @@ -1300,7 +1300,7 @@ proc genTryGoto(p: BProc; t: PNode; d: var TLoc) = let memberName = if p.module.compileToCpp: "m_type" else: "Sup.m_type" if optTinyRtti in p.config.globalOptions: let checkFor = $getObjDepth(t[i][j].typ) - appcg(p.module, orExpr, "#isObjDisplayCheck(#nimBorrowCurrentException()->$1, $2, $3)", [memberName, checkFor, $genDisplayElem(MD5Digest(hashType(t[i][j].typ)))]) + appcg(p.module, orExpr, "#isObjDisplayCheck(#nimBorrowCurrentException()->$1, $2, $3)", [memberName, checkFor, $genDisplayElem(MD5Digest(hashType(t[i][j].typ, p.config)))]) else: let checkFor = genTypeInfoV1(p.module, t[i][j].typ, t[i][j].info) appcg(p.module, orExpr, "#isObj(#nimBorrowCurrentException()->$1, $2)", [memberName, checkFor]) @@ -1445,7 +1445,7 @@ proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) = let memberName = if p.module.compileToCpp: "m_type" else: "Sup.m_type" if optTinyRtti in p.config.globalOptions: let checkFor = $getObjDepth(t[i][j].typ) - appcg(p.module, orExpr, "#isObjDisplayCheck(#nimBorrowCurrentException()->$1, $2, $3)", [memberName, checkFor, $genDisplayElem(MD5Digest(hashType(t[i][j].typ)))]) + appcg(p.module, orExpr, "#isObjDisplayCheck(#nimBorrowCurrentException()->$1, $2, $3)", [memberName, checkFor, $genDisplayElem(MD5Digest(hashType(t[i][j].typ, p.config)))]) else: let checkFor = genTypeInfoV1(p.module, t[i][j].typ, t[i][j].info) appcg(p.module, orExpr, "#isObj(#nimBorrowCurrentException()->$1, $2)", [memberName, checkFor]) diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index e7677f2f2..98c1fc55f 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -43,14 +43,14 @@ proc fillBackendName(m: BModule; s: PSym) = result.add rope s.itemId.item if m.hcrOn: result.add "_" - result.add(idOrSig(s, m.module.name.s.mangle, m.sigConflicts)) + result.add(idOrSig(s, m.module.name.s.mangle, m.sigConflicts, m.config)) s.loc.r = result writeMangledName(m.ndi, s, m.config) proc fillParamName(m: BModule; s: PSym) = if s.loc.r == "": var res = s.name.s.mangle - res.add idOrSig(s, res, m.sigConflicts) + res.add idOrSig(s, res, m.sigConflicts, m.config) # Take into account if HCR is on because of the following scenario: # if a module gets imported and it has some more importc symbols in it, # some param names might receive the "_0" suffix to distinguish from what @@ -302,7 +302,7 @@ proc getSimpleTypeDesc(m: BModule, typ: PType): Rope = else: result = "" if result != "" and typ.isImportedType(): - let sig = hashType typ + let sig = hashType(typ, m.config) if cacheGetType(m.typeCache, sig) == "": m.typeCache[sig] = result @@ -362,10 +362,10 @@ proc getTypeDescWeak(m: BModule; t: PType; check: var IntSet; kind: TSymKind): R if isImportedCppType(etB) and t.kind == tyGenericInst: result = getTypeDescAux(m, t, check, kind) else: - result = getTypeForward(m, t, hashType(t)) + result = getTypeForward(m, t, hashType(t, m.config)) pushType(m, t) of tySequence: - let sig = hashType(t) + let sig = hashType(t, m.config) if optSeqDestructors in m.config.globalOptions: if skipTypes(etB[0], typedescInst).kind == tyEmpty: internalError(m.config, "cannot map the empty seq type to a C type") @@ -398,7 +398,7 @@ proc getSeqPayloadType(m: BModule; t: PType): Rope = #result = getTypeForward(m, t, hashType(t)) & "_Content" proc seqV2ContentType(m: BModule; t: PType; check: var IntSet) = - let sig = hashType(t) + let sig = hashType(t, m.config) let result = cacheGetType(m.typeCache, sig) if result == "": discard getTypeDescAux(m, t, check, skVar) @@ -664,7 +664,7 @@ proc resolveStarsInCppType(typ: PType, idx, stars: int): PType = else: result.elemType proc getOpenArrayDesc(m: BModule, t: PType, check: var IntSet; kind: TSymKind): Rope = - let sig = hashType(t) + let sig = hashType(t, m.config) if kind == skParam: result = getTypeDescWeak(m, t[0], check, kind) & "*" else: @@ -688,7 +688,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin # C type generation into an analysis and a code generation phase somehow. if t.sym != nil: useHeader(m, t.sym) if t != origTyp and origTyp.sym != nil: useHeader(m, origTyp.sym) - let sig = hashType(origTyp) + let sig = hashType(origTyp, m.config) defer: # defer is the simplest in this case if isImportedType(t) and not m.typeABICache.containsOrIncl(sig): @@ -717,7 +717,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin result = getTypeDescAux(m, et, check, kind) & star else: # no restriction! We have a forward declaration for structs - let name = getTypeForward(m, et, hashType et) + let name = getTypeForward(m, et, hashType(et, m.config)) result = name & star m.typeCache[sig] = result of tySequence: @@ -726,7 +726,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin m.typeCache[sig] = result else: # no restriction! We have a forward declaration for structs - let name = getTypeForward(m, et, hashType et) + let name = getTypeForward(m, et, hashType(et, m.config)) result = name & seqStar(m) & star m.typeCache[sig] = result pushType(m, et) @@ -893,7 +893,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin # Don't use the imported name as it may be scoped: 'Foo::SomeKind' result = rope("tySet_") t.lastSon.typeName(result) - result.add $t.lastSon.hashType + result.add $t.lastSon.hashType(m.config) m.typeCache[sig] = result if not isImportedType(t): let s = int(getSize(m.config, t)) @@ -1066,7 +1066,7 @@ proc discriminatorTableName(m: BModule, objtype: PType, d: PSym): Rope = objtype = objtype[0].skipTypes(abstractPtrs) if objtype.sym == nil: internalError(m.config, d.info, "anonymous obj with discriminator") - result = "NimDT_$1_$2" % [rope($hashType(objtype)), rope(d.name.s.mangle)] + result = "NimDT_$1_$2" % [rope($hashType(objtype, m.config)), rope(d.name.s.mangle)] proc rope(arg: Int128): Rope = rope($arg) @@ -1285,7 +1285,7 @@ proc genTypeInfo2Name(m: BModule; t: PType): Rope = result.add m.name.s & "." result.add it.sym.name.s else: - result = $hashType(it) + result = $hashType(it, m.config) result = makeCString(result) proc isTrivialProc(g: ModuleGraph; s: PSym): bool {.inline.} = getBody(g, s).len == 0 @@ -1329,14 +1329,14 @@ proc genDisplayElem(d: MD5Digest): uint32 = result += uint32(d[i]) result = result shl 8 -proc genDisplay(t: PType, depth: int): Rope = +proc genDisplay(m: BModule, t: PType, depth: int): Rope = result = Rope"{" var x = t var seqs = newSeq[string](depth+1) var i = 0 while x != nil: x = skipTypes(x, skipPtrs) - seqs[i] = $genDisplayElem(MD5Digest(hashType(x))) + seqs[i] = $genDisplayElem(MD5Digest(hashType(x, m.config))) x = x[0] inc i @@ -1376,7 +1376,7 @@ proc genTypeInfoV2OldImpl(m: BModule; t, origType: PType, name: Rope; info: TLin [name, getTypeDesc(m, t), rope(objDepth), rope(flags)]) if objDepth >= 0: - let objDisplay = genDisplay(t, objDepth) + let objDisplay = genDisplay(m, t, objDepth) let objDisplayStore = getTempName(m) m.s[cfsVars].addf("static $1 $2[$3] = $4;$n", [getTypeDesc(m, getSysType(m.g.graph, unknownLineInfo, tyUInt32), skVar), objDisplayStore, rope(objDepth+1), objDisplay]) addf(typeEntry, "$1.display = $2;$n", [name, rope(objDisplayStore)]) @@ -1408,7 +1408,7 @@ proc genTypeInfoV2Impl(m: BModule; t, origType: PType, name: Rope; info: TLineIn [getTypeDesc(m, t), rope(objDepth)]) if objDepth >= 0: - let objDisplay = genDisplay(t, objDepth) + let objDisplay = genDisplay(m, t, objDepth) let objDisplayStore = getTempName(m) m.s[cfsVars].addf("static NIM_CONST $1 $2[$3] = $4;$n", [getTypeDesc(m, getSysType(m.g.graph, unknownLineInfo, tyUInt32), skVar), objDisplayStore, rope(objDepth+1), objDisplay]) addf(typeEntry, ", .display = $1", [rope(objDisplayStore)]) @@ -1435,7 +1435,7 @@ proc genTypeInfoV2(m: BModule, t: PType; info: TLineInfo): Rope = let prefixTI = if m.hcrOn: "(" else: "(&" - let sig = hashType(origType) + let sig = hashType(origType, m.config) result = m.typeInfoMarkerV2.getOrDefault(sig) if result != "": return prefixTI.rope & result & ")".rope @@ -1509,7 +1509,7 @@ proc genTypeInfoV1(m: BModule, t: PType; info: TLineInfo): Rope = let prefixTI = if m.hcrOn: "(" else: "(&" - let sig = hashType(origType) + let sig = hashType(origType, m.config) result = m.typeInfoMarker.getOrDefault(sig) if result != "": return prefixTI.rope & result & ")".rope diff --git a/compiler/cgen.nim b/compiler/cgen.nim index c346a0b78..ab6e6559f 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1630,7 +1630,7 @@ proc registerModuleToMain(g: BModuleList; m: BModule) = hcrModuleMeta.addf("\t\"\"};$n", []) hcrModuleMeta.addf("$nN_LIB_EXPORT N_NIMCALL(void**, HcrGetImportedModules)() { return (void**)hcr_module_list; }$n", []) hcrModuleMeta.addf("$nN_LIB_EXPORT N_NIMCALL(char*, HcrGetSigHash)() { return \"$1\"; }$n$n", - [($sigHash(m.module)).rope]) + [($sigHash(m.module, m.config)).rope]) if sfMainModule in m.module.flags: g.mainModProcs.add(hcrModuleMeta) g.mainModProcs.addf("static void* hcr_handle;$N", []) diff --git a/compiler/injectdestructors.nim b/compiler/injectdestructors.nim index 13141b765..d58f28289 100644 --- a/compiler/injectdestructors.nim +++ b/compiler/injectdestructors.nim @@ -217,7 +217,7 @@ proc genOp(c: var Con; t: PType; kind: TTypeAttachedOp; dest, ri: PNode): PNode var op = getAttachedOp(c.graph, t, kind) if op == nil or op.ast.isGenericRoutine: # give up and find the canonical type instead: - let h = sighashes.hashType(t, {CoType, CoConsiderOwned, CoDistinct}) + let h = sighashes.hashType(t, c.graph.config, {CoType, CoConsiderOwned, CoDistinct}) let canon = c.graph.canonTypes.getOrDefault(h) if canon != nil: op = getAttachedOp(c.graph, canon, kind) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 572d19b13..6619963b0 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -262,7 +262,7 @@ proc mangleName(m: BModule, s: PSym): Rope = if m.config.hcrOn: # When hot reloading is enabled, we must ensure that the names # of functions and types will be preserved across rebuilds: - result.add(idOrSig(s, m.module.name.s, m.sigConflicts)) + result.add(idOrSig(s, m.module.name.s, m.sigConflicts, m.config)) else: result.add("_") result.add(rope(s.id)) @@ -2813,7 +2813,7 @@ proc genModule(p: PProc, n: PNode) = if p.config.hcrOn and n.kind == nkStmtList: let moduleSym = p.module.module var moduleLoadedVar = rope(moduleSym.name.s) & "_loaded" & - idOrSig(moduleSym, moduleSym.name.s, p.module.sigConflicts) + idOrSig(moduleSym, moduleSym.name.s, p.module.sigConflicts, p.config) lineF(p, "var $1;$n", [moduleLoadedVar]) var inGuardedBlock = false diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim index c7464d39e..d7b1f2daa 100644 --- a/compiler/liftdestructors.nim +++ b/compiler/liftdestructors.nim @@ -553,7 +553,7 @@ proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = # operation here: var t = t if t.assignment == nil or t.destructor == nil: - let h = sighashes.hashType(t, {CoType, CoConsiderOwned, CoDistinct}) + let h = sighashes.hashType(t,c.g.config, {CoType, CoConsiderOwned, CoDistinct}) let canon = c.g.canonTypes.getOrDefault(h) if canon != nil: t = canon @@ -1113,7 +1113,7 @@ proc createTypeBoundOps(g: ModuleGraph; c: PContext; orig: PType; info: TLineInf let skipped = orig.skipTypes({tyGenericInst, tyAlias, tySink}) if isEmptyContainer(skipped) or skipped.kind == tyStatic: return - let h = sighashes.hashType(skipped, {CoType, CoConsiderOwned, CoDistinct}) + let h = sighashes.hashType(skipped, g.config, {CoType, CoConsiderOwned, CoDistinct}) var canon = g.canonTypes.getOrDefault(h) if canon == nil: g.canonTypes[h] = skipped diff --git a/compiler/pathutils.nim b/compiler/pathutils.nim index 1ef0143d5..d9779deab 100644 --- a/compiler/pathutils.nim +++ b/compiler/pathutils.nim @@ -10,7 +10,7 @@ ## Path handling utilities for Nim. Strictly typed code in order ## to avoid the never ending time sink in getting path handling right. -import os, pathnorm +import os, pathnorm, strutils when defined(nimPreviewSlimSystem): import std/[syncio, assertions] @@ -102,3 +102,52 @@ when true: proc addFileExt*(x: RelativeFile; ext: string): RelativeFile {.borrow.} proc writeFile*(x: AbsoluteFile; content: string) {.borrow.} + +proc skipHomeDir(x: string): int = + when defined(windows): + if x.continuesWith("Users/", len("C:/")): + result = 3 + else: + result = 0 + else: + if x.startsWith("/home/") or x.startsWith("/Users/"): + result = 3 + elif x.startsWith("/mnt/") and x.continuesWith("/Users/", len("/mnt/c")): + result = 5 + else: + result = 0 + +proc relevantPart(s: string; afterSlashX: int): string = + result = newStringOfCap(s.len - 8) + var slashes = afterSlashX + for i in 0.. + # //user/ + if not isAbsolute(x): + result = customPathImpl(canonSlashes(getCurrentDir() / x)) + else: + let slashes = skipHomeDir(x) + if slashes > 0: + result = "//user/" & relevantPart(x, slashes) + else: + result = x + +proc customPath*(x: string): string = + customPathImpl canonSlashes x diff --git a/compiler/sighashes.nim b/compiler/sighashes.nim index 3508a146a..7e2a0b660 100644 --- a/compiler/sighashes.nim +++ b/compiler/sighashes.nim @@ -9,10 +9,12 @@ ## Computes hash values for routine (proc, method etc) signatures. -import ast, tables, ropes, md5, modulegraphs +import ast, tables, ropes, md5, modulegraphs, options, msgs, packages, pathutils from hashes import Hash import types +import std/os + when defined(nimPreviewSlimSystem): import std/assertions @@ -42,8 +44,7 @@ type CoDistinct CoHashTypeInsideNode -proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]) - +proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]; conf: ConfigRef) proc hashSym(c: var MD5Context, s: PSym) = if sfAnon in s.flags or s.kind == skGenericParam: c &= ":anon" @@ -54,20 +55,21 @@ proc hashSym(c: var MD5Context, s: PSym) = c &= "." it = it.owner -proc hashTypeSym(c: var MD5Context, s: PSym) = +proc hashTypeSym(c: var MD5Context, s: PSym; conf: ConfigRef) = if sfAnon in s.flags or s.kind == skGenericParam: c &= ":anon" else: var it = s + c &= customPath(conf.toFullPath(s.info)) while it != nil: if sfFromGeneric in it.flags and it.kind in routineKinds and it.typ != nil: - hashType c, it.typ, {CoProc} + hashType c, it.typ, {CoProc}, conf c &= it.name.s c &= "." it = it.owner -proc hashTree(c: var MD5Context, n: PNode; flags: set[ConsiderFlag]) = +proc hashTree(c: var MD5Context, n: PNode; flags: set[ConsiderFlag]; conf: ConfigRef) = if n == nil: c &= "\255" return @@ -82,7 +84,7 @@ proc hashTree(c: var MD5Context, n: PNode; flags: set[ConsiderFlag]) = of nkSym: hashSym(c, n.sym) if CoHashTypeInsideNode in flags and n.sym.typ != nil: - hashType(c, n.sym.typ, flags) + hashType(c, n.sym.typ, flags, conf) of nkCharLit..nkUInt64Lit: let v = n.intVal lowlevel v @@ -92,9 +94,9 @@ proc hashTree(c: var MD5Context, n: PNode; flags: set[ConsiderFlag]) = of nkStrLit..nkTripleStrLit: c &= n.strVal else: - for i in 0.. 0 and t[0] != nil: - hashType c, t[0], flags + hashType c, t[0], flags, conf of tyRef, tyPtr, tyGenericBody, tyVar: c &= char(t.kind) if t.sons.len > 0: - c.hashType t.lastSon, flags + c.hashType t.lastSon, flags, conf if tfVarIsPtr in t.flags: c &= ".varisptr" of tyFromExpr: c &= char(t.kind) - c.hashTree(t.n, {}) + c.hashTree(t.n, {}, conf) of tyTuple: c &= char(t.kind) if t.n != nil and CoType notin flags: @@ -199,19 +201,19 @@ proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]) = assert(t.n[i].kind == nkSym) c &= t.n[i].sym.name.s c &= ':' - c.hashType(t[i], flags+{CoIgnoreRange}) + c.hashType(t[i], flags+{CoIgnoreRange}, conf) c &= ',' else: - for i in 0.. 1) order by hash; -proc hashType*(t: PType; flags: set[ConsiderFlag] = {CoType}): SigHash = +proc hashType*(t: PType; conf: ConfigRef; flags: set[ConsiderFlag] = {CoType}): SigHash = var c: MD5Context md5Init c - hashType c, t, flags+{CoOwnerSig} + hashType c, t, flags+{CoOwnerSig}, conf md5Final c, result.MD5Digest when defined(debugSigHashes): db.exec(sql"INSERT OR IGNORE INTO sighashes(type, hash) VALUES (?, ?)", typeToString(t), $result) -proc hashProc*(s: PSym): SigHash = +proc hashProc*(s: PSym; conf: ConfigRef): SigHash = var c: MD5Context md5Init c - hashType c, s.typ, {CoProc} + hashType c, s.typ, {CoProc}, conf var m = s while m.kind != skModule: m = m.owner @@ -315,9 +317,9 @@ proc hashOwner*(s: PSym): SigHash = md5Final c, result.MD5Digest -proc sigHash*(s: PSym): SigHash = +proc sigHash*(s: PSym; conf: ConfigRef): SigHash = if s.kind in routineKinds and s.typ != nil: - result = hashProc(s) + result = hashProc(s, conf) else: result = hashNonProc(s) @@ -378,7 +380,7 @@ proc symBodyDigest*(graph: ModuleGraph, sym: PSym): SigHash = var c: MD5Context md5Init(c) - c.hashType(sym.typ, {CoProc}) + c.hashType(sym.typ, {CoProc}, graph.config) c &= char(sym.kind) c.md5Final(result.MD5Digest) graph.symBodyHashes[sym.id] = result # protect from recursion in the body @@ -391,12 +393,12 @@ proc symBodyDigest*(graph: ModuleGraph, sym: PSym): SigHash = graph.symBodyHashes[sym.id] = result proc idOrSig*(s: PSym, currentModule: string, - sigCollisions: var CountTable[SigHash]): Rope = + sigCollisions: var CountTable[SigHash]; conf: ConfigRef): Rope = if s.kind in routineKinds and s.typ != nil: # signatures for exported routines are reliable enough to # produce a unique name and this means produced C++ is more stable regarding # Nim changes: - let sig = hashProc(s) + let sig = hashProc(s, conf) result = rope($sig) #let m = if s.typ.callConv != ccInline: findPendingModule(m, s) else: m let counter = sigCollisions.getOrDefault(sig) diff --git a/compiler/vm.nim b/compiler/vm.nim index 38b856277..18d840613 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -1855,7 +1855,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = if regs[rb].node.kind != nkSym: stackTrace(c, tos, pc, "node is not a symbol") else: - regs[ra].node.strVal = $sigHash(regs[rb].node.sym) + regs[ra].node.strVal = $sigHash(regs[rb].node.sym, c.config) of opcSlurp: decodeB(rkNode) createStr regs[ra] diff --git a/tests/ccgbugs/m1/defs.nim b/tests/ccgbugs/m1/defs.nim new file mode 100644 index 000000000..ed78d8b72 --- /dev/null +++ b/tests/ccgbugs/m1/defs.nim @@ -0,0 +1,4 @@ +type MyObj* = object + field1*: int + s*: string + ch*: char diff --git a/tests/ccgbugs/m2/defs.nim b/tests/ccgbugs/m2/defs.nim new file mode 100644 index 000000000..798d1fea8 --- /dev/null +++ b/tests/ccgbugs/m2/defs.nim @@ -0,0 +1,4 @@ +type MyObj* = object + s*: string + field1*: int + ch*: char diff --git a/tests/ccgbugs/t20139.nim b/tests/ccgbugs/t20139.nim new file mode 100644 index 000000000..4592b994d --- /dev/null +++ b/tests/ccgbugs/t20139.nim @@ -0,0 +1,10 @@ +discard """ + joinable: false +""" + +# bug #20139 +import m1/defs as md1 +import m2/defs as md2 + +doAssert $(md1.MyObj(field1: 1)) == """(field1: 1, s: "", ch: '\x00')""" +doAssert $(md2.MyObj(field1: 1)) == """(s: "", field1: 1, ch: '\x00')""" From d4d28f2ffe522d2509dccfa7be0eef709732d93f Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Fri, 3 Mar 2023 06:27:10 +1100 Subject: [PATCH 2450/2472] Allow `futureLogging` in release builds (#21448) * Add test case * Fixes 21447: Keeps stackTrace around when using futureLogging * Remove extra whitespace --- lib/pure/asyncfutures.nim | 2 +- tests/async/t21447.nim | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 tests/async/t21447.nim diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index edf5f7277..51aaaca36 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -29,7 +29,7 @@ type finished: bool error*: ref Exception ## Stored exception errorStackTrace*: string - when not defined(release): + when not defined(release) or defined(futureLogging): stackTrace: seq[StackTraceEntry] ## For debugging purposes only. id: int fromProc: string diff --git a/tests/async/t21447.nim b/tests/async/t21447.nim new file mode 100644 index 000000000..e4f7ae31f --- /dev/null +++ b/tests/async/t21447.nim @@ -0,0 +1,6 @@ +discard """ + action: "compile" + cmd: "nim c -d:release -d:futureLogging $file" +""" + +import std/asyncdispatch From d51a392149df1c0783fa526eabbc904ce0cd0cbd Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 3 Mar 2023 14:36:38 +0800 Subject: [PATCH 2451/2472] replaces implicit passes array registed at runtime with explicit function calls; simplify compilation pipeline (#21444) * abolish using passes in the compiler; simplify compilation pipeline * duplicate code * Really cool to have the same signature... * haul * unify other backends * refactor process * introduce PipelinePhase * refactor compiler * fixes passes * fixes nimsuggest * add a sentinel * enable docs checkj * activate doc testing * clean up * complete cleanups --- compiler/cgen.nim | 25 +--- compiler/depends.nim | 9 +- compiler/docgen2.nim | 25 ++-- compiler/jsgen.nim | 19 ++- compiler/main.nim | 67 +++++----- compiler/modulegraphs.nim | 13 ++ compiler/modules.nim | 96 +------------- compiler/nimeval.nim | 29 +++-- compiler/passes.nim | 187 +++++++++++++++++---------- compiler/pipelines.nim | 254 +++++++++++++++++++++++++++++++++++++ compiler/pipelineutils.nim | 26 ++++ compiler/scriptconfig.nim | 15 +-- compiler/sem.nim | 63 +++++---- compiler/suggest.nim | 2 +- compiler/vm.nim | 11 +- doc/refc.md | 1 + nimsuggest/nimsuggest.nim | 2 +- 17 files changed, 531 insertions(+), 313 deletions(-) create mode 100644 compiler/pipelines.nim create mode 100644 compiler/pipelineutils.nim diff --git a/compiler/cgen.nim b/compiler/cgen.nim index ab6e6559f..d051934b5 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -12,11 +12,13 @@ import ast, astalgo, hashes, trees, platform, magicsys, extccomp, options, intsets, nversion, nimsets, msgs, bitsets, idents, types, - ccgutils, os, ropes, math, passes, wordrecg, treetab, cgmeth, + ccgutils, os, ropes, math, wordrecg, treetab, cgmeth, rodutils, renderer, cgendata, aliases, lowerings, tables, sets, ndi, lineinfos, pathutils, transf, injectdestructors, astmsgs, modulepaths, backendpragmas +import pipelineutils + when defined(nimPreviewSlimSystem): import std/assertions @@ -1938,8 +1940,7 @@ template injectG() {.dirty.} = graph.backend = newModuleList(graph) let g = BModuleList(graph.backend) - -proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} = +proc setupCgen*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} = injectG() result = newModule(g, module, graph.config) result.idgen = idgen @@ -2007,7 +2008,7 @@ proc addHcrInitGuards(p: BProc, n: PNode, inInitGuard: var bool) = proc genTopLevelStmt*(m: BModule; n: PNode) = ## Also called from `ic/cbackend.nim`. - if passes.skipCodegen(m.config, n): return + if pipelineutils.skipCodegen(m.config, n): return m.initProc.options = initProcOptions(m) #softRnl = if optLineDir in m.config.options: noRnl else: rnl # XXX replicate this logic! @@ -2020,12 +2021,6 @@ proc genTopLevelStmt*(m: BModule; n: PNode) = else: genProcBody(m.initProc, transformedN) -proc myProcess(b: PPassContext, n: PNode): PNode = - result = n - if b != nil: - var m = BModule(b) - genTopLevelStmt(m, n) - proc shouldRecompile(m: BModule; code: Rope, cfile: Cfile): bool = if optForceFullMake notin m.config.globalOptions: if not moduleHasChanged(m.g.graph, m.module): @@ -2102,7 +2097,7 @@ proc finalCodegenActions*(graph: ModuleGraph; m: BModule; n: PNode) = if {optGenStaticLib, optGenDynLib, optNoMain} * m.config.globalOptions == {}: for i in countdown(high(graph.globalDestructors), 0): n.add graph.globalDestructors[i] - if passes.skipCodegen(m.config, n): return + if pipelineutils.skipCodegen(m.config, n): return if moduleHasChanged(graph, m.module): # if the module is cached, we don't regenerate the main proc # nor the dispatchers? But if the dispatchers changed? @@ -2143,12 +2138,6 @@ proc finalCodegenActions*(graph: ModuleGraph; m: BModule; n: PNode) = let mm = m m.g.modulesClosed.add mm - -proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode = - result = n - if b == nil: return - finalCodegenActions(graph, BModule(b), n) - proc genForwardedProcs(g: BModuleList) = # Forward declared proc:s lack bodies when first encountered, so they're given # a second pass here @@ -2175,5 +2164,3 @@ proc cgenWriteModules*(backend: RootRef, config: ConfigRef) = m.writeModule(pending=true) writeMapping(config, g.mapping) if g.generatedHeader != nil: writeHeader(g.generatedHeader) - -const cgenPass* = makePass(myOpen, myProcess, myClose) diff --git a/compiler/depends.nim b/compiler/depends.nim index 59cdc0330..2087198f2 100644 --- a/compiler/depends.nim +++ b/compiler/depends.nim @@ -9,7 +9,7 @@ # This module implements a dependency file generator. -import options, ast, ropes, passes, pathutils, msgs, lineinfos +import options, ast, ropes, pathutils, msgs, lineinfos import modulegraphs @@ -79,7 +79,7 @@ proc addDependency(c: PPassContext, g: PGen, b: Backend, n: PNode) = let child = nativeToUnixPath(path.dir / path.name).toNimblePath(belongsToStdlib(g.graph, n.sym)) addDependencyAux(b, parent, child) -proc addDotDependency(c: PPassContext, n: PNode): PNode = +proc addDotDependency*(c: PPassContext, n: PNode): PNode = result = n let g = PGen(c) let b = Backend(g.graph.backend) @@ -100,7 +100,7 @@ proc generateDot*(graph: ModuleGraph; project: AbsoluteFile) = rope(project.splitFile.name), b.dotGraph], changeFileExt(project, "dot")) -proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} = +proc setupDependPass*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} = var g: PGen new(g) g.module = module @@ -109,6 +109,3 @@ proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext if graph.backend == nil: graph.backend = Backend(dotGraph: "") result = g - -const gendependPass* = makePass(open = myOpen, process = addDotDependency) - diff --git a/compiler/docgen2.nim b/compiler/docgen2.nim index e5669937f..0941be78a 100644 --- a/compiler/docgen2.nim +++ b/compiler/docgen2.nim @@ -11,7 +11,7 @@ # semantic checking. import - options, ast, msgs, passes, docgen, lineinfos, pathutils, packages + options, ast, msgs, docgen, lineinfos, pathutils, packages from modulegraphs import ModuleGraph, PPassContext @@ -38,21 +38,21 @@ template closeImpl(body: untyped) {.dirty.} = except IOError: discard -proc close(graph: ModuleGraph; p: PPassContext, n: PNode): PNode = +proc closeDoc*(graph: ModuleGraph; p: PPassContext, n: PNode): PNode = closeImpl: writeOutput(g.doc, useWarning, groupedToc) -proc closeJson(graph: ModuleGraph; p: PPassContext, n: PNode): PNode = +proc closeJson*(graph: ModuleGraph; p: PPassContext, n: PNode): PNode = closeImpl: writeOutputJson(g.doc, useWarning) -proc processNode(c: PPassContext, n: PNode): PNode = +proc processNode*(c: PPassContext, n: PNode): PNode = result = n var g = PGen(c) if shouldProcess(g): generateDoc(g.doc, n, n, g.config) -proc processNodeJson(c: PPassContext, n: PNode): PNode = +proc processNodeJson*(c: PPassContext, n: PNode): PNode = result = n var g = PGen(c) if shouldProcess(g): @@ -68,20 +68,11 @@ template myOpenImpl(ext: untyped) {.dirty.} = g.doc = d result = g -proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext = +proc openHtml*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext = myOpenImpl(HtmlExt) -proc myOpenTex(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext = +proc openTex*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext = myOpenImpl(TexExt) -proc myOpenJson(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext = +proc openJson*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext = myOpenImpl(JsonExt) - -const docgen2Pass* = makePass(open = myOpen, process = processNode, close = close) -const docgen2TexPass* = makePass(open = myOpenTex, process = processNode, - close = close) -const docgen2JsonPass* = makePass(open = myOpenJson, process = processNodeJson, - close = closeJson) - -proc finishDoc2Pass*(project: string) = - discard diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 6619963b0..5df40b996 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -31,10 +31,12 @@ implements the required case distinction. import ast, trees, magicsys, options, nversion, msgs, idents, types, - ropes, passes, ccgutils, wordrecg, renderer, + ropes, ccgutils, wordrecg, renderer, cgmeth, lowerings, sighashes, modulegraphs, lineinfos, rodutils, transf, injectdestructors, sourcemap, astmsgs, backendpragmas +import pipelineutils + import json, sets, math, tables, intsets import strutils except addf @@ -2830,11 +2832,11 @@ proc genModule(p: PProc, n: PNode) = if optStackTrace in p.options: p.body.add(frameDestroy(p)) -proc myProcess(b: PPassContext, n: PNode): PNode = +proc processJSCodeGen*(b: PPassContext, n: PNode): PNode = ## Generate JS code for a node. result = n let m = BModule(b) - if passes.skipCodegen(m.config, n): return n + if pipelineutils.skipCodegen(m.config, n): return n if m.module == nil: internalError(m.config, n.info, "myProcess") let globals = PGlobals(m.graph.backend) var p = newInitProc(globals, m) @@ -2869,7 +2871,7 @@ proc getClassName(t: PType): Rope = if s.loc.r != "": result = s.loc.r else: result = rope(s.name.s) -proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode = +proc finalJSCodeGen*(graph: ModuleGraph; b: PPassContext, n: PNode): PNode = ## Finalize JS code generation of a Nim module. ## Param `n` may contain nodes returned from the last module close call. var m = BModule(b) @@ -2879,14 +2881,14 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode = for i in countdown(high(graph.globalDestructors), 0): n.add graph.globalDestructors[i] # Process any nodes left over from the last call to `myClose`. - result = myProcess(b, n) + result = processJSCodeGen(b, n) # Some codegen is different (such as no stacktraces; see `initProcOptions`) # when `std/system` is being processed. if sfSystemModule in m.module.flags: PGlobals(graph.backend).inSystem = false # Check if codegen should continue before any files are generated. # It may bail early is if too many errors have been raised. - if passes.skipCodegen(m.config, n): return n + if pipelineutils.skipCodegen(m.config, n): return n # Nim modules are compiled into a single JS file. # If this is the main module, then this is the final call to `myClose`. if sfMainModule in m.module.flags: @@ -2904,9 +2906,6 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode = if not writeRope(code, outFile): rawMessage(m.config, errCannotOpenFile, outFile.string) -proc myOpen(graph: ModuleGraph; s: PSym; idgen: IdGenerator): PPassContext = - ## Create the JS backend pass context `BModule` for a Nim module. +proc setupJSgen*(graph: ModuleGraph; s: PSym; idgen: IdGenerator): PPassContext = result = newModule(graph, s) result.idgen = idgen - -const JSgenPass* = makePass(myOpen, myProcess, myClose) diff --git a/compiler/main.nim b/compiler/main.nim index b8e0e2f12..d37d26478 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -16,9 +16,9 @@ import std/[strutils, os, times, tables, sha1, with, json], llstream, ast, lexer, syntaxes, options, msgs, condsyms, - sem, idents, passes, extccomp, + idents, extccomp, cgen, nversion, - platform, nimconf, passaux, depends, vm, + platform, nimconf, depends, modules, modulegraphs, lineinfos, pathutils, vmprofiler @@ -29,12 +29,10 @@ when defined(nimPreviewSlimSystem): import ic / [cbackend, integrity, navigator] from ic / ic import rodViewer -when not defined(leanCompiler): - import jsgen, docgen, docgen2 +import pipelines -proc semanticPasses(g: ModuleGraph) = - registerPass g, verbosePass - registerPass g, semPass +when not defined(leanCompiler): + import docgen proc writeDepsFile(g: ModuleGraph) = let fname = g.config.nimcacheDir / RelativeFile(g.config.projectName & ".deps") @@ -68,9 +66,8 @@ proc writeCMakeDepsFile(conf: ConfigRef) = fl.close() proc commandGenDepend(graph: ModuleGraph) = - semanticPasses(graph) - registerPass(graph, gendependPass) - compileProject(graph) + setPipeLinePass(graph, GenDependPass) + compilePipelineProject(graph) let project = graph.config.projectFull writeDepsFile(graph) generateDot(graph, project) @@ -87,8 +84,8 @@ proc commandCheck(graph: ModuleGraph) = defineSymbol(conf.symbols, "nimconfig") elif conf.backend == backendJs: setTarget(conf.target, osJS, cpuJS) - semanticPasses(graph) # use an empty backend for semantic checking only - compileProject(graph) + setPipeLinePass(graph, SemPass) + compilePipelineProject(graph) if conf.symbolFiles != disabledSf: case conf.ideCmd @@ -102,22 +99,20 @@ when not defined(leanCompiler): proc commandDoc2(graph: ModuleGraph; ext: string) = handleDocOutputOptions graph.config graph.config.setErrorMaxHighMaybe - semanticPasses(graph) case ext: - of TexExt: registerPass(graph, docgen2TexPass) - of JsonExt: registerPass(graph, docgen2JsonPass) - of HtmlExt: registerPass(graph, docgen2Pass) + of TexExt: + setPipeLinePass(graph, Docgen2TexPass) + of JsonExt: + setPipeLinePass(graph, Docgen2JsonPass) + of HtmlExt: + setPipeLinePass(graph, Docgen2Pass) else: doAssert false, $ext - compileProject(graph) - finishDoc2Pass(graph.config.projectName) + compilePipelineProject(graph) proc commandCompileToC(graph: ModuleGraph) = let conf = graph.config extccomp.initVars(conf) - semanticPasses(graph) if conf.symbolFiles == disabledSf: - registerPass(graph, cgenPass) - if {optRun, optForceFullMake} * conf.globalOptions == {optRun} or isDefined(conf, "nimBetterRun"): if not changeDetectedViaJsonBuildInstructions(conf, conf.jsonBuildInstructionsFile): # nothing changed @@ -127,7 +122,11 @@ proc commandCompileToC(graph: ModuleGraph) = if not extccomp.ccHasSaneOverflow(conf): conf.symbols.defineSymbol("nimEmulateOverflowChecks") - compileProject(graph) + if conf.symbolFiles == disabledSf: + setPipeLinePass(graph, CgenPass) + else: + setPipeLinePass(graph, SemPass) + compilePipelineProject(graph) if graph.config.errorCounter > 0: return # issue #9933 if conf.symbolFiles == disabledSf: @@ -160,33 +159,27 @@ proc commandCompileToJS(graph: ModuleGraph) = conf.exc = excCpp setTarget(conf.target, osJS, cpuJS) defineSymbol(conf.symbols, "ecmascript") # For backward compatibility - semanticPasses(graph) - registerPass(graph, JSgenPass) - compileProject(graph) + setPipeLinePass(graph, JSgenPass) + compilePipelineProject(graph) if optGenScript in conf.globalOptions: writeDepsFile(graph) -proc interactivePasses(graph: ModuleGraph) = +proc commandInteractive(graph: ModuleGraph) = + graph.config.setErrorMaxHighMaybe initDefines(graph.config.symbols) defineSymbol(graph.config.symbols, "nimscript") # note: seems redundant with -d:nimHasLibFFI when hasFFI: defineSymbol(graph.config.symbols, "nimffi") - registerPass(graph, verbosePass) - registerPass(graph, semPass) - registerPass(graph, evalPass) - -proc commandInteractive(graph: ModuleGraph) = - graph.config.setErrorMaxHighMaybe - interactivePasses(graph) - compileSystemModule(graph) + setPipeLinePass(graph, InterpreterPass) + compilePipelineSystemModule(graph) if graph.config.commandArgs.len > 0: - discard graph.compileModule(fileInfoIdx(graph.config, graph.config.projectFull), {}) + discard graph.compilePipelineModule(fileInfoIdx(graph.config, graph.config.projectFull), {}) else: var m = graph.makeStdinModule() incl(m.flags, sfMainModule) var idgen = IdGenerator(module: m.itemId.module, symId: m.itemId.item, typeId: 0) let s = llStreamOpenStdIn(onPrompt = proc() = flushDot(graph.config)) - processModule(graph, m, idgen, s) + discard processPipelineModule(graph, m, idgen, s) proc commandScan(cache: IdentCache, config: ConfigRef) = var f = addFileExt(AbsoluteFile mainCommandArg(config), NimExt) @@ -241,8 +234,6 @@ proc mainCommand*(graph: ModuleGraph) = let conf = graph.config let cache = graph.cache - # In "nim serve" scenario, each command must reset the registered passes - clearPasses(graph) conf.lastCmdTime = epochTime() conf.searchPaths.add(conf.libpath) diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index 598959a37..8ffbe20a5 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -57,6 +57,18 @@ type sym*: PSym info*: TLineInfo + PipelinePass* = enum + NonePass + SemPass + JSgenPass + CgenPass + EvalPass + InterpreterPass + GenDependPass + Docgen2TexPass + Docgen2JsonPass + Docgen2Pass + ModuleGraph* {.acyclic.} = ref object ifaces*: seq[Iface] ## indexed by int32 fileIdx packed*: PackedModuleGraph @@ -104,6 +116,7 @@ type cacheCounters*: Table[string, BiggestInt] # IC: implemented cacheTables*: Table[string, BTree[string, PNode]] # IC: implemented passes*: seq[TPass] + pipelinePass*: PipelinePass onDefinition*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.} onDefinitionResolveForward*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.} onUsage*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.} diff --git a/compiler/modules.nim b/compiler/modules.nim index 838a89d83..0aa1c8930 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -11,13 +11,8 @@ import ast, magicsys, msgs, options, - idents, lexer, passes, syntaxes, llstream, modulegraphs, - lineinfos, pathutils, tables, packages - -when defined(nimPreviewSlimSystem): - import std/[syncio, assertions] - -import ic / replayer + idents, lexer, syntaxes, modulegraphs, + lineinfos, pathutils proc resetSystemArtifacts*(g: ModuleGraph) = magicsys.resetSysTypes(g) @@ -25,12 +20,12 @@ proc resetSystemArtifacts*(g: ModuleGraph) = template getModuleIdent(graph: ModuleGraph, filename: AbsoluteFile): PIdent = getIdent(graph.cache, splitFile(filename).name) -proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; filename: AbsoluteFile) = +proc partialInitModule*(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; filename: AbsoluteFile) = let packSym = getPackage(graph, fileIdx) result.owner = packSym result.position = int fileIdx -proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym = +proc newModule*(graph: ModuleGraph; fileIdx: FileIndex): PSym = let filename = AbsoluteFile toFullPath(graph.config, fileIdx) # We cannot call ``newSym`` here, because we have to circumvent the ID # mechanism, which we do in order to assign each module a persistent ID. @@ -43,99 +38,16 @@ proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym = partialInitModule(result, graph, fileIdx, filename) graph.registerModule(result) -proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags, fromModule: PSym = nil): PSym = - var flags = flags - if fileIdx == graph.config.projectMainIdx2: flags.incl sfMainModule - result = graph.getModule(fileIdx) - - template processModuleAux(moduleStatus) = - onProcessing(graph, fileIdx, moduleStatus, fromModule = fromModule) - var s: PLLStream - if sfMainModule in flags: - if graph.config.projectIsStdin: s = stdin.llStreamOpen - elif graph.config.projectIsCmd: s = llStreamOpen(graph.config.cmdInput) - discard processModule(graph, result, idGeneratorFromModule(result), s) - if result == nil: - var cachedModules: seq[FileIndex] - result = moduleFromRodFile(graph, fileIdx, cachedModules) - let filename = AbsoluteFile toFullPath(graph.config, fileIdx) - if result == nil: - result = newModule(graph, fileIdx) - result.flags.incl flags - registerModule(graph, result) - processModuleAux("import") - else: - if sfSystemModule in flags: - graph.systemModule = result - partialInitModule(result, graph, fileIdx, filename) - for m in cachedModules: - registerModuleById(graph, m) - replayStateChanges(graph.packed[m.int].module, graph) - replayGenericCacheInformation(graph, m.int) - elif graph.isDirty(result): - result.flags.excl sfDirty - # reset module fields: - initStrTables(graph, result) - result.ast = nil - processModuleAux("import(dirty)") - graph.markClientsDirty(fileIdx) - -proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym = - # this is called by the semantic checking phase - assert graph.config != nil - result = compileModule(graph, fileIdx, {}, s) - graph.addDep(s, fileIdx) - # keep track of import relationships - if graph.config.hcrOn: - graph.importDeps.mgetOrPut(FileIndex(s.position), @[]).add(fileIdx) - #if sfSystemModule in result.flags: - # localError(result.info, errAttemptToRedefine, result.name.s) - # restore the notes for outer module: - graph.config.notes = - if graph.config.belongsToProjectPackage(s) or isDefined(graph.config, "booting"): graph.config.mainPackageNotes - else: graph.config.foreignPackageNotes - proc includeModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PNode = result = syntaxes.parseFile(fileIdx, graph.cache, graph.config) graph.addDep(s, fileIdx) graph.addIncludeDep(s.position.FileIndex, fileIdx) -proc connectCallbacks*(graph: ModuleGraph) = - graph.includeFileCallback = includeModule - graph.importModuleCallback = importModule - -proc compileSystemModule*(graph: ModuleGraph) = - if graph.systemModule == nil: - connectCallbacks(graph) - graph.config.m.systemFileIdx = fileInfoIdx(graph.config, - graph.config.libpath / RelativeFile"system.nim") - discard graph.compileModule(graph.config.m.systemFileIdx, {sfSystemModule}) - proc wantMainModule*(conf: ConfigRef) = if conf.projectFull.isEmpty: fatal(conf, gCmdLineInfo, "command expects a filename") conf.projectMainIdx = fileInfoIdx(conf, addFileExt(conf.projectFull, NimExt)) -proc compileProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) = - connectCallbacks(graph) - let conf = graph.config - wantMainModule(conf) - configComplete(graph) - - let systemFileIdx = fileInfoIdx(conf, conf.libpath / RelativeFile"system.nim") - let projectFile = if projectFileIdx == InvalidFileIdx: conf.projectMainIdx else: projectFileIdx - conf.projectMainIdx2 = projectFile - - let packSym = getPackage(graph, projectFile) - graph.config.mainPackageId = packSym.getPackageId - graph.importStack.add projectFile - - if projectFile == systemFileIdx: - discard graph.compileModule(projectFile, {sfMainModule, sfSystemModule}) - else: - graph.compileSystemModule() - discard graph.compileModule(projectFile, {sfMainModule}) - proc makeModule*(graph: ModuleGraph; filename: AbsoluteFile): PSym = result = graph.newModule(fileInfoIdx(graph.config, filename)) registerModule(graph, result) diff --git a/compiler/nimeval.nim b/compiler/nimeval.nim index 82e2f0812..8e8f4ac7b 100644 --- a/compiler/nimeval.nim +++ b/compiler/nimeval.nim @@ -9,10 +9,16 @@ ## exposes the Nim VM to clients. import - ast, astalgo, modules, passes, condsyms, - options, sem, llstream, lineinfos, vm, + ast, modules, condsyms, + options, llstream, lineinfos, vm, vmdef, modulegraphs, idents, os, pathutils, - passaux, scriptconfig, std/compilesettings + scriptconfig, std/compilesettings + +import pipelines + + +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] type Interpreter* = ref object ## Use Nim as an interpreter with this object @@ -76,7 +82,7 @@ proc evalScript*(i: Interpreter; scriptStream: PLLStream = nil) = let s = if scriptStream != nil: scriptStream else: llStreamOpen(findFile(i.graph.config, i.scriptName), fmRead) - processModule(i.graph, i.mainModule, i.idgen, s) + discard processPipelineModule(i.graph, i.mainModule, i.idgen, s) proc findNimStdLib*(): string = ## Tries to find a path to a valid "system.nim" file. @@ -109,12 +115,10 @@ proc createInterpreter*(scriptName: string; var conf = newConfigRef() var cache = newIdentCache() var graph = newModuleGraph(cache, conf) - connectCallbacks(graph) + connectPipelineCallbacks(graph) initDefines(conf.symbols) for define in defines: defineSymbol(conf.symbols, define[0], define[1]) - registerPass(graph, semPass) - registerPass(graph, evalPass) for p in searchPaths: conf.searchPaths.add(AbsoluteDir p) @@ -129,7 +133,8 @@ proc createInterpreter*(scriptName: string; if registerOps: vm.registerAdditionalOps() # Required to register parts of stdlib modules graph.vm = vm - graph.compileSystemModule() + setPipeLinePass(graph, EvalPass) + graph.compilePipelineSystemModule() result = Interpreter(mainModule: m, graph: graph, scriptName: scriptName, idgen: idgen) proc destroyInterpreter*(i: Interpreter) = @@ -159,13 +164,11 @@ proc runRepl*(r: TLLRepl; defineSymbol(conf.symbols, "nimscript") if supportNimscript: defineSymbol(conf.symbols, "nimconfig") when hasFFI: defineSymbol(graph.config.symbols, "nimffi") - registerPass(graph, verbosePass) - registerPass(graph, semPass) - registerPass(graph, evalPass) var m = graph.makeStdinModule() incl(m.flags, sfMainModule) var idgen = idGeneratorFromModule(m) if supportNimscript: graph.vm = setupVM(m, cache, "stdin", graph, idgen) - graph.compileSystemModule() - processModule(graph, m, idgen, llStreamOpenStdIn(r)) + setPipeLinePass(graph, InterpreterPass) + graph.compilePipelineSystemModule() + discard processPipelineModule(graph, m, idgen, llStreamOpenStdIn(r)) diff --git a/compiler/passes.nim b/compiler/passes.nim index 74c5ff77c..38c133d69 100644 --- a/compiler/passes.nim +++ b/compiler/passes.nim @@ -14,13 +14,22 @@ import options, ast, llstream, msgs, idents, syntaxes, modulegraphs, reorder, - lineinfos, pathutils, packages + lineinfos, + pipelineutils, + modules, pathutils, packages, + sem, semdata + +import ic/replayer + +export skipCodegen, resolveMod, prepareConfigNotes when defined(nimsuggest): import std/sha1 when defined(nimPreviewSlimSystem): - import std/syncio + import std/[syncio, assertions] + +import std/tables type TPassData* = tuple[input: PNode, closeOutput: PNode] @@ -38,12 +47,6 @@ proc makePass*(open: TPassOpen = nil, result.process = process result.isFrontend = isFrontend -proc skipCodegen*(config: ConfigRef; n: PNode): bool {.inline.} = - # can be used by codegen passes to determine whether they should do - # something with `n`. Currently, this ignores `n` and uses the global - # error count instead. - result = config.errorCounter > 0 - const maxPasses = 10 @@ -80,13 +83,6 @@ proc processTopLevelStmt(graph: ModuleGraph, n: PNode, a: var TPassContextArray) if isNil(m): return false result = true -proc resolveMod(conf: ConfigRef; module, relativeTo: string): FileIndex = - let fullPath = findModule(conf, module, relativeTo) - if fullPath.isEmpty: - result = InvalidFileIdx - else: - result = fileInfoIdx(conf, fullPath) - proc processImplicits(graph: ModuleGraph; implicits: seq[string], nodeKind: TNodeKind, a: var TPassContextArray; m: PSym) = # XXX fixme this should actually be relative to the config file! @@ -100,23 +96,6 @@ proc processImplicits(graph: ModuleGraph; implicits: seq[string], nodeKind: TNod importStmt.add str if not processTopLevelStmt(graph, importStmt, a): break -const - imperativeCode = {low(TNodeKind)..high(TNodeKind)} - {nkTemplateDef, nkProcDef, nkMethodDef, - nkMacroDef, nkConverterDef, nkIteratorDef, nkFuncDef, nkPragma, - nkExportStmt, nkExportExceptStmt, nkFromStmt, nkImportStmt, nkImportExceptStmt} - -proc prepareConfigNotes(graph: ModuleGraph; module: PSym) = - # don't be verbose unless the module belongs to the main package: - if graph.config.belongsToProjectPackage(module): - graph.config.notes = graph.config.mainPackageNotes - else: - if graph.config.mainPackageNotes == {}: graph.config.mainPackageNotes = graph.config.notes - graph.config.notes = graph.config.foreignPackageNotes - -proc moduleHasChanged*(graph: ModuleGraph; module: PSym): bool {.inline.} = - result = true - #module.id >= 0 or isDefined(graph.config, "nimBackendAssumesChange") - proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator; stream: PLLStream): bool {.discardable.} = if graph.stopCompile(): return true @@ -153,42 +132,22 @@ proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator; processImplicits graph, graph.config.implicitIncludes, nkIncludeStmt, a, module checkFirstLineIndentation(p) - while true: - if graph.stopCompile(): break - var n = parseTopLevelStmt(p) # todo merge it - if n.kind == nkEmpty: break + block processCode: + if graph.stopCompile(): break processCode + var n = parseTopLevelStmt(p) + if n.kind == nkEmpty: break processCode - if true: - # read everything, no streaming possible - var sl = newNodeI(nkStmtList, n.info) + # read everything, no streaming possible + var sl = newNodeI(nkStmtList, n.info) + sl.add n + while true: + var n = parseTopLevelStmt(p) + if n.kind == nkEmpty: break sl.add n - while true: - var n = parseTopLevelStmt(p) - if n.kind == nkEmpty: break - sl.add n - if sfReorder in module.flags or codeReordering in graph.config.features: - sl = reorder(graph, sl, module) - discard processTopLevelStmt(graph, sl, a) - break - elif n.kind in imperativeCode: - # read everything until the next proc declaration etc. - var sl = newNodeI(nkStmtList, n.info) - sl.add n - var rest: PNode = nil - while true: - var n = parseTopLevelStmt(p) - if n.kind == nkEmpty or n.kind notin imperativeCode: - rest = n - break - sl.add n - #echo "-----\n", sl - if not processTopLevelStmt(graph, sl, a): break - if rest != nil: - #echo "-----\n", rest - if not processTopLevelStmt(graph, rest, a): break - else: - #echo "----- single\n", n - if not processTopLevelStmt(graph, n, a): break + if sfReorder in module.flags or codeReordering in graph.config.features: + sl = reorder(graph, sl, module) + discard processTopLevelStmt(graph, sl, a) + closeParser(p) if s.kind != llsStdIn: break closePasses(graph, a) @@ -198,3 +157,99 @@ proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator; # They are responsible for closing the rod files. See `cbackend.nim`. closeRodFile(graph, module) result = true + +proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags, fromModule: PSym = nil): PSym = + var flags = flags + if fileIdx == graph.config.projectMainIdx2: flags.incl sfMainModule + result = graph.getModule(fileIdx) + + template processModuleAux(moduleStatus) = + onProcessing(graph, fileIdx, moduleStatus, fromModule = fromModule) + var s: PLLStream + if sfMainModule in flags: + if graph.config.projectIsStdin: s = stdin.llStreamOpen + elif graph.config.projectIsCmd: s = llStreamOpen(graph.config.cmdInput) + discard processModule(graph, result, idGeneratorFromModule(result), s) + if result == nil: + var cachedModules: seq[FileIndex] + result = moduleFromRodFile(graph, fileIdx, cachedModules) + let filename = AbsoluteFile toFullPath(graph.config, fileIdx) + if result == nil: + result = newModule(graph, fileIdx) + result.flags.incl flags + registerModule(graph, result) + processModuleAux("import") + else: + if sfSystemModule in flags: + graph.systemModule = result + partialInitModule(result, graph, fileIdx, filename) + for m in cachedModules: + registerModuleById(graph, m) + replayStateChanges(graph.packed[m.int].module, graph) + replayGenericCacheInformation(graph, m.int) + elif graph.isDirty(result): + result.flags.excl sfDirty + # reset module fields: + initStrTables(graph, result) + result.ast = nil + processModuleAux("import(dirty)") + graph.markClientsDirty(fileIdx) + +proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym = + # this is called by the semantic checking phase + assert graph.config != nil + result = compileModule(graph, fileIdx, {}, s) + graph.addDep(s, fileIdx) + # keep track of import relationships + if graph.config.hcrOn: + graph.importDeps.mgetOrPut(FileIndex(s.position), @[]).add(fileIdx) + #if sfSystemModule in result.flags: + # localError(result.info, errAttemptToRedefine, result.name.s) + # restore the notes for outer module: + graph.config.notes = + if graph.config.belongsToProjectPackage(s) or isDefined(graph.config, "booting"): graph.config.mainPackageNotes + else: graph.config.foreignPackageNotes + +proc connectCallbacks*(graph: ModuleGraph) = + graph.includeFileCallback = modules.includeModule + graph.importModuleCallback = importModule + +proc compileSystemModule*(graph: ModuleGraph) = + if graph.systemModule == nil: + connectCallbacks(graph) + graph.config.m.systemFileIdx = fileInfoIdx(graph.config, + graph.config.libpath / RelativeFile"system.nim") + discard graph.compileModule(graph.config.m.systemFileIdx, {sfSystemModule}) + +proc compileProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) = + connectCallbacks(graph) + let conf = graph.config + wantMainModule(conf) + configComplete(graph) + + let systemFileIdx = fileInfoIdx(conf, conf.libpath / RelativeFile"system.nim") + let projectFile = if projectFileIdx == InvalidFileIdx: conf.projectMainIdx else: projectFileIdx + conf.projectMainIdx2 = projectFile + + let packSym = getPackage(graph, projectFile) + graph.config.mainPackageId = packSym.getPackageId + graph.importStack.add projectFile + + if projectFile == systemFileIdx: + discard graph.compileModule(projectFile, {sfMainModule, sfSystemModule}) + else: + graph.compileSystemModule() + discard graph.compileModule(projectFile, {sfMainModule}) + +proc mySemOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} = + result = preparePContext(graph, module, idgen) + +proc mySemClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode = + var c = PContext(context) + closePContext(graph, c, n) + +proc mySemProcess(context: PPassContext, n: PNode): PNode {.nosinks.} = + result = semWithPContext(PContext(context), n) + +const semPass* = makePass(mySemOpen, mySemProcess, mySemClose, + isFrontend = true) diff --git a/compiler/pipelines.nim b/compiler/pipelines.nim new file mode 100644 index 000000000..b80681c4b --- /dev/null +++ b/compiler/pipelines.nim @@ -0,0 +1,254 @@ +import sem, cgen, modulegraphs, ast, llstream, parser, msgs, + lineinfos, reorder, options, semdata, cgendata, modules, pathutils, + packages, syntaxes, depends, vm + +import pipelineutils + +when not defined(leanCompiler): + import jsgen, docgen2 + +import std/[syncio, objectdollar, assertions, tables] +import renderer +import ic/replayer + + +proc setPipeLinePass*(graph: ModuleGraph; pass: PipelinePass) = + graph.pipelinePass = pass + +proc processPipeline(graph: ModuleGraph; semNode: PNode; bModule: PPassContext): PNode = + case graph.pipelinePass + of CgenPass: + result = semNode + if bModule != nil: + genTopLevelStmt(BModule(bModule), result) + of JSgenPass: + when not defined(leanCompiler): + result = processJSCodeGen(bModule, semNode) + of GenDependPass: + result = addDotDependency(bModule, semNode) + of SemPass: + result = graph.emptyNode + of Docgen2Pass, Docgen2TexPass: + when not defined(leanCompiler): + result = processNode(bModule, semNode) + of Docgen2JsonPass: + when not defined(leanCompiler): + result = processNodeJson(bModule, semNode) + of EvalPass, InterpreterPass: + result = interpreterCode(bModule, semNode) + of NonePass: + doAssert false, "use setPipeLinePass to set a proper PipelinePass" + +proc processImplicitImports(graph: ModuleGraph; implicits: seq[string], nodeKind: TNodeKind, + m: PSym, ctx: PContext, bModule: PPassContext, idgen: IdGenerator, + ) = + # XXX fixme this should actually be relative to the config file! + let relativeTo = toFullPath(graph.config, m.info) + for module in items(implicits): + # implicit imports should not lead to a module importing itself + if m.position != resolveMod(graph.config, module, relativeTo).int32: + var importStmt = newNodeI(nodeKind, m.info) + var str = newStrNode(nkStrLit, module) + str.info = m.info + importStmt.add str + message(graph.config, importStmt.info, hintProcessingStmt, $idgen[]) + let semNode = semWithPContext(ctx, importStmt) + if semNode == nil or processPipeline(graph, semNode, bModule) == nil: + break + +proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator; + stream: PLLStream): bool = + if graph.stopCompile(): return true + var + p: Parser + s: PLLStream + fileIdx = module.fileIdx + + prepareConfigNotes(graph, module) + let ctx = preparePContext(graph, module, idgen) + let bModule: PPassContext = + case graph.pipelinePass + of CgenPass: + setupCgen(graph, module, idgen) + of JSgenPass: + when not defined(leanCompiler): + setupJSgen(graph, module, idgen) + else: + nil + of EvalPass, InterpreterPass: + setupEvalGen(graph, module, idgen) + of GenDependPass: + setupDependPass(graph, module, idgen) + of Docgen2Pass: + when not defined(leanCompiler): + openHtml(graph, module, idgen) + else: + nil + of Docgen2TexPass: + when not defined(leanCompiler): + openTex(graph, module, idgen) + else: + nil + of Docgen2JsonPass: + when not defined(leanCompiler): + openJson(graph, module, idgen) + else: + nil + of SemPass: + nil + of NonePass: + doAssert false, "use setPipeLinePass to set a proper PipelinePass" + nil + + if stream == nil: + let filename = toFullPathConsiderDirty(graph.config, fileIdx) + s = llStreamOpen(filename, fmRead) + if s == nil: + rawMessage(graph.config, errCannotOpenFile, filename.string) + return false + else: + s = stream + + while true: + syntaxes.openParser(p, fileIdx, s, graph.cache, graph.config) + + if not belongsToStdlib(graph, module) or (belongsToStdlib(graph, module) and module.name.s == "distros"): + # XXX what about caching? no processing then? what if I change the + # modules to include between compilation runs? we'd need to track that + # in ROD files. I think we should enable this feature only + # for the interactive mode. + if module.name.s != "nimscriptapi": + processImplicitImports graph, graph.config.implicitImports, nkImportStmt, module, ctx, bModule, idgen + processImplicitImports graph, graph.config.implicitIncludes, nkIncludeStmt, module, ctx, bModule, idgen + + checkFirstLineIndentation(p) + block processCode: + if graph.stopCompile(): break processCode + var n = parseTopLevelStmt(p) + if n.kind == nkEmpty: break processCode + # read everything, no streaming possible + var sl = newNodeI(nkStmtList, n.info) + sl.add n + while true: + var n = parseTopLevelStmt(p) + if n.kind == nkEmpty: break + sl.add n + if sfReorder in module.flags or codeReordering in graph.config.features: + sl = reorder(graph, sl, module) + if graph.pipelinePass != EvalPass: + message(graph.config, sl.info, hintProcessingStmt, $idgen[]) + var semNode = semWithPContext(ctx, sl) + discard processPipeline(graph, semNode, bModule) + + closeParser(p) + if s.kind != llsStdIn: break + let finalNode = closePContext(graph, ctx, nil) + case graph.pipelinePass + of CgenPass: + if bModule != nil: + finalCodegenActions(graph, BModule(bModule), finalNode) + of JSgenPass: + when not defined(leanCompiler): + discard finalJSCodeGen(graph, bModule, finalNode) + of EvalPass, InterpreterPass: + discard interpreterCode(bModule, finalNode) + of SemPass, GenDependPass: + discard + of Docgen2Pass, Docgen2TexPass: + when not defined(leanCompiler): + discard closeDoc(graph, bModule, finalNode) + of Docgen2JsonPass: + when not defined(leanCompiler): + discard closeJson(graph, bModule, finalNode) + of NonePass: + doAssert false, "use setPipeLinePass to set a proper PipelinePass" + + if graph.config.backend notin {backendC, backendCpp, backendObjc}: + # We only write rod files here if no C-like backend is active. + # The C-like backends have been patched to support the IC mechanism. + # They are responsible for closing the rod files. See `cbackend.nim`. + closeRodFile(graph, module) + result = true + +proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags, fromModule: PSym = nil): PSym = + var flags = flags + if fileIdx == graph.config.projectMainIdx2: flags.incl sfMainModule + result = graph.getModule(fileIdx) + + template processModuleAux(moduleStatus) = + onProcessing(graph, fileIdx, moduleStatus, fromModule = fromModule) + var s: PLLStream + if sfMainModule in flags: + if graph.config.projectIsStdin: s = stdin.llStreamOpen + elif graph.config.projectIsCmd: s = llStreamOpen(graph.config.cmdInput) + discard processPipelineModule(graph, result, idGeneratorFromModule(result), s) + if result == nil: + var cachedModules: seq[FileIndex] + result = moduleFromRodFile(graph, fileIdx, cachedModules) + let filename = AbsoluteFile toFullPath(graph.config, fileIdx) + if result == nil: + result = newModule(graph, fileIdx) + result.flags.incl flags + registerModule(graph, result) + processModuleAux("import") + else: + if sfSystemModule in flags: + graph.systemModule = result + partialInitModule(result, graph, fileIdx, filename) + for m in cachedModules: + registerModuleById(graph, m) + replayStateChanges(graph.packed[m.int].module, graph) + replayGenericCacheInformation(graph, m.int) + elif graph.isDirty(result): + result.flags.excl sfDirty + # reset module fields: + initStrTables(graph, result) + result.ast = nil + processModuleAux("import(dirty)") + graph.markClientsDirty(fileIdx) + +proc importPipelineModule(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym = + # this is called by the semantic checking phase + assert graph.config != nil + result = compilePipelineModule(graph, fileIdx, {}, s) + graph.addDep(s, fileIdx) + # keep track of import relationships + if graph.config.hcrOn: + graph.importDeps.mgetOrPut(FileIndex(s.position), @[]).add(fileIdx) + #if sfSystemModule in result.flags: + # localError(result.info, errAttemptToRedefine, result.name.s) + # restore the notes for outer module: + graph.config.notes = + if graph.config.belongsToProjectPackage(s) or isDefined(graph.config, "booting"): graph.config.mainPackageNotes + else: graph.config.foreignPackageNotes + +proc connectPipelineCallbacks*(graph: ModuleGraph) = + graph.includeFileCallback = modules.includeModule + graph.importModuleCallback = importPipelineModule + +proc compilePipelineSystemModule*(graph: ModuleGraph) = + if graph.systemModule == nil: + connectPipelineCallbacks(graph) + graph.config.m.systemFileIdx = fileInfoIdx(graph.config, + graph.config.libpath / RelativeFile"system.nim") + discard graph.compilePipelineModule(graph.config.m.systemFileIdx, {sfSystemModule}) + +proc compilePipelineProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) = + connectPipelineCallbacks(graph) + let conf = graph.config + wantMainModule(conf) + configComplete(graph) + + let systemFileIdx = fileInfoIdx(conf, conf.libpath / RelativeFile"system.nim") + let projectFile = if projectFileIdx == InvalidFileIdx: conf.projectMainIdx else: projectFileIdx + conf.projectMainIdx2 = projectFile + + let packSym = getPackage(graph, projectFile) + graph.config.mainPackageId = packSym.getPackageId + graph.importStack.add projectFile + + if projectFile == systemFileIdx: + discard graph.compilePipelineModule(projectFile, {sfMainModule, sfSystemModule}) + else: + graph.compilePipelineSystemModule() + discard graph.compilePipelineModule(projectFile, {sfMainModule}) diff --git a/compiler/pipelineutils.nim b/compiler/pipelineutils.nim new file mode 100644 index 000000000..75ba33f14 --- /dev/null +++ b/compiler/pipelineutils.nim @@ -0,0 +1,26 @@ +import ast, options, lineinfos, pathutils, msgs, modulegraphs, packages + +proc skipCodegen*(config: ConfigRef; n: PNode): bool {.inline.} = + # can be used by codegen passes to determine whether they should do + # something with `n`. Currently, this ignores `n` and uses the global + # error count instead. + result = config.errorCounter > 0 + +proc resolveMod*(conf: ConfigRef; module, relativeTo: string): FileIndex = + let fullPath = findModule(conf, module, relativeTo) + if fullPath.isEmpty: + result = InvalidFileIdx + else: + result = fileInfoIdx(conf, fullPath) + +proc prepareConfigNotes*(graph: ModuleGraph; module: PSym) = + # don't be verbose unless the module belongs to the main package: + if graph.config.belongsToProjectPackage(module): + graph.config.notes = graph.config.mainPackageNotes + else: + if graph.config.mainPackageNotes == {}: graph.config.mainPackageNotes = graph.config.notes + graph.config.notes = graph.config.foreignPackageNotes + +proc moduleHasChanged*(graph: ModuleGraph; module: PSym): bool {.inline.} = + result = true + #module.id >= 0 or isDefined(graph.config, "nimBackendAssumesChange") diff --git a/compiler/scriptconfig.nim b/compiler/scriptconfig.nim index 009e90fd5..27ea94aae 100644 --- a/compiler/scriptconfig.nim +++ b/compiler/scriptconfig.nim @@ -11,10 +11,10 @@ ## language. import - ast, modules, idents, passes, condsyms, - options, sem, llstream, vm, vmdef, commands, + ast, modules, idents, condsyms, + options, llstream, vm, vmdef, commands, os, times, osproc, wordrecg, strtabs, modulegraphs, - pathutils + pathutils, pipelines when defined(nimPreviewSlimSystem): import std/syncio @@ -197,13 +197,11 @@ proc runNimScript*(cache: IdentCache; scriptName: AbsoluteFile; conf.symbolFiles = disabledSf let graph = newModuleGraph(cache, conf) - connectCallbacks(graph) + connectPipelineCallbacks(graph) if freshDefines: initDefines(conf.symbols) defineSymbol(conf.symbols, "nimscript") defineSymbol(conf.symbols, "nimconfig") - registerPass(graph, semPass) - registerPass(graph, evalPass) conf.searchPaths.add(conf.libpath) @@ -218,8 +216,9 @@ proc runNimScript*(cache: IdentCache; scriptName: AbsoluteFile; var vm = setupVM(m, cache, scriptName.string, graph, idgen) graph.vm = vm - graph.compileSystemModule() - discard graph.processModule(m, vm.idgen, stream) + graph.setPipeLinePass(EvalPass) + graph.compilePipelineSystemModule() + discard graph.processPipelineModule(m, vm.idgen, stream) # watch out, "newruntime" can be set within NimScript itself and then we need # to remember this: diff --git a/compiler/sem.nim b/compiler/sem.nim index 78596d5b2..a5d06f3fe 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -13,7 +13,7 @@ import ast, strutils, options, astalgo, trees, wordrecg, ropes, msgs, idents, renderer, types, platform, math, magicsys, nversion, nimsets, semfold, modulepaths, importer, - procfind, lookups, pragmas, passes, semdata, semtypinst, sigmatch, + procfind, lookups, pragmas, semdata, semtypinst, sigmatch, intsets, transf, vmdef, vm, aliases, cgmeth, lambdalifting, evaltempl, patterns, parampatterns, sempass2, linter, semmacrosanity, lowerings, plugins/active, lineinfos, strtabs, int128, @@ -666,38 +666,37 @@ proc addCodeForGenerics(c: PContext, n: PNode) = n.add prc.ast c.lastGenericIdx = c.generics.len -proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} = - var c = newContext(graph, module) - c.idgen = idgen - c.enforceVoidContext = newType(tyTyped, nextTypeId(idgen), nil) - c.voidType = newType(tyVoid, nextTypeId(idgen), nil) +proc preparePContext*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PContext {.nosinks.} = + result = newContext(graph, module) + result.idgen = idgen + result.enforceVoidContext = newType(tyTyped, nextTypeId(idgen), nil) + result.voidType = newType(tyVoid, nextTypeId(idgen), nil) - if c.p != nil: internalError(graph.config, module.info, "sem.myOpen") - c.semConstExpr = semConstExpr - c.semExpr = semExpr - c.semTryExpr = tryExpr - c.semTryConstExpr = tryConstExpr - c.computeRequiresInit = computeRequiresInit - c.semOperand = semOperand - c.semConstBoolExpr = semConstBoolExpr - c.semOverloadedCall = semOverloadedCall - c.semInferredLambda = semInferredLambda - c.semGenerateInstance = generateInstance - c.semTypeNode = semTypeNode - c.instTypeBoundOp = sigmatch.instTypeBoundOp - c.hasUnresolvedArgs = hasUnresolvedArgs - c.templInstCounter = new int + if result.p != nil: internalError(graph.config, module.info, "sem.preparePContext") + result.semConstExpr = semConstExpr + result.semExpr = semExpr + result.semTryExpr = tryExpr + result.semTryConstExpr = tryConstExpr + result.computeRequiresInit = computeRequiresInit + result.semOperand = semOperand + result.semConstBoolExpr = semConstBoolExpr + result.semOverloadedCall = semOverloadedCall + result.semInferredLambda = semInferredLambda + result.semGenerateInstance = generateInstance + result.semTypeNode = semTypeNode + result.instTypeBoundOp = sigmatch.instTypeBoundOp + result.hasUnresolvedArgs = hasUnresolvedArgs + result.templInstCounter = new int - pushProcCon(c, module) - pushOwner(c, c.module) + pushProcCon(result, module) + pushOwner(result, result.module) - c.moduleScope = openScope(c) - c.moduleScope.addSym(module) # a module knows itself + result.moduleScope = openScope(result) + result.moduleScope.addSym(module) # a module knows itself if sfSystemModule in module.flags: graph.systemModule = module - c.topLevelScope = openScope(c) - result = c + result.topLevelScope = openScope(result) proc isImportSystemStmt(g: ModuleGraph; n: PNode): bool = if g.systemModule == nil: return false @@ -771,8 +770,7 @@ proc recoverContext(c: PContext) = while getCurrOwner(c).kind != skModule: popOwner(c) while c.p != nil and c.p.owner.kind != skModule: c.p = c.p.next -proc myProcess(context: PPassContext, n: PNode): PNode {.nosinks.} = - var c = PContext(context) +proc semWithPContext*(c: PContext, n: PNode): PNode {.nosinks.} = # no need for an expensive 'try' if we stop after the first error anyway: if c.config.errorMax <= 1: result = semStmtAndGenerateGenerics(c, n) @@ -793,13 +791,13 @@ proc myProcess(context: PPassContext, n: PNode): PNode {.nosinks.} = #if c.config.cmd == cmdIdeTools: findSuggest(c, n) storeRodNode(c, result) + proc reportUnusedModules(c: PContext) = for i in 0..high(c.unusedImports): if sfUsed notin c.unusedImports[i][0].flags: message(c.config, c.unusedImports[i][1], warnUnusedImportX, c.unusedImports[i][0].name.s) -proc myClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode = - var c = PContext(context) +proc closePContext*(graph: ModuleGraph; c: PContext, n: PNode): PNode = if c.config.cmd == cmdIdeTools and not c.suggestionsMade: suggestSentinel(c) closeScope(c) # close module's scope @@ -814,6 +812,3 @@ proc myClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode = popOwner(c) popProcCon(c) sealRodFile(c) - -const semPass* = makePass(myOpen, myProcess, myClose, - isFrontend = true) diff --git a/compiler/suggest.nim b/compiler/suggest.nim index 637010ad5..384ceb271 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -36,7 +36,7 @@ import algorithm, sets, prefixmatches, parseutils, tables from wordrecg import wDeprecated, wError, wAddr, wYield when defined(nimsuggest): - import passes, tables, pathutils # importer + import tables, pathutils # importer const sep = '\t' diff --git a/compiler/vm.nim b/compiler/vm.nim index 18d840613..96f05c0f9 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -13,7 +13,7 @@ import semmacrosanity import std/[strutils, tables, parseutils], - msgs, vmdef, vmgen, nimsets, types, passes, + msgs, vmdef, vmgen, nimsets, types, parser, vmdeps, idents, trees, renderer, options, transf, gorgeimpl, lineinfos, btrees, macrocacheimpl, modulegraphs, sighashes, int128, vmprofiler @@ -2309,7 +2309,7 @@ proc setupGlobalCtx*(module: PSym; graph: ModuleGraph; idgen: IdGenerator) = else: refresh(PCtx graph.vm, module, idgen) -proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} = +proc setupEvalGen*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} = #var c = newEvalContext(module, emRepl) #c.features = {allowCast, allowInfiniteLoops} #pushStackFrame(c, newStackFrame()) @@ -2318,7 +2318,7 @@ proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext setupGlobalCtx(module, graph, idgen) result = PCtx graph.vm -proc myProcess(c: PPassContext, n: PNode): PNode = +proc interpreterCode*(c: PPassContext, n: PNode): PNode = let c = PCtx(c) # don't eval errornous code: if c.oldErrorCount == c.config.errorCounter: @@ -2328,11 +2328,6 @@ proc myProcess(c: PPassContext, n: PNode): PNode = result = n c.oldErrorCount = c.config.errorCounter -proc myClose(graph: ModuleGraph; c: PPassContext, n: PNode): PNode = - result = myProcess(c, n) - -const evalPass* = makePass(myOpen, myProcess, myClose) - proc evalConstExprAux(module: PSym; idgen: IdGenerator; g: ModuleGraph; prc: PSym, n: PNode, mode: TEvalMode): PNode = diff --git a/doc/refc.md b/doc/refc.md index 6e4672ac5..4023748e6 100644 --- a/doc/refc.md +++ b/doc/refc.md @@ -138,6 +138,7 @@ The garbage collector won't try to free them, you need to call their respective when you are done with them or they will leak. + Heap dump ========= diff --git a/nimsuggest/nimsuggest.nim b/nimsuggest/nimsuggest.nim index a0a3b8e82..ce9681975 100644 --- a/nimsuggest/nimsuggest.nim +++ b/nimsuggest/nimsuggest.nim @@ -23,7 +23,7 @@ import strutils, os, parseopt, parseutils, sequtils, net, rdstdin, sexp # Do NOT import suggest. It will lead to weird bugs with # suggestionResultHook, because suggest.nim is included by sigmatch. # So we import that one instead. -import compiler / [options, commands, modules, sem, +import compiler / [options, commands, modules, passes, passaux, msgs, sigmatch, ast, idents, modulegraphs, prefixmatches, lineinfos, cmdlinehelper, From 2e2affb13ce13723e5ab1996d05f7b82c184a090 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 3 Mar 2023 23:13:27 +0800 Subject: [PATCH 2452/2472] test DLL generation with ORC (#21445) * test DLL generation with ORC * fixes * fixes refc * Update testament/categories.nim --- testament/categories.nim | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/testament/categories.nim b/testament/categories.nim index cae699327..c428ffc04 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -47,21 +47,24 @@ proc isTestFile*(file: string): bool = # --------------------- DLL generation tests ---------------------------------- -proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) = +proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string, isOrc = false) = const rpath = when defined(macosx): " --passL:-rpath --passL:@loader_path" else: "" - var test1 = makeTest("lib/nimrtl.nim", options & " --outdir:tests/dll", cat) - test1.spec.action = actionCompile - testSpec c, test1 + if not defined(windows) or not isOrc: # todo fix me on windows + var test1 = makeTest("lib/nimrtl.nim", options & " --outdir:tests/dll", cat) + test1.spec.action = actionCompile + testSpec c, test1 var test2 = makeTest("tests/dll/server.nim", options & " --threads:on" & rpath, cat) test2.spec.action = actionCompile testSpec c, test2 - var test3 = makeTest("lib/nimhcr.nim", options & " --threads:off --outdir:tests/dll" & rpath, cat) - test3.spec.action = actionCompile - testSpec c, test3 + + if not isOrc: + var test3 = makeTest("lib/nimhcr.nim", options & " --threads:off --outdir:tests/dll" & rpath, cat) + test3.spec.action = actionCompile + testSpec c, test3 var test4 = makeTest("tests/dll/visibility.nim", options & " --threads:off --app:lib" & rpath, cat) test4.spec.action = actionCompile testSpec c, test4 @@ -76,8 +79,9 @@ proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) = putEnv(libpathenv, "tests/dll" & (if libpath.len > 0: ":" & libpath else: "")) defer: putEnv(libpathenv, libpath) - testSpec r, makeTest("tests/dll/client.nim", options & " --threads:on" & rpath, cat) - testSpec r, makeTest("tests/dll/nimhcr_unit.nim", options & " --threads:off" & rpath, cat) + if not isOrc: + testSpec r, makeTest("tests/dll/client.nim", options & " --threads:on" & rpath, cat) + testSpec r, makeTest("tests/dll/nimhcr_unit.nim", options & " --threads:off" & rpath, cat) testSpec r, makeTest("tests/dll/visibility.nim", options & " --threads:off" & rpath, cat) if "boehm" notin options: @@ -96,6 +100,8 @@ proc dllTests(r: var TResults, cat: Category, options: string) = runBasicDLLTest c, r, cat, options & " --mm:refc" runBasicDLLTest c, r, cat, options & " -d:release --mm:refc" + runBasicDLLTest c, r, cat, options, isOrc = true + runBasicDLLTest c, r, cat, options & " -d:release", isOrc = true when not defined(windows): # still cannot find a recent Windows version of boehm.dll: runBasicDLLTest c, r, cat, options & " --gc:boehm" From b2c5f8a05f2f9db6ed06663a077882eb112b3e3f Mon Sep 17 00:00:00 2001 From: Arnaud Moura Date: Fri, 3 Mar 2023 23:37:12 +0100 Subject: [PATCH 2453/2472] fixes #21461 (#21463) * fixes #21461; Move nim-gdb.py and add nim-lldb.py * fixes bad path for nim-gdb.py --- bin/nim-gdb | 2 +- bin/nim-gdb.bat | 2 +- compiler/installer.ini | 2 +- doc/packaging.md | 2 +- .../untestable/gdb/gdb_pretty_printer_test.py | 2 +- tools/{ => debug}/nim-gdb.py | 0 tools/debug/nim-lldb.py | 1039 +++++++++++++++++ 7 files changed, 1044 insertions(+), 5 deletions(-) rename tools/{ => debug}/nim-gdb.py (100%) create mode 100644 tools/debug/nim-lldb.py diff --git a/bin/nim-gdb b/bin/nim-gdb index 13b361f93..d4d60e432 100755 --- a/bin/nim-gdb +++ b/bin/nim-gdb @@ -14,7 +14,7 @@ else fi # Find out where the pretty printer Python module is -GDB_PYTHON_MODULE_PATH="$NIM_SYSROOT/tools/nim-gdb.py" +GDB_PYTHON_MODULE_PATH="$NIM_SYSROOT/tools/debug/nim-gdb.py" # Run GDB with the additional arguments that load the pretty printers # Set the environment variable `NIM_GDB` to overwrite the call to a diff --git a/bin/nim-gdb.bat b/bin/nim-gdb.bat index e98a2063c..b5537dd9d 100644 --- a/bin/nim-gdb.bat +++ b/bin/nim-gdb.bat @@ -3,7 +3,7 @@ for %%i in (nim.exe) do (set NIM_BIN=%%~dp$PATH:i) for %%i in ("%NIM_BIN%\..\") do (set NIM_ROOT=%%~fi) -set @GDB_PYTHON_MODULE_PATH=%NIM_ROOT%\tools\nim-gdb.py +set @GDB_PYTHON_MODULE_PATH=%NIM_ROOT%\tools\debug\nim-gdb.py set @NIM_GDB=gdb.exe @echo source %@GDB_PYTHON_MODULE_PATH%> wingdbcommand.txt diff --git a/compiler/installer.ini b/compiler/installer.ini index f67ca5c25..226682715 100644 --- a/compiler/installer.ini +++ b/compiler/installer.ini @@ -65,7 +65,7 @@ Files: "compiler" Files: "doc" Files: "doc/html" Files: "tools" -Files: "tools/nim-gdb.py" +Files: "tools/debug/nim-gdb.py" Files: "nimpretty" Files: "testament" Files: "nimsuggest" diff --git a/doc/packaging.md b/doc/packaging.md index 522ca6d44..b742bef28 100644 --- a/doc/packaging.md +++ b/doc/packaging.md @@ -71,7 +71,7 @@ What to install: - The expected stdlib location is `/usr/lib/nim/lib`, previously it was just `/usr/lib/nim` - `nimdoc.css` and `nimdoc.cls` from the `doc` folder should go into `/usr/lib/nim/doc/` -- `tools/nim-gdb.py` should go into `/usr/lib/nim/tools/` +- `tools/debug/nim-gdb.py` should go into `/usr/lib/nim/tools/` - `tools/dochack/dochack.js` should be installed to `/usr/lib/nim/tools/dochack/` - Global configuration files under `/etc/nim` - Optionally: manpages, documentation, shell completion diff --git a/tests/untestable/gdb/gdb_pretty_printer_test.py b/tests/untestable/gdb/gdb_pretty_printer_test.py index a96df9992..aed0cfeb0 100644 --- a/tests/untestable/gdb/gdb_pretty_printer_test.py +++ b/tests/untestable/gdb/gdb_pretty_printer_test.py @@ -8,7 +8,7 @@ import sys # frontends might still be broken. gdb.execute("set python print-stack full") -gdb.execute("source ../../../tools/nim-gdb.py") +gdb.execute("source ../../../tools/debug/nim-gdb.py") # debug all instances of the generic function `myDebug`, should be 14 gdb.execute("rbreak myDebug") gdb.execute("run") diff --git a/tools/nim-gdb.py b/tools/debug/nim-gdb.py similarity index 100% rename from tools/nim-gdb.py rename to tools/debug/nim-gdb.py diff --git a/tools/debug/nim-lldb.py b/tools/debug/nim-lldb.py new file mode 100644 index 000000000..625b75988 --- /dev/null +++ b/tools/debug/nim-lldb.py @@ -0,0 +1,1039 @@ +import lldb +from collections import OrderedDict +from typing import Union + + +def sbvaluegetitem(self: lldb.SBValue, name: Union[int, str]) -> lldb.SBValue: + if isinstance(name, str): + return self.GetChildMemberWithName(name) + else: + return self.GetChildAtIndex(name) + + +# Make this easier to work with +lldb.SBValue.__getitem__ = sbvaluegetitem + + +def colored(in_str, *args, **kwargs): + # TODO: Output in color if user is in terminal + return in_str + + +def reprEnum(val, typ): + """ + this is a port of the nim runtime function `reprEnum` to python + NOTE: DOES NOT WORK WITH ORC + """ + val = int(val) + n = typ["node"] + sons_type = n["sons"].type.GetPointeeType().GetPointeeType() + sons = n["sons"].deref.Cast(sons_type.GetPointerType().GetArrayType(3)) + flags = int(typ["flags"].unsigned) + # 1 << 6 is {ntfEnumHole} + if ((1 << 6) & flags) == 0: + offset = val - sons[0]["offset"].unsigned + if offset >= 0 and 0 < n["len"].unsigned: + return NCSTRING(sons[offset]["name"])[1:-1] + else: + # ugh we need a slow linear search: + for i in range(n["len"].unsigned): + if sons[i]["offset"].unsigned == val: + return NCSTRING(sons[i]["name"])[1:-1] + + return str(val) + " (invalid data!)" + + +def get_nti(value, nim_name=None): + """DOES NOT WORK WITH ORC""" + name_split = value.type.name.split("_") + type_nim_name = nim_name or name_split[1] + id_string = name_split[-1].split(" ")[0] + + type_info_name = "NTI" + type_nim_name.lower() + "__" + id_string + "_" + print("TYPEINFONAME: ", type_info_name) + nti = value.target.FindFirstGlobalVariable(type_info_name) + if nti is None: + type_info_name = "NTI" + "__" + id_string + "_" + nti = value.target.FindFirstGlobalVariable(type_info_name) + if nti is None: + print( + f"NimEnumPrinter: lookup global symbol: '{type_info_name}' failed for {value.type.name}.\n" + ) + return type_nim_name, nti + + +def enum_to_string(value): + type_nim_name, nti = get_nti(value) + if nti is None: + return type_nim_name + "(" + str(value.unsigned) + ")" + return reprEnum(value.signed, nti), nti + + +def to_string(value): + # For getting NimStringDesc * value + data = value["data"] + try: + size = int(value["Sup"]["len"].unsigned) + if size > 2**14: + return None + except TypeError: + return None + + cast = data.Cast(value.target.FindFirstType("char").GetArrayType(size)) + return bytearray(cast.data.uint8s).decode("utf-8") + + +def to_stringV2(value: lldb.SBValue): + # For getting NimStringDesc * value + data = value["p"]["data"] + try: + size = int(value["len"].signed) + if size > 2**14: + return "... (too long)" + except TypeError: + return "" + + base_data_type = data.type.GetArrayElementType().GetTypedefedType() + cast = data.Cast(base_data_type.GetArrayType(size)) + return bytearray(cast.data.uint8s).decode("utf-8") + + +def NimStringDesc(value, internal_dict): + res = to_string(value) + if res: + return colored('"' + res + '"', "red") + else: + return str(value) + + +def NimStringV2(value: lldb.SBValue, internal_dict): + res = to_stringV2(value.GetNonSyntheticValue()) + if res is not None: + return colored('"' + res + '"', "red") + else: + return str(value) + + +def NCSTRING(value: lldb.SBValue, internal_dict=None): + ty = value.Dereference().type + val = value.target.CreateValueFromAddress( + value.name or "temp", lldb.SBAddress(value.unsigned, value.target), ty + ).AddressOf() + return val.summary + + +def ObjectV1(value, internal_dict): + if not value.num_children and not value.value: + return "" + + ignore_fields = set() + if "colonObjectType" in value.type.name: + value = value.Dereference() + ignore_fields.add("Sup") + + if not value.type.name: + return "" + + summary = value.summary + if summary is not None: + return summary + + if "_" in value.type.name: + obj_name = value.type.name.split("_")[1].replace("colonObjectType", "") + else: + obj_name = value.type.name + + obj_name = colored(obj_name, "green") + + num_children = value.num_children + + fields = ", ".join( + [ + value[i].name + + ": " + + (value[i].summary or value[i].value or value[i].type.name or "not found") + for i in range(num_children) + if value[i].name not in ignore_fields + ] + ) + + res = f"{obj_name}({fields})" + return res + + +def ObjectV2(value: lldb.SBValue, internal_dict): + custom_summary = get_summary(value) + if not custom_summary is None: + return custom_summary + + orig_value = value.GetNonSyntheticValue() + while orig_value.type.is_pointer: + orig_value = orig_value.Dereference() + + if "_" in orig_value.type.name: + obj_name = orig_value.type.name.split("_")[1].replace("colonObjectType", "") + else: + obj_name = orig_value.type.name + + num_children = value.num_children + fields = [] + + for i in range(num_children): + fields.append(f"{value[i].name}: {value[i].summary}") + + res = f"{obj_name}(" + ", ".join(fields) + ")" + return res + + +def Number(value: lldb.SBValue, internal_dict): + while value.type.is_pointer: + value = value.Dereference() + return colored(str(value.signed), "yellow") + + +def Float(value: lldb.SBValue, internal_dict): + while value.type.is_pointer: + value = value.Dereference() + return colored(str(value.value), "yellow") + + +def UnsignedNumber(value: lldb.SBValue, internal_dict): + while value.type.is_pointer: + value = value.Dereference() + return colored(str(value.unsigned), "yellow") + + +def Bool(value: lldb.SBValue, internal_dict): + while value.type.is_pointer: + value = value.Dereference() + return colored(str(value.GetValue()), "red") + + +def CharArray(value: lldb.SBValue, internal_dict): + return str([colored(f"'{char}'", "red") for char in value.uint8s]) + + +def Array(value: lldb.SBValue, internal_dict): + value = value.GetNonSyntheticValue() + return "[" + ", ".join([value[i].summary for i in range(value.num_children)]) + "]" + + +def Tuple(value: lldb.SBValue, internal_dict): + while value.type.is_pointer: + value = value.Dereference() + + num_children = value.num_children + + fields = [] + + for i in range(num_children): + key = value[i].name + val = value[i].summary + if key.startswith("Field"): + fields.append(f"{val}") + else: + fields.append(f"{key}: {val}") + + return "(" + ", ".join(fields) + f")" + + +def Enum(value, internal_dict): + tname = value.type.name.split("_")[1] + return colored(f"{tname}." + str(value.signed), "blue") + + +def EnumSet(value, internal_dict): + type_nim_name = value.type.name.split("_")[2] + # type_nim_name, nti = get_nti(value, type_nim_name) + + val = int(value.signed) + # if nti: + # enum_strings = [] + # i = 0 + # while val > 0: + # if (val & 1) == 1: + # enum_strings.append(reprEnum(i, nti)) + # val = val >> 1 + # i += 1 + + # return '{' + ', '.join(enum_strings) + '}' + return colored(f"{type_nim_name}." + str(val), "blue") + + +def Set(value, internal_dict): + vals = [] + max_vals = 7 + for child in value.children: + vals.append(child.value) + if len(vals) > max_vals: + vals.append("...") + break + + return "{" + ", ".join(vals) + "}" + + +def Table(value: lldb.SBValue, internal_dict): + fields = [] + + for i in range(value.num_children): + key = value[i].name + val = value[i].summary + fields.append(f"{key}: {val}") + + table_suffix = "Table" + return "{" + ", ".join(fields) + f"}}.{table_suffix}" + + +def HashSet(value: lldb.SBValue, internal_dict): + fields = [] + + for i in range(value.num_children): + fields.append(f"{value[i].summary}") + + table_suffix = "HashSet" + + return "{" + ", ".join(fields) + f"}}.{table_suffix}" + + +def StringTable(value: lldb.SBValue, internal_dict): + table = value.GetNonSyntheticValue() + mode = table["mode"].unsigned + + table_suffix = "StringTable" + + table_mode = "" + if mode == 0: + table_mode = "Case Sensitive" + elif mode == 1: + table_mode = "Case Insensitive" + elif mode == 2: + table_mode = "Style Insensitive" + + fields = [] + + for i in range(value.num_children): + key = value[i].name + val = value[i].summary + fields.append(f"{key}: {val}") + + return "{" + ", ".join(fields) + f"}}.{table_suffix}({table_mode})" + + +def Sequence(value: lldb.SBValue, internal_dict): + value = value.GetNonSyntheticValue() + + data_len = int(value["len"].unsigned) + data = value["p"]["data"] + base_data_type = data.type.GetArrayElementType() + + cast = data.Cast(base_data_type.GetArrayType(data_len)) + + return ( + "@[" + + ", ".join([cast[i].summary or cast[i].type.name for i in range(data_len)]) + + "]" + ) + + +class StringChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.data_type: lldb.SBType + self.first_element: lldb.SBValue + self.update() + self.count = 0 + + def num_children(self): + return self.count + + def get_child_index(self, name): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + offset = index * self.data_size + return self.first_element.CreateChildAtOffset( + "[" + str(index) + "]", offset, self.data_type + ) + + def update(self): + data = self.value["p"]["data"] + size = int(self.value["len"].unsigned) + + self.count = size + self.first_element = data + + self.data_type = data.type.GetArrayElementType().GetTypedefedType() + self.data_size = self.data_type.GetByteSize() + + def has_children(self): + return bool(self.num_children()) + + +class ArrayChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.data_type: lldb.SBType + self.first_element: lldb.SBValue + self.update() + + def num_children(self): + return self.value.num_children + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + offset = index * self.value[index].GetByteSize() + return self.first_element.CreateChildAtOffset( + "[" + str(index) + "]", offset, self.data_type + ) + + def update(self): + if not self.has_children(): + return + + self.first_element = self.value[0] + self.data_type = self.value.type.GetArrayElementType() + + def has_children(self): + return bool(self.num_children()) + + +class SeqChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.data_type: lldb.SBType + self.first_element: lldb.SBValue + self.data: lldb.SBValue + self.update() + + def num_children(self): + return int(self.value["len"].unsigned) + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + offset = index * self.data[index].GetByteSize() + return self.first_element.CreateChildAtOffset( + "[" + str(index) + "]", offset, self.data_type + ) + + def update(self): + if not self.has_children(): + return + + data = self.value["p"]["data"] + self.data_type = data.type.GetArrayElementType() + + self.data = data.Cast(self.data_type.GetArrayType(self.num_children())) + self.first_element = self.data + + def has_children(self): + return bool(self.num_children()) + + +class ObjectChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.data_type: lldb.SBType + self.first_element: lldb.SBValue + self.data: lldb.SBValue + self.children: OrderedDict[str, int] = OrderedDict() + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.children) + + def get_child_index(self, name: str): + return self.children[name] + + def get_child_at_index(self, index): + return self.child_list[index] + + def populate_children(self): + self.children.clear() + self.child_list = [] + stack = [self.value] + + index = 0 + + while stack: + cur_val = stack.pop() + while cur_val.type.is_pointer: + cur_val = cur_val.Dereference() + + if cur_val.num_children > 0 and cur_val[0].name == "m_type": + if "_" in cur_val.type.name: + tname = cur_val.type.name.split("_")[1].replace( + "colonObjectType", "" + ) + else: + tname = cur_val.type.name + if tname == "TNimTypeV2": + # We've reached the end + break + + if ( + cur_val.num_children > 0 + and cur_val[0].name == "Sup" + and cur_val[0].type.name.startswith("tyObject") + ): + stack.append(cur_val[0]) + + for i in range(cur_val.num_children): + child = cur_val[i] + if child.name == "Sup": + continue + self.children[child.name] = index + self.child_list.append(child) + index += 1 + + def update(self): + self.populate_children() + + def has_children(self): + return bool(self.num_children()) + + +class HashSetChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + return self.child_list[index] + + def update(self): + self.child_list = [] + data = self.value["data"] + + tuple_len = int(data["len"].unsigned) + tuple = data["p"]["data"] + + base_data_type = tuple.type.GetArrayElementType() + + cast = tuple.Cast(base_data_type.GetArrayType(tuple_len)) + + index = 0 + for i in range(tuple_len): + el = cast[i] + field0 = int(el["Field0"].unsigned) + if field0 == 0: + continue + key = el["Field1"] + child = key.CreateValueFromAddress( + f"[{str(index)}]", key.GetLoadAddress(), key.GetType() + ) + index += 1 + + self.child_list.append(child) + + def has_children(self): + return bool(self.num_children()) + + +class SetCharChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.ty = self.value.target.FindFirstType("char") + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + return self.child_list[index] + + def update(self): + self.child_list = [] + cur_pos = 0 + for child in self.value.children: + child_val = child.signed + if child_val != 0: + temp = child_val + num_bits = 8 + while temp != 0: + is_set = temp & 1 + if is_set == 1: + data = lldb.SBData.CreateDataFromInt(cur_pos) + child = self.value.synthetic_child_from_data( + f"[{len(self.child_list)}]", data, self.ty + ) + self.child_list.append(child) + temp = temp >> 1 + cur_pos += 1 + num_bits -= 1 + cur_pos += num_bits + else: + cur_pos += 8 + + def has_children(self): + return bool(self.num_children()) + + +class SetIntChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.ty = self.value.target.FindFirstType(f"NI64") + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + return self.child_list[index] + + def update(self): + self.child_list = [] + bits = self.value.GetByteSize() * 8 + + cur_pos = -(bits // 2) + + if self.value.num_children > 0: + children = self.value.children + else: + children = [self.value] + + for child in children: + child_val = child.signed + if child_val != 0: + temp = child_val + num_bits = 8 + while temp != 0: + is_set = temp & 1 + if is_set == 1: + data = lldb.SBData.CreateDataFromInt(cur_pos) + child = self.value.synthetic_child_from_data( + f"[{len(self.child_list)}]", data, self.ty + ) + self.child_list.append(child) + temp = temp >> 1 + cur_pos += 1 + num_bits -= 1 + cur_pos += num_bits + else: + cur_pos += 8 + + def has_children(self): + return bool(self.num_children()) + + +class SetUIntChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.ty = self.value.target.FindFirstType(f"NU64") + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + return self.child_list[index] + + def update(self): + self.child_list = [] + + cur_pos = 0 + if self.value.num_children > 0: + children = self.value.children + else: + children = [self.value] + + for child in children: + child_val = child.signed + if child_val != 0: + temp = child_val + num_bits = 8 + while temp != 0: + is_set = temp & 1 + if is_set == 1: + data = lldb.SBData.CreateDataFromInt(cur_pos) + child = self.value.synthetic_child_from_data( + f"[{len(self.child_list)}]", data, self.ty + ) + self.child_list.append(child) + temp = temp >> 1 + cur_pos += 1 + num_bits -= 1 + cur_pos += num_bits + else: + cur_pos += 8 + + def has_children(self): + return bool(self.num_children()) + + +class SetEnumChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.ty = self.value.target.FindFirstType(f"NU64") + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + return self.child_list[index] + + def update(self): + self.child_list = [] + + cur_pos = 0 + if self.value.num_children > 0: + children = self.value.children + else: + children = [self.value] + + for child in children: + child_val = child.unsigned + if child_val != 0: + temp = child_val + num_bits = 8 + while temp != 0: + is_set = temp & 1 + if is_set == 1: + data = lldb.SBData.CreateDataFromInt(cur_pos) + child = self.value.synthetic_child_from_data( + f"[{len(self.child_list)}]", data, self.ty + ) + self.child_list.append(child) + temp = temp >> 1 + cur_pos += 1 + num_bits -= 1 + cur_pos += num_bits + else: + cur_pos += 8 + + def has_children(self): + return bool(self.num_children()) + + +class TableChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.children: OrderedDict[str, int] = OrderedDict() + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return self.children[name] + + def get_child_at_index(self, index): + return self.child_list[index] + + def update(self): + self.child_list = [] + data = self.value["data"] + + tuple_len = int(data["len"].unsigned) + tuple = data["p"]["data"] + + base_data_type = tuple.type.GetArrayElementType() + + cast = tuple.Cast(base_data_type.GetArrayType(tuple_len)) + + index = 0 + for i in range(tuple_len): + el = cast[i] + field0 = int(el["Field0"].unsigned) + if field0 == 0: + continue + key = el["Field1"] + val = el["Field2"] + child = val.CreateValueFromAddress( + key.summary, val.GetLoadAddress(), val.GetType() + ) + self.child_list.append(child) + self.children[key.summary] = index + index += 1 + + def has_children(self): + return bool(self.num_children()) + + +class StringTableChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.children: OrderedDict[str, int] = OrderedDict() + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return self.children[name] + + def get_child_at_index(self, index): + return self.child_list[index] + + def update(self): + self.children.clear() + self.child_list = [] + data = self.value["data"] + + tuple_len = int(data["len"].unsigned) + tuple = data["p"]["data"] + + base_data_type = tuple.type.GetArrayElementType() + + cast = tuple.Cast(base_data_type.GetArrayType(tuple_len)) + + index = 0 + for i in range(tuple_len): + el = cast[i] + field0 = int(el["Field2"].unsigned) + if field0 == 0: + continue + key = el["Field0"] + val = el["Field1"] + child = val.CreateValueFromAddress( + key.summary, val.GetLoadAddress(), val.GetType() + ) + self.child_list.append(child) + self.children[key.summary] = index + index += 1 + + def has_children(self): + return bool(self.num_children()) + + +class CustomObjectChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + print("CUSTOMOBJ: ", value.name) + self.value: lldb.SBValue = get_synthetic(value) or value + for child in self.value.children: + print(child) + + def num_children(self): + return self.value.num_children + + def get_child_index(self, name: str): + return self.value.GetIndexOfChildWithName(name) + + def get_child_at_index(self, index): + return self.value.GetChildAtIndex(index) + + def update(self): + pass + + def has_children(self): + return self.num_children() > 0 + + +def echo(debugger: lldb.SBDebugger, command: str, result, internal_dict): + debugger.HandleCommand("po " + command) + + +SUMMARY_FUNCTIONS: dict[str, lldb.SBFunction] = {} +SYNTHETIC_FUNCTIONS: dict[str, lldb.SBFunction] = {} + + +def get_summary(value: lldb.SBValue) -> Union[str, None]: + base_type = get_base_type(value.type) + + fn = SUMMARY_FUNCTIONS.get(base_type.name) + if fn is None: + return None + + res = executeCommand( + f"{fn.name}(*({base_type.GetPointerType().name})" + str(value.GetLoadAddress()) + ");" + ) + + if res.error.fail: + return None + + return res.summary.strip('"') + + +def get_synthetic(value: lldb.SBValue) -> Union[lldb.SBValue, None]: + base_type = get_base_type(value.type) + + fn = SYNTHETIC_FUNCTIONS.get(base_type.name) + if fn is None: + return None + + res = executeCommand( + f"{fn.name}(*({base_type.GetPointerType().name})" + str(value.GetLoadAddress()) + ");" + ) + + if res.error.fail: + return None + + return res + + +def get_base_type(ty: lldb.SBType) -> lldb.SBType: + temp = ty + while temp.IsPointerType(): + temp = temp.GetPointeeType() + return temp + + +def breakpoint_function_wrapper(frame: lldb.SBFrame, bp_loc, internal_dict): + """This allows function calls to Nim for custom object summaries and synthetic children""" + debugger = lldb.debugger + + global SUMMARY_FUNCTIONS + global SYNTHETIC_FUNCTIONS + for tname, fn in SYNTHETIC_FUNCTIONS.items(): + print("DELETING SYNTH: ", tname) + debugger.HandleCommand(f"type synthetic delete -w nim {tname}") + + SUMMARY_FUNCTIONS = {} + SYNTHETIC_FUNCTIONS = {} + + target: lldb.SBTarget = debugger.GetSelectedTarget() + print("BREAKPOINT") + module = frame.GetSymbolContext(lldb.eSymbolContextModule).module + + for sym in module: + if not sym.name.startswith("lldbDebugSummary") and not sym.name.startswith( + "lldbDebugSynthetic" + ): + continue + + print("SYM: ", sym.name) + + fn_syms: lldb.SBSymbolContextList = target.FindFunctions(sym.name) + if not fn_syms.GetSize() > 0: + continue + fn_sym: lldb.SBSymbolContext = fn_syms.GetContextAtIndex(0) + + print("fn found!") + + fn: lldb.SBFunction = fn_sym.function + fn_type: lldb.SBType = fn.type + arg_types: lldb.SBTypeList = fn_type.GetFunctionArgumentTypes() + + if not arg_types.GetSize() > 0: + continue + arg_type: lldb.SBType = get_base_type(arg_types.GetTypeAtIndex(0)) + + print("FIRST ARG TYPE: ", arg_type.name) + + if sym.name.startswith("lldbDebugSummary"): + SUMMARY_FUNCTIONS[arg_type.name] = fn + elif sym.name.startswith("lldbDebugSynthetic"): + SYNTHETIC_FUNCTIONS[arg_type.name] = fn + debugger.HandleCommand( + f"type synthetic add -w nim -l {__name__}.CustomObjectChildrenProvider -x {arg_type.name}$" + ) + + +def executeCommand(command, *args): + debugger = lldb.debugger + process = debugger.GetSelectedTarget().GetProcess() + frame: lldb.SBFrame = process.GetSelectedThread().GetSelectedFrame() + # module = frame.GetSymbolContext(lldb.eSymbolContextModule).module + # for sym in module: + # print("SYM: ", sym.name) + # target = debugger.GetSelectedTarget() + + expr_options = lldb.SBExpressionOptions() + expr_options.SetIgnoreBreakpoints(False) + expr_options.SetFetchDynamicValue(lldb.eDynamicCanRunTarget) + expr_options.SetTimeoutInMicroSeconds(30 * 1000 * 1000) # 30 second timeout + expr_options.SetTryAllThreads(True) + expr_options.SetUnwindOnError(False) + expr_options.SetGenerateDebugInfo(True) + expr_options.SetLanguage(lldb.eLanguageTypeC) + expr_options.SetCoerceResultToId(True) + res = frame.EvaluateExpression(command, expr_options) + # if res.error.fail: + # print("ERROR: ", res.error.GetError()) + # return str(res.error) + return res + + +def __lldb_init_module(debugger, internal_dict): + # debugger.HandleCommand(f"type summary add -w nim -n any -F {__name__}.CatchAll -x .*") + debugger.HandleCommand(f"type summary add -w nim -n sequence -F {__name__}.Sequence -x tySequence_+[[:alnum:]]+$") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.SeqChildrenProvider -x tySequence_+[[:alnum:]]+$") + + debugger.HandleCommand(f"type summary add -w nim -n chararray -F {__name__}.CharArray -x char\s+[\d+]") + debugger.HandleCommand(f"type summary add -w nim -n array -F {__name__}.Array -x tyArray_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.ArrayChildrenProvider -x tyArray_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n string -F {__name__}.NimStringDesc NimStringDesc") + + debugger.HandleCommand(f"type summary add -w nim -n stringv2 -F {__name__}.NimStringV2 -x NimStringV2$") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.StringChildrenProvider -x NimStringV2$") + + debugger.HandleCommand(f"type summary add -w nim -n cstring -F {__name__}.NCSTRING NCSTRING") + + debugger.HandleCommand(f"type summary add -w nim -n object -F {__name__}.ObjectV2 -x tyObject_+[[:alnum:]]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.ObjectChildrenProvider -x tyObject_+[[:alnum:]]+_+[[:alnum:]]+$") + + debugger.HandleCommand(f"type summary add -w nim -n tframe -F {__name__}.ObjectV2 -x TFrame$") + + debugger.HandleCommand(f"type summary add -w nim -n rootobj -F {__name__}.ObjectV2 -x RootObj$") + + debugger.HandleCommand(f"type summary add -w nim -n enum -F {__name__}.Enum -x tyEnum_+[[:alnum:]]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n hashset -F {__name__}.HashSet -x tyObject_+HashSet_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.HashSetChildrenProvider -x tyObject_+HashSet_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n setuint -F {__name__}.Set -x tySet_+tyInt_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.SetIntChildrenProvider -x tySet_+tyInt[0-9]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n setint -F {__name__}.Set -x tySet_+tyInt[0-9]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n setuint2 -F {__name__}.Set -x tySet_+tyUInt[0-9]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.SetUIntChildrenProvider -x tySet_+tyUInt[0-9]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.SetUIntChildrenProvider -x tySet_+tyInt_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n setenum -F {__name__}.EnumSet -x tySet_+tyEnum_+[[:alnum:]]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.SetUIntChildrenProvider -x tySet_+tyEnum_+[[:alnum:]]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n setchar -F {__name__}.Set -x tySet_+tyChar_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.SetCharChildrenProvider -x tySet_+tyChar_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n table -F {__name__}.Table -x tyObject_+Table_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.TableChildrenProvider -x tyObject_+Table_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n stringtable -F {__name__}.StringTable -x tyObject_+StringTableObj_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.StringTableChildrenProvider -x tyObject_+StringTableObj_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n tuple2 -F {__name__}.Tuple -x tyObject_+Tuple_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n tuple -F {__name__}.Tuple -x tyTuple_+[[:alnum:]]+") + # debugger.HandleCommand(f"type summary add -w nim -n TNimType -F {__name__}.Object TNimType") + debugger.HandleCommand(f"type summary add -w nim -n TNimTypeV2 -F {__name__}.ObjectV2 TNimTypeV2") + # debugger.HandleCommand(f"type summary add -w nim -n TNimNode -F {__name__}.Object TNimNode") + debugger.HandleCommand(f"type summary add -w nim -n float -F {__name__}.Float NF") + debugger.HandleCommand(f"type summary add -w nim -n float32 -F {__name__}.Float NF32") + debugger.HandleCommand(f"type summary add -w nim -n float64 -F {__name__}.Float NF64") + debugger.HandleCommand(f"type summary add -w nim -n integer -F {__name__}.Number -x NI") + debugger.HandleCommand(f"type summary add -w nim -n integer8 -F {__name__}.Number -x NI8") + debugger.HandleCommand(f"type summary add -w nim -n integer16 -F {__name__}.Number -x NI16") + debugger.HandleCommand(f"type summary add -w nim -n integer32 -F {__name__}.Number -x NI32") + debugger.HandleCommand(f"type summary add -w nim -n integer64 -F {__name__}.Number -x NI64") + debugger.HandleCommand(f"type summary add -w nim -n bool -F {__name__}.Bool -x bool") + debugger.HandleCommand(f"type summary add -w nim -n bool2 -F {__name__}.Bool -x NIM_BOOL") + debugger.HandleCommand(f"type summary add -w nim -n uinteger -F {__name__}.UnsignedNumber -x NU") + debugger.HandleCommand(f"type summary add -w nim -n uinteger8 -F {__name__}.UnsignedNumber -x NU8") + debugger.HandleCommand(f"type summary add -w nim -n uinteger16 -F {__name__}.UnsignedNumber -x NU16") + debugger.HandleCommand(f"type summary add -w nim -n uinteger32 -F {__name__}.UnsignedNumber -x NU32") + debugger.HandleCommand(f"type summary add -w nim -n uinteger64 -F {__name__}.UnsignedNumber -x NU64") + debugger.HandleCommand("type category enable nim") + debugger.HandleCommand(f"command script add -f {__name__}.echo echo") + debugger.HandleCommand(f"command script add -f {__name__}.handle_command ddp") + debugger.HandleCommand(f"breakpoint command add -F {__name__}.breakpoint_function_wrapper --script-type python 1") \ No newline at end of file From 6994e1b1d7962f9c61887c7e606aeccbebf9bef6 Mon Sep 17 00:00:00 2001 From: quantimnot <54247259+quantimnot@users.noreply.github.com> Date: Sat, 4 Mar 2023 01:57:30 -0500 Subject: [PATCH 2454/2472] `--embedsrc` for JavaScript (#21467) Co-authored-by: quantimnot --- compiler/jsgen.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 5df40b996..d0a143793 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -735,6 +735,8 @@ proc genLineDir(p: PProc, n: PNode) = let line = toLinenumber(n.info) if line < 0: return + if optEmbedOrigSrc in p.config.globalOptions: + lineF(p, "//$1$n", [sourceLine(p.config, n.info)]) if optLineDir in p.options or optLineDir in p.config.options: lineF(p, "$1", [lineDir(p.config, n.info, line)]) if hasFrameInfo(p): From d950e5f3a8efe0c3be8e773cce4a89807833ffea Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sat, 4 Mar 2023 20:07:33 +0800 Subject: [PATCH 2455/2472] remove nosinks hacks from compiler (#21469) --- compiler/cgen.nim | 2 +- compiler/depends.nim | 2 +- compiler/passes.nim | 4 ++-- compiler/sem.nim | 4 ++-- compiler/semcall.nim | 2 +- compiler/seminst.nim | 2 +- compiler/semstmts.nim | 2 +- compiler/sigmatch.nim | 2 +- compiler/vm.nim | 6 +++--- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index d051934b5..e24a2cb1e 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1940,7 +1940,7 @@ template injectG() {.dirty.} = graph.backend = newModuleList(graph) let g = BModuleList(graph.backend) -proc setupCgen*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} = +proc setupCgen*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext = injectG() result = newModule(g, module, graph.config) result.idgen = idgen diff --git a/compiler/depends.nim b/compiler/depends.nim index 2087198f2..823e0f970 100644 --- a/compiler/depends.nim +++ b/compiler/depends.nim @@ -100,7 +100,7 @@ proc generateDot*(graph: ModuleGraph; project: AbsoluteFile) = rope(project.splitFile.name), b.dotGraph], changeFileExt(project, "dot")) -proc setupDependPass*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} = +proc setupDependPass*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext = var g: PGen new(g) g.module = module diff --git a/compiler/passes.nim b/compiler/passes.nim index 38c133d69..536a64714 100644 --- a/compiler/passes.nim +++ b/compiler/passes.nim @@ -241,14 +241,14 @@ proc compileProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) = graph.compileSystemModule() discard graph.compileModule(projectFile, {sfMainModule}) -proc mySemOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} = +proc mySemOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext = result = preparePContext(graph, module, idgen) proc mySemClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode = var c = PContext(context) closePContext(graph, c, n) -proc mySemProcess(context: PPassContext, n: PNode): PNode {.nosinks.} = +proc mySemProcess(context: PPassContext, n: PNode): PNode = result = semWithPContext(PContext(context), n) const semPass* = makePass(mySemOpen, mySemProcess, mySemClose, diff --git a/compiler/sem.nim b/compiler/sem.nim index a5d06f3fe..48a7d56c8 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -666,7 +666,7 @@ proc addCodeForGenerics(c: PContext, n: PNode) = n.add prc.ast c.lastGenericIdx = c.generics.len -proc preparePContext*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PContext {.nosinks.} = +proc preparePContext*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PContext = result = newContext(graph, module) result.idgen = idgen result.enforceVoidContext = newType(tyTyped, nextTypeId(idgen), nil) @@ -770,7 +770,7 @@ proc recoverContext(c: PContext) = while getCurrOwner(c).kind != skModule: popOwner(c) while c.p != nil and c.p.owner.kind != skModule: c.p = c.p.next -proc semWithPContext*(c: PContext, n: PNode): PNode {.nosinks.} = +proc semWithPContext*(c: PContext, n: PNode): PNode = # no need for an expensive 'try' if we stop after the first error anyway: if c.config.errorMax <= 1: result = semStmtAndGenerateGenerics(c, n) diff --git a/compiler/semcall.nim b/compiler/semcall.nim index 2147a9645..ba7908aa5 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -601,7 +601,7 @@ proc tryDeref(n: PNode): PNode = result.add n proc semOverloadedCall(c: PContext, n, nOrig: PNode, - filter: TSymKinds, flags: TExprFlags): PNode {.nosinks.} = + filter: TSymKinds, flags: TExprFlags): PNode = var errors: CandidateErrors = @[] # if efExplain in flags: @[] else: nil var r = resolveOverloads(c, n, nOrig, filter, flags, errors, efExplain in flags) if r.state == csMatch: diff --git a/compiler/seminst.nim b/compiler/seminst.nim index 80d455d1e..b5fe244b0 100644 --- a/compiler/seminst.nim +++ b/compiler/seminst.nim @@ -314,7 +314,7 @@ proc fillMixinScope(c: PContext) = p = p.next proc generateInstance(c: PContext, fn: PSym, pt: TIdTable, - info: TLineInfo): PSym {.nosinks.} = + info: TLineInfo): PSym = ## Generates a new instance of a generic procedure. ## The `pt` parameter is a type-unsafe mapping table used to link generic ## parameters to their concrete types within the generic instance. diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 6237e6eb0..6d1ce388d 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -1692,7 +1692,7 @@ proc semProcAnnotation(c: PContext, prc: PNode; return result -proc semInferredLambda(c: PContext, pt: TIdTable, n: PNode): PNode {.nosinks.} = +proc semInferredLambda(c: PContext, pt: TIdTable, n: PNode): PNode = ## used for resolving 'auto' in lambdas based on their callsite var n = n let original = n[namePos].sym diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index f8c7fa870..87a4fdf1c 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -2674,7 +2674,7 @@ proc argtypeMatches*(c: PContext, f, a: PType, fromHlo = false): bool = proc instTypeBoundOp*(c: PContext; dc: PSym; t: PType; info: TLineInfo; - op: TTypeAttachedOp; col: int): PSym {.nosinks.} = + op: TTypeAttachedOp; col: int): PSym = var m = newCandidate(c, dc.typ) if col >= dc.typ.len: localError(c.config, info, "cannot instantiate: '" & dc.name.s & "'") diff --git a/compiler/vm.nim b/compiler/vm.nim index 96f05c0f9..e00f0f02e 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -1893,7 +1893,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = var error: string let ast = parseString(regs[rb].node.strVal, c.cache, c.config, regs[rc].node.strVal, 0, - proc (conf: ConfigRef; info: TLineInfo; msg: TMsgKind; arg: string) {.nosinks.} = + proc (conf: ConfigRef; info: TLineInfo; msg: TMsgKind; arg: string) = if error.len == 0 and msg <= errMax: error = formatMsg(conf, info, msg, arg)) if error.len > 0: @@ -1908,7 +1908,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = var error: string let ast = parseString(regs[rb].node.strVal, c.cache, c.config, regs[rc].node.strVal, 0, - proc (conf: ConfigRef; info: TLineInfo; msg: TMsgKind; arg: string) {.nosinks.} = + proc (conf: ConfigRef; info: TLineInfo; msg: TMsgKind; arg: string) = if error.len == 0 and msg <= errMax: error = formatMsg(conf, info, msg, arg)) if error.len > 0: @@ -2309,7 +2309,7 @@ proc setupGlobalCtx*(module: PSym; graph: ModuleGraph; idgen: IdGenerator) = else: refresh(PCtx graph.vm, module, idgen) -proc setupEvalGen*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} = +proc setupEvalGen*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext = #var c = newEvalContext(module, emRepl) #c.features = {allowCast, allowInfiniteLoops} #pushStackFrame(c, newStackFrame()) From 04a494f8cfe960104150dd288d587d7e2cf3bd8a Mon Sep 17 00:00:00 2001 From: Sultan Al Isaiee Date: Sat, 4 Mar 2023 18:53:57 +0400 Subject: [PATCH 2456/2472] Add warning to specify timeout value in milliseconds, Fix #21449 (#21471) a warning message been added to the documentation to remind users that the timeout parameter is expressed in milliseconds, not seconds. to help prevent confusion and unexpected behaviours. --- lib/pure/osproc.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 4cdd98c0d..915337f12 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -220,6 +220,10 @@ proc waitForExit*(p: Process, timeout: int = -1): int {.rtl, ## ## On posix, if the process has exited because of a signal, 128 + signal ## number will be returned. + ## + ## .. warning:: When working with `timeout` parameters, remember that the value is + ## typically expressed in milliseconds, and ensure that the correct unit of time + ## is used to avoid unexpected behavior. proc peekExitCode*(p: Process): int {.rtl, extern: "nosp$1", raises: [OSError], tags: [].} ## Return `-1` if the process is still running. Otherwise the process' exit code. From 7bde421e4dae1f078ad5940dda8199df00b725cc Mon Sep 17 00:00:00 2001 From: SirOlaf <34164198+SirOlaf@users.noreply.github.com> Date: Sun, 5 Mar 2023 11:56:51 +0100 Subject: [PATCH 2457/2472] Fix #21272: Rewrite parts of pickBestCandidate (#21465) * Make pickBestCandidate store syms * Remove useless cursor * Try making pickBestCandidate more readable * Fix advance order * Revert back to seq with lots of comments --------- Co-authored-by: SirOlaf <> --- compiler/semcall.nim | 86 +++++++++++++++++++++++++------------------- 1 file changed, 50 insertions(+), 36 deletions(-) diff --git a/compiler/semcall.nim b/compiler/semcall.nim index ba7908aa5..54f03026f 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -43,6 +43,7 @@ proc initCandidateSymbols(c: PContext, headSymbol: PNode, best, alt: var TCandidate, o: var TOverloadIter, diagnostics: bool): seq[tuple[s: PSym, scope: int]] = + ## puts all overloads into a seq and prepares best+alt result = @[] var symx = initOverloadIter(o, c, headSymbol) while symx != nil: @@ -64,36 +65,35 @@ proc pickBestCandidate(c: PContext, headSymbol: PNode, errors: var CandidateErrors, diagnosticsFlag: bool, errorsEnabled: bool, flags: TExprFlags) = + # `matches` may find new symbols, so keep track of count + var symCount = c.currentScope.symbols.counter + var o: TOverloadIter - var sym = initOverloadIter(o, c, headSymbol) - var scope = o.lastOverloadScope - # Thanks to the lazy semchecking for operands, we need to check whether - # 'initCandidate' modifies the symbol table (via semExpr). - # This can occur in cases like 'init(a, 1, (var b = new(Type2); b))' - let counterInitial = c.currentScope.symbols.counter - var syms: seq[tuple[s: PSym, scope: int]] - var noSyms = true - var nextSymIndex = 0 - while sym != nil: - if sym.kind in filter: - # Initialise 'best' and 'alt' with the first available symbol - initCandidate(c, best, sym, initialBinding, scope, diagnosticsFlag) - initCandidate(c, alt, sym, initialBinding, scope, diagnosticsFlag) - best.state = csNoMatch - break - else: - sym = nextOverloadIter(o, c, headSymbol) - scope = o.lastOverloadScope - var z: TCandidate - while sym != nil: - if sym.kind notin filter: - sym = nextOverloadIter(o, c, headSymbol) - scope = o.lastOverloadScope - continue + # https://github.com/nim-lang/Nim/issues/21272 + # prevent mutation during iteration by storing them in a seq + # luckily `initCandidateSymbols` does just that + var syms = initCandidateSymbols(c, headSymbol, initialBinding, filter, + best, alt, o, diagnosticsFlag) + if len(syms) == 0: + return + # current overload being considered + var sym = syms[0].s + var scope = syms[0].scope + + # starts at 1 because 0 is already done with setup, only needs checking + var nextSymIndex = 1 + var z: TCandidate # current candidate + while true: determineType(c, sym) initCandidate(c, z, sym, initialBinding, scope, diagnosticsFlag) - if c.currentScope.symbols.counter == counterInitial or syms.len != 0: + + # this is kinda backwards as without a check here the described + # problems in recalc would not happen, but instead it 100% + # does check forever in some cases + if c.currentScope.symbols.counter == symCount: + # may introduce new symbols with caveats described in recalc branch matches(c, n, orig, z) + if z.state == csMatch: # little hack so that iterators are preferred over everything else: if sym.kind == skIterator: @@ -113,22 +113,36 @@ proc pickBestCandidate(c: PContext, headSymbol: PNode, firstMismatch: z.firstMismatch, diagnostics: z.diagnostics)) else: + # this branch feels like a ticking timebomb + # one of two bad things could happen + # 1) new symbols are discovered but the loop ends before we recalc + # 2) new symbols are discovered and resemmed forever + # not 100% sure if these are possible though as they would rely + # on somehow introducing a new overload during overload resolution + # Symbol table has been modified. Restart and pre-calculate all syms # before any further candidate init and compare. SLOW, but rare case. syms = initCandidateSymbols(c, headSymbol, initialBinding, filter, best, alt, o, diagnosticsFlag) - noSyms = false - if noSyms: - sym = nextOverloadIter(o, c, headSymbol) - scope = o.lastOverloadScope - elif nextSymIndex < syms.len: - # rare case: retrieve the next pre-calculated symbol - sym = syms[nextSymIndex].s - scope = syms[nextSymIndex].scope - nextSymIndex += 1 - else: + + # reset counter because syms may be in a new order + symCount = c.currentScope.symbols.counter + nextSymIndex = 0 + + # just in case, should be impossible though + if syms.len == 0: + break + + if nextSymIndex > high(syms): + # we have reached the end break + # advance to next sym + sym = syms[nextSymIndex].s + scope = syms[nextSymIndex].scope + inc(nextSymIndex) + + proc effectProblem(f, a: PType; result: var string; c: PContext) = if f.kind == tyProc and a.kind == tyProc: if tfThread in f.flags and tfThread notin a.flags: From 05e8a60bb4fe40d450aaec4603209e310ea490a4 Mon Sep 17 00:00:00 2001 From: Joey Date: Sun, 5 Mar 2023 12:14:59 -0700 Subject: [PATCH 2458/2472] Add line directives for C code variables (#21466) * Add line directives for C code variables * Refactor genCLineDir to only use toFullPath if necessary --- compiler/cgen.nim | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index e24a2cb1e..d2095016c 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -273,7 +273,8 @@ proc genCLineDir(r: var Rope, filename: string, line: int; conf: ConfigRef) = [rope(makeSingleLineCString(filename)), rope(line)]) proc genCLineDir(r: var Rope, info: TLineInfo; conf: ConfigRef) = - genCLineDir(r, toFullPath(conf, info), info.safeLineNm, conf) + if optLineDir in conf.options: + genCLineDir(r, toFullPath(conf, info), info.safeLineNm, conf) proc freshLineInfo(p: BProc; info: TLineInfo): bool = if p.lastLineInfo.line != info.line or @@ -287,7 +288,7 @@ proc genLineDir(p: BProc, t: PNode) = if optEmbedOrigSrc in p.config.globalOptions: p.s(cpsStmts).add("//" & sourceLine(p.config, t.info) & "\L") - genCLineDir(p.s(cpsStmts), toFullPath(p.config, t.info), line, p.config) + genCLineDir(p.s(cpsStmts), t.info, p.config) if ({optLineTrace, optStackTrace} * p.options == {optLineTrace, optStackTrace}) and (p.prc == nil or sfPure notin p.prc.flags) and t.info.fileIndex != InvalidFileIdx: if freshLineInfo(p, t.info): @@ -566,6 +567,9 @@ proc localVarDecl(p: BProc; n: PNode): Rope = if s.kind == skLet: incl(s.loc.flags, lfNoDeepCopy) if s.kind in {skLet, skVar, skField, skForVar} and s.alignment > 0: result.addf("NIM_ALIGN($1) ", [rope(s.alignment)]) + + genCLineDir(result, n.info, p.config) + result.add getTypeDesc(p.module, s.typ, skVar) if s.constraint.isNil: if sfRegister in s.flags: result.add(" register") From 4d76725299e524e19d515c04d923bd408607a966 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Mon, 6 Mar 2023 14:12:47 +0800 Subject: [PATCH 2459/2472] closes #16654; add a test case (#21478) --- tests/errmsgs/t16654.nim | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 tests/errmsgs/t16654.nim diff --git a/tests/errmsgs/t16654.nim b/tests/errmsgs/t16654.nim new file mode 100644 index 000000000..749707c06 --- /dev/null +++ b/tests/errmsgs/t16654.nim @@ -0,0 +1,12 @@ +discard """ + cmd: "nim check $options $file" + errormsg: "type mismatch: got but expected 'float'" +""" + +when true: # bug #16654 + func fn[T](a: T, op: proc(a: T): float) = discard + proc main() = + let v = 1 + proc bar(r: auto): auto = v + fn(1, bar) + main() From 2d9af2bd559d03c4f6b2a5f89be316f2465a091b Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Mon, 6 Mar 2023 17:34:41 +0800 Subject: [PATCH 2460/2472] closes #20704; add a test case (#21480) --- tests/stdlib/tsugar.nim | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/stdlib/tsugar.nim b/tests/stdlib/tsugar.nim index 6ef3ae519..1b629165a 100644 --- a/tests/stdlib/tsugar.nim +++ b/tests/stdlib/tsugar.nim @@ -3,7 +3,7 @@ discard """ x + y = 30 ''' """ -import std/[sugar, algorithm, random, sets, tables, strutils] +import std/[sugar, algorithm, random, sets, tables, strutils, sequtils] import std/[syncio, assertions] type # for capture test, ref #20679 @@ -287,6 +287,17 @@ proc mainProc() = doAssertRaises(AssertionDefect): doAssert false doAssert "except AssertionDefect" in s2 + block: # bug #20704 + proc test() = + var xs, ys: seq[int] + for i in 0..5: + xs.add(i) + + xs.apply(d => ys.add(d)) + doAssert ys == @[0, 1, 2, 3, 4, 5] + + test() + static: main() mainProc() From 64a0355f3f95c9ec8ca3597f8028819b489f08c8 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Mon, 6 Mar 2023 22:30:22 +0800 Subject: [PATCH 2461/2472] fixes #21377; fixes `@[]` and `{}` type inference as returns in generics (#21475) * fixes `@[]` type inference in generics * add issue links * fixes macros and iterators * refactor * add one more test --- compiler/semexprs.nim | 2 +- compiler/seminst.nim | 11 +++++++-- compiler/semmagic.nim | 6 ++++- tests/types/ttopdowninference.nim | 37 ++++++++++++++++++++++++++++++- 4 files changed, 51 insertions(+), 5 deletions(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 9cf206b4c..976d7c757 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -998,7 +998,7 @@ proc afterCallActions(c: PContext; n, orig: PNode, flags: TExprFlags; expectedTy fixAbstractType(c, result) analyseIfAddressTakenInCall(c, result) if callee.magic != mNone: - result = magicsAfterOverloadResolution(c, result, flags) + result = magicsAfterOverloadResolution(c, result, flags, expectedType) when false: if result.typ != nil and not (result.typ.kind == tySequence and result.typ[0].kind == tyEmpty): diff --git a/compiler/seminst.nim b/compiler/seminst.nim index b5fe244b0..25e5b267e 100644 --- a/compiler/seminst.nim +++ b/compiler/seminst.nim @@ -127,11 +127,18 @@ proc instantiateBody(c: PContext, n, params: PNode, result, orig: PSym) = if sfGenSym in param.flags: idTablePut(symMap, params[i].sym, result.typ.n[param.position+1].sym) freshGenSyms(c, b, result, orig, symMap) - + if sfBorrow notin orig.flags: # We do not want to generate a body for generic borrowed procs. # As body is a sym to the borrowed proc. - b = semProcBody(c, b) + let resultType = # todo probably refactor it into a function + if result.kind == skMacro: + sysTypeFromName(c.graph, n.info, "NimNode") + elif not isInlineIterator(result.typ): + result.typ[0] + else: + nil + b = semProcBody(c, b, resultType) result.ast[bodyPos] = hloBody(c, b) excl(result.flags, sfForward) trackProc(c, result, result.ast[bodyPos]) diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim index 751ca8fe7..b4c6cd275 100644 --- a/compiler/semmagic.nim +++ b/compiler/semmagic.nim @@ -526,7 +526,7 @@ proc checkDefault(c: PContext, n: PNode): PNode = message(c.config, n.info, warnUnsafeDefault, typeToString(constructed)) proc magicsAfterOverloadResolution(c: PContext, n: PNode, - flags: TExprFlags): PNode = + flags: TExprFlags; expectedType: PType = nil): PNode = ## This is the preferred code point to implement magics. ## ``c`` the current module, a symbol table to a very good approximation ## ``n`` the ast like it would be passed to a real macro @@ -635,5 +635,9 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode, result = n of mPrivateAccess: result = semPrivateAccess(c, n) + of mArrToSeq: + result = n + if result.typ != nil and expectedType != nil and result.typ.kind == tySequence and expectedType.kind == tySequence and result.typ[0].kind == tyEmpty: + result.typ = expectedType # type inference for empty sequence # bug #21377 else: result = n diff --git a/tests/types/ttopdowninference.nim b/tests/types/ttopdowninference.nim index 310dbb2ad..cabc798ed 100644 --- a/tests/types/ttopdowninference.nim +++ b/tests/types/ttopdowninference.nim @@ -245,7 +245,7 @@ block: # bug #11777 var s: S = {1, 2} doAssert 1 in s -block: # regression #20807 +block: # bug #20807 var s: seq[string] template fail = s = @[] @@ -255,3 +255,38 @@ block: # regression #20807 test: fail() doAssert not (compiles do: let x: seq[int] = `@`[string]([])) + +block: # bug #21377 + proc b[T](v: T): seq[int] = + let x = 0 + @[] + + doAssert b(0) == @[] + +block: # bug #21377 + proc b[T](v: T): seq[T] = + let x = 0 + @[] + + doAssert b(0) == @[] + +block: # bug #21377 + proc b[T](v: T): set[bool] = + let x = 0 + {} + + doAssert b(0) == {} + +block: # bug #21377 + proc b[T](v: T): array[0, int] = + let x = 0 + [] + + doAssert b(0) == [] + +block: # bug #21377 + proc b[T](v: T): array[0, (string, string)] = + let x = 0 + {:} + + doAssert b(0) == {:} From 25eef64fe1969581436696556e0bf10ef25aa3ec Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Tue, 7 Mar 2023 01:31:53 +1100 Subject: [PATCH 2462/2472] Remove Defect from raises list in `std/times` (#21473) * Remove Defect from raises list Since defects aren't tracked anymore this causes a hint to pop up mentioning it * Still track Defect when getting ran with an older Nim version The raises followed a pattern so moving them into a pragma didn't seem to cause any extra problems --- lib/pure/times.nim | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 70af466d9..138f2d9ec 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -408,6 +408,16 @@ const unitWeights: array[FixedTimeUnit, int64] = [ 7 * secondsInDay * 1e9.int64, ] +when (NimMajor, NimMinor) >= (1, 4): + # Newer versions of Nim don't track defects + {.pragma: parseFormatRaises, raises: [TimeParseError, TimeFormatParseError].} + {.pragma: parseRaises, raises: [TimeParseError].} +else: + # Still track when using older versions + {.pragma: parseFormatRaises, raises: [TimeParseError, TimeFormatParseError, Defect].} + {.pragma: parseRaises, raises: [TimeParseError, Defect].} + + # # Helper procs # @@ -2134,8 +2144,7 @@ proc format*(time: Time, f: static[string], zone: Timezone = local()): string result = time.inZone(zone).format(f2) proc parse*(input: string, f: TimeFormat, zone: Timezone = local(), - loc: DateTimeLocale = DefaultLocale): DateTime - {.raises: [TimeParseError, Defect].} = + loc: DateTimeLocale = DefaultLocale): DateTime {.parseRaises.} = ## Parses `input` as a `DateTime` using the format specified by `f`. ## If no UTC offset was parsed, then `input` is assumed to be specified in ## the `zone` timezone. If a UTC offset was parsed, the result will be @@ -2178,8 +2187,7 @@ proc parse*(input: string, f: TimeFormat, zone: Timezone = local(), result = toDateTime(parsed, zone, f, input) proc parse*(input, f: string, tz: Timezone = local(), - loc: DateTimeLocale = DefaultLocale): DateTime - {.raises: [TimeParseError, TimeFormatParseError, Defect].} = + loc: DateTimeLocale = DefaultLocale): DateTime {.parseFormatRaises.} = ## Shorthand for constructing a `TimeFormat` and using it to parse ## `input` as a `DateTime`. ## @@ -2192,14 +2200,12 @@ proc parse*(input, f: string, tz: Timezone = local(), result = input.parse(dtFormat, tz, loc = loc) proc parse*(input: string, f: static[string], zone: Timezone = local(), - loc: DateTimeLocale = DefaultLocale): - DateTime {.raises: [TimeParseError, Defect].} = + loc: DateTimeLocale = DefaultLocale): DateTime {.parseRaises.} = ## Overload that validates `f` at compile time. const f2 = initTimeFormat(f) result = input.parse(f2, zone, loc = loc) -proc parseTime*(input, f: string, zone: Timezone): Time - {.raises: [TimeParseError, TimeFormatParseError, Defect].} = +proc parseTime*(input, f: string, zone: Timezone): Time {.parseFormatRaises.} = ## Shorthand for constructing a `TimeFormat` and using it to parse ## `input` as a `DateTime`, then converting it a `Time`. ## @@ -2211,7 +2217,7 @@ proc parseTime*(input, f: string, zone: Timezone): Time parse(input, f, zone).toTime() proc parseTime*(input: string, f: static[string], zone: Timezone): Time - {.raises: [TimeParseError, Defect].} = + {.parseRaises.} = ## Overload that validates `format` at compile time. const f2 = initTimeFormat(f) result = input.parse(f2, zone).toTime() From 6ef94301538f0a17461630c08c456112954e66bb Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Tue, 7 Mar 2023 17:34:02 +0800 Subject: [PATCH 2463/2472] closes #6231; add a test case (#21485) --- tests/generics/tgenerics_various.nim | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/generics/tgenerics_various.nim b/tests/generics/tgenerics_various.nim index 37b6e43a1..4cfa47809 100644 --- a/tests/generics/tgenerics_various.nim +++ b/tests/generics/tgenerics_various.nim @@ -262,3 +262,8 @@ proc doSomething[A, B](t: tuple[a: A, b: B]) = discard discard identity((c: 1, d: 2)) doSomething(identity((1, 2))) + +# bug #6231 +proc myProc[T, U](x: T or U) = discard + +myProc[int, string](x = 2) From 2f89f1eb780ab32be31dbe4074af70b587ad2f36 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Tue, 7 Mar 2023 20:17:05 +0800 Subject: [PATCH 2464/2472] closes #8295; add a test case (#21486) --- tests/generics/tgeneric0.nim | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/generics/tgeneric0.nim b/tests/generics/tgeneric0.nim index 44c34917d..b5e1c4bb4 100644 --- a/tests/generics/tgeneric0.nim +++ b/tests/generics/tgeneric0.nim @@ -153,3 +153,16 @@ proc unzip*[T,U](xs: List[tuple[t: T, u: U]]): (List[T], List[U]) = discard proc unzip2*[T,U](xs: List[(T,U)]): (List[T], List[U]) = discard +type + AtomicType = pointer|ptr|int + + Atomic[T: AtomicType] = distinct T + + Block[T: AtomicType] = object + + AtomicContainer[T: AtomicType] = object + b: Atomic[ptr Block[T]] + +# bug #8295 +var x = AtomicContainer[int]() +doAssert (ptr Block[int])(x.b) == nil From 8e9fc5e63968177c75d93af08d32717bf6c97693 Mon Sep 17 00:00:00 2001 From: ghost <97684920+sslime336@users.noreply.github.com> Date: Thu, 9 Mar 2023 05:34:59 +0800 Subject: [PATCH 2465/2472] a better message if graphviz's dot/nodejs is not found in PATH (#21488) * finish issue #21474: a better message if dot is not found locally when using gendepend * fix a typo in compiler * trim empty path reported in `findNodeJs` * compiler/main.nim: switch raise to simply quit --- compiler/main.nim | 7 +++++++ compiler/nodejs.nim | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/compiler/main.nim b/compiler/main.nim index d37d26478..ff870a14a 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -71,6 +71,13 @@ proc commandGenDepend(graph: ModuleGraph) = let project = graph.config.projectFull writeDepsFile(graph) generateDot(graph, project) + + # dot in graphivz tool kit is required + let graphvizDotPath = findExe("dot") + if graphvizDotPath.len == 0: + quit("gendepend: Graphviz's tool dot is required," & + "see https://graphviz.org/download for downloading") + execExternalProgram(graph.config, "dot -Tpng -o" & changeFileExt(project, "png").string & ' ' & changeFileExt(project, "dot").string) diff --git a/compiler/nodejs.nim b/compiler/nodejs.nim index 283643e8d..c1feb196a 100644 --- a/compiler/nodejs.nim +++ b/compiler/nodejs.nim @@ -7,4 +7,4 @@ proc findNodeJs*(): string {.inline.} = result = findExe("node") if result.len == 0: echo "Please install NodeJS first, see https://nodejs.org/en/download" - raise newException(IOError, "NodeJS not found in PATH: " & result) + raise newException(IOError, "NodeJS not found in PATH") From 72e262666bdf2bb3c239183dd32b48bb05d113aa Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 9 Mar 2023 15:09:08 +0800 Subject: [PATCH 2466/2472] fixes quoted variables with typedesc types (#21493) --- compiler/semexprs.nim | 2 +- tests/stdlib/tmacros.nim | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 976d7c757..cfa34fcdc 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -2206,7 +2206,7 @@ proc semQuoteAst(c: PContext, n: PNode): PNode = dummyTemplate[paramsPos].add newTreeI(nkIdentDefs, n.info, ids[0], getSysSym(c.graph, n.info, "typed").newSymNode, c.graph.emptyNode) for i in 1.. Date: Fri, 10 Mar 2023 16:28:51 +0800 Subject: [PATCH 2467/2472] fixes #21023; Segfault when mixing seqs, orc, variants and futures (#21497) * fixes #21023; Segfault when mixing seqs, orc, variants and futures * fixes none of the branches were explicitly selected * add one more test * one more test --- compiler/semobjconstr.nim | 61 +++++++++++++++++-------- tests/arc/tcaseobj.nim | 61 +++++++++++++++++++++++++ tests/objects/tobject_default_value.nim | 32 +++++++++++++ 3 files changed, 136 insertions(+), 18 deletions(-) diff --git a/compiler/semobjconstr.nim b/compiler/semobjconstr.nim index 15e53a639..29067cfa4 100644 --- a/compiler/semobjconstr.nim +++ b/compiler/semobjconstr.nim @@ -142,15 +142,45 @@ proc fieldsPresentInInitExpr(c: PContext, fieldsRecList, initExpr: PNode): strin if result.len != 0: result.add ", " result.add field.sym.name.s.quoteStr +proc locateFieldInDefaults(sym: PSym, defaults: seq[PNode]): bool = + result = false + for d in defaults: + if sym.id == d[0].sym.id: + return true + proc collectMissingFields(c: PContext, fieldsRecList: PNode, - constrCtx: var ObjConstrContext) = - for r in directFieldsInRecList(fieldsRecList): - if constrCtx.needsFullInit or - sfRequiresInit in r.sym.flags or - r.sym.typ.requiresInit: + constrCtx: var ObjConstrContext, defaults: seq[PNode] + ): seq[PSym] = + for r in directFieldsInRecList(fieldsRecList): let assignment = locateFieldInInitExpr(c, r.sym, constrCtx.initExpr) - if assignment == nil: - constrCtx.missingFields.add r.sym + if assignment == nil and not locateFieldInDefaults(r.sym, defaults): + if constrCtx.needsFullInit or + sfRequiresInit in r.sym.flags or + r.sym.typ.requiresInit: + constrCtx.missingFields.add r.sym + else: + result.add r.sym + +proc collectMissingCaseFields(c: PContext, branchNode: PNode, + constrCtx: var ObjConstrContext, defaults: seq[PNode]): seq[PSym] = + if branchNode != nil: + let fieldsRecList = branchNode[^1] + result = collectMissingFields(c, fieldsRecList, constrCtx, defaults) + +proc collectOrAddMissingCaseFields(c: PContext, branchNode: PNode, + constrCtx: var ObjConstrContext, defaults: var seq[PNode]) = + let res = collectMissingCaseFields(c, branchNode, constrCtx, defaults) + for sym in res: + let asgnType = newType(tyTypeDesc, nextTypeId(c.idgen), sym.typ.owner) + let recTyp = sym.typ.skipTypes(defaultFieldsSkipTypes) + rawAddSon(asgnType, recTyp) + let asgnExpr = newTree(nkCall, + newSymNode(getSysMagic(c.graph, constrCtx.initExpr.info, "zeroDefault", mZeroDefault)), + newNodeIT(nkType, constrCtx.initExpr.info, asgnType) + ) + asgnExpr.flags.incl nfUseDefaultField + asgnExpr.typ = recTyp + defaults.add newTree(nkExprColonExpr, newSymNode(sym), asgnExpr) proc semConstructFields(c: PContext, n: PNode, constrCtx: var ObjConstrContext, flags: TExprFlags): tuple[status: InitStatus, defaults: seq[PNode]] = @@ -166,11 +196,6 @@ proc semConstructFields(c: PContext, n: PNode, constrCtx: var ObjConstrContext, let fields = branch[^1] fieldsPresentInInitExpr(c, fields, constrCtx.initExpr) - template collectMissingFields(branchNode: PNode) = - if branchNode != nil: - let fields = branchNode[^1] - collectMissingFields(c, fields, constrCtx) - let discriminator = n[0] internalAssert c.config, discriminator.kind == nkSym var selectedBranch = -1 @@ -288,8 +313,7 @@ proc semConstructFields(c: PContext, n: PNode, constrCtx: var ObjConstrContext, # When a branch is selected with a partial match, some of the fields # that were not initialized may be mandatory. We must check for this: if result.status == initPartial: - collectMissingFields branchNode - + collectOrAddMissingCaseFields(c, branchNode, constrCtx, result.defaults) else: result.status = initNone let discriminatorVal = semConstrField(c, flags + {efPreferStatic}, @@ -302,7 +326,7 @@ proc semConstructFields(c: PContext, n: PNode, constrCtx: var ObjConstrContext, # a result: let defaultValue = newIntLit(c.graph, constrCtx.initExpr.info, 0) let matchedBranch = n.pickCaseBranch defaultValue - collectMissingFields matchedBranch + discard collectMissingCaseFields(c, matchedBranch, constrCtx, @[]) else: result.status = initPartial if discriminatorVal.kind == nkIntLit: @@ -312,11 +336,12 @@ proc semConstructFields(c: PContext, n: PNode, constrCtx: var ObjConstrContext, if matchedBranch != nil: let (_, defaults) = semConstructFields(c, matchedBranch[^1], constrCtx, flags) result.defaults.add defaults - collectMissingFields matchedBranch + collectOrAddMissingCaseFields(c, matchedBranch, constrCtx, result.defaults) else: # All bets are off. If any of the branches has a mandatory # fields we must produce an error: - for i in 1.. Date: Fri, 10 Mar 2023 21:19:31 +0800 Subject: [PATCH 2468/2472] fixes #21306; fixes #20485; don't transform yields in the var section when introducing new local vars [backport: 1.6] (#21489) * fixes #21306; don't transform yields in the var section when introducing new local vars * adds `inVarSection` so the var section in the var section is freshed * use `isIntroducingNewLocalVars` to avoid yield transformations in var sections * fixes comments --- compiler/transf.nim | 5 +- tests/iter/t21306.nim | 114 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 tests/iter/t21306.nim diff --git a/compiler/transf.nim b/compiler/transf.nim index 7277e6898..445bc1df1 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -50,6 +50,7 @@ type module: PSym transCon: PTransCon # top of a TransCon stack inlining: int # > 0 if we are in inlining context (copy vars) + isIntroducingNewLocalVars: bool # true if we are in `introducingNewLocalVars` (don't transform yields) contSyms, breakSyms: seq[PSym] # to transform 'continue' and 'break' deferDetected, tooEarly: bool graph: ModuleGraph @@ -450,7 +451,9 @@ proc transformYield(c: PTransf, n: PNode): PNode = result.add(c.transCon.forLoopBody) else: # we need to introduce new local variables: + c.isIntroducingNewLocalVars = true # don't transform yields when introducing new local vars result.add(introduceNewLocalVars(c, c.transCon.forLoopBody)) + c.isIntroducingNewLocalVars = false for idx in 0 ..< result.len: var changeNode = result[idx] @@ -1036,7 +1039,7 @@ proc transform(c: PTransf, n: PNode): PNode = else: result = transformSons(c, n) of nkYieldStmt: - if c.inlining > 0: + if c.inlining > 0 and not c.isIntroducingNewLocalVars: result = transformYield(c, n) else: result = transformSons(c, n) diff --git a/tests/iter/t21306.nim b/tests/iter/t21306.nim new file mode 100644 index 000000000..43fea9c80 --- /dev/null +++ b/tests/iter/t21306.nim @@ -0,0 +1,114 @@ +# bug #21306 +type + FutureState {.pure.} = enum + Pending, Finished, Cancelled, Failed + + FutureBase = ref object of RootObj + state: FutureState + error: ref CatchableError + id: uint + + Future[T] = ref object of FutureBase + closure: iterator(f: Future[T]): FutureBase {.raises: [Defect, CatchableError, Exception], gcsafe.} + value: T + +template setupFutureBase() = + new(result) + result.state = FutureState.Pending + +proc newFutureImpl[T](): Future[T] = + setupFutureBase() + +template newFuture[T](fromProc: static[string] = ""): Future[T] = + newFutureImpl[T]() + +proc internalRead[T](fut: Future[T]): T = + when T isnot void: + return fut.value + +template await[T](f: Future[T]): untyped = + when declared(chronosInternalRetFuture): + when not declaredInScope(chronosInternalTmpFuture): + var chronosInternalTmpFuture {.inject.}: FutureBase = f + else: + chronosInternalTmpFuture = f + + yield chronosInternalTmpFuture + + when T isnot void: + cast[type(f)](chronosInternalTmpFuture).internalRead() + +type + VerifierError {.pure.} = enum + Invalid + MissingParent + UnviableFork + Duplicate + ProcessingCallback = proc() {.gcsafe, raises: [Defect].} + BlockVerifier = + proc(signedBlock: int): + Future[VerifierError] {.gcsafe, raises: [Defect].} + + SyncQueueKind {.pure.} = enum + Forward, Backward + + SyncRequest[T] = object + kind: SyncQueueKind + index: uint64 + slot: uint64 + count: uint64 + item: T + + SyncResult[T] = object + request: SyncRequest[T] + data: seq[ref int] + + SyncQueue[T] = ref object + kind: SyncQueueKind + readyQueue: seq[SyncResult[T]] + blockVerifier: BlockVerifier + +iterator blocks[T](sq: SyncQueue[T], + sr: SyncResult[T]): ref int = + case sq.kind + of SyncQueueKind.Forward: + for i in countup(0, len(sr.data) - 1): + yield sr.data[i] + of SyncQueueKind.Backward: + for i in countdown(len(sr.data) - 1, 0): + yield sr.data[i] + +proc push[T](sq: SyncQueue[T]; sr: SyncRequest[T]; data: seq[ref int]; + processingCb: ProcessingCallback = nil): Future[void] {. + stackTrace: off, gcsafe.} = + iterator push_436208182(chronosInternalRetFuture: Future[void]): FutureBase {. + closure, gcsafe, raises: [Defect, CatchableError, Exception].} = + block: + template result(): auto {.used.} = + {.fatal: "You should not reference the `result` variable inside" & + " a void async proc".} + + let item = default(SyncResult[T]) + for blk in sq.blocks(item): + let res = await sq.blockVerifier(blk[]) + + var resultFuture = newFuture[void]("push") + resultFuture.closure = push_436208182 + return resultFuture + +type + SomeTPeer = ref object + score: int + +proc getSlice(): seq[ref int] = + discard + +template smokeTest(kkind: SyncQueueKind, start, finish: uint64, + chunkSize: uint64) = + var queue: SyncQueue[SomeTPeer] + var request: SyncRequest[SomeTPeer] + discard queue.push(request, getSlice()) + +for k in {SyncQueueKind.Forward}: + for item in [(uint64(1181), uint64(1399), 41'u64)]: + smokeTest(k, item[0], item[1], item[2]) \ No newline at end of file From 46d2161c23c2aa1905571512b9a1ef7d61ae670e Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 10 Mar 2023 14:20:30 +0100 Subject: [PATCH 2469/2472] minor refactoring (#21499) --- compiler/liftdestructors.nim | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim index d7b1f2daa..467f015d8 100644 --- a/compiler/liftdestructors.nim +++ b/compiler/liftdestructors.nim @@ -1085,13 +1085,7 @@ proc inst(g: ModuleGraph; c: PContext; t: PType; kind: TTypeAttachedOp; idgen: I let op = getAttachedOp(g, t, kind) if op != nil and op.ast != nil and op.ast.isGenericRoutine: if t.typeInst != nil: - var a: TLiftCtx - a.info = info - a.g = g - a.kind = kind - a.c = c - a.idgen = idgen - + var a = TLiftCtx(info: info, g: g, kind: kind, c: c, idgen: idgen) let opInst = instantiateGeneric(a, op, t, t.typeInst) if opInst.ast != nil: patchBody(g, c, opInst.ast, info, a.idgen) From af086b68f2f718a4ad20e37bf3a00d9a57abec4d Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sun, 12 Mar 2023 13:01:12 +0800 Subject: [PATCH 2470/2472] fixes sinkinference documentation, which has been disabled (#21470) since https://github.com/nim-lang/Nim/pull/15105 --- doc/advopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/advopt.txt b/doc/advopt.txt index 122a7c6ea..4db89062b 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -172,6 +172,6 @@ Advanced options: --useVersion:1.0|1.2|1.6 emulate Nim version X of the Nim compiler, for testing --benchmarkVM:on|off turn benchmarking of VM code with cpuTime() on|off --profileVM:on|off turn compile time VM profiler on|off - --sinkInference:on|off turn sink parameter inference on|off (default: on) + --sinkInference:on|off turn sink parameter inference on|off (default: off) --panics:on|off turn panics into process terminations (default: off) --deepcopy:on|off enable 'system.deepCopy' for ``--mm:arc|orc`` From ffadc75afead6baed0885877edb2c3b1c9ef1b2e Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Mon, 13 Mar 2023 03:03:14 +0800 Subject: [PATCH 2471/2472] fixes #21496; Ambiguous calls compiles when module name are equal (#21500) * fixes #21496; Ambiguous calls compiles when module name are equal * add a test case --- compiler/lookups.nim | 3 ++- tests/import/buzz/m21496.nim | 1 + tests/import/fizz/m21496.nim | 1 + tests/import/t21496.nim | 9 +++++++++ 4 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 tests/import/buzz/m21496.nim create mode 100644 tests/import/fizz/m21496.nim create mode 100644 tests/import/t21496.nim diff --git a/compiler/lookups.nim b/compiler/lookups.nim index e7bca08bc..3f028a52f 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -343,7 +343,8 @@ proc addDeclAt*(c: PContext; scope: PScope, sym: PSym, info: TLineInfo) = if sym.name.s == "_": return let conflict = scope.addUniqueSym(sym) if conflict != nil: - if sym.kind == skModule and conflict.kind == skModule and sym.owner == conflict.owner: + if sym.kind == skModule and conflict.kind == skModule and + sym.position == conflict.position: # e.g.: import foo; import foo # xxx we could refine this by issuing a different hint for the case # where a duplicate import happens inside an include. diff --git a/tests/import/buzz/m21496.nim b/tests/import/buzz/m21496.nim new file mode 100644 index 000000000..7c87e2c04 --- /dev/null +++ b/tests/import/buzz/m21496.nim @@ -0,0 +1 @@ +proc fb* = echo "buzz!" \ No newline at end of file diff --git a/tests/import/fizz/m21496.nim b/tests/import/fizz/m21496.nim new file mode 100644 index 000000000..834c11eae --- /dev/null +++ b/tests/import/fizz/m21496.nim @@ -0,0 +1 @@ +proc fb* = echo "fizz!" \ No newline at end of file diff --git a/tests/import/t21496.nim b/tests/import/t21496.nim new file mode 100644 index 000000000..568f2ac51 --- /dev/null +++ b/tests/import/t21496.nim @@ -0,0 +1,9 @@ +discard """ + errormsg: "redefinition of 'm21496'; previous declaration here: t21496.nim(5, 12)" +""" + +import fizz/m21496, buzz/m21496 + +# bug #21496 + +m21496.fb() From b2c1dcbbc9b097c9c13dda4951e824cdb5f16225 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Mon, 13 Mar 2023 03:03:46 +0800 Subject: [PATCH 2472/2472] fixes explicit globals in macros (#21502) --- compiler/vmgen.nim | 3 ++- tests/vm/tvmmisc.nim | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index be7938e53..937d4b095 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1909,7 +1909,8 @@ proc genVarSection(c: PCtx; n: PNode) = c.genAdditionalCopy(a[2], opcWrDeref, tmp, 0, val) c.freeTemp(val) c.freeTemp(tmp) - elif not importcCondVar(s) and not (s.typ.kind == tyProc and s.typ.callConv == ccClosure): # fixes #10938 + elif not importcCondVar(s) and not (s.typ.kind == tyProc and s.typ.callConv == ccClosure) and + sfPure notin s.flags: # fixes #10938 # there is a pre-existing issue with closure types in VM # if `(var s: proc () = default(proc ()); doAssert s == nil)` works for you; # you might remove the second condition. diff --git a/tests/vm/tvmmisc.nim b/tests/vm/tvmmisc.nim index 673e3e965..14818375d 100644 --- a/tests/vm/tvmmisc.nim +++ b/tests/vm/tvmmisc.nim @@ -643,3 +643,16 @@ const b = block: doAssert a == @[@[0, 1, 2], @[0, 1, 2], @[0, 1, 2]] doAssert b == @[@[0, 1, 2], @[0, 1, 2], @[0, 1, 2]] + +macro m1(s: string): int = + var ProcID {.global, compileTime.}: int + inc(ProcID) + result = newLit(ProcID) + +proc macroGlobal = + doAssert m1("Macro argument") == 1 + doAssert m1("Macro argument") == 2 + doAssert m1("Macro argument") == 3 + +static: macroGlobal() +macroGlobal()